fuhouyin/easyexcel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

* 优化大文件内存存储，减少内存占用 [Issue #2657](https://github.com/alibaba/easyexcel/issues/2657)

Browse Source
This commit is contained in:
Jiaju Zhuang 2023-02-10 15:02:57 +08:00
parent d58f51db0f
commit 58e5ca1739
3 changed files with 75 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
easyexcel-core/src/main/java/com/alibaba/excel/cache/Ehcache.java vendored
View File

@ -15,6 +15,7 @@ import org.ehcache.config.CacheConfiguration;
import org.ehcache.config.builders.CacheConfigurationBuilder;
import org.ehcache.config.builders.CacheManagerBuilder;
import org.ehcache.config.builders.ResourcePoolsBuilder;
import org.ehcache.config.units.EntryUnit;
import org.ehcache.config.units.MemoryUnit;
/**
@ -50,12 +51,27 @@ public class Ehcache implements ReadCache {
*/
private int cacheMiss = 0;
public Ehcache(int maxCacheActivateSize) {
activeCacheConfiguration = CacheConfigurationBuilder
.newCacheConfigurationBuilder(Integer.class, ArrayList.class,
ResourcePoolsBuilder.newResourcePoolsBuilder().heap(maxCacheActivateSize, MemoryUnit.MB))
.withSizeOfMaxObjectGraph(1000 * 1000L).withSizeOfMaxObjectSize(maxCacheActivateSize, MemoryUnit.MB)
.build();
@Deprecated
public Ehcache(Integer maxCacheActivateSize) {
this(maxCacheActivateSize, null);
}
public Ehcache(Integer maxCacheActivateSize, Integer maxCacheActivateBatchCount) {
// In order to be compatible with the code
// If the user set up `maxCacheActivateSize`, then continue using it
if (maxCacheActivateSize != null) {
this.activeCacheConfiguration = CacheConfigurationBuilder
.newCacheConfigurationBuilder(Integer.class, ArrayList.class,
ResourcePoolsBuilder.newResourcePoolsBuilder()
.heap(maxCacheActivateSize, MemoryUnit.MB))
.build();
} else {
this.activeCacheConfiguration = CacheConfigurationBuilder
.newCacheConfigurationBuilder(Integer.class, ArrayList.class,
ResourcePoolsBuilder.newResourcePoolsBuilder()
.heap(maxCacheActivateBatchCount, EntryUnit.ENTRIES))
.build();
}
}
static {
@ -64,9 +80,8 @@ public class Ehcache implements ReadCache {
CacheManagerBuilder.newCacheManagerBuilder().with(CacheManagerBuilder.persistence(cacheFile)).build(true);
ACTIVE_CACHE_MANAGER = CacheManagerBuilder.newCacheManagerBuilder().build(true);
FILE_CACHE_CONFIGURATION = CacheConfigurationBuilder
.newCacheConfigurationBuilder(Integer.class, ArrayList.class,
ResourcePoolsBuilder.newResourcePoolsBuilder().disk(10, MemoryUnit.GB))
.withSizeOfMaxObjectGraph(1000 * 1000L).withSizeOfMaxObjectSize(10, MemoryUnit.GB).build();
.newCacheConfigurationBuilder(Integer.class, ArrayList.class, ResourcePoolsBuilder.newResourcePoolsBuilder()
.disk(20, MemoryUnit.GB)).build();
}
@Override

66
easyexcel-core/src/main/java/com/alibaba/excel/cache/selector/SimpleReadCacheSelector.java vendored
View File

@ -2,6 +2,10 @@ package com.alibaba.excel.cache.selector;
import java.io.IOException;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import org.apache.poi.openxml4j.opc.PackagePart;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -15,6 +19,9 @@ import com.alibaba.excel.cache.ReadCache;
*
* @author Jiaju Zhuang
**/
@Getter
@Setter
@EqualsAndHashCode
public class SimpleReadCacheSelector implements ReadCacheSelector {
private static final Logger LOGGER = LoggerFactory.getLogger(SimpleReadCacheSelector.class);
/**
@ -24,37 +31,46 @@ public class SimpleReadCacheSelector implements ReadCacheSelector {
/**
* If it's less than 5M, use map cache, or use ehcache.unit MB.
*/
private static final int DEFAULT_MAX_USE_MAP_CACHE_SIZE = 5;
private static final long DEFAULT_MAX_USE_MAP_CACHE_SIZE = 5;
/**
* Maximum size of cache activation.unit MB.
* Maximum batch of `SharedStrings` stored in memory.
* The batch size is 100.{@link Ehcache#BATCH_COUNT}
*/
private static final int DEFAULT_MAX_EHCACHE_ACTIVATE_SIZE = 20;
private static final int DEFAULT_MAX_EHCACHE_ACTIVATE_BATCH_COUNT = 20;
/**
* Shared strings exceeding this value will use {@link Ehcache},or use {@link MapCache}.unit MB.
*/
private final long maxUseMapCacheSize;
private Long maxUseMapCacheSize;
/**
* Maximum size of cache activation.unit MB.
*
* @deprecated Please use maxCacheActivateBatchCount to control the size of the occupied memory
*/
private final int maxCacheActivateSize;
@Deprecated
private Integer maxCacheActivateSize;
/**
* Maximum batch of `SharedStrings` stored in memory.
* The batch size is 100.{@link Ehcache#BATCH_COUNT}
*/
private Integer maxCacheActivateBatchCount;
public SimpleReadCacheSelector() {
this(DEFAULT_MAX_USE_MAP_CACHE_SIZE, DEFAULT_MAX_EHCACHE_ACTIVATE_SIZE);
}
public SimpleReadCacheSelector(long maxUseMapCacheSize, int maxCacheActivateSize) {
if (maxUseMapCacheSize <= 0) {
this.maxUseMapCacheSize = DEFAULT_MAX_USE_MAP_CACHE_SIZE;
} else {
this.maxUseMapCacheSize = maxUseMapCacheSize;
}
if (maxCacheActivateSize <= 0) {
this.maxCacheActivateSize = DEFAULT_MAX_EHCACHE_ACTIVATE_SIZE;
} else {
this.maxCacheActivateSize = maxCacheActivateSize;
}
/**
* Parameter maxCacheActivateSize has already been abandoned
*
* @param maxUseMapCacheSize
* @param maxCacheActivateSize
*/
@Deprecated
public SimpleReadCacheSelector(Long maxUseMapCacheSize, Integer maxCacheActivateSize) {
this.maxUseMapCacheSize = maxUseMapCacheSize;
this.maxCacheActivateSize = maxCacheActivateSize;
}
@Override
@ -68,6 +84,9 @@ public class SimpleReadCacheSelector implements ReadCacheSelector {
return new MapCache();
}
}
if (maxUseMapCacheSize == null) {
maxUseMapCacheSize = DEFAULT_MAX_USE_MAP_CACHE_SIZE;
}
if (size < maxUseMapCacheSize * B2M) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Use map cache.size:{}", size);
@ -77,6 +96,17 @@ public class SimpleReadCacheSelector implements ReadCacheSelector {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Use ehcache.size:{}", size);
}
return new Ehcache(maxCacheActivateSize);
// In order to be compatible with the code
// If the user set up `maxCacheActivateSize`, then continue using it
if (maxCacheActivateSize != null) {
return new Ehcache(maxCacheActivateSize, maxCacheActivateBatchCount);
} else {
if (maxCacheActivateBatchCount == null) {
maxCacheActivateBatchCount = DEFAULT_MAX_EHCACHE_ACTIVATE_BATCH_COUNT;
}
return new Ehcache(maxCacheActivateSize, maxCacheActivateBatchCount);
}
}
}

4
update.md
View File

@ -1,6 +1,7 @@
# 3.2.1
* 兼容`LocalDate` [Issue #2908](https://github.com/alibaba/easyexcel/issues/2908)
* 优化大文件内存存储，减少内存占用 [Issue #2657](https://github.com/alibaba/easyexcel/issues/2657)
# 3.2.0
@ -14,7 +15,8 @@
# 3.1.4
* 提高xlsx读取兼容性：在存在第一行很多空列的情况下，忽略空列
* 提高xlsx读取兼容性：在存在第一行很多空
* 列的情况下，忽略空列
# 3.1.3