fuhouyin/fastjson2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize read csv

Browse Source
This commit is contained in:
wenshao 2025-02-09 12:05:55 +08:00
parent c033ccf6c6
commit 0fc60bafd3
9 changed files with 495 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
benchmark/src/test/java/com/alibaba/fastjson2/benchmark/CSVPersonTest.java
View File

@ -15,8 +15,8 @@ public class CSVPersonTest {
long millis = System.currentTimeMillis() - start;
System.out.println("fastjson2 millis : " + millis);
// zulu8.68.0.21 : 3621 3270 2788 2424 2431 2123
// zulu11.62.17 : 2576 1848 2155 1567 1522
// zulu17.40.19 : 3501 3134 2586 2788 2733 2682
// zulu11.62.17 : 2576 1848 2155 1567 1522 1307
// zulu17.40.19 : 3501 3134 2586 2788 2733 2682 2494
}
}

4
core/src/main/java/com/alibaba/fastjson2/stream/JSONStreamReaderUTF8.java
View File

@ -106,7 +106,7 @@ class JSONStreamReaderUTF8<T>
}
if (!lineTerminated) {
if (input != null && !inputEnd) {
if (input != null && !(inputEnd && off >= end)) {
int len = end - off;
if (off > 0) {
if (len > 0) {
@ -145,7 +145,7 @@ class JSONStreamReaderUTF8<T>
public <T> T readLineObject() {
try {
if (inputEnd) {
if (inputEnd && off >= end) {
return null;
}

4
core/src/main/java/com/alibaba/fastjson2/stream/StreamReader.java
View File

@ -584,7 +584,7 @@ public abstract class StreamReader<T> {
throw new IllegalArgumentException("action must not be null");
}
T object = next();
if (streamReader.inputEnd || object == null) {
if ((streamReader.inputEnd && streamReader.off >= streamReader.end) || object == null) {
return false;
} else {
action.accept(object);
@ -611,7 +611,7 @@ public abstract class StreamReader<T> {
@Override
public long estimateSize() {
return streamReader.inputEnd ? 0L : Long.MAX_VALUE;
return (streamReader.inputEnd && streamReader.off >= streamReader.end) ? 0L : Long.MAX_VALUE;
}
@Override

2
core/src/main/java/com/alibaba/fastjson2/support/csv/CSVReader.java
View File

@ -345,7 +345,7 @@ public abstract class CSVReader<T>
if (!objectSupport) {
throw new UnsupportedOperationException("this method should not be called, try specify objectClass or method readLineValues instead ?");
}
if (inputEnd) {
if (inputEnd && off == end) {
return null;
}

437
core/src/main/java/com/alibaba/fastjson2/support/csv/CSVReaderUTF8.java
View File

@ -24,6 +24,9 @@ import static com.alibaba.fastjson2.util.DateUtils.DEFAULT_ZONE_ID;
final class CSVReaderUTF8<T>
extends CSVReader<T> {
static final int ESCAPE_INDEX_NOT_SET = -2;
int nextEscapeIndex = ESCAPE_INDEX_NOT_SET;
static final Map<Long, Function<Consumer, ByteArrayValueConsumer>> valueConsumerCreators
= new ConcurrentHashMap<>();
@ -107,34 +110,99 @@ final class CSVReaderUTF8<T>
return ((x22 | x0a | x0d) & 0x8080808080808080L) != 0;
}
protected boolean seekLine() throws IOException {
private byte[] getBuf() throws IOException {
byte[] buf = this.buf;
int off = this.off;
if (buf == null) {
if (input != null) {
buf = this.buf = new byte[SIZE_512K];
int cnt = input.read(buf);
if (cnt == -1) {
inputEnd = true;
return false;
}
this.end = cnt;
if (buf != null) {
return buf;
}
return initBuf();
}
if (end > 4 && IOUtils.isUTF8BOM(buf, 0)) {
off = 3;
lineNextStart = off;
private byte[] initBuf() throws IOException {
if (input == null) {
throw new JSONException("init buf error");
}
byte[] buf = new byte[SIZE_512K];
int end = 0;
while (end < buf.length) {
int cnt = input.read(buf, off, buf.length - off);
if (cnt == -1) {
inputEnd = true;
break;
}
end += cnt;
}
this.end = end;
if (end > 4 && IOUtils.isUTF8BOM(buf, 0)) {
off = 3;
lineNextStart = 3;
}
this.buf = buf;
return buf;
}
protected boolean seekLine() throws IOException {
byte[] buf = getBuf();
int off = this.off, end = this.end;
int slashIndex = this.nextEscapeIndex;
if (slashIndex == ESCAPE_INDEX_NOT_SET || (slashIndex != -1 && slashIndex < off)) {
nextEscapeIndex = slashIndex = IOUtils.indexOfDoubleQuote(buf, off, end);
}
int lineSeparatorIndex = IOUtils.indexOfLineSeparator(buf, off, end);
if (lineSeparatorIndex == -1) {
if (input != null) {
if (!inputEnd) {
int rest = end - off;
if (rest > 0) {
System.arraycopy(buf, off, buf, 0, rest);
}
end = rest;
while (end < buf.length) {
int cnt = input.read(buf, end, buf.length - end);
if (cnt != -1) {
end += cnt;
} else {
inputEnd = true;
break;
}
}
this.nextEscapeIndex = slashIndex >= off
? slashIndex - off
: IOUtils.indexOfDoubleQuote(buf, rest, end);
lineSeparatorIndex = IOUtils.indexOfLineSeparator(buf, rest, end);
this.off = off = 0;
this.end = end;
}
}
if (lineSeparatorIndex == -1 && inputEnd) {
if (off < end) {
lineSeparatorIndex = end;
} else {
return false;
}
}
}
if ((lineSeparatorIndex != -1) && (slashIndex == -1 || slashIndex > lineSeparatorIndex)) {
this.lineTerminated = true;
this.lineStart = off;
this.lineEnd = lineSeparatorIndex != off && buf[lineSeparatorIndex - 1] == '\r'
? lineSeparatorIndex - 1
: lineSeparatorIndex;
this.off = lineSeparatorIndex + 1;
return true;
}
return seekLine0(buf, off, end);
}
private boolean seekLine0(byte[] buf, int off, int end) throws IOException {
int lineNextStart = off;
for (int k = 0; k < 3; ++k) {
lineTerminated = false;
int i = off, end = this.end;
while (i + 8 < end && !containsQuoteOrLineSeparator(IOUtils.getLongUnaligned(buf, i))) {
i += 8;
}
for (; i < end; i++) {
for (int i = off; i < end; i++) {
byte ch = buf[i];
if (ch == '"') {
lineSize++;
@ -160,18 +228,7 @@ final class CSVReaderUTF8<T>
continue;
}
if (ch == '\n') {
if (lineSize > 0 || (features & Feature.IgnoreEmptyLine.mask) == 0) {
rowCount++;
}
lineTerminated = true;
lineSize = 0;
lineEnd = i;
lineStart = lineNextStart;
lineNextStart = off = i + 1;
break;
} else if (ch == '\r') {
if (ch == '\r' || ch == '\n') {
if (lineSize > 0 || (features & Feature.IgnoreEmptyLine.mask) == 0) {
rowCount++;
}
@ -180,12 +237,14 @@ final class CSVReaderUTF8<T>
lineSize = 0;
lineEnd = i;
int n = i + 1;
if (n >= end) {
break;
}
if (buf[n] == '\n') {
i++;
if (ch == '\r') {
int n = i + 1;
if (n >= end) {
break;
}
if (buf[n] == '\n') {
i++;
}
}
lineStart = lineNextStart;
@ -200,9 +259,11 @@ final class CSVReaderUTF8<T>
if (!lineTerminated) {
if (input != null && !inputEnd) {
int len = end - off;
int slashIndex = this.nextEscapeIndex;
if (off > 0) {
if (len > 0) {
System.arraycopy(buf, off, buf, 0, len);
this.nextEscapeIndex = slashIndex >= off ? slashIndex - off : ESCAPE_INDEX_NOT_SET;
}
lineStart = lineNextStart = 0;
off = 0;
@ -283,7 +344,7 @@ final class CSVReaderUTF8<T>
public Object[] readLineValues(boolean strings) {
try {
if (inputEnd) {
if (inputEnd && off == end) {
return null;
}
@ -302,21 +363,113 @@ final class CSVReaderUTF8<T>
throw new JSONException("seekLine error", e);
}
int escapedIndex = this.nextEscapeIndex;
return escapedIndex == -1 || escapedIndex >= lineEnd
? readLineValue(strings)
: readLineValueEscaped(strings);
}
private Object[] readLineValue(boolean strings) {
List<String> columns = this.columns;
Object[] values = null;
List<Object> valueList = null;
if (columns != null) {
if (strings) {
values = new String[columns.size()];
} else {
values = new Object[columns.size()];
int size = columns.size();
values = strings ? new String[size] : new Object[size];
} else {
valueList = new ArrayList<>();
}
int valueStart = lineStart, lineEnd = this.lineEnd;
int valueSize = 0;
int columnIndex = 0;
byte[] buf = this.buf;
for (int i = lineStart; i < lineEnd; ++i) {
if (buf[i] == ',') {
readValue(strings, columnIndex, valueSize, buf, valueStart, values, valueList);
valueStart = i + 1;
valueSize = 0;
columnIndex++;
continue;
}
valueSize++;
}
if (valueSize > 0) {
readValue(strings, columnIndex, valueSize, buf, valueStart, values, valueList);
}
if (values == null) {
if (valueList != null) {
int size = valueList.size();
valueList.toArray(
values = strings ? new String[size] : new Object[size]
);
}
}
if (input == null && off == end) {
inputEnd = true;
}
return values;
}
private void readValue(boolean strings,
int columnIndex,
int valueSize,
byte[] buf,
int valueStart,
Object[] values,
List<Object> valueList) {
Type type = types != null && columnIndex < types.length ? types[columnIndex] : null;
Object value;
if (type == null || type == String.class || type == Object.class || strings) {
byte c0, c1;
if (valueSize == 1 && (c0 = buf[valueStart]) >= 0) {
value = TypeUtils.toString((char) c0);
} else if (valueSize == 2
&& (c0 = buf[valueStart]) >= 0
&& (c1 = buf[valueStart + 1]) >= 0
) {
value = TypeUtils.toString((char) c0, (char) c1);
} else {
value = new String(buf, valueStart, valueSize, charset);
}
} else {
try {
value = readValue(buf, valueStart, valueSize, type);
} catch (Exception e) {
value = error(columnIndex, e);
}
}
if (values != null) {
if (columnIndex < values.length) {
values[columnIndex] = value;
}
} else {
valueList.add(value);
}
}
private Object[] readLineValueEscaped(boolean strings) {
List<String> columns = this.columns;
Object[] values = null;
List<Object> valueList = null;
if (columns != null) {
int size = columns.size();
values = strings ? new String[size] : new Object[size];
}
boolean quote = false;
int valueStart = lineStart;
int valueStart = lineStart, lineEnd = this.lineEnd;
int valueSize = 0;
int escapeCount = 0;
int columnIndex = 0;
byte[] buf = this.buf;
for (int i = lineStart; i < lineEnd; ++i) {
byte ch = buf[i];
@ -375,7 +528,7 @@ final class CSVReaderUTF8<T>
}
if (type == null || type == String.class || type == Object.class) {
value = new String(bytes, 0, bytes.length, charset);
value = new String(bytes, charset);
} else {
try {
value = readValue(bytes, 0, bytes.length, type);
@ -500,12 +653,10 @@ final class CSVReaderUTF8<T>
if (values == null) {
if (valueList != null) {
if (strings) {
values = new String[valueList.size()];
} else {
values = new Object[valueList.size()];
}
valueList.toArray(values);
int size = valueList.size();
valueList.toArray(
values = strings ? new String[size] : new Object[size]
);
}
}
@ -649,7 +800,7 @@ final class CSVReaderUTF8<T>
for (int r = 0; r < maxRows || maxRows < 0; ++r) {
try {
if (inputEnd) {
if (inputEnd & off == end) {
break;
}
@ -670,80 +821,78 @@ final class CSVReaderUTF8<T>
consumer.beforeRow(rowCount);
boolean quote = false;
int valueStart = lineStart;
int valueSize = 0;
int escapeCount = 0;
int columnIndex = 0;
for (int i = lineStart; i < lineEnd; ++i) {
byte ch = buf[i];
if (quote) {
if (ch == '"') {
int n = i + 1;
if (n < lineEnd) {
byte c1 = buf[n];
if (c1 == '"') {
valueSize += 2;
escapeCount++;
++i;
continue;
} else if (c1 == ',') {
++i;
ch = c1;
}
} else if (n == lineEnd) {
break;
}
} else {
valueSize++;
continue;
}
} else {
if (ch == '"') {
quote = true;
continue;
}
}
if (ch == ',') {
byte[] columnBuf = buf;
int columnStart = 0;
int columnSize = valueSize;
if (quote) {
if (escapeCount == 0) {
columnStart = valueStart + 1;
} else {
byte[] bytes = new byte[valueSize - escapeCount];
int valueEnd = valueStart + valueSize;
for (int j = valueStart + 1, k = 0; j < valueEnd; ++j) {
byte c = buf[j];
bytes[k++] = c;
if (c == '"' && buf[j + 1] == '"') {
++j;
}
}
columnBuf = bytes;
columnSize = bytes.length;
}
} else {
columnStart = valueStart;
}
consumer.accept(rowCount, columnIndex, columnBuf, columnStart, columnSize, charset);
quote = false;
valueStart = i + 1;
valueSize = 0;
escapeCount = 0;
columnIndex++;
continue;
}
valueSize++;
int escapedIndex = this.nextEscapeIndex;
if (escapedIndex == -1 || escapedIndex >= lineEnd) {
readLine(consumer);
} else {
readLineEscaped(consumer);
}
if (valueSize > 0) {
consumer.afterRow(rowCount);
}
consumer.end();
}
private void readLine(ByteArrayValueConsumer consumer) {
int valueStart = lineStart;
int valueSize = 0;
int columnIndex = 0;
byte[] buf = this.buf;
for (int i = lineStart; i < lineEnd; ++i) {
if (buf[i] == ',') {
consumer.accept(rowCount, columnIndex, buf, valueStart, valueSize, charset);
valueStart = i + 1;
valueSize = 0;
columnIndex++;
continue;
}
valueSize++;
}
if (valueSize > 0) {
consumer.accept(rowCount, columnIndex, buf, valueStart, valueSize, charset);
}
}
private void readLineEscaped(ByteArrayValueConsumer consumer) {
boolean quote = false;
int valueStart = lineStart;
int valueSize = 0;
int escapeCount = 0;
int columnIndex = 0;
for (int i = lineStart; i < lineEnd; ++i) {
byte ch = buf[i];
if (quote) {
if (ch == '"') {
int n = i + 1;
if (n < lineEnd) {
byte c1 = buf[n];
if (c1 == '"') {
valueSize += 2;
escapeCount++;
++i;
continue;
} else if (c1 == ',') {
++i;
ch = c1;
}
} else if (n == lineEnd) {
break;
}
} else {
valueSize++;
continue;
}
} else {
if (ch == '"') {
quote = true;
continue;
}
}
if (ch == ',') {
byte[] columnBuf = buf;
int columnStart = 0;
int columnSize = valueSize;
@ -752,7 +901,7 @@ final class CSVReaderUTF8<T>
columnStart = valueStart + 1;
} else {
byte[] bytes = new byte[valueSize - escapeCount];
int valueEnd = lineEnd;
int valueEnd = valueStart + valueSize;
for (int j = valueStart + 1, k = 0; j < valueEnd; ++j) {
byte c = buf[j];
bytes[k++] = c;
@ -768,9 +917,43 @@ final class CSVReaderUTF8<T>
columnStart = valueStart;
}
consumer.accept(rowCount, columnIndex, columnBuf, columnStart, columnSize, charset);
quote = false;
valueStart = i + 1;
valueSize = 0;
escapeCount = 0;
columnIndex++;
continue;
}
consumer.afterRow(rowCount);
valueSize++;
}
if (valueSize > 0) {
byte[] columnBuf = buf;
int columnStart = 0;
int columnSize = valueSize;
if (quote) {
if (escapeCount == 0) {
columnStart = valueStart + 1;
} else {
byte[] bytes = new byte[valueSize - escapeCount];
int valueEnd = lineEnd;
for (int j = valueStart + 1, k = 0; j < valueEnd; ++j) {
byte c = buf[j];
bytes[k++] = c;
if (c == '"' && buf[j + 1] == '"') {
++j;
}
}
columnBuf = bytes;
columnSize = bytes.length;
}
} else {
columnStart = valueStart;
}
consumer.accept(rowCount, columnIndex, columnBuf, columnStart, columnSize, charset);
}
consumer.end();
}
}

152
core/src/main/java/com/alibaba/fastjson2/util/IOUtils.java
View File

@ -1483,6 +1483,10 @@ public class IOUtils {
}
}
public static boolean isTRUE(byte[] buf, int pos) {
return getIntUnaligned(buf, pos) == TRUE;
}
public static boolean isALSE(byte[] buf, int pos) {
return getIntUnaligned(buf, pos) == ALSE;
}
@ -1525,6 +1529,75 @@ public class IOUtils {
);
}
public static int digit5(byte[] bytes, int off) {
int d4 = digit4(bytes, off);
int d1 = digit2(bytes, off + 4);
return (d4 | d1) == -1 ? -1 : (d4 * 10 + d1);
}
public static int digit6(byte[] bytes, int off) {
int d4 = digit4(bytes, off);
int d2 = digit2(bytes, off + 4);
return (d4 | d2) == -1 ? -1 : (d4 * 100 + d2);
}
public static int digit7(byte[] bytes, int off) {
int d4 = digit4(bytes, off);
int d2 = digit2(bytes, off + 4);
int d1 = digit1(bytes, off + 6);
return (d4 | d2 | d1) == -1 ? -1 : (d4 * 1000 + d2 * 10 + d1);
}
public static int digit8(byte[] bytes, int off) {
return digit8(
getLongLE(bytes, off)
);
}
private static int digit8(long x) {
/*
Here we are doing a 4-Byte Vector operation on the Int type.
x & 0xF0 != 0xC0
---------------
0 0b0011_0000 & 0b1111_0000 = 0b0011_0000
1 0b0011_0001 & 0b1111_0000 = 0b0011_0000
2 0b0011_0010 & 0b1111_0000 = 0b0011_0000
3 0b0011_0011 & 0b1111_0000 = 0b0011_0000
4 0b0011_0100 & 0b1111_0000 = 0b0011_0000
5 0b0011_0101 & 0b1111_0000 = 0b0011_0000
6 0b0011_0110 & 0b1111_0000 = 0b0011_0000
7 0b0011_0111 & 0b1111_0000 = 0b0011_0000
8 0b0011_1000 & 0b1111_0000 = 0b0011_0000
9 0b0011_1001 & 0b1111_0000 = 0b0011_0000
(((d = x & 0x0F) + 0x06) & 0xF0) != 0
---------------
0 ((0b0011_0000) & 0b0000_1111 + 0b0110_0000) & 0b1111_0000 = 0b0110_0000
1 ((0b0011_0001) & 0b0000_1111 + 0b0110_0000) & 0b1111_0000 = 0b0110_0000
2 ((0b0011_0010) & 0b0000_1111 + 0b0110_0000) & 0b1111_0000 = 0b0110_0000
3 ((0b0011_0011) & 0b0000_1111 + 0b0110_0000) & 0b1111_0000 = 0b0110_0000
4 ((0b0011_0100) & 0b0000_1111 + 0b0110_0000) & 0b1111_0000 = 0b0110_0000
5 ((0b0011_0101) & 0b0000_1111 + 0b0110_0000) & 0b1111_0000 = 0b0110_0000
6 ((0b0011_0110) & 0b0000_1111 + 0b0110_0000) & 0b1111_0000 = 0b0110_0000
7 ((0b0011_0111) & 0b0000_1111 + 0b0110_0000) & 0b1111_0000 = 0b0110_0000
8 ((0b0011_1000) & 0b0000_1111 + 0b0110_0000) & 0b1111_0000 = 0b0110_0000
9 ((0b0011_1001) & 0b0000_1111 + 0b0110_0000) & 0b1111_0000 = 0b0110_0000
*/
long d;
if ((((x & 0xF0F0F0F0F0F0F0F0L) - 0x3030303030303030L) | (((d = x & 0x0F0F0F0F0F0F0F0FL) + 0x0606060606060606L) & 0xF0F0F0F0F0F0F0F0L)) != 0) {
return -1;
}
return (int) ((d & 0x0FL) * 10000000
+ ((d & 0x0F00L) >> 8) * 1000000
+ ((d & 0x0F0000L) >> 16) * 100000
+ ((d & 0x0F000000L) >> 24) * 10000
+ ((d & 0x0F00000000L) >> 32) * 1000
+ ((d & 0x0F0000000000L) >> 40) * 100
+ ((d & 0x0F000000000000L) >> 48) * 10
+ (d >> 56));
}
private static int digit4(int x) {
/*
Here we are doing a 4-Byte Vector operation on the Int type.
@ -1639,6 +1712,7 @@ public class IOUtils {
throw new JSONException(e.getMessage());
}
}
static int indexOfQuote0(byte[] value, int quote, int fromIndex, int max) {
int i = fromIndex;
long address = ARRAY_BYTE_BASE_OFFSET + fromIndex;
@ -1651,6 +1725,50 @@ public class IOUtils {
return indexOfChar0(value, quote, i, max);
}
public static int indexOfDoubleQuote(byte[] value, int fromIndex, int max) {
if (INDEX_OF_CHAR_LATIN1 == null) {
return indexOfDoubleQuoteV(value, fromIndex, max);
}
try {
return (int) INDEX_OF_CHAR_LATIN1.invokeExact(value, (int) '"', fromIndex, max);
} catch (Throwable e) {
throw new JSONException(e.getMessage());
}
}
public static int indexOfDoubleQuoteV(byte[] value, int fromIndex, int max) {
int i = fromIndex;
long address = ARRAY_BYTE_BASE_OFFSET + fromIndex;
int upperBound = fromIndex + ((max - fromIndex) & ~7);
while (i < upperBound && notContains(UNSAFE.getLong(value, address), 0x2222_2222_2222_2222L)) {
i += 8;
address += 8;
}
return indexOfChar0(value, '"', i, max);
}
public static int indexOfLineSeparator(byte[] value, int fromIndex, int max) {
if (INDEX_OF_CHAR_LATIN1 == null) {
return indexOfLineSeparatorV(value, fromIndex, max);
}
try {
return (int) INDEX_OF_CHAR_LATIN1.invokeExact(value, (int) '\n', fromIndex, max);
} catch (Throwable e) {
throw new JSONException(e.getMessage());
}
}
public static int indexOfLineSeparatorV(byte[] value, int fromIndex, int max) {
int i = fromIndex;
long address = ARRAY_BYTE_BASE_OFFSET + fromIndex;
int upperBound = fromIndex + ((max - fromIndex) & ~7);
while (i < upperBound && notContains(UNSAFE.getLong(value, address), 0x0A0A0A0A0A0A0A0AL)) {
i += 8;
address += 8;
}
return indexOfChar0(value, '\n', i, max);
}
public static int indexOfSlash(byte[] value, int fromIndex, int max) {
if (INDEX_OF_CHAR_LATIN1 == null) {
return indexOfSlashV(value, fromIndex, max);
@ -1906,4 +2024,38 @@ public class IOUtils {
}
return true;
}
private static boolean isDigitLatin1(int c) {
return c >= '0' && c <= '9';
}
public static int parseInt(byte[] bytes, int off, int len) {
int fc = bytes[off];
int result = isDigitLatin1(fc)
? '0' - fc
: len != 1 && (fc == '-' || fc == '+')
? 0
: 1; // or any value > 0
int end = off + len;
off++;
int d;
while (off + 1 < end
&& (d = IOUtils.digit2(bytes, off)) != -1
&& Integer.MIN_VALUE / 100 <= result & result <= 0) {
result = result * 100 - d; // overflow from d => result > 0
off += 2;
}
if (off < end
&& isDigitLatin1(d = bytes[off])
&& Integer.MIN_VALUE / 10 <= result & result <= 0) {
result = result * 10 + '0' - d; // overflow from '0' - d => result > 0
off += 1;
}
if (off == end
& result <= 0
& (Integer.MIN_VALUE < result || fc == '-')) {
return fc == '-' ? result : -result;
}
throw new NumberFormatException(new String(bytes, off, len));
}
}

146
core/src/main/java/com/alibaba/fastjson2/util/TypeUtils.java
View File

@ -2076,25 +2076,21 @@ public class TypeUtils {
return null;
case 1: {
byte b0 = bytes[off];
if (b0 == '1' || b0 == 'Y') {
if (b0 == '1' | b0 == 'Y') {
return Boolean.TRUE;
}
if (b0 == '0' || b0 == 'N') {
if (b0 == '0' | b0 == 'N') {
return Boolean.FALSE;
}
break;
}
case 4:
if (bytes[off] == 't' && bytes[off + 1] == 'r' && bytes[off + 2] == 'u' && bytes[off + 3] == 'e') {
if (IOUtils.isTRUE(bytes, off)) {
return Boolean.TRUE;
}
break;
case 5:
if (bytes[off] == 'f'
&& bytes[off + 1] == 'a'
&& bytes[off + 2] == 'l'
&& bytes[off + 3] == 's'
&& bytes[off + 4] == 'e') {
if (bytes[off] == 'f' && IOUtils.isALSE(bytes, off + 1)) {
return Boolean.FALSE;
}
break;
@ -2143,155 +2139,47 @@ public class TypeUtils {
}
public static int parseInt(byte[] bytes, int off, int len) {
int digit;
switch (len) {
case 1: {
byte b0 = bytes[off];
if (b0 >= '0' && b0 <= '9') {
return b0 - '0';
}
digit = IOUtils.digit1(bytes, off);
break;
}
case 2: {
byte b0 = bytes[off];
byte b1 = bytes[off + 1];
if (b0 >= '0' && b0 <= '9'
&& b1 >= '0' && b1 <= '9') {
return (b0 - '0') * 10
+ (b1 - '0');
}
digit = IOUtils.digit2(bytes, off);
break;
}
case 3: {
byte b0 = bytes[off];
byte b1 = bytes[off + 1];
byte b2 = bytes[off + 2];
if (b0 >= '0' && b0 <= '9'
&& b1 >= '0' && b1 <= '9'
&& b2 >= '0' && b2 <= '9') {
return (b0 - '0') * 100
+ (b1 - '0') * 10
+ (b2 - '0');
}
digit = IOUtils.digit3(bytes, off);
break;
}
case 4: {
byte b0 = bytes[off];
byte b1 = bytes[off + 1];
byte b2 = bytes[off + 2];
byte b3 = bytes[off + 3];
if (b0 >= '0' && b0 <= '9'
&& b1 >= '0' && b1 <= '9'
&& b2 >= '0' && b2 <= '9'
&& b3 >= '0' && b3 <= '9'
) {
return (b0 - '0') * 1000
+ (b1 - '0') * 100
+ (b2 - '0') * 10
+ (b3 - '0');
}
digit = IOUtils.digit4(bytes, off);
break;
}
case 5: {
byte b0 = bytes[off];
byte b1 = bytes[off + 1];
byte b2 = bytes[off + 2];
byte b3 = bytes[off + 3];
byte b4 = bytes[off + 4];
if (b0 >= '0' && b0 <= '9'
&& b1 >= '0' && b1 <= '9'
&& b2 >= '0' && b2 <= '9'
&& b3 >= '0' && b3 <= '9'
&& b4 >= '0' && b4 <= '9'
) {
return (b0 - '0') * 10_000
+ (b1 - '0') * 1000
+ (b2 - '0') * 100
+ (b3 - '0') * 10
+ (b4 - '0');
}
digit = IOUtils.digit5(bytes, off);
break;
}
case 6: {
byte b0 = bytes[off];
byte b1 = bytes[off + 1];
byte b2 = bytes[off + 2];
byte b3 = bytes[off + 3];
byte b4 = bytes[off + 4];
byte b5 = bytes[off + 5];
if (b0 >= '0' && b0 <= '9'
&& b1 >= '0' && b1 <= '9'
&& b2 >= '0' && b2 <= '9'
&& b3 >= '0' && b3 <= '9'
&& b4 >= '0' && b4 <= '9'
&& b5 >= '0' && b5 <= '9'
) {
return (b0 - '0') * 100_000
+ (b1 - '0') * 10_000
+ (b2 - '0') * 1_000
+ (b3 - '0') * 100
+ (b4 - '0') * 10
+ (b5 - '0');
}
digit = IOUtils.digit6(bytes, off);
break;
}
case 7: {
byte b0 = bytes[off];
byte b1 = bytes[off + 1];
byte b2 = bytes[off + 2];
byte b3 = bytes[off + 3];
byte b4 = bytes[off + 4];
byte b5 = bytes[off + 5];
byte b6 = bytes[off + 6];
if (b0 >= '0' && b0 <= '9'
&& b1 >= '0' && b1 <= '9'
&& b2 >= '0' && b2 <= '9'
&& b3 >= '0' && b3 <= '9'
&& b4 >= '0' && b4 <= '9'
&& b5 >= '0' && b5 <= '9'
&& b6 >= '0' && b6 <= '9'
) {
return (b0 - '0') * 1_000_000
+ (b1 - '0') * 100_000
+ (b2 - '0') * 10_000
+ (b3 - '0') * 1_000
+ (b4 - '0') * 100
+ (b5 - '0') * 10
+ (b6 - '0');
}
digit = IOUtils.digit7(bytes, off);
break;
}
case 8: {
byte b0 = bytes[off];
byte b1 = bytes[off + 1];
byte b2 = bytes[off + 2];
byte b3 = bytes[off + 3];
byte b4 = bytes[off + 4];
byte b5 = bytes[off + 5];
byte b6 = bytes[off + 6];
byte b7 = bytes[off + 7];
if (b0 >= '0' && b0 <= '9'
&& b1 >= '0' && b1 <= '9'
&& b2 >= '0' && b2 <= '9'
&& b3 >= '0' && b3 <= '9'
&& b4 >= '0' && b4 <= '9'
&& b5 >= '0' && b5 <= '9'
&& b6 >= '0' && b6 <= '9'
&& b7 >= '0' && b7 <= '9'
) {
return (b0 - '0') * 10_000_000
+ (b1 - '0') * 1_000_000
+ (b2 - '0') * 100_000
+ (b3 - '0') * 10_000
+ (b4 - '0') * 1_000
+ (b5 - '0') * 100
+ (b6 - '0') * 10
+ (b7 - '0');
}
digit = IOUtils.digit8(bytes, off);
break;
}
default:
digit = -1;
break;
}
if (digit != -1) {
return digit;
}
String str = new String(bytes, off, len);
return Integer.parseInt(str);

3
core/src/test/java/com/alibaba/fastjson2/support/csv/CSVTest4.java
View File

@ -56,7 +56,8 @@ public class CSVTest4 {
in = new ByteArrayInputStream(bytes);
CSVReader<Integer[]> parser2 = CSVReader.of(in, types);
Object object = parser2.stream(Object[].class).findFirst().get();
Object object = parser2.stream(Object[].class)
.findFirst().get();
assertTrue(object instanceof Object[]);
assertFalse(object instanceof Integer[]);
assertFalse(object instanceof String[]);

7
core/src/test/java/com/alibaba/fastjson2/util/IOUtilsTest.java
View File

@ -294,6 +294,13 @@ public class IOUtilsTest {
}
}
@Test
public void digit8() {
byte[] bytes = "12345678".getBytes(StandardCharsets.ISO_8859_1);
int d8 = IOUtils.digit8(bytes, 0);
System.out.println(d8);
}
@Test
public void printDigit() {
System.out.println("0xF " + Integer.toBinaryString(0xF));