Mercurial > hg > monetdb-java
changeset 94:4bb4e988164d embedded
Less memory usage while retrieving some native types on a JDBC MAPI connection.
| author | Pedro Ferreira <pedro.ferreira@monetdbsolutions.com> |
|---|---|
| date | Fri, 06 Jan 2017 15:46:52 +0000 (2017-01-06) |
| parents | eeb71f7d36bf |
| children | c9db0fdbfc53 |
| files | src/main/java/nl/cwi/monetdb/jdbc/MonetClob.java src/main/java/nl/cwi/monetdb/mcl/connection/helpers/BufferReallocator.java src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiProtocol.java src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiStartOfHeaderParser.java src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiTupleLineParser.java src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiTupleLineParserHelper.java |
| diffstat | 6 files changed, 221 insertions(+), 79 deletions(-) [+] |
line wrap: on
line diff
--- a/src/main/java/nl/cwi/monetdb/jdbc/MonetClob.java +++ b/src/main/java/nl/cwi/monetdb/jdbc/MonetClob.java @@ -30,6 +30,10 @@ public class MonetClob implements Clob, buffer = new StringBuilder(in); } + public MonetClob(char[] toParse, int startPosition, int count) { + buffer = new StringBuilder(new String(toParse, startPosition, count)); + } + //== begin interface Clob /**
--- a/src/main/java/nl/cwi/monetdb/mcl/connection/helpers/BufferReallocator.java +++ b/src/main/java/nl/cwi/monetdb/mcl/connection/helpers/BufferReallocator.java @@ -33,9 +33,16 @@ public final class BufferReallocator { public static CharBuffer ReallocateBuffer(CharBuffer oldBuffer) { int newCapacity = GetNewCapacity(oldBuffer); - CharBuffer newBuffer = CharBuffer.allocate(newCapacity); + CharBuffer newBuffer = CharBuffer.wrap(new char[newCapacity]); oldBuffer.flip(); newBuffer.put(oldBuffer.array()); return newBuffer; } + + public static CharBuffer EnsureCapacity(CharBuffer oldBuffer, int newCapacity) { + if(newCapacity > oldBuffer.capacity()) { + oldBuffer = CharBuffer.wrap(new char[newCapacity]); + } + return oldBuffer; + } }
--- a/src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiProtocol.java +++ b/src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiProtocol.java @@ -32,12 +32,12 @@ public class OldMapiProtocol extends Abs CharBuffer lineBuffer; - private final StringBuilder tupleLineBuilder; + CharBuffer tupleLineBuffer; public OldMapiProtocol(OldMapiSocket socket) { this.socket = socket; this.lineBuffer = CharBuffer.wrap(new char[OldMapiSocket.BLOCK]); - this.tupleLineBuilder = new StringBuilder(OldMapiSocket.BLOCK); + this.tupleLineBuffer = CharBuffer.wrap(new char[1024]); } public OldMapiSocket getSocket() { @@ -132,8 +132,7 @@ public class OldMapiProtocol extends Abs @Override public int parseTupleLines(int firstLineNumber, int[] typesMap, Object[] data, boolean[][] nulls) throws ProtocolException { - OldMapiTupleLineParser.OldMapiParseTupleLine(firstLineNumber, this.lineBuffer, - this.tupleLineBuilder, typesMap, data, nulls); + OldMapiTupleLineParser.OldMapiParseTupleLine(this, firstLineNumber, typesMap, data, nulls); return firstLineNumber; }
--- a/src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiStartOfHeaderParser.java +++ b/src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiStartOfHeaderParser.java @@ -55,13 +55,14 @@ final class OldMapiStartOfHeaderParser { if (currentPointer >= limit) { throw new ProtocolException("unexpected end of string", currentPointer - 1); } - int tmp = 0, negative = 1;; + int tmp = 0; + boolean positive = true; char chr = array[currentPointer++]; // note: don't use Character.isDigit() here, because we only want ISO-LATIN-1 digits if (chr >= '0' && chr <= '9') { tmp = (int)chr - (int)'0'; } else if(chr == '-') { - negative = -1; + positive = false; } else { throw new ProtocolException("expected a digit", currentPointer - 1); } @@ -78,9 +79,8 @@ final class OldMapiStartOfHeaderParser { throw new ProtocolException("expected a digit", currentPointer - 1); } } - tmp *= negative; protocol.lineBuffer.position(currentPointer); - return tmp; + return positive ? tmp : -tmp; } static String GetNextResponseDataAsString(OldMapiProtocol protocol) throws ProtocolException {
--- a/src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiTupleLineParser.java +++ b/src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiTupleLineParser.java @@ -10,6 +10,7 @@ package nl.cwi.monetdb.mcl.protocol.oldm import nl.cwi.monetdb.jdbc.MonetBlob; import nl.cwi.monetdb.jdbc.MonetClob; +import nl.cwi.monetdb.mcl.connection.helpers.BufferReallocator; import nl.cwi.monetdb.mcl.connection.helpers.GregorianCalendarParser; import nl.cwi.monetdb.mcl.protocol.ProtocolException; @@ -22,39 +23,13 @@ import java.util.Calendar; final class OldMapiTupleLineParser { - private static final char[] NULL_STRING = "NULL".toCharArray(); - - private static int CharIndexOf(char[] source, int sourceCount, char[] target, int targetCount) { - if (targetCount == 0) { - return 0; - } - - char first = target[0]; - int max = sourceCount - targetCount; - - for (int i = 0; i <= max; i++) { - /* Look for first character. */ - if (source[i] != first) { - while (++i <= max && source[i] != first); - } + private static final char[] NULL_STRING = new char[]{'N','U','L','L'}; - /* Found first character, now look at the rest of v2 */ - if (i <= max) { - int j = i + 1; - int end = j + targetCount - 1; - for (int k = 1; j < end && source[j] == target[k]; j++, k++); + static int OldMapiParseTupleLine(OldMapiProtocol protocol, int lineNumber, int[] typesMap, Object[] values, + boolean[][] nulls) throws ProtocolException { + CharBuffer lineBuffer = protocol.lineBuffer; + CharBuffer tupleLineBuffer = protocol.tupleLineBuffer; - if (j == end) { - /* Found whole string. */ - return i; - } - } - } - return -1; - } - - static int OldMapiParseTupleLine(int lineNumber, CharBuffer lineBuffer, StringBuilder helper, int[] typesMap, - Object[] values, boolean[][] nulls) throws ProtocolException { int len = lineBuffer.limit(); char[] array = lineBuffer.array(); @@ -64,7 +39,7 @@ final class OldMapiTupleLineParser { throw new ProtocolException(typesMap.length + " columns expected, but only single value found"); } // return the whole string but the leading = - OldMapiStringToJavaObjectConverter(new String(array, 1, len - 1), lineNumber, values[0], typesMap[0]); + OldMapiStringToJavaDataConversion(array, 1, len - 1, lineNumber, values[0], typesMap[0]); return 1; } @@ -102,26 +77,26 @@ final class OldMapiTupleLineParser { if (!inString && (i > 0 && array[i - 1] == ',') || (i + 1 == len - 1 && array[++i] == ']')) { // dirty // split! if (array[cursor] == '"' && array[i - 2] == '"') { - // reuse the StringBuilder by cleaning it - helper.setLength(0); - // prevent capacity increases - helper.ensureCapacity((i - 2) - (cursor + 1)); + // reuse the CharBuffer by cleaning it and ensure the capacity + tupleLineBuffer.clear(); + tupleLineBuffer = BufferReallocator.EnsureCapacity(tupleLineBuffer, (i - 2) - (cursor + 1)); + for (int pos = cursor + 1; pos < i - 2; pos++) { if (array[cursor] == '\\' && pos + 1 < i - 2) { pos++; // strToStr and strFromStr in gdk_atoms.mx only support \t \n \\ \" and \377 switch (array[pos]) { case '\\': - helper.append('\\'); + tupleLineBuffer.put('\\'); break; case 'n': - helper.append('\n'); + tupleLineBuffer.put('\n'); break; case 't': - helper.append('\t'); + tupleLineBuffer.put('\t'); break; case '"': - helper.append('"'); + tupleLineBuffer.put('"'); break; case '0': case '1': case '2': case '3': // this could be an octal number, let's check it out @@ -129,7 +104,7 @@ final class OldMapiTupleLineParser { array[pos + 2] >= '0' && array[pos + 2] <= '7') { // we got the number! try { - helper.append((char)(Integer.parseInt("" + array[pos] + array[pos + 1] + array[pos + 2], 8))); + tupleLineBuffer.put((char)(Integer.parseInt("" + array[pos] + array[pos + 1] + array[pos + 2], 8))); pos += 2; } catch (NumberFormatException e) { // hmmm, this point should never be reached actually... @@ -137,26 +112,27 @@ final class OldMapiTupleLineParser { } } else { // do default action if number seems not to be correct - helper.append(array[pos]); + tupleLineBuffer.put(array[pos]); } break; default: // this is wrong, just ignore the escape, and print the char - helper.append(array[pos]); + tupleLineBuffer.put(array[pos]); break; } } else { - helper.append(array[pos]); + tupleLineBuffer.put(array[pos]); } } // put the unescaped string in the right place - OldMapiStringToJavaObjectConverter(helper.toString(), lineNumber, values[column], typesMap[column]); + tupleLineBuffer.flip(); + OldMapiStringToJavaDataConversion(tupleLineBuffer.array(), 0, tupleLineBuffer.limit(), lineNumber, values[column], typesMap[column]); nulls[column][lineNumber] = false; - } else if ((i - 1) - cursor == 4 && CharIndexOf(array, array.length, NULL_STRING, NULL_STRING.length) == cursor) { + } else if ((i - 1) - cursor == 4 && OldMapiTupleLineParserHelper.CharIndexOf(array, array.length, NULL_STRING, 4) == cursor) { SetNullValue(lineNumber, values[column], typesMap[column]); nulls[column][lineNumber] = true; } else { - OldMapiStringToJavaObjectConverter(new String(array, cursor, i - 1 - cursor), lineNumber, values[column], typesMap[column]); + OldMapiStringToJavaDataConversion(array, cursor, i - 1 - cursor, lineNumber, values[column], typesMap[column]); nulls[column][lineNumber] = false; } column++; @@ -167,82 +143,84 @@ final class OldMapiTupleLineParser { break; } } + + protocol.tupleLineBuffer = tupleLineBuffer; // check if this result is of the size we expected it to be if (column != typesMap.length) throw new ProtocolException("illegal result length: " + column + "\nlast read: " + (column > 0 ? values[column - 1] : "<none>")); return column; } - private static byte[] BinaryBlobConverter(String toParse) { - int len = toParse.length() / 2; + private static byte[] BinaryBlobConverter(char[] toParse, int startPosition, int count) { + int len = (startPosition + count) / 2; byte[] res = new byte[len]; for (int i = 0; i < len; i++) { - res[i] = (byte) Integer.parseInt(toParse.substring(2 * i, (2 * i) + 2), 16); + res[i] = (byte) Integer.parseInt(new String(toParse, 2 * i, (2 * i) + 2), 16); } return res; } private static final ParsePosition Ppos = new ParsePosition(0); - private static void OldMapiStringToJavaObjectConverter(String toParse, int lineNumber, Object columnArray, - int jDBCMapping) throws ProtocolException { + private static void OldMapiStringToJavaDataConversion(char[] toParse, int startPosition, int count, int lineNumber, + Object columnArray, int jDBCMapping) throws ProtocolException { switch (jDBCMapping) { case Types.BOOLEAN: - ((boolean[]) columnArray)[lineNumber] = Boolean.parseBoolean(toParse); + ((boolean[]) columnArray)[lineNumber] = OldMapiTupleLineParserHelper.CharArrayToBoolean(toParse, startPosition, count); break; case Types.TINYINT: - ((byte[]) columnArray)[lineNumber] = Byte.parseByte(toParse); + ((byte[]) columnArray)[lineNumber] = OldMapiTupleLineParserHelper.CharArrayToByte(toParse, startPosition, count); break; case Types.SMALLINT: - ((short[]) columnArray)[lineNumber] = Short.parseShort(toParse); + ((short[]) columnArray)[lineNumber] = OldMapiTupleLineParserHelper.CharArrayToShort(toParse, startPosition, count); break; case Types.INTEGER: - ((int[]) columnArray)[lineNumber] = Integer.parseInt(toParse.replace(".", "")); //intervals :( + ((int[]) columnArray)[lineNumber] = OldMapiTupleLineParserHelper.CharArrayToInt(toParse, startPosition, count); break; case Types.BIGINT: - ((long[]) columnArray)[lineNumber] = Long.parseLong(toParse.replace(".", "")); //intervals :( + ((long[]) columnArray)[lineNumber] = OldMapiTupleLineParserHelper.CharArrayToLong(toParse, startPosition, count); break; case Types.REAL: - ((float[]) columnArray)[lineNumber] = Float.parseFloat(toParse); + ((float[]) columnArray)[lineNumber] = Float.parseFloat(new String(toParse, startPosition, count)); break; case Types.DOUBLE: - ((double[]) columnArray)[lineNumber] = Double.parseDouble(toParse); + ((double[]) columnArray)[lineNumber] = Double.parseDouble(new String(toParse, startPosition, count)); break; case Types.DECIMAL: - ((BigDecimal[]) columnArray)[lineNumber] = new BigDecimal(toParse); + ((BigDecimal[]) columnArray)[lineNumber] = new BigDecimal(toParse, startPosition, count); break; case Types.NUMERIC: - ((BigInteger[]) columnArray)[lineNumber] = new BigInteger(toParse); + ((BigInteger[]) columnArray)[lineNumber] = new BigInteger(new String(toParse, startPosition, count)); break; case Types.CHAR: case Types.VARCHAR: case Types.LONGVARCHAR: case Types.OTHER: - ((String[]) columnArray)[lineNumber] = toParse; + ((String[]) columnArray)[lineNumber] = new String(toParse, startPosition, count); break; case Types.DATE: - ((Calendar[]) columnArray)[lineNumber] = GregorianCalendarParser.ParseDate(toParse, Ppos); + ((Calendar[]) columnArray)[lineNumber] = GregorianCalendarParser.ParseDate(new String(toParse, startPosition, count), Ppos); break; case Types.TIME: - ((Calendar[]) columnArray)[lineNumber] = GregorianCalendarParser.ParseTime(toParse, Ppos, false); + ((Calendar[]) columnArray)[lineNumber] = GregorianCalendarParser.ParseTime(new String(toParse, startPosition, count), Ppos, false); break; case Types.TIME_WITH_TIMEZONE: - ((Calendar[]) columnArray)[lineNumber] = GregorianCalendarParser.ParseTime(toParse, Ppos, true); + ((Calendar[]) columnArray)[lineNumber] = GregorianCalendarParser.ParseTime(new String(toParse, startPosition, count), Ppos, true); break; case Types.TIMESTAMP: - ((Calendar[]) columnArray)[lineNumber] = GregorianCalendarParser.ParseTimestamp(toParse, Ppos, false); + ((Calendar[]) columnArray)[lineNumber] = GregorianCalendarParser.ParseTimestamp(new String(toParse, startPosition, count), Ppos, false); break; case Types.TIMESTAMP_WITH_TIMEZONE: - ((Calendar[]) columnArray)[lineNumber] = GregorianCalendarParser.ParseTimestamp(toParse, Ppos, true); + ((Calendar[]) columnArray)[lineNumber] = GregorianCalendarParser.ParseTimestamp(new String(toParse, startPosition, count), Ppos, true); break; case Types.CLOB: - ((MonetClob[]) columnArray)[lineNumber] = new MonetClob(toParse); + ((MonetClob[]) columnArray)[lineNumber] = new MonetClob(toParse, startPosition, count); break; case Types.BLOB: - ((MonetBlob[]) columnArray)[lineNumber] = new MonetBlob(BinaryBlobConverter(toParse)); + ((MonetBlob[]) columnArray)[lineNumber] = new MonetBlob(BinaryBlobConverter(toParse, startPosition, count)); break; case Types.LONGVARBINARY: - ((byte[][]) columnArray)[lineNumber] = BinaryBlobConverter(toParse); + ((byte[][]) columnArray)[lineNumber] = BinaryBlobConverter(toParse, startPosition, count); break; default: throw new ProtocolException("Unknown type!");
new file mode 100644 --- /dev/null +++ b/src/main/java/nl/cwi/monetdb/mcl/protocol/oldmapi/OldMapiTupleLineParserHelper.java @@ -0,0 +1,154 @@ +package nl.cwi.monetdb.mcl.protocol.oldmapi; + +final class OldMapiTupleLineParserHelper { + + static int CharIndexOf(char[] source, int sourceCount, char[] target, int targetCount) { + if (targetCount == 0) { + return 0; + } + + char first = target[0]; + int max = sourceCount - targetCount; + + for (int i = 0; i <= max; i++) { + /* Look for first character. */ + if (source[i] != first) { + while (++i <= max && source[i] != first); + } + + /* Found first character, now look at the rest of v2 */ + if (i <= max) { + int j = i + 1; + int end = j + targetCount - 1; + for (int k = 1; j < end && source[j] == target[k]; j++, k++); + + if (j == end) { + /* Found whole string. */ + return i; + } + } + } + return -1; + } + + private static final char[] TrueConstant = new char[]{'t','r','u','e'}; + + static boolean CharArrayToBoolean(char[] data, int start, int count) { + return CharIndexOf(data, start + count, TrueConstant, 4) == start; + } + + static byte CharArrayToByte(char[] data, int start, int count) { + byte tmp = 0; + int limit = start + count; + boolean positive = true; + char chr = data[start++]; + // note: don't use Character.isDigit() here, because we only want ISO-LATIN-1 digits + if (chr >= '0' && chr <= '9') { + tmp = (byte)(chr - '0'); + } else if(chr == '-') { + positive = false; + } else { + throw new NumberFormatException(); + } + while (start < limit) { + chr = data[start++]; + if(chr == ' ') { + break; + } + tmp *= 10; + if (chr >= '0' && chr <= '9') { + tmp += chr - '0'; + } else { + throw new NumberFormatException(); + } + } + return positive ? tmp : (byte) -tmp; + } + + static short CharArrayToShort(char[] data, int start, int count) { + short tmp = 0; + int limit = start + count; + boolean positive = true; + char chr = data[start++]; + // note: don't use Character.isDigit() here, because we only want ISO-LATIN-1 digits + if (chr >= '0' && chr <= '9') { + tmp = (short)(chr - '0'); + } else if(chr == '-') { + positive = false; + } else { + throw new NumberFormatException(); + } + while (start < limit) { + chr = data[start++]; + if(chr == ' ') { + break; + } + tmp *= 10; + if (chr >= '0' && chr <= '9') { + tmp += chr - '0'; + } else { + throw new NumberFormatException(); + } + } + return positive ? tmp : (short) -tmp; + } + + static int CharArrayToInt(char[] data, int start, int count) { + int tmp = 0, limit = start + count; + boolean positive = true; + char chr = data[start++]; + // note: don't use Character.isDigit() here, because we only want ISO-LATIN-1 digits + if (chr >= '0' && chr <= '9') { + tmp = chr - '0'; + } else if(chr == '-') { + positive = false; + } else { + throw new NumberFormatException(); + } + while (start < limit) { + chr = data[start++]; + if(chr == ' ') { + break; + } else if(chr == '.') { //for intervals + continue; + } + tmp *= 10; + if (chr >= '0' && chr <= '9') { + tmp += (int)chr - (int)'0'; + } else { + throw new NumberFormatException(); + } + } + return positive ? tmp : -tmp; + } + + static long CharArrayToLong(char[] data, int start, int count) { + long tmp = 0; + int limit = start + count; + boolean positive = true; + char chr = data[start++]; + // note: don't use Character.isDigit() here, because we only want ISO-LATIN-1 digits + if (chr >= '0' && chr <= '9') { + tmp = chr - '0'; + } else if(chr == '-') { + positive = false; + } else { + throw new NumberFormatException(); + } + while (start < limit) { + chr = data[start++]; + if(chr == ' ') { + break; + } else if(chr == '.') { //for intervals + continue; + } + tmp *= 10; + if (chr >= '0' && chr <= '9') { + tmp += chr - '0'; + } else { + throw new NumberFormatException(); + } + } + return positive ? tmp : -tmp; + } +}