Mercurial > hg > monetdb-java
changeset 551:5ce6a942aff3 onclient
Last minute API fix: 'int offset' -> 'long linesToSkip'
Int -> long because there might be many lines
Offset -> linesToSkip to save users some work.
The COPY INTO statement has an OFFSET modifier which is 1-based
but also allows 0. This means both 0 and 1 mean 'upload the whole
file' and any N > 1 means skip N-1 lines and upload the rest.
To avoid having to deal with this over and over in every implementation
of MonetConnection.UploadHandler#handleUpload(), parameter 'offset'
has been replaced with 'linesToSkip' where the adjustment has already
been performed.
author | Joeri van Ruth <joeri.van.ruth@monetdbsolutions.com> |
---|---|
date | Tue, 14 Sep 2021 10:52:04 +0200 (2021-09-14) |
parents | c5cf3f00c4c5 |
children | 7b320303b579 |
files | example/OnClientExample.java src/main/java/org/monetdb/jdbc/MonetConnection.java src/main/java/org/monetdb/util/FileTransferHandler.java tests/OnClientTester.java |
diffstat | 4 files changed, 35 insertions(+), 38 deletions(-) [+] |
line wrap: on
line diff
--- a/example/OnClientExample.java +++ b/example/OnClientExample.java @@ -96,15 +96,14 @@ public class OnClientExample { } @Override - public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException { + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { // COPY OFFSET line numbers are 1-based but 0 is also allowed. // Compute the number of lines to skip - long toSkip = offset <= 1 ? 0 : offset - 1; // We can upload data read from the file system but also make up our own data if (name.equals("generated.csv")) { - uploadGenerated(handle, toSkip); + uploadGenerated(handle, linesToSkip); return; } @@ -124,12 +123,12 @@ public class OnClientExample { boolean binary = !textMode; if (binary) { uploadBinary(handle, path); - } else if (toSkip == 0 && filesAreUtf8) { + } else if (linesToSkip == 0 && filesAreUtf8) { // Avoid unnecessary character set conversions by pretending it's binary uploadBinary(handle, path); } else { // Charset and skip handling really necessary - uploadTextFile(handle, path, toSkip); + uploadTextFile(handle, path, linesToSkip); } }
--- a/src/main/java/org/monetdb/jdbc/MonetConnection.java +++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java @@ -3213,9 +3213,9 @@ public class MonetConnection private String handleTransfer(String transferCommand) throws IOException { String[] parts = transferCommand.split(" ", 3); if (transferCommand.startsWith("r ") && parts.length == 3) { - final int offset; + final long offset; try { - offset = Integer.parseInt(parts[1]); + offset = Long.parseLong(parts[1]); } catch (NumberFormatException e) { return e.toString(); } @@ -3229,15 +3229,16 @@ public class MonetConnection } } - private String handleUpload(String path, boolean textMode, int offset) throws IOException { + private String handleUpload(String path, boolean textMode, long offset) throws IOException { if (uploadHandler == null) { return "No file upload handler has been registered with the JDBC driver"; } + long linesToSkip = offset >= 1 ? offset - 1 : 0; Upload handle = new Upload(server); boolean wasFaking = server.setInsertFakePrompts(false); try { - uploadHandler.handleUpload(handle, path, textMode, offset); + uploadHandler.handleUpload(handle, path, textMode, linesToSkip); if (!handle.hasBeenUsed()) { String message = String.format("Call to %s.handleUpload for path '%s' sent neither data nor an error message", uploadHandler.getClass().getCanonicalName(), path); @@ -3280,16 +3281,16 @@ public class MonetConnection * Called if the server sends a request to read file data. * * Use the given handle to receive data or send errors to the server. - * - * @param handle Handle to communicate with the server + * @param handle Handle to communicate with the server * @param name Name of the file the server would like to read. Make sure * to validate this before reading from the file system * @param textMode Whether this is text or binary data. - * @param offset line number of the first line to upload. Both 0 and 1 - * mean 'upload the whole file', 2 means 'skip line 1', - * etc. + * @param linesToSkip In text mode, number of initial lines to skip. + * 0 means upload everything, 1 means skip the first line, etc. + * Note: this is different from the OFFSET option of the COPY INTO, + * where both 0 and 1 mean 'upload everything' */ - void handleUpload(Upload handle, String name, boolean textMode, int offset) throws IOException; + void handleUpload(Upload handle, String name, boolean textMode, long linesToSkip) throws IOException; } /** @@ -3413,17 +3414,12 @@ public class MonetConnection /** * Read data from the given buffered reader and send it to the server * @param reader reader to read from - * @param offset start uploading at line {@code offset}. Value 0 and 1 + * @param linesToSkip start uploading at line {@code offset}. Value 0 and 1 * both mean upload the whole file, value 2 means skip the first line, etc.q * @throws IOException */ - public void uploadFrom(BufferedReader reader, int offset) throws IOException { - // we're 1-based but also accept 0 - if (offset > 0) { - offset -= 1; - } - - for (int i = 0; i < offset; i++) { + public void uploadFrom(BufferedReader reader, long linesToSkip) throws IOException { + for (int i = 0; i < linesToSkip; i++) { String line = reader.readLine(); if (line == null) { return;
--- a/src/main/java/org/monetdb/util/FileTransferHandler.java +++ b/src/main/java/org/monetdb/util/FileTransferHandler.java @@ -52,7 +52,7 @@ public class FileTransferHandler impleme this(FileSystems.getDefault().getPath(dir), utf8Encoded); } - public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException { + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { Path path = root.resolve(name).normalize(); if (!path.startsWith(root)) { handle.sendError("File is not in upload directory"); @@ -62,10 +62,10 @@ public class FileTransferHandler impleme handle.sendError("Cannot read " + name); return; } - if (textMode && (offset > 1 || !utf8Encoded)) { + if (textMode && (linesToSkip > 0 || !utf8Encoded)) { Charset encoding = utf8Encoded ? StandardCharsets.UTF_8 : Charset.defaultCharset(); BufferedReader reader = Files.newBufferedReader(path, encoding); - handle.uploadFrom(reader, offset); + handle.uploadFrom(reader, linesToSkip); } else { handle.uploadFrom(Files.newInputStream(path)); }
--- a/tests/OnClientTester.java +++ b/tests/OnClientTester.java @@ -164,7 +164,8 @@ public final class OnClientTester extend final String data = "1|one\n2|two\n3|three\n"; @Override - public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException { + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { + // ignoring linesToSkip as it's not used in this test ByteArrayInputStream s = new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8)); handle.uploadFrom(s); } @@ -180,7 +181,8 @@ public final class OnClientTester extend final String data = "1|one\n2|two\n3|three\n"; @Override - public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException { + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { + // ignoring linesToSkip as it's not used in this test StringReader r = new StringReader(data); handle.uploadFrom(r); } @@ -196,9 +198,9 @@ public final class OnClientTester extend final String data = "1|one\n2|two\n3|three\n"; @Override - public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException { + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { BufferedReader r = new BufferedReader(new StringReader(data)); - handle.uploadFrom(r, offset); + handle.uploadFrom(r, linesToSkip); } }; conn.setUploadHandler(handler); @@ -222,7 +224,8 @@ public final class OnClientTester extend prepare(); UploadHandler handler = new UploadHandler() { @Override - public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException { + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { + // ignoring linesToSkip as it's not used in this test PrintStream stream = handle.getStream(); for (int i = 1; i <= 20_000; i++) stream.println(); @@ -248,20 +251,20 @@ public final class OnClientTester extend } static class MyUploadHandler implements UploadHandler { - private final int rows; - private final int errorAt; + private final long rows; + private final long errorAt; private final String errorMessage; private boolean encounteredWriteError; private int chunkSize = 100; // small number to trigger more bugs - MyUploadHandler(int rows, int errorAt, String errorMessage) { + MyUploadHandler(long rows, long errorAt, String errorMessage) { this.rows = rows; this.errorAt = errorAt; this.errorMessage = errorMessage; } - MyUploadHandler(int rows) { + MyUploadHandler(long rows) { this(rows, -1, null); } @@ -274,15 +277,14 @@ public final class OnClientTester extend } @Override - public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException { - int toSkip = offset > 0 ? offset - 1 : 0; + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { if (errorAt == -1 && errorMessage != null) { handle.sendError(errorMessage); return; } handle.setChunkSize(chunkSize); PrintStream stream = handle.getStream(); - for (int i = toSkip; i < rows; i++) { + for (long i = linesToSkip; i < rows; i++) { if (i == errorAt) { throw new IOException(errorMessage); }