changeset 551:5ce6a942aff3 onclient

Last minute API fix: 'int offset' -> 'long linesToSkip' Int -> long because there might be many lines Offset -> linesToSkip to save users some work. The COPY INTO statement has an OFFSET modifier which is 1-based but also allows 0. This means both 0 and 1 mean 'upload the whole file' and any N > 1 means skip N-1 lines and upload the rest. To avoid having to deal with this over and over in every implementation of MonetConnection.UploadHandler#handleUpload(), parameter 'offset' has been replaced with 'linesToSkip' where the adjustment has already been performed.
author Joeri van Ruth <joeri.van.ruth@monetdbsolutions.com>
date Tue, 14 Sep 2021 10:52:04 +0200 (2021-09-14)
parents c5cf3f00c4c5
children 7b320303b579
files example/OnClientExample.java src/main/java/org/monetdb/jdbc/MonetConnection.java src/main/java/org/monetdb/util/FileTransferHandler.java tests/OnClientTester.java
diffstat 4 files changed, 35 insertions(+), 38 deletions(-) [+]
line wrap: on
line diff
--- a/example/OnClientExample.java
+++ b/example/OnClientExample.java
@@ -96,15 +96,14 @@ public class OnClientExample {
 		}
 
 		@Override
-		public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException {
+		public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException {
 
 			// COPY OFFSET line numbers are 1-based but 0 is also allowed.
 			// Compute the number of lines to skip
-			long toSkip = offset <= 1 ? 0 : offset - 1;
 
 			// We can upload data read from the file system but also make up our own data
 			if (name.equals("generated.csv")) {
-				uploadGenerated(handle, toSkip);
+				uploadGenerated(handle, linesToSkip);
 				return;
 			}
 
@@ -124,12 +123,12 @@ public class OnClientExample {
 			boolean binary = !textMode;
 			if (binary) {
 				uploadBinary(handle, path);
-			} else if (toSkip == 0 && filesAreUtf8) {
+			} else if (linesToSkip == 0 && filesAreUtf8) {
 				// Avoid unnecessary character set conversions by pretending it's binary
 				uploadBinary(handle, path);
 			} else {
 				// Charset and skip handling really necessary
-				uploadTextFile(handle, path, toSkip);
+				uploadTextFile(handle, path, linesToSkip);
 			}
 		}
 
--- a/src/main/java/org/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java
@@ -3213,9 +3213,9 @@ public class MonetConnection
 	private String handleTransfer(String transferCommand) throws IOException {
 		String[] parts = transferCommand.split(" ", 3);
 		if (transferCommand.startsWith("r ") && parts.length == 3) {
-			final int offset;
+			final long offset;
 			try {
-				offset = Integer.parseInt(parts[1]);
+				offset = Long.parseLong(parts[1]);
 			} catch (NumberFormatException e) {
 				return e.toString();
 			}
@@ -3229,15 +3229,16 @@ public class MonetConnection
 		}
 	}
 
-	private String handleUpload(String path, boolean textMode, int offset) throws IOException {
+	private String handleUpload(String path, boolean textMode, long offset) throws IOException {
 		if (uploadHandler == null) {
 			return "No file upload handler has been registered with the JDBC driver";
 		}
 
+		long linesToSkip = offset >= 1 ? offset - 1 : 0;
 		Upload handle = new Upload(server);
 		boolean wasFaking = server.setInsertFakePrompts(false);
 		try {
-			uploadHandler.handleUpload(handle, path, textMode, offset);
+			uploadHandler.handleUpload(handle, path, textMode, linesToSkip);
 			if (!handle.hasBeenUsed()) {
 				String message = String.format("Call to %s.handleUpload for path '%s' sent neither data nor an error message",
 						uploadHandler.getClass().getCanonicalName(), path);
@@ -3280,16 +3281,16 @@ public class MonetConnection
 		 * Called if the server sends a request to read file data.
 		 *
 		 * Use the given handle to receive data or send errors to the server.
-		 *
-		 * @param handle Handle to communicate with the server
+		 *  @param handle Handle to communicate with the server
 		 * @param name Name of the file the server would like to read. Make sure
 		 *             to validate this before reading from the file system
 		 * @param textMode Whether this is text or binary data.
-		 * @param offset line number of the first line to upload. Both 0 and 1
-		 *               mean 'upload the whole file', 2 means 'skip line 1',
-		 *               etc.
+		 * @param linesToSkip In text mode, number of initial lines to skip.
+		 *                    0 means upload everything, 1 means skip the first line, etc.
+		 *                    Note: this is different from the OFFSET option of the COPY INTO,
+		 *                    where both 0 and 1 mean 'upload everything'
 		 */
-		void handleUpload(Upload handle, String name, boolean textMode, int offset) throws IOException;
+		void handleUpload(Upload handle, String name, boolean textMode, long linesToSkip) throws IOException;
 	}
 
 	/**
@@ -3413,17 +3414,12 @@ public class MonetConnection
 		/**
 		 * Read data from the given buffered reader and send it to the server
 		 * @param reader reader to read from
-		 * @param offset start uploading at line {@code offset}. Value 0 and 1
+		 * @param linesToSkip start uploading at line {@code offset}. Value 0 and 1
 		 * both mean upload the whole file, value 2 means skip the first line, etc.q
 		 * @throws IOException
 		 */
-		public void uploadFrom(BufferedReader reader, int offset) throws IOException {
-			// we're 1-based but also accept 0
-			if (offset > 0) {
-				offset -= 1;
-			}
-
-			for (int i = 0; i < offset; i++) {
+		public void uploadFrom(BufferedReader reader, long linesToSkip) throws IOException {
+			for (int i = 0; i < linesToSkip; i++) {
 				String line = reader.readLine();
 				if (line == null) {
 					return;
--- a/src/main/java/org/monetdb/util/FileTransferHandler.java
+++ b/src/main/java/org/monetdb/util/FileTransferHandler.java
@@ -52,7 +52,7 @@ public class FileTransferHandler impleme
 		this(FileSystems.getDefault().getPath(dir), utf8Encoded);
 	}
 
-	public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException {
+	public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException {
 		Path path = root.resolve(name).normalize();
 		if (!path.startsWith(root)) {
 			handle.sendError("File is not in upload directory");
@@ -62,10 +62,10 @@ public class FileTransferHandler impleme
 			handle.sendError("Cannot read " + name);
 			return;
 		}
-		if (textMode && (offset > 1 || !utf8Encoded)) {
+		if (textMode && (linesToSkip > 0 || !utf8Encoded)) {
 			Charset encoding = utf8Encoded ? StandardCharsets.UTF_8 : Charset.defaultCharset();
 			BufferedReader reader = Files.newBufferedReader(path, encoding);
-			handle.uploadFrom(reader, offset);
+			handle.uploadFrom(reader, linesToSkip);
 		} else {
 			handle.uploadFrom(Files.newInputStream(path));
 		}
--- a/tests/OnClientTester.java
+++ b/tests/OnClientTester.java
@@ -164,7 +164,8 @@ public final class OnClientTester extend
 			final String data = "1|one\n2|two\n3|three\n";
 
 			@Override
-			public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException {
+			public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException {
+				// ignoring linesToSkip as it's not used in this test
 				ByteArrayInputStream s = new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8));
 				handle.uploadFrom(s);
 			}
@@ -180,7 +181,8 @@ public final class OnClientTester extend
 			final String data = "1|one\n2|two\n3|three\n";
 
 			@Override
-			public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException {
+			public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException {
+				// ignoring linesToSkip as it's not used in this test
 				StringReader r = new StringReader(data);
 				handle.uploadFrom(r);
 			}
@@ -196,9 +198,9 @@ public final class OnClientTester extend
 			final String data = "1|one\n2|two\n3|three\n";
 
 			@Override
-			public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException {
+			public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException {
 				BufferedReader r = new BufferedReader(new StringReader(data));
-				handle.uploadFrom(r, offset);
+				handle.uploadFrom(r, linesToSkip);
 			}
 		};
 		conn.setUploadHandler(handler);
@@ -222,7 +224,8 @@ public final class OnClientTester extend
 		prepare();
 		UploadHandler handler = new UploadHandler() {
 			@Override
-			public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException {
+			public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException {
+				// ignoring linesToSkip as it's not used in this test
 				PrintStream stream = handle.getStream();
 				for (int i = 1; i <= 20_000; i++)
 					stream.println();
@@ -248,20 +251,20 @@ public final class OnClientTester extend
 	}
 
 	static class MyUploadHandler implements UploadHandler {
-		private final int rows;
-		private final int errorAt;
+		private final long rows;
+		private final long errorAt;
 		private final String errorMessage;
 		private boolean encounteredWriteError;
 
 		private int chunkSize = 100; // small number to trigger more bugs
 
-		MyUploadHandler(int rows, int errorAt, String errorMessage) {
+		MyUploadHandler(long rows, long errorAt, String errorMessage) {
 			this.rows = rows;
 			this.errorAt = errorAt;
 			this.errorMessage = errorMessage;
 		}
 
-		MyUploadHandler(int rows) {
+		MyUploadHandler(long rows) {
 			this(rows, -1, null);
 		}
 
@@ -274,15 +277,14 @@ public final class OnClientTester extend
 		}
 
 		@Override
-		public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, int offset) throws IOException {
-			int toSkip = offset > 0 ? offset - 1 : 0;
+		public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException {
 			if (errorAt == -1 && errorMessage != null) {
 				handle.sendError(errorMessage);
 				return;
 			}
 			handle.setChunkSize(chunkSize);
 			PrintStream stream = handle.getStream();
-			for (int i = toSkip; i < rows; i++) {
+			for (long i = linesToSkip; i < rows; i++) {
 				if (i == errorAt) {
 					throw new IOException(errorMessage);
 				}