Mercurial > hg > monetdb-java
changeset 587:fca22b72ce7e
Merge onclient into default
| author | Joeri van Ruth <joeri.van.ruth@monetdbsolutions.com> |
|---|---|
| date | Fri, 05 Nov 2021 16:17:55 +0100 (2021-11-05) |
| parents | ac5e97fc812d (current diff) 1c5e59760ff8 (diff) |
| children | 9f658a97666f |
| files | tests/JDBC_API_Tester.java |
| diffstat | 20 files changed, 2438 insertions(+), 171 deletions(-) [+] |
line wrap: on
line diff
--- a/build.xml +++ b/build.xml @@ -108,6 +108,7 @@ Copyright 1997 - July 2008 CWI, August 2 <include name="${nl-cwi-jdbc-package}/types/*.class" /> <include name="${mcl-package}/**/*.class" /> <include name="${nl-cwi-mcl-package}/net/MapiSocket.class" /> + <include name="${util-package}/FileTransferHandler.class" /> </fileset> </jar> </target> @@ -203,6 +204,7 @@ Copyright 1997 - July 2008 CWI, August 2 <include name="${nl-cwi-jdbc-package}/MonetDriver.java" /> <include name="${nl-cwi-jdbc-package}/types/*.java" /> <include name="${nl-cwi-mcl-package}/net/MapiSocket.java" /> + <include name="${util-package}/FileTransferHandler.java" /> <compilerarg line="${javac.flags}" /> </javac> </target>
new file mode 100644 --- /dev/null +++ b/example/OnClientExample.java @@ -0,0 +1,190 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V. + */ + +import org.monetdb.jdbc.MonetConnection; +import org.monetdb.jdbc.MonetConnection.UploadHandler; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.*; + +public class OnClientExample { + + public static void main(String[] args) { + int status; + try { + // Ideally this would not be hardcoded.. + final String dbUrl = "jdbc:monetdb://localhost:50000/demo"; + final String userName = "monetdb"; + final String password = "monetdb"; + final String uploadDir = "/home/jvr/mydata"; + final boolean filesAreUtf8 = false; + String[] queries = { + "DROP TABLE IF EXISTS mytable", + "CREATE TABLE mytable(i INT, t TEXT)", + "COPY INTO mytable FROM 'generated.csv' ON CLIENT", + "COPY 20 RECORDS OFFSET 5 INTO mytable FROM 'generated.csv' ON CLIENT", + "COPY INTO mytable FROM 'nonexistentfilethatdoesnotexist.csv' ON CLIENT", + "SELECT COUNT(*) FROM mytable", + }; + + status = run(dbUrl, userName, password, uploadDir, filesAreUtf8, queries); + + } catch (Exception e) { + status = 1; + e.printStackTrace(); + } + System.exit(status); + } + + private static int run(String dbUrl, String userName, String password, String uploadDir, boolean filesAreUtf8, String[] queries) throws ClassNotFoundException, SQLException { + int status = 0; + + // Connect + Class.forName("org.monetdb.jdbc.MonetDriver"); + Connection conn = DriverManager.getConnection(dbUrl, userName, password); + + // Register upload handler + MyUploader handler = new MyUploader(uploadDir, filesAreUtf8); + conn.unwrap(MonetConnection.class).setUploadHandler(handler); + + Statement stmt = conn.createStatement(); + for (String q : queries) { + System.out.println(q); + try { + boolean hasResultSet = stmt.execute(q); + if (hasResultSet) { + ResultSet rs = stmt.getResultSet(); + long count = 0; + while (rs.next()) { + count++; + } + System.out.printf(" OK, returned %d rows%n", count); + } else { + System.out.printf(" OK, updated %d rows%n", stmt.getUpdateCount()); + } + } catch (SQLNonTransientException e) { + throw e; + } catch (SQLException e) { + System.out.println(" => SQL ERROR " + e.getMessage()); + status = 1; + } + + } + + return status; + } + + + private static class MyUploader implements UploadHandler { + private final Path uploadDir; + private final boolean filesAreUtf8; + private boolean stopUploading = false; + + public MyUploader(String uploadDir, boolean filesAreUtf8) { + this.uploadDir = FileSystems.getDefault().getPath(uploadDir).normalize(); + this.filesAreUtf8 = filesAreUtf8; + } + + @Override + public void uploadCancelled() { + System.out.println(" CANCELLATION CALLBACK: server cancelled the upload"); + stopUploading = true; + } + + @Override + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { + + // We can upload data read from the file system but also make up our own data + if (name.equals("generated.csv")) { + uploadGeneratedData(handle, linesToSkip); + return; + } + + // Validate the path, demonstrating two ways of dealing with errors + Path path = securityCheck(name); + if (path == null || !Files.exists(path)) { + // This makes the COPY command fail but keeps the connection + // alive. Can only be used if we haven't sent any data yet + handle.sendError("Invalid path"); + return; + } + if (!Files.isReadable(path)) { + // As opposed to handle.sendError(), we can throw an IOException + // at any time. Unfortunately, the file upload protocol does not + // provide a way to indicate to the server that the data sent so + // far is incomplete, so for the time being throwing an + // IOException from {@handleUpload} terminates the connection. + throw new IOException("Unreadable: " + path); + } + + boolean binary = !textMode; + if (binary) { + uploadAsBinary(handle, path); + } else if (linesToSkip == 0 && filesAreUtf8) { + // Avoid unnecessary UTF-8 -> Java String -> UTF-8 conversions + // by pretending the data is binary. + uploadAsBinary(handle, path); + } else { + // Charset and skip handling really necessary + uploadAsText(handle, path, linesToSkip); + } + } + + private Path securityCheck(String name) { + Path p = uploadDir.resolve(name).normalize(); + if (p.startsWith(uploadDir)) { + return p; + } else { + return null; + } + } + + private void uploadGeneratedData(MonetConnection.Upload handle, long toSkip) throws IOException { + // Set the chunk size to a tiny amount so we can demonstrate + // cancellation handling. The default chunk size is one megabyte. + // DO NOT DO THIS IN PRODUCTION! + handle.setChunkSize(50); + + // Make up some data and upload it. + PrintStream stream = handle.getStream(); + long n = 100; + System.out.printf(" HANDLER: uploading %d generated lines, numbered %d to %d%n", n - toSkip, toSkip + 1, n); + long i; + for (i = toSkip + 1; i <= n; i++) { + if (stopUploading) { + System.out.printf(" HANDLER: at line %d we noticed the server asked us to stop sending%n", i); + break; + } + stream.printf("%d|the number is %d%n", i, i); + } + System.out.println(" HANDLER: done uploading"); + stream.close(); + } + + private void uploadAsText(MonetConnection.Upload handle, Path path, long toSkip) throws IOException { + BufferedReader reader = Files.newBufferedReader(path);// Converts from system encoding to Java text + for (long i = 0; i < toSkip; i++) { + reader.readLine(); + } + handle.uploadFrom(reader); // Converts from Java text to UTF-8 as required by MonetDB + } + + private void uploadAsBinary(MonetConnection.Upload handle, Path path) throws IOException { + // No charset conversion whatsoever.. + // Use this for binary data or when you are certain the file is UTF-8 encoded. + InputStream stream = Files.newInputStream(path); + handle.uploadFrom(stream); + } + } +}
new file mode 100644 --- /dev/null +++ b/onclient.txt @@ -0,0 +1,94 @@ +COPY ... ON CLIENT support in the MonetDB JDBC driver and JdbcClient program. + +MonetDB provides the nonstandard COPY INTO statement to perform bulk inserts and +retrievals, see also +https://www.monetdb.org/Documentation/ServerAdministration/LoadingBulkData/CSVBulkLoads +https://www.monetdb.org/Documentation/ServerAdministration/ExportingBulkData + +By default, COPY INTO accesses files on the server but it also has a mode to +access files on the client. This is supported by the command line tool +mclient(1) and now also as an extension to the MonetDB JDBC driver. + +This is how it works: The JDBC client automatically announces that it is capable +of file transfers. If you execute, for example, + + COPY INTO mytable FROM 'data.csv' ON CLIENT; + +the server will send a request for file 'data.csv' to the JDBC driver. +By default, the JDBC driver will refuse with an error message: + + 'No file upload handler has been registered with the JDBC driver' + +or in JdbcClient: + 'Error [22000] data.csv: No file upload handler has been registered with the JDBC driver' + +This is for security reasons. However, you can register a callback to handle +these requests from the server: + + Connection conn = DriverManager.getConnection(dbUrl, userName, password); + MyUploader handler = new MyUploadHandler(); + conn.unwrap(MonetConnection.class).setUploadHandler(handler); + +or provide the JdbcClient startup argument: --csvdir "/path/to/csvfilesdir" + +Here, MyUploadHandler is an implementation of the interface MonetConnection.UploadHandler, +which looks like this: + + public interface UploadHandler { + /** + * Called if the server sends a request to read file data. + * + * Use the given handle to receive data or send errors to the server. + * @param handle Handle to communicate with the server + * @param name Name of the file the server would like to read. Make sure + * to validate this before reading from the file system + * @param textMode Whether to open the file as text or binary data. + * @param linesToSkip In text mode, number of initial lines to skip. + * 0 means upload everything, 1 means skip the first line, etc. + * Note: this is different from the OFFSET option of the COPY INTO, + * where both 0 and 1 mean 'upload everything' + */ + void handleUpload(Upload handle, String name, boolean textMode, long linesToSkip) throws IOException; + + /** + * Called when the upload is cancelled halfway by the server. + * + * The default implementation does nothing. + */ + default void uploadCancelled() {} + } + +In your implementation of handleUpload(), you can use the 'handle' object to +communicate with the server, for example: + +- handle.getStream() to obtain a stream object to which you can write. + This is useful if you want to generate the data on the fly. + +- void uploadFrom(InputStream stream) to have the JDBC driver read data from the + stream and send it to the server as-is. For text mode uploads this means the + text must be UTF-8 encoded. + +- handle.uploadFrom(Reader reader) to have the JDBC driver read text from the given + Reader and upload it. + +- handle.uploadFrom(BufferedReader reader, long linesToSkip) to have the JDBC + driver read from the given BufferedReader and upload the text, skipping the first + 'linesToSkip' lines. Typically you would use the value passed to handleUpload in + parameter 'linesToSkip'. + +- handle.sendError(String errorMessage) to refuse the upload. + +If you use sendError to refuse the upload, the COPY INTO statement will fail but +the connection will remain usable. On the other hand, if you implementation of +handleUpload throws an IO Exception, the connection will be closed because there +is currently no way to signal errors to the server once the transfer has begun. + +The interface for downloading is similar to that for uploading. + +Class org.monetdb.util.FileTransferHandler provides a default implementation +of both MonetConnection.UploadHandler and MonetConnection.DownloadHandler for +reading from and writing to local csv files. You pass the FileTransferHandler +constructor a directory name and a flag indicating whether the contents of +the csv files data are UTF-8 encoded. FileTransferHandler is intended for +situations where you do not need to generate or transform data while uploading. +
--- a/release.txt +++ b/release.txt @@ -1,6 +1,6 @@ RELEASE NOTES -MonetDB JDBC driver version 3.1 (Liberica/MCL-1.20) -Release date: 2021-06-14 +MonetDB JDBC driver version 3.2 (Liberica/MCL-1.21) +Release date: 2021-10-15 The Java Database Connectivity (JDBC) API provides universal data access from the Java programming language. @@ -186,6 +186,28 @@ The following java.sql.* interfaces are * java.sql.Struct +Since release 3.2 (monetdb-jdbc-3.2.jre8.jar), the MonetDB JDBC driver has +support for the ON CLIENT clause of the COPY statement. To make use of +this functionality you must register handlers for upload and download of data. +The MonetConnection class has been extended with 2 methods: + public void setUploadHandler(UploadHandler uploadHandler) + public void setDownloadHandler(DownloadHandler downloadHandler) +The API has been extended with interfaces and utility class: + public interface org.monetdb.jdbc.MonetConnection.UploadHandler + public interface org.monetdb.jdbc.MonetConnection.DownloadHandler + public class org.monetdb.util.FileTransferHandler + which implements MonetConnection.UploadHandler, MonetConnection.DownloadHandler +See onclient.txt for more information on how to use these from Java. + +The JdbcClient application has also been extended to support COPY ... +ON CLIENT functionality. However for security reasons you must provide an +explicit new startup argument + --csvdir "/path/to/csvdatafiles" +or on MS Windows + --csvdir "C:\\path\\to\\csvdatafiles" +in order to allow the JdbcClient to down/up load data to/from local csv files. + + Notes and Tips for Java Programmers using MonetDB JDBC driver: - Close JDBC ResultSet, Statement, PreparedStatement, CallableStatement and Connection objects immediately (via close()) when they are no longer needed, @@ -248,4 +270,4 @@ Warning: Note: as of Febr 2021 (monetdb-jdbc-3.0.jre8.jar) we compile all the java sources to target: Java SE 8 (profile compact2), so you need a JRE/JDK JVM of version 8 or higher to use it. - +
--- a/src/main/java/org/monetdb/client/JdbcClient.java +++ b/src/main/java/org/monetdb/client/JdbcClient.java @@ -9,8 +9,10 @@ package org.monetdb.client; import org.monetdb.jdbc.MonetDriver; +import org.monetdb.jdbc.MonetConnection; import org.monetdb.util.CmdLineOpts; import org.monetdb.util.Exporter; +import org.monetdb.util.FileTransferHandler; import org.monetdb.util.MDBvalidator; import org.monetdb.util.OptionsException; import org.monetdb.util.SQLExporter; @@ -25,6 +27,7 @@ import java.io.File; import java.io.PrintWriter; import java.net.HttpURLConnection; import java.net.URL; +import java.nio.charset.Charset; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.DriverManager; // this import is required as it will trigger loading the org.monetdb.jdbc.MonetDriver class @@ -65,7 +68,8 @@ public class JdbcClient { /* cannot (yet * * <pre>Usage java -jar jdbcclient.jre8.jar * [-h host[:port]] [-p port] [-f file] [-u user] - * [-l language] [-d database] [-e] [-D [table]] [-X<opt>] + * [-l language] [-d database] [-e] [-D [table]] + * [--csvdir /path/to/csvfiles] [-X<opt>] * | [--help] | [--version] * or using long option equivalents --host --port --file --user --language * --dump --echo --database. @@ -98,6 +102,8 @@ public class JdbcClient { /* cannot (yet * -e --echo Also outputs the contents of the input file, if any. * -q --quiet Suppress printing the welcome header. * -D --dump Dumps the given table(s), or the complete database if none given. + * --csvdir The directory path where csv data files wil be read from or + ^ written to when COPY ... ON CLIENT commands are executed. * -Xoutput The output mode when dumping. Default is sql, xml may be used for * an experimental XML output. * -Xhash Use the given hash algorithm during challenge response. @@ -142,6 +148,9 @@ public class JdbcClient { /* cannot (yet "if connecting to monetdbd)."); copts.addOption("l", "language", CmdLineOpts.CAR_ONE, "sql", "Use the given language, defaults to 'sql'."); + copts.addOption(null, "csvdir", CmdLineOpts.CAR_ONE, null, + "The directory path where csv data files are read or " + + "written when using ON CLIENT clause of COPY command."); // arguments which have no argument(s) copts.addOption(null, "help", CmdLineOpts.CAR_ZERO, null, @@ -220,7 +229,8 @@ public class JdbcClient { /* cannot (yet System.out.print( "Usage java -jar jdbcclient.jre8.jar\n" + "\t\t[-h host[:port]] [-p port] [-f file] [-u user]\n" + - "\t\t[-l language] [-d database] [-e] [-D [table]] [-X<opt>]\n" + + "\t\t[-l language] [-d database] [-e] [-D [table]]\n" + + "\t\t[--csvdir /path/to/csvfiles]] [-X<opt>]\n" + "\t\t| [--help] | [--version]\n" + "or using long option equivalents --host --port --file --user --language\n" + "--dump --echo --database.\n" + @@ -337,6 +347,28 @@ public class JdbcClient { /* cannot (yet dbmd = null; } + oc = copts.getOption("csvdir"); + if (oc.isPresent()) { + final String csvdir = oc.getArgument(); + if (csvdir != null) { + // check if provided csvdir is an existing dir + // else a download of data into file will terminate the JDBC connection!! + if (java.nio.file.Files.isDirectory(java.nio.file.Paths.get(csvdir))) { + final FileTransferHandler FThandler = new FileTransferHandler(csvdir, Charset.defaultCharset()); + + // register file data uploadHandler to allow support + // for: COPY INTO mytable FROM 'data.csv' ON CLIENT; + ((MonetConnection) con).setUploadHandler(FThandler); + + // register file data downloadHandler to allow support + // for: COPY select_query INTO 'data.csv' ON CLIENT; + ((MonetConnection) con).setDownloadHandler(FThandler); + } else { + System.err.println("Warning: provided csvdir \"" + csvdir + "\" does not exist. Ignoring csvdir setting."); + } + } + } + stmt = con.createStatement(); // is used by processInteractive(), processBatch(), doDump() in = new BufferedReader(new InputStreamReader(System.in));
--- a/src/main/java/org/monetdb/jdbc/MonetCallableStatement.java +++ b/src/main/java/org/monetdb/jdbc/MonetCallableStatement.java @@ -77,7 +77,7 @@ public class MonetCallableStatement * which deals with most of the required stuff of this class. * * @param connection the connection that created this Statement - * @param resultSetType type of {@link ResultSet} to produce + * @param resultSetType type of {@link java.sql.ResultSet} to produce * @param resultSetConcurrency concurrency of ResultSet to produce * @param callQuery - an SQL CALL statement that may contain one or more '?' parameter placeholders. * Typically this statement is specified using JDBC call escape syntax:
--- a/src/main/java/org/monetdb/jdbc/MonetConnection.java +++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java @@ -8,10 +8,10 @@ package org.monetdb.jdbc; -import java.io.File; -import java.io.IOException; +import java.io.*; import java.net.SocketException; import java.net.SocketTimeoutException; +import java.nio.charset.StandardCharsets; import java.sql.CallableStatement; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -35,6 +35,7 @@ import java.util.concurrent.Executor; import org.monetdb.mcl.io.BufferedMCLReader; import org.monetdb.mcl.io.BufferedMCLWriter; +import org.monetdb.mcl.io.LineType; import org.monetdb.mcl.net.HandshakeOptions; import org.monetdb.mcl.net.MapiSocket; import org.monetdb.mcl.parser.HeaderLineParser; @@ -67,7 +68,7 @@ import org.monetdb.mcl.parser.StartOfHea * * @author Fabian Groffen * @author Martin van Dinther - * @version 1.6 + * @version 1.7 */ public class MonetConnection extends MonetWrapper @@ -150,7 +151,6 @@ public class MonetConnection /** A cache to reduce the number of DatabaseMetaData objects created by getMetaData() to maximum 1 per connection */ private DatabaseMetaData dbmd; - /** * Constructor of a Connection for MonetDB. At this moment the * current implementation limits itself to storing the given host, @@ -1238,7 +1238,6 @@ public class MonetConnection throw newSQLFeatureNotSupportedException("createArrayOf"); } - /** * Constructs an object that implements the Clob interface. The * object returned initially contains no data. The setAsciiStream, @@ -1674,6 +1673,43 @@ public class MonetConnection //== internal helper methods which do not belong to the JDBC interface + /** Handler for COPY ... INTO ... FROM 'data-file-name' ON CLIENT requests */ + private UploadHandler uploadHandler; + /** Handler for COPY ... INTO 'data-file-name' ON CLIENT requests */ + private DownloadHandler downloadHandler; + + /** + * Registers a {@link UploadHandler} to support for example COPY INTO mytable FROM 'data.csv' ON CLIENT + * + * @param uploadHandler the handler to register, or null to deregister + */ + public void setUploadHandler(final UploadHandler uploadHandler) { + this.uploadHandler = uploadHandler; + } + + /** + * Returns the currently registerered {@link UploadHandler}, or null + */ + public UploadHandler getUploadHandler() { + return uploadHandler; + } + + /** + * Registers a {@link DownloadHandler} to support for example COPY select_result INTO 'data.csv' ON CLIENT + * + * @param downloadHandler the handler to register, or null to deregister + */ + public void setDownloadHandler(final DownloadHandler downloadHandler) { + this.downloadHandler = downloadHandler; + } + + /** + * Returns the currently registerered {@link DownloadHandler} handler, or null + */ + public DownloadHandler getDownloadHandler() { + return downloadHandler; + } + /** * Local helper method to test whether the Connection object is closed * When closed it throws an SQLException @@ -2100,7 +2136,7 @@ public class MonetConnection * @return a non-null String if the line is invalid, * or additional lines are not allowed. */ - public abstract String addLine(String line, int linetype); + String addLine(String line, LineType linetype); /** * Returns whether this Response expects more lines to be added @@ -2108,7 +2144,7 @@ public class MonetConnection * * @return true if a next line should be added, false otherwise */ - public abstract boolean wantsMore(); + boolean wantsMore(); /** * Indicates that no more header lines will be added to this @@ -2118,14 +2154,14 @@ public class MonetConnection * consistent or sufficient. */ /* MvD: disabled not used/needed code - public abstract void complete() throws SQLException; + void complete() throws SQLException; */ /** * Instructs the Response implementation to close and do the * necessary clean up procedures. */ - public abstract void close(); + void close(); } // }}} @@ -2259,12 +2295,12 @@ public class MonetConnection */ // {{{ addLine @Override - public String addLine(final String tmpLine, final int linetype) { + public String addLine(final String tmpLine, final LineType linetype) { if (isSet[LENS] && isSet[TYPES] && isSet[TABLES] && isSet[NAMES]) { return resultBlocks[0].addLine(tmpLine, linetype); } - if (linetype != BufferedMCLReader.HEADER) + if (linetype != LineType.HEADER) return "header expected, got: " + tmpLine; // depending on the name of the header, we continue @@ -2634,8 +2670,8 @@ public class MonetConnection * or additional lines are not allowed. */ @Override - public String addLine(final String line, final int linetype) { - if (linetype != BufferedMCLReader.RESULT) + public String addLine(final String line, final LineType linetype) { + if (linetype != LineType.RESULT) return "protocol violation: unexpected line in data block: " + line; // add to the backing array data[++pos] = line; @@ -2725,7 +2761,7 @@ public class MonetConnection } @Override - public String addLine(final String line, final int linetype) { + public String addLine(final String line, final LineType linetype) { return "Header lines are not supported for an UpdateResponse"; } @@ -2762,7 +2798,7 @@ public class MonetConnection public final int state = Statement.SUCCESS_NO_INFO; @Override - public String addLine(final String line, final int linetype) { + public String addLine(final String line, final LineType linetype) { return "Header lines are not supported for a SchemaResponse"; } @@ -2946,7 +2982,7 @@ public class MonetConnection * Internal executor of queries. * * @param templ the template to fill in - * @param the query to execute + * @param query the query to execute * @throws SQLException if a database error occurs */ @SuppressWarnings("fallthrough") @@ -2989,12 +3025,12 @@ public class MonetConnection // go for new results String tmpLine = in.readLine(); - int linetype = in.getLineType(); + LineType linetype = in.getLineType(); Response res = null; - while (linetype != BufferedMCLReader.PROMPT) { + while (linetype != LineType.PROMPT) { // each response should start with a start of header (or error) switch (linetype) { - case BufferedMCLReader.SOHEADER: + case SOHEADER: // make the response object, and fill it try { switch (sohp.parse(tmpLine)) { @@ -3014,7 +3050,7 @@ public class MonetConnection if (rowcount < tuplecount) { if (rsresponses == null) rsresponses = new HashMap<Integer, ResultSetResponse>(); - rsresponses.put(Integer.valueOf(id), (ResultSetResponse) res); + rsresponses.put(id, (ResultSetResponse) res); } } break; case StartOfHeaderParser.Q_UPDATE: @@ -3042,7 +3078,7 @@ public class MonetConnection final int offset = sohp.getNextAsInt(); final ResultSetResponse t; if (rsresponses != null) - t = rsresponses.get(Integer.valueOf(id)); + t = rsresponses.get(id); else t = null; if (t == null) { @@ -3099,18 +3135,37 @@ public class MonetConnection tmpLine = in.readLine(); linetype = in.getLineType(); break; - case BufferedMCLReader.INFO: + case INFO: addWarning(tmpLine.substring(1), "01000"); // read the next line (can be prompt, new result, error, etc.) // before we start the loop over tmpLine = in.readLine(); linetype = in.getLineType(); break; + case FILETRANSFER: + // Consume the command + String transferCommand = in.readLine(); + // Consume the fake prompt inserted by MapiSocket. + String dummy = in.readLine(); + // Handle the request + if (transferCommand != null) + error = handleTransfer(transferCommand); + else + error = "Protocol violation, expected transfer command, got nothing"; + // Then prepare for the next iteration + if (error != null) { + out.writeLine(error + "\n"); + error = in.waitForPrompt(); + } else { + tmpLine = in.readLine(); + } + linetype = in.getLineType(); + break; default: // Yeah... in Java this is correct! // we have something we don't expect/understand, let's make it an error message - tmpLine = "!M0M10!protocol violation, unexpected line: " + tmpLine; + tmpLine = "!M0M10!protocol violation, unexpected " + linetype + " line: " + tmpLine; // don't break; fall through... - case BufferedMCLReader.ERROR: + case ERROR: // read everything till the prompt (should be // error) we don't know if we ignore some // garbage here... but the log should reveal that @@ -3154,4 +3209,362 @@ public class MonetConnection } } // }}} + + private String handleTransfer(final String transferCommand) throws IOException { + if (transferCommand.startsWith("r ")) { + final String[] parts = transferCommand.split(" ", 3); + if (parts.length == 3) { + final long offset; + try { + offset = Long.parseLong(parts[1]); + } catch (NumberFormatException e) { + return e.toString(); + } + return handleUpload(parts[2], true, offset); + } + } else if (transferCommand.startsWith("rb ")) { + return handleUpload(transferCommand.substring(3), false, 0); + } else if (transferCommand.startsWith("w ")) { + return handleDownload(transferCommand.substring(2)); + } + return "JDBC does not support this file transfer yet: " + transferCommand; + } + + private String handleUpload(final String path, final boolean textMode, final long offset) throws IOException { + if (uploadHandler == null) { + return "No file upload handler has been registered with the JDBC driver"; + } + + final long linesToSkip = offset >= 1 ? offset - 1 : 0; + final Upload handle = new Upload(server, uploadHandler::uploadCancelled); + final boolean wasFaking = server.setInsertFakePrompts(false); + try { + uploadHandler.handleUpload(handle, path, textMode, linesToSkip); + if (!handle.hasBeenUsed()) { + throw new IOException("Call to " + uploadHandler.getClass().getCanonicalName() + ".handleUpload for path '" + path + "' sent neither data nor an error message"); + } + handle.close(); + } finally { + server.setInsertFakePrompts(wasFaking); + } + return handle.getError(); + } + + private String handleDownload(final String path) throws IOException { + if (downloadHandler == null) { + return "No file download handler has been registered with the JDBC driver"; + } + + final Download handle = new Download(server); + try { + downloadHandler.handleDownload(handle, path, true); + if (!handle.hasBeenUsed()) { + handle.sendError("Call to " + downloadHandler.getClass().getSimpleName() + ".handleDownload sent neither data nor error"); + } + } finally { + handle.close(); + } + return handle.getError(); + } + + /** + * Callback for sending files for COPY INTO "table" FROM 'file-name' ON CLIENT commands + * + * To be registered with {@link MonetConnection#setUploadHandler(UploadHandler)} + * + * An example implementation can be found at ../util/FileTransferHandler.java + */ + + public interface UploadHandler { + /** + * Called if the server sends a request to read file data. + * + * Use the given handle to receive data or send errors to the server. + * @param handle Handle to communicate with the server + * @param name Name of the file the server would like to read. Make sure + * to validate this before reading from the file system + * @param textMode Whether to open the file as text or binary data. + * @param linesToSkip In text mode, number of initial lines to skip. + * 0 means upload everything, 1 means skip the first line, etc. + * Note: this is different from the OFFSET option of the COPY INTO, + * where both 0 and 1 mean 'upload everything' + */ + void handleUpload(Upload handle, String name, boolean textMode, long linesToSkip) throws IOException; + + /** + * Called when the upload is cancelled halfway by the server. + * + * The default implementation does nothing. + */ + default void uploadCancelled() {} + } + + /** + * Callback for receiving files from COPY .. INTO 'file-name' ON CLIENT commands + * + * To be registered with {@link MonetConnection#setDownloadHandler(DownloadHandler)} + * + * An example implementation can be found at ../util/FileTransferHandler.java + */ + public interface DownloadHandler { + /** + * Called if the server sends a request to write a file. + * + * Use the given handle to send data or errors to the server. + * + * @param handle Handle to communicate with the server + * @param name Name of the file the server would like to write. Make sure to validate this before writing to + * the file system + * @param textMode Whether this is text or binary data. + */ + void handleDownload(Download handle, String name, boolean textMode) throws IOException; + } + + /** + * Handle passed to {@link UploadHandler} to allow communication with the server + */ + public static class Upload { + private final MapiSocket server; + private final Runnable cancellationCallback; + private PrintStream print = null; + private String error = null; + private int customChunkSize = -1; + + Upload(MapiSocket server, Runnable cancellationCallback) { + this.server = server; + this.cancellationCallback = cancellationCallback; + } + + /** + * Send an error message to the server + * + * The server will generally let the currently executing statement fail + * with this error message. The connection will remain usable. + * + * This method can only be sent if no data has been sent to the server + * yet. After data has been sent, you can still throw an + * {@link IOException} but this will terminate the connection. + * @param errorMessage error message to send + */ + public void sendError(final String errorMessage) throws IOException { + if (error != null) { + throw new IOException("another error has already been sent: " + error); + } + error = errorMessage; + } + + /** + * After every {@code chunkSize} bytes, the server gets the opportunity to + * terminate the upload. + */ + public void setChunkSize(final int chunkSize) { + this.customChunkSize = chunkSize; + } + + /** + * Get a {@link PrintStream} to write data to. + * + * For text mode uploads, the data MUST be validly UTF-8 encoded. + */ + public PrintStream getStream() throws IOException { + if (error != null) { + throw new IOException("Cannot send data after an error has been sent"); + } + if (print == null) { + try { + final MapiSocket.UploadStream up = customChunkSize >= 0 ? server.uploadStream(customChunkSize) : server.uploadStream(); + up.setCancellationCallback(cancellationCallback); + print = new PrintStream(up, false, "UTF-8"); + up.write('\n'); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("The system is guaranteed to support the UTF-8 encoding but apparently it doesn't", e); + } + } + return print; + } + + /** + * @return true if data or an error has been sent. + */ + public boolean hasBeenUsed() { + return print != null || error != null; + } + + /** + * @return the error that was sent, if any + */ + public String getError() { + return error; + } + + /** + * Read from the given input stream and write it to the server. + * + * For text mode uploads, the data MUST be validly UTF-8 encoded. + */ + public void uploadFrom(final InputStream inputStream) throws IOException { + final OutputStream s = getStream(); + final byte[] buffer = new byte[64 * 1024]; + while (true) { + int nread = inputStream.read(buffer); + if (nread < 0) { + break; + } + s.write(buffer, 0, nread); + } + } + + /** + * Read data from the given buffered reader and send it to the server + * @param reader reader to read from + * @param linesToSkip start uploading at line {@code offset}. Value 0 and 1 + * both mean upload the whole file, value 2 means skip the first line, etc.q + */ + public void uploadFrom(final BufferedReader reader, final long linesToSkip) throws IOException { + for (int i = 0; i < linesToSkip; i++) { + String line = reader.readLine(); + if (line == null) { + return; + } + } + + uploadFrom(reader); + } + + /** + * Read data from the given buffered reader and send it to the server + * @param reader reader to read from + */ + public void uploadFrom(final Reader reader) throws IOException { + final OutputStream s = getStream(); + final OutputStreamWriter writer = new OutputStreamWriter(s, StandardCharsets.UTF_8); + final char[] buffer = new char[64 * 1024]; + while (true) { + int nread = reader.read(buffer, 0, buffer.length); + if (nread < 0) { + break; + } + writer.write(buffer, 0, nread); + writer.close(); + } + } + + /** + * Close opened {@link PrintStream}. + */ + public void close() { + if (print != null) { + print.close(); + print = null; + } + } + } + + /** + * Handle passed to {@link DownloadHandler} to allow communication with the server + */ + public static class Download { + private final MapiSocket server; + private MapiSocket.DownloadStream stream = null; + private String error = null; + + Download(MapiSocket server) { + this.server = server; + } + + /** + * Send an error message to the server + * + * The server will generally let the currently executing statement fail + * with this error message. The connection will remain usable. + * + * This method can only be sent if no data has been received from the server + * yet. After data has been received, you can still throw an + * {@link IOException} but this will terminate the connection. + * + * Note: as of MonetDB version Jul2021 the server always terminates the connection + * when this error is used. This will probably change in the future. + */ + public void sendError(String errorMessage) throws IOException { + if (error != null) { + throw new IOException("another error has already been sent: " + error); + } + error = errorMessage; + } + + /** + * Get an {@link InputStream} to read data from. + * + * Textual data is UTF-8 encoded. + */ + public InputStream getStream() throws IOException { + if (error != null) { + throw new IOException("cannot receive data after error has been sent"); + } + if (stream == null) { + stream = server.downloadStream(); + server.getOutputStream().flush(); + } + return stream; + } + + /** + * Write the data from the server to the given {@link OutputStream}. + */ + public void downloadTo(final OutputStream stream) throws IOException { + final InputStream s = getStream(); + final byte[] buffer = new byte[65536]; + while (true) { + int nread = s.read(buffer); + if (nread < 0) { + break; + } + stream.write(buffer, 0, nread); + } + } + + /** + * Write the textual data from the server to the given {@link Writer} + * @param writer + */ + public void downloadTo(final Writer writer) throws IOException { + final InputStream s = getStream(); + final InputStreamReader r = new InputStreamReader(s, StandardCharsets.UTF_8); + final char[] buffer = new char[65536]; + while (true) { + final int nread = r.read(buffer); + if (nread < 0) + break; + writer.write(buffer, 0, nread); + } + } + + /** + * @return true if data has been received or an error has been sent. + */ + public boolean hasBeenUsed() { + return error != null || stream != null; + } + + /** + * @return the error that was sent, if any + */ + public String getError() { + return error; + } + + /** + * Close opened stream. + */ + public void close() { + if (stream != null) { + try { + stream.close(); + stream = null; + } catch (IOException e) { + /* ignore close error */ + } + } + } + } }
--- a/src/main/java/org/monetdb/jdbc/MonetPreparedStatement.java +++ b/src/main/java/org/monetdb/jdbc/MonetPreparedStatement.java @@ -720,7 +720,7 @@ public class MonetPreparedStatement /** * Gets the designated column's name * - * @param column the first column is 1, the second is 2, ... + * @param colnr column number, the first column is 1, the second is 2, ... * @return the column name * @throws SQLException if there is no such column */
--- a/src/main/java/org/monetdb/jdbc/MonetResultSet.java +++ b/src/main/java/org/monetdb/jdbc/MonetResultSet.java @@ -1267,7 +1267,7 @@ public class MonetResultSet /** * A private utility method to check validity of column index number - * @throws an SQLDataException when invalid column index number + * @throws SQLDataException when invalid column index number */ private final void checkColumnIndexValidity(final int column) throws SQLDataException { if (column < 1 || column > columns.length) @@ -2717,7 +2717,7 @@ public class MonetResultSet * The dates are parsed with the given Calendar. * * @param cal the Calendar to use/fill when parsing the date/time - * @param col the column to parse + * @param columnIndex the column to parse * @param type the corresponding java.sql.Types type of the calling function * @return the fractional seconds (nanos) or -1 if the value is NULL * @throws SQLException if a database error occurs
--- a/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java +++ b/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java @@ -41,27 +41,9 @@ import java.io.UnsupportedEncodingExcept * @see org.monetdb.mcl.io.BufferedMCLWriter */ public final class BufferedMCLReader extends BufferedReader { - /** "there is currently no line", or the the type is unknown is represented by UNKNOWN */ - public final static int UNKNOWN = 0; - /** a line starting with ! indicates ERROR */ - public final static int ERROR = '!'; - /** a line starting with % indicates HEADER */ - public final static int HEADER = '%'; - /** a line starting with [ indicates RESULT */ - public final static int RESULT = '['; - /** a line which matches the pattern of prompt1 is a PROMPT */ - public final static int PROMPT = '.'; - /** a line which matches the pattern of prompt2 is a MORE */ - public final static int MORE = ','; - /** a line starting with & indicates the start of a header block */ - public final static int SOHEADER = '&'; - /** a line starting with ^ indicates REDIRECT */ - public final static int REDIRECT = '^'; - /** a line starting with # indicates INFO */ - public final static int INFO = '#'; /** The type of the last line read */ - private int lineType = UNKNOWN; + private LineType lineType = LineType.UNKNOWN; /** * Create a buffering character-input stream that uses a @@ -107,7 +89,7 @@ public final class BufferedMCLReader ext public String readLine() throws IOException { String r = super.readLine(); setLineType(r); - if (lineType == ERROR && r != null && !r.matches("^![0-9A-Z]{5}!.+")) { + if (lineType == LineType.ERROR && r != null && !r.matches("^![0-9A-Z]{5}!.+")) { r = "!22000!" + r.substring(1); } return r; @@ -120,39 +102,7 @@ public final class BufferedMCLReader ext * @param line the string to examine */ public void setLineType(final String line) { - if (line == null || line.isEmpty()) { - lineType = UNKNOWN; - return; - } - switch (line.charAt(0)) { - case '.': - lineType = PROMPT; - break; - case ',': - lineType = MORE; - break; - case '[': /* multi field result */ - case '=': /* single value result */ - lineType = RESULT; - break; - case '%': - lineType = HEADER; - break; - case '&': - lineType = SOHEADER; - break; - case '#': - lineType = INFO; - break; - case '!': - lineType = ERROR; - break; - case '^': - lineType = REDIRECT; - break; - default: - lineType = UNKNOWN; - } + lineType = LineType.classify(line); } /** @@ -162,7 +112,7 @@ public final class BufferedMCLReader ext * following constants: UNKNOWN, HEADER, ERROR, PROMPT, MORE, * RESULT, SOHEADER, REDIRECT, INFO */ - public int getLineType() { + public LineType getLineType() { return lineType; } @@ -184,11 +134,11 @@ public final class BufferedMCLReader ext final StringBuilder ret = new StringBuilder(128); String tmp; - while (lineType != PROMPT) { + while (lineType != LineType.PROMPT) { tmp = readLine(); if (tmp == null) throw new IOException("Connection to server lost!"); - if (lineType == ERROR) + if (lineType == LineType.ERROR) ret.append('\n').append(tmp.substring(1)); } return ret.length() == 0 ? null : ret.toString().trim();
new file mode 100644 --- /dev/null +++ b/src/main/java/org/monetdb/mcl/io/LineType.java @@ -0,0 +1,115 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V. + */ + +package org.monetdb.mcl.io; + +/** + * Enumeration of the various message types used in the MAPI protocol. + */ +public enum LineType { + /** "there is currently no line", or the type is unknown is represented by UNKNOWN */ + UNKNOWN(null), + + /** a line starting with ! indicates ERROR */ + ERROR(new byte[] { '!' }), + + /** a line starting with % indicates HEADER */ + HEADER(new byte[] { '%' }), + + /** a line starting with [ indicates RESULT */ + RESULT(new byte[] { '[' }), + + /** a line which matches the pattern of prompt1 is a PROMPT */ + PROMPT(new byte[] { 1, 1 }), + + /** a line which matches the pattern of prompt2 is a MORE */ + MORE(new byte[] { 1, 2 }), + + /** a line which matches the pattern of prompt3 is a FILETRANSFER */ + FILETRANSFER(new byte[] { 1, 3 }), + + /** a line starting with & indicates the start of a header block */ + SOHEADER(new byte[] { '&' }), + + /** a line starting with ^ indicates REDIRECT */ + REDIRECT(new byte[] { '^' }), + + /** a line starting with # indicates INFO */ + INFO(new byte[] { '#' }); + + private final byte[] bytes; + + LineType(byte[] bytes) { + this.bytes = bytes; + } + + public final byte[] bytes() { + return this.bytes; + } + + /** + * Look at a mapi message and decide the LineType + */ + public static final LineType classify(String line) { + if (line != null) { + if (line.length() > 1) { + return classify(line.charAt(0), line.charAt(1)); + } else if (line.length() == 1) { + return classify(line.charAt(0), 0); + } + } + return UNKNOWN; + } + + /** + * Look at a mapi message and decide the LineType + */ + public static final LineType classify(byte[] line) { + if (line != null) { + if (line.length > 1) { + return classify(line[0], line[1]); + } else if (line.length == 1) { + return classify(line[0], 0); + } + } + return UNKNOWN; + } + + private static final LineType classify(int ch0, int ch1) { + switch (ch0) { + case '!': + return ERROR; + case '%': + return HEADER; + case '[': + return RESULT; + case '&': + return SOHEADER; + case '^': + return REDIRECT; + case '#': + return INFO; + case 1: + // prompts, see below + break; + default: + return UNKNOWN; + } + + switch (ch1) { + case 1: + return PROMPT; + case 2: + return MORE; + case 3: + return FILETRANSFER; + default: + return UNKNOWN; + } + } +}
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java +++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java @@ -22,6 +22,7 @@ import java.net.Socket; import java.net.SocketException; import java.net.UnknownHostException; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -31,6 +32,7 @@ import java.util.List; import org.monetdb.mcl.MCLException; import org.monetdb.mcl.io.BufferedMCLReader; import org.monetdb.mcl.io.BufferedMCLWriter; +import org.monetdb.mcl.io.LineType; import org.monetdb.mcl.parser.MCLParseException; /** @@ -79,7 +81,7 @@ import org.monetdb.mcl.parser.MCLParseEx * geared towards the format of the data. * * @author Fabian Groffen - * @version 4.1 + * @version 4.2 * @see org.monetdb.mcl.io.BufferedMCLReader * @see org.monetdb.mcl.io.BufferedMCLWriter */ @@ -89,7 +91,7 @@ public class MapiSocket { /* cannot (yet /** The TCP Socket timeout in milliseconds. Default is 0 meaning the timeout is disabled (i.e., timeout of infinity) */ private int soTimeout = 0; /** Stream from the Socket for reading */ - private InputStream fromMonet; + private BlockInputStream fromMonet; /** Stream from the Socket for writing */ private OutputStream toMonet; /** MCLReader on the InputStream */ @@ -293,7 +295,7 @@ public class MapiSocket { /* cannot (yet final ArrayList<String> redirects = new ArrayList<String>(); final List<String> warns = new ArrayList<String>(); String err = "", tmp; - int lineType; + LineType lineType; do { tmp = reader.readLine(); if (tmp == null) @@ -301,14 +303,14 @@ public class MapiSocket { /* cannot (yet con.getInetAddress().getHostName() + ":" + con.getPort() + ": End of stream reached"); lineType = reader.getLineType(); - if (lineType == BufferedMCLReader.ERROR) { + if (lineType == LineType.ERROR) { err += "\n" + tmp.substring(7); - } else if (lineType == BufferedMCLReader.INFO) { + } else if (lineType == LineType.INFO) { warns.add(tmp.substring(1)); - } else if (lineType == BufferedMCLReader.REDIRECT) { + } else if (lineType == LineType.REDIRECT) { redirects.add(tmp.substring(1)); } - } while (lineType != BufferedMCLReader.PROMPT); + } while (lineType != LineType.PROMPT); if (err.length() > 0) { close(); @@ -344,26 +346,32 @@ public class MapiSocket { /* cannot (yet int pos = args[i].indexOf("="); if (pos > 0) { tmp = args[i].substring(0, pos); - if (tmp.equals("database")) { - tmp = args[i].substring(pos + 1); - if (!tmp.equals(database)) { - warns.add("redirect points to different database: " + tmp); - setDatabase(tmp); - } - } else if (tmp.equals("language")) { - tmp = args[i].substring(pos + 1); - warns.add("redirect specifies use of different language: " + tmp); - setLanguage(tmp); - } else if (tmp.equals("user")) { - tmp = args[i].substring(pos + 1); - if (!tmp.equals(user)) - warns.add("ignoring different username '" + tmp + "' set by " + - "redirect, what are the security implications?"); - } else if (tmp.equals("password")) { - warns.add("ignoring different password set by redirect, " + - "what are the security implications?"); - } else { - warns.add("ignoring unknown argument '" + tmp + "' from redirect"); + switch (tmp) { + case "database": + tmp = args[i].substring(pos + 1); + if (!tmp.equals(database)) { + warns.add("redirect points to different database: " + tmp); + setDatabase(tmp); + } + break; + case "language": + tmp = args[i].substring(pos + 1); + warns.add("redirect specifies use of different language: " + tmp); + setLanguage(tmp); + break; + case "user": + tmp = args[i].substring(pos + 1); + if (!tmp.equals(user)) + warns.add("ignoring different username '" + tmp + "' set by " + + "redirect, what are the security implications?"); + break; + case "password": + warns.add("ignoring different password set by redirect, " + + "what are the security implications?"); + break; + default: + warns.add("ignoring unknown argument '" + tmp + "' from redirect"); + break; } } else { warns.add("ignoring illegal argument from redirect: " + args[i]); @@ -423,6 +431,7 @@ public class MapiSocket { /* cannot (yet * @param language the language to use * @param database the database to connect to * @param hash the hash method(s) to use, or NULL for all supported hashes + * @return the response string for the server */ private String getChallengeResponse( final String chalstr, @@ -454,27 +463,30 @@ public class MapiSocket { /* cannot (yet String pwhash = chaltok[5]; /* NOTE: Java doesn't support RIPEMD160 :( */ /* see: https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#MessageDigest */ - if (pwhash.equals("SHA512")) { - algo = "SHA-512"; - } else if (pwhash.equals("SHA384")) { - algo = "SHA-384"; - } else if (pwhash.equals("SHA256")) { - algo = "SHA-256"; - /* NOTE: Java 7 doesn't support SHA-224. Java 8 does but we have not tested it. It is also not requested yet. */ - } else if (pwhash.equals("SHA1")) { - algo = "SHA-1"; - } else { - /* Note: MD5 has been deprecated by security experts and support is removed from Oct 2020 release */ - throw new MCLException("Unsupported password hash: " + pwhash); + switch (pwhash) { + case "SHA512": + algo = "SHA-512"; + break; + case "SHA384": + algo = "SHA-384"; + break; + case "SHA256": + algo = "SHA-256"; + /* NOTE: Java 7 doesn't support SHA-224. Java 8 does but we have not tested it. It is also not requested yet. */ + break; + case "SHA1": + algo = "SHA-1"; + break; + default: + /* Note: MD5 has been deprecated by security experts and support is removed from Oct 2020 release */ + throw new MCLException("Unsupported password hash: " + pwhash); } try { final MessageDigest md = MessageDigest.getInstance(algo); - md.update(password.getBytes("UTF-8")); + md.update(password.getBytes(StandardCharsets.UTF_8)); password = toHex(md.digest()); } catch (NoSuchAlgorithmException e) { - throw new MCLException("This JVM does not support password hash: " + pwhash + "\n" + e.toString()); - } catch (UnsupportedEncodingException e) { - throw new MCLException("This JVM does not support UTF-8 encoding\n" + e.toString()); + throw new MCLException("This JVM does not support password hash: " + pwhash + "\n" + e); } // proto 7 (finally) used the challenge and works with a @@ -517,13 +529,11 @@ public class MapiSocket { /* cannot (yet } try { final MessageDigest md = MessageDigest.getInstance(algo); - md.update(password.getBytes("UTF-8")); - md.update(chaltok[0].getBytes("UTF-8")); // salt/key + md.update(password.getBytes(StandardCharsets.UTF_8)); + md.update(chaltok[0].getBytes(StandardCharsets.UTF_8)); // salt/key pwhash += toHex(md.digest()); } catch (NoSuchAlgorithmException e) { - throw new MCLException("This JVM does not support password hash: " + pwhash + "\n" + e.toString()); - } catch (UnsupportedEncodingException e) { - throw new MCLException("This JVM does not support UTF-8 encoding\n" + e.toString()); + throw new MCLException("This JVM does not support password hash: " + pwhash + "\n" + e); } // TODO: some day when we need this, we should store this @@ -540,11 +550,9 @@ public class MapiSocket { /* cannot (yet + username + ":" + pwhash + ":" + language + ":" - + (database == null ? "" : database) + ":"; + + (database == null ? "" : database) + ":" + + "FILETRANS:"; // this capability is added in monetdb-jdbc-3.2.jre8.jar if (chaltok.length > 6) { - // this ':' delimits the FILETRANS field, currently empty because we don't support it. - response += ":"; - // if supported, send handshake options for (String part : chaltok[6].split(",")) { if (part.startsWith("sql=") && handshakeOptions != null) { @@ -560,7 +568,6 @@ public class MapiSocket { /* cannot (yet } // this ':' delimits the handshake options field. response += ":"; - } return response; default: @@ -591,7 +598,6 @@ public class MapiSocket { /* cannot (yet : (char) ('0' + n); } - /** * Returns an InputStream that reads from this open connection on * the MapiSocket. @@ -721,6 +727,14 @@ public class MapiSocket { /* cannot (yet } /** + * For internal use + */ + public boolean setInsertFakePrompts(boolean b) { + return fromMonet.setInsertFakePrompts(b); + } + + + /** * Inner class that is used to write data on a normal stream as a * blocked stream. A call to the flush() method will write a * "final" block to the underlying stream. Non-final blocks are @@ -797,7 +811,7 @@ public class MapiSocket { /* cannot (yet } else { log("TD ", "write block: " + writePos + " bytes", false); } - log("TX ", new String(block, 0, writePos, "UTF-8"), true); + log("TX ", new String(block, 0, writePos, StandardCharsets.UTF_8), true); } writePos = 0; @@ -818,9 +832,8 @@ public class MapiSocket { /* cannot (yet @Override public void write(final byte[] b, int off, int len) throws IOException { - int t = 0; while (len > 0) { - t = BLOCK - writePos; + int t = BLOCK - writePos; if (len > t) { System.arraycopy(b, off, block, writePos, t); off += t; @@ -851,7 +864,9 @@ public class MapiSocket { /* cannot (yet final class BlockInputStream extends FilterInputStream { private int readPos = 0; private int blockLen = 0; + private boolean wasEndBlock = false; private final byte[] block = new byte[BLOCK + 3]; // \n.\n + private boolean insertFakePrompts = true; /** * Constructs this BlockInputStream, backed by the given @@ -864,6 +879,12 @@ public class MapiSocket { /* cannot (yet super(new BufferedInputStream(in)); } + public boolean setInsertFakePrompts(boolean doFake) { + boolean old = insertFakePrompts; + insertFakePrompts = doFake; + return old; + } + @Override public int available() { return blockLen - readPos; @@ -907,7 +928,7 @@ public class MapiSocket { /* cannot (yet if (off > 0) { if (debug) { log("RD ", "the following incomplete block was received:", false); - log("RX ", new String(b, 0, off, "UTF-8"), true); + log("RX ", new String(b, 0, off, StandardCharsets.UTF_8), true); } throw new IOException("Read from " + con.getInetAddress().getHostName() + ":" + @@ -957,10 +978,12 @@ public class MapiSocket { /* cannot (yet (blklen[0] & 0xFF) >> 1 | (blklen[1] & 0xFF) << 7 ); + wasEndBlock = (blklen[0] & 0x1) == 1; + readPos = 0; if (debug) { - if ((blklen[0] & 0x1) == 1) { + if (wasEndBlock) { log("RD ", "read final block: " + blockLen + " bytes", false); } else { log("RD ", "read new block: " + blockLen + " bytes", false); @@ -976,19 +999,24 @@ public class MapiSocket { /* cannot (yet return -1; if (debug) - log("RX ", new String(block, 0, blockLen, "UTF-8"), true); + log("RX ", new String(block, 0, blockLen, StandardCharsets.UTF_8), true); // if this is the last block, make it end with a newline and prompt - if ((blklen[0] & 0x1) == 1) { - if (blockLen > 0 && block[blockLen - 1] != '\n') { - // to terminate the block in a Reader + if (wasEndBlock) { + // insert 'fake' newline and prompt + if (insertFakePrompts) { + if (blockLen > 0 && block[blockLen - 1] != '\n') { + // to terminate the block in a Reader + block[blockLen++] = '\n'; + } + for (byte b : LineType.PROMPT.bytes()) { + block[blockLen++] = b; + } block[blockLen++] = '\n'; + if (debug) { + log("RD ", "inserting prompt", true); + } } - // insert 'fake' flush - block[blockLen++] = BufferedMCLReader.PROMPT; - block[blockLen++] = '\n'; - if (debug) - log("RD ", "inserting prompt", true); } return blockLen; @@ -1002,7 +1030,7 @@ public class MapiSocket { /* cannot (yet } if (debug) - log("RX ", new String(block, readPos, 1, "UTF-8"), true); + log("RX ", new String(block, readPos, 1, StandardCharsets.UTF_8), true); return (int)block[readPos++]; } @@ -1047,9 +1075,8 @@ public class MapiSocket { /* cannot (yet @Override public long skip(final long n) throws IOException { long skip = n; - int t = 0; while (skip > 0) { - t = available(); + int t = available(); if (skip > t) { skip -= t; readPos += t; @@ -1061,6 +1088,47 @@ public class MapiSocket { /* cannot (yet } return n; } + + /** + * For internal use + */ + Raw getRaw() { + return new Raw(); + } + + /** An alternative I/O interface that exposes the block based nature of the MAPI protocol */ + final class Raw { + byte[] getBytes() { + return block; + } + + int getLength() { + return blockLen; + } + + int getPosition() { + return readPos; + } + + int consume(int delta) { + int pos = readPos; + readPos += delta; + return pos; + } + + int readBlock() throws IOException { + boolean wasFaking = setInsertFakePrompts(false); + try { + return BlockInputStream.this.readBlock(); + } finally { + setInsertFakePrompts(wasFaking); + } + } + + boolean wasEndBlock() { + return wasEndBlock; + } + } } /** @@ -1108,6 +1176,34 @@ public class MapiSocket { /* cannot (yet } /** + * Return an UploadStream for use with for example COPY FROM filename ON CLIENT. + * + * Building block for {@link org.monetdb.jdbc.MonetConnection.UploadHandler}. + * @param chunkSize chunk size for the upload stream + */ + public UploadStream uploadStream(int chunkSize) { + return new UploadStream(chunkSize); + } + + /** + * Return an UploadStream for use with for example COPY FROM filename ON CLIENT. + * + * Building block for {@link org.monetdb.jdbc.MonetConnection.UploadHandler}. + */ + public UploadStream uploadStream() { + return new UploadStream(); + } + + /** + * Return a DownloadStream for use with for example COPY INTO filename ON CLIENT + * + * Building block for {@link org.monetdb.jdbc.MonetConnection.DownloadHandler}. + */ + public DownloadStream downloadStream() { + return new DownloadStream(fromMonet.getRaw(), toMonet); + } + + /** * Destructor called by garbage collector before destroying this * object tries to disconnect the MonetDB connection if it has not * been disconnected already. @@ -1120,4 +1216,233 @@ public class MapiSocket { /* cannot (yet close(); super.finalize(); } + + + /** + * Stream of data sent to the server + * + * Building block for {@link org.monetdb.jdbc.MonetConnection.UploadHandler}. + * + * An UploadStream has a chunk size. Every chunk size bytes, the server gets + * the opportunity to abort the upload. + */ + public class UploadStream extends FilterOutputStream { + public final static int DEFAULT_CHUNK_SIZE = 1024 * 1024; + private final int chunkSize; + private boolean closed = false; + private boolean serverCancelled = false; + private int chunkLeft; + private byte[] promptBuffer; + private Runnable cancellationCallback = null; + + /** Create an UploadStream with the given chunk size */ + UploadStream(final int chunkSize) { + super(toMonet); + if (chunkSize <= 0) { + throw new IllegalArgumentException("chunk size must be positive"); + } + this.chunkSize = chunkSize; + assert LineType.MORE.bytes().length == LineType.FILETRANSFER.bytes().length; + int promptLen = LineType.MORE.bytes().length; + promptBuffer = new byte[promptLen + 1]; + chunkLeft = this.chunkSize; + } + + /** Create an UploadStream with the default chunk size */ + UploadStream() { + this(DEFAULT_CHUNK_SIZE); + } + + /** Set a callback to be invoked if the server cancels the upload + */ + public void setCancellationCallback(final Runnable cancellationCallback) { + this.cancellationCallback = cancellationCallback; + } + + @Override + public void write(final int b) throws IOException { + if (serverCancelled) { + // We have already thrown an exception and apparently that has been ignored. + // Probably because they're calling print methods instead of write. + // Throw another one, maybe they'll catch this one. + throw new IOException("Server aborted the upload"); + } + handleChunking(); + super.write(b); + wrote(1); + } + + @Override + public void write(final byte[] b) throws IOException { + write(b, 0, b.length); + } + + @Override + public void write(final byte[] b, int off, int len) throws IOException { + if (serverCancelled) { + // We have already thrown an exception and apparently that has been ignored. + // Probably because they're calling print methods instead of write. + // Throw another one, maybe they'll catch this one. + throw new IOException("Server aborted the upload"); + } + while (len > 0) { + handleChunking(); + int toWrite = Integer.min(len, chunkLeft); + super.write(b, off, toWrite); + off += toWrite; + len -= toWrite; + wrote(toWrite); + } + } + + @Override + public void flush() throws IOException { + // suppress flushes + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + + if (serverCancelled) + closeAfterServerCancelled(); + else + closeAfterSuccesfulUpload(); + } + + private void closeAfterSuccesfulUpload() throws IOException { + if (chunkLeft != chunkSize) { + // flush pending data + flushAndReadPrompt(); + } + // send empty block + out.flush(); + final LineType acknowledgement = readPrompt(); + if (acknowledgement != LineType.FILETRANSFER) { + throw new IOException("Expected server to acknowledge end of file"); + } + } + + private void closeAfterServerCancelled() { + // nothing to do here, we have already read the error prompt. + } + + private void wrote(final int i) { + chunkLeft -= i; + } + + private void handleChunking() throws IOException { + if (chunkLeft > 0) { + return; + } + flushAndReadPrompt(); + } + + private void flushAndReadPrompt() throws IOException { + out.flush(); + chunkLeft = chunkSize; + final LineType lineType = readPrompt(); + switch (lineType) { + case MORE: + return; + case FILETRANSFER: + // Note, if the caller is calling print methods instead of write, the IO exception gets hidden. + // This is unfortunate but there's nothing we can do about it. + serverCancelled = true; + if (cancellationCallback != null) { + cancellationCallback.run(); + } + throw new IOException("Server aborted the upload"); + default: + throw new IOException("Expected MORE/DONE from server, got " + lineType); + } + } + + private LineType readPrompt() throws IOException { + final int nread = fromMonet.read(promptBuffer); + if (nread != promptBuffer.length || promptBuffer[promptBuffer.length - 1] != '\n') { + throw new IOException("server return incomplete prompt"); + } + return LineType.classify(promptBuffer); + } + } + + + /** + * Stream of data received from the server + * + * Building block for {@link org.monetdb.jdbc.MonetConnection.DownloadHandler}. + */ + public static class DownloadStream extends InputStream { + private final BlockInputStream.Raw rawIn; + private final OutputStream out; + private boolean endBlockSeen = false; + private boolean closed = false; + + DownloadStream(BlockInputStream.Raw rawIn, OutputStream out) { + this.rawIn = rawIn; + this.out = out; + } + + void nextBlock() throws IOException { + if (endBlockSeen || closed) + return; + final int ret = rawIn.readBlock(); + if (ret < 0 || rawIn.wasEndBlock()) { + endBlockSeen = true; + } + } + + @Override + public void close() throws IOException { + if (closed) + return; + closed = true; + while (!endBlockSeen) { + nextBlock(); + } + // Send acknowledgement to server + out.write('\n'); + out.flush(); + // Do whatever super has to do + super.close(); + } + + @Override + public int read() throws IOException { + final byte[] buf = { 0 }; + final int nread = read(buf, 0, 1); + if (nread == 1) + return buf[0]; + else + return -1; + } + + @Override + public int read(final byte[] b, int off, int len) throws IOException { + final int origOff = off; + while (len > 0) { + int chunk = Integer.min(len, rawIn.getLength() - rawIn.getPosition()); + if (chunk > 0) { + // make progress copying some bytes + System.arraycopy(rawIn.getBytes(), rawIn.getPosition(), b, off, chunk); + off += chunk; + rawIn.consume(chunk); + len -= chunk; + } else { + // make progress fetching data + if (endBlockSeen) + break; + nextBlock(); + } + } + if (off == origOff && endBlockSeen) + return -1; + else + return off - origOff; + } + } }
--- a/src/main/java/org/monetdb/merovingian/Control.java +++ b/src/main/java/org/monetdb/merovingian/Control.java @@ -207,17 +207,17 @@ public class Control { mout.writeLine(database + " " + command + "\n"); ArrayList<String> l = new ArrayList<String>(); String tmpLine = min.readLine(); - int linetype = min.getLineType(); - if (linetype == BufferedMCLReader.ERROR) + LineType linetype = min.getLineType(); + if (linetype == LineType.ERROR) throw new MerovingianException(tmpLine.substring(6)); - if (linetype != BufferedMCLReader.RESULT) + if (linetype != LineType.RESULT) throw new MerovingianException("unexpected line: " + tmpLine); if (!tmpLine.substring(1).equals(RESPONSE_OK)) throw new MerovingianException(tmpLine.substring(1)); tmpLine = min.readLine(); linetype = min.getLineType(); - while (linetype != BufferedMCLReader.PROMPT) { - if (linetype != BufferedMCLReader.RESULT) + while (linetype != LineType.PROMPT) { + if (linetype != LineType.RESULT) throw new MerovingianException("unexpected line: " + tmpLine);
new file mode 100644 --- /dev/null +++ b/src/main/java/org/monetdb/util/FileTransferHandler.java @@ -0,0 +1,107 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V. + */ + +package org.monetdb.util; + +import org.monetdb.jdbc.MonetConnection; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +/** + * Sample implementation of COPY ... INTO 'file-name' ON CLIENT handling + * + * Can be registered with {@link MonetConnection#setUploadHandler(MonetConnection.UploadHandler)} + * and {@link MonetConnection#setDownloadHandler(MonetConnection.DownloadHandler)}. + * Implements uploads and downloads by reading and writing files on the file system. + */ +public class FileTransferHandler implements MonetConnection.UploadHandler, MonetConnection.DownloadHandler { + private final Path root; + private final Charset encoding; + + /** + * Create a new FileTransferHandler which serves the given directory. + * @param dir directory to read and write files from + * @param encoding set this to true if all files in the directory are known to be utf-8 encoded. + */ + public FileTransferHandler(final Path dir, final Charset encoding) { + this.root = dir.toAbsolutePath().normalize(); + this.encoding = encoding != null ? encoding: Charset.defaultCharset(); + } + + /** + * Create a new FileTransferHandler which serves the given directory. + * + * @param dir directory to read and write files from + * @param utf8Encoded set this to true if all files in the directory are known to be utf-8 encoded. + */ + public FileTransferHandler(final String dir, final Charset encoding) { + this(FileSystems.getDefault().getPath(dir), encoding); + } + + public void handleUpload(final MonetConnection.Upload handle, final String name, final boolean textMode, final long linesToSkip) throws IOException { + final Path path = root.resolve(name).normalize(); + if (!path.startsWith(root)) { + handle.sendError("File is not in upload directory"); + return; + } + if (!Files.isReadable(path)) { + handle.sendError("Cannot read " + name); + return; + } + if (!textMode) { + // must upload as a byte stream + handle.uploadFrom(Files.newInputStream(path)); + } else if (linesToSkip == 0 && utf8Encoded()) { + // more efficient to upload as a byte stream + handle.uploadFrom(Files.newInputStream(path)); + } else { + // cannot upload as a byte stream, must deal with encoding + final BufferedReader reader = Files.newBufferedReader(path, encoding); + handle.uploadFrom(reader, linesToSkip); + } + } + + public void handleDownload(final MonetConnection.Download handle, final String name, final boolean textMode) throws IOException { + final Path path = root.resolve(name).normalize(); + if (!path.startsWith(root)) { + handle.sendError("File is not in download directory"); + return; + } + if (Files.exists(path)) { + handle.sendError("File already exists: " + name); + return; + } + if (!textMode) { + // must download as a byte stream + final OutputStream outputStream = Files.newOutputStream(path, StandardOpenOption.CREATE_NEW); + handle.downloadTo(outputStream); + } else if (utf8Encoded()) { + // more efficient to download as a byte stream + final OutputStream outputStream = Files.newOutputStream(path, StandardOpenOption.CREATE_NEW); + handle.downloadTo(outputStream); + } else { + // cannot download as a byte stream, must deal with encoding + final BufferedWriter writer = Files.newBufferedWriter(path, encoding, StandardOpenOption.CREATE_NEW); + handle.downloadTo(writer); + writer.close(); + } + } + + public boolean utf8Encoded() { + return encoding.equals(StandardCharsets.UTF_8); + } +}
--- a/src/main/java/org/monetdb/util/SQLExporter.java +++ b/src/main/java/org/monetdb/util/SQLExporter.java @@ -427,7 +427,6 @@ public final class SQLExporter extends E * format. * * @param rs the ResultSet to convert into INSERT INTO statements - * @param absolute if true, dumps table name prepended with schema name * @throws SQLException if a database related error occurs */ private void resultSetToSQL(final ResultSet rs)
--- a/src/main/java/org/monetdb/util/SQLRestore.java +++ b/src/main/java/org/monetdb/util/SQLRestore.java @@ -15,6 +15,7 @@ import java.util.concurrent.atomic.Atomi import org.monetdb.mcl.io.BufferedMCLReader; import org.monetdb.mcl.io.BufferedMCLWriter; +import org.monetdb.mcl.io.LineType; import org.monetdb.mcl.net.MapiSocket; /** @@ -53,9 +54,9 @@ public final class SQLRestore { final String line = _is.readLine(); if (line == null) break; - final int result = _is.getLineType(); + final LineType result = _is.getLineType(); switch (result) { - case BufferedMCLReader.ERROR: + case ERROR: _errorMessage = line; _errorState.set(true); return;
--- a/tests/JDBC_API_Tester.java +++ b/tests/JDBC_API_Tester.java @@ -104,6 +104,11 @@ final public class JDBC_API_Tester { if (jt.foundDifferences) System.exit(-1); + + OnClientTester oct = new OnClientTester(con_URL, 0, true); + int failures = oct.runTests(""); + if (failures > 0) + System.exit(-1); } private void Test_Cautocommit(String arg0) {
new file mode 100644 --- /dev/null +++ b/tests/OnClientTester.java @@ -0,0 +1,576 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V. + */ + +import org.monetdb.jdbc.MonetConnection; +import org.monetdb.jdbc.MonetConnection.UploadHandler; +import org.monetdb.jdbc.MonetConnection.DownloadHandler; +import org.monetdb.util.FileTransferHandler; + +import java.io.*; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.List; + +import static java.nio.file.StandardOpenOption.CREATE_NEW; + +public final class OnClientTester extends TestRunner { + + public OnClientTester(String jdbcUrl, int verbosity, boolean watchDogEnabled) { + super(jdbcUrl, verbosity, watchDogEnabled); + } + + public static void main(String[] args) throws SQLException, NoSuchMethodException, ClassNotFoundException { + String jdbcUrl = null; + String requiredPrefix = null; + int verbosity = 0; + boolean watchDogEnabled = true; + + // Don't know why I need this all of a sudden.. is it only on my system? + Class.forName("org.monetdb.jdbc.MonetDriver"); + + for (String arg : args) { + if (arg.equals("-v")) + verbosity++; + else if (arg.equals("-vv")) + verbosity += 2; + else if (arg.equals("-w")) + watchDogEnabled = false; + else if (jdbcUrl == null) + jdbcUrl = arg; + else if (requiredPrefix == null) + requiredPrefix = arg; + else { + System.err.println("Unexpected argument " + arg); + System.exit(2); + } + } + + OnClientTester tester = new OnClientTester(jdbcUrl, verbosity, watchDogEnabled); + int failures = tester.runTests(requiredPrefix); + + if (failures > 0) + System.exit(1); + } + + /// Some tests have to work limitations of the protocol or bugs in the server. + /// This Enum is used to indicate the possibilities. + public enum BugFixLevel { + /// Only those tests that work with older MonetDB versions + Baseline(0, 0, 0), + + /// Connection keeps working after download request has been refused by client + CanRefuseDownload(11, 41, 12), + + ; + + private final int major; + private final int minor; + private final int micro; + + BugFixLevel(int major, int minor, int micro) { + this.major = major; + this.minor = minor; + this.micro = micro; + } + + boolean includesVersion(int major, int minor, int micro) { + if (major > this.major) + return true; + if (major < this.major) + return false; + if (minor > this.minor) + return true; + if (minor < this.minor) + return false; + return micro >= this.micro; + } + + static BugFixLevel forVersion(String version) { + String[] parts = version.split("[.]", 3); + assert parts.length == 3; + int major = Integer.parseInt(parts[0]); + int minor = Integer.parseInt(parts[1]); + int micro = Integer.parseInt(parts[2]); + + return BugFixLevel.forVersion(major, minor, micro); + } + + static BugFixLevel forVersion(int major, int minor, int micro) { + BugFixLevel lastValid = Baseline; + for (BugFixLevel level: BugFixLevel.values()) { + if (level.includesVersion(major, minor, micro)) + lastValid = level; + else + break; + } + return lastValid; + } + } + + void prepare() throws SQLException { + execute("DROP TABLE IF EXISTS foo"); + execute("CREATE TABLE foo (i INT, t TEXT)"); + } + + private BugFixLevel getLevel() throws SQLException, Failure { + String version = queryString("SELECT value FROM environment WHERE name = 'monet_version'"); + BugFixLevel level = BugFixLevel.forVersion(version); + out.println(" NOTE: version " + version + " means level = " + level); + return level; + } + + public void test_BugFixLevel() throws Failure { + assertEq("Baseline includes 0.0.0", true, BugFixLevel.Baseline.includesVersion(0, 0, 0)); + assertEq("Baseline includes 11.41.11", true, BugFixLevel.Baseline.includesVersion(11, 41, 11)); + assertEq("Baseline includes 11.41.12", true, BugFixLevel.Baseline.includesVersion(11, 41, 12)); + + assertEq("CanRefuseDownload includes 0.0.0", false, BugFixLevel.CanRefuseDownload.includesVersion(0, 0, 0)); + + assertEq("CanRefuseDownload includes 11.0.0", false, BugFixLevel.CanRefuseDownload.includesVersion(11, 0, 0)); + assertEq("CanRefuseDownload includes 12.0.0", true, BugFixLevel.CanRefuseDownload.includesVersion(12, 0, 0)); + + assertEq("CanRefuseDownload includes 11.41.0", false, BugFixLevel.CanRefuseDownload.includesVersion(11, 41, 0)); + assertEq("CanRefuseDownload includes 11.42.0", true, BugFixLevel.CanRefuseDownload.includesVersion(11, 42, 0)); + + assertEq("CanRefuseDownload includes 11.41.11", false, BugFixLevel.CanRefuseDownload.includesVersion(11, 41, 11)); + assertEq("CanRefuseDownload includes 11.41.12", true, BugFixLevel.CanRefuseDownload.includesVersion(11, 41, 12)); + + assertEq("Level for 0.0.0", BugFixLevel.Baseline, BugFixLevel.forVersion(0, 0, 0)); + assertEq("Level for 11.0.0", BugFixLevel.Baseline, BugFixLevel.forVersion(11, 0, 0)); + assertEq("Level for 11.41.0", BugFixLevel.Baseline, BugFixLevel.forVersion(11, 41, 0)); + assertEq("Level for 11.41.11", BugFixLevel.Baseline, BugFixLevel.forVersion(11, 41, 11)); + assertEq("Level for 11.41.12", BugFixLevel.CanRefuseDownload, BugFixLevel.forVersion(11, 41, 12)); + assertEq("Level for 11.42.0", BugFixLevel.CanRefuseDownload, BugFixLevel.forVersion(11, 42, 0)); + assertEq("Level for 12.0.0", BugFixLevel.CanRefuseDownload, BugFixLevel.forVersion(12, 0, 0)); + + assertEq("Level for \"11.41.11\"", BugFixLevel.Baseline, BugFixLevel.forVersion("11.41.11")); + assertEq("Level for \"11.41.12\"", BugFixLevel.CanRefuseDownload, BugFixLevel.forVersion("11.41.12")); + } + + public void test_Upload() throws SQLException, Failure { + prepare(); + MyUploadHandler handler = new MyUploadHandler(100); + conn.setUploadHandler(handler); + update("COPY INTO foo FROM 'banana' ON CLIENT"); + assertEq("cancellation callback called", false, handler.isCancelled()); + assertQueryInt("SELECT COUNT(*) FROM foo", 100); + } + + public void test_ClientRefusesUpload() throws SQLException, Failure { + prepare(); + MyUploadHandler handler = new MyUploadHandler("immediate error"); + conn.setUploadHandler(handler); + expectError("COPY INTO foo FROM 'banana' ON CLIENT", "immediate error"); + assertEq("cancellation callback called", false, handler.isCancelled()); + assertQueryInt("SELECT COUNT(*) FROM foo", 0); + } + + public void test_Offset0() throws SQLException, Failure { + prepare(); + MyUploadHandler handler = new MyUploadHandler(100); + conn.setUploadHandler(handler); + update("COPY OFFSET 0 INTO foo FROM 'banana' ON CLIENT"); + assertEq("cancellation callback called", false, handler.isCancelled()); + assertQueryInt("SELECT MIN(i) FROM foo", 1); + assertQueryInt("SELECT MAX(i) FROM foo", 100); + } + + public void test_Offset1() throws SQLException, Failure { + prepare(); + MyUploadHandler handler = new MyUploadHandler(100); + conn.setUploadHandler(handler); + update("COPY OFFSET 1 INTO foo FROM 'banana' ON CLIENT"); + assertEq("cancellation callback called", false, handler.isCancelled()); + assertQueryInt("SELECT MIN(i) FROM foo", 1); + assertQueryInt("SELECT MAX(i) FROM foo", 100); + } + + public void test_Offset5() throws SQLException, Failure { + prepare(); + MyUploadHandler handler = new MyUploadHandler(100); + conn.setUploadHandler(handler); + update("COPY OFFSET 5 INTO foo FROM 'banana' ON CLIENT"); + assertEq("cancellation callback called", false, handler.isCancelled()); + assertQueryInt("SELECT MIN(i) FROM foo", 5); + assertQueryInt("SELECT MAX(i) FROM foo", 100); + } + + public void test_ServerStopsReading() throws SQLException, Failure { + prepare(); + MyUploadHandler handler = new MyUploadHandler(100); + conn.setUploadHandler(handler); + update("COPY 10 RECORDS INTO foo FROM 'banana' ON CLIENT"); + assertEq("cancellation callback called", true, handler.isCancelled()); + assertEq("handler encountered write error", true, handler.encounteredWriteError()); + // connection is still alive + assertQueryInt("SELECT COUNT(i) FROM foo", 10); + } + + public void test_Download(int n) throws SQLException, Failure { + prepare(); + MyDownloadHandler handler = new MyDownloadHandler(); + conn.setDownloadHandler(handler); + String q = "INSERT INTO foo SELECT value as i, 'number' || value AS t FROM sys.generate_series(0, " + n + ")"; + update(q); + update("COPY (SELECT * FROM foo) INTO 'banana' ON CLIENT"); + assertEq("download attempts", 1, handler.countAttempts()); + assertEq("lines downloaded", n, handler.lineCount()); + // connection is still alive + assertQueryInt("SELECT COUNT(*) FROM foo", n); + } + + public void test_Download() throws SQLException, Failure { + test_Download(100); + } + + public void test_ClientRefusesDownload() throws SQLException, Failure { + prepare(); + BugFixLevel level = getLevel(); + MyDownloadHandler handler = new MyDownloadHandler("download refused"); + conn.setDownloadHandler(handler); + update("INSERT INTO foo SELECT value as i, 'number' || value AS t FROM sys.generate_series(0, 100)"); + expectError("COPY (SELECT * FROM foo) INTO 'banana' ON CLIENT", "download refused"); + // Wish it were different but the server closes the connection + expectError("SELECT 42 -- check if the connection still works", "Connection to server lost!"); + if (level.compareTo(BugFixLevel.CanRefuseDownload) >= 0) { + // connection is still alive + assertQueryInt("SELECT COUNT(*) FROM foo", 100); + } + } + + public void test_LargeUpload() throws SQLException, Failure { + watchDog.setDuration(25_000); + prepare(); + int n = 4_000_000; + MyUploadHandler handler = new MyUploadHandler(n); + conn.setUploadHandler(handler); + handler.setChunkSize(1024 * 1024); + update("COPY INTO foo FROM 'banana' ON CLIENT"); + assertEq("cancellation callback called", false, handler.isCancelled()); + // connection is still alive + assertQueryInt("SELECT COUNT(DISTINCT i) FROM foo", n); + } + + public void test_LargeDownload() throws SQLException, Failure { + watchDog.setDuration(25_000); + test_Download(4_000_000); + } + + public void test_UploadFromStream() throws SQLException, Failure { + prepare(); + UploadHandler handler = new UploadHandler() { + final String data = "1|one\n2|two\n3|three\n"; + + @Override + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { + // ignoring linesToSkip as it's not used in this test + ByteArrayInputStream s = new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8)); + handle.uploadFrom(s); + } + }; + conn.setUploadHandler(handler); + update("COPY INTO foo FROM 'banana' ON CLIENT"); + // connection is still alive + assertQueryInt("SELECT i FROM foo WHERE t = 'three'", 3); + } + + public void test_UploadFromReader() throws SQLException, Failure { + prepare(); + UploadHandler handler = new UploadHandler() { + final String data = "1|one\n2|two\n3|three\n"; + + @Override + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { + // ignoring linesToSkip as it's not used in this test + StringReader r = new StringReader(data); + handle.uploadFrom(r); + } + }; + conn.setUploadHandler(handler); + update("COPY INTO foo FROM 'banana' ON CLIENT"); + assertQueryInt("SELECT i FROM foo WHERE t = 'three'", 3); + } + + public void test_UploadFromReaderOffset() throws SQLException, Failure { + prepare(); + UploadHandler handler = new UploadHandler() { + final String data = "1|one\n2|two\n3|three\n"; + + @Override + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { + BufferedReader r = new BufferedReader(new StringReader(data)); + handle.uploadFrom(r, linesToSkip); + } + }; + conn.setUploadHandler(handler); + update("COPY OFFSET 2 INTO foo FROM 'banana' ON CLIENT"); + assertQueryInt("SELECT i FROM foo WHERE t = 'three'", 3); + } + + public void test_FailUploadLate() throws SQLException, Failure { + prepare(); + MyUploadHandler handler = new MyUploadHandler(100, 50, "i don't like line 50"); + conn.setUploadHandler(handler); + expectError("COPY INTO foo FROM 'banana' ON CLIENT", "i don't like"); + assertEq("cancellation callback called", false, handler.isCancelled()); + assertEq("connection is closed", true, conn.isClosed()); + } + + + public void test_FailUploadLate2() throws SQLException, Failure { + // Here we send empty lines only, to check if the server detects is properly instead + // of simply complaining about an incomplete file. + prepare(); + UploadHandler handler = new UploadHandler() { + @Override + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { + // ignoring linesToSkip as it's not used in this test + PrintStream stream = handle.getStream(); + for (int i = 1; i <= 20_000; i++) + stream.println(); + stream.flush(); + throw new IOException("exception after all"); + } + }; + conn.setUploadHandler(handler); + expectError("COPY INTO foo(t) FROM 'banana'(t) ON CLIENT", "after all"); + assertEq("connection is closed", true, conn.isClosed()); + // Cannot check the server log, but at the time I checked, it said "prematurely stopped client", which is fine. + } + + public void test_FailDownloadLate() throws SQLException, Failure { + prepare(); + MyDownloadHandler handler = new MyDownloadHandler(200, "download refused"); + conn.setDownloadHandler(handler); + update("INSERT INTO foo SELECT value as i, 'number' || value AS t FROM sys.generate_series(0, 100)"); + expectError("COPY (SELECT * FROM sys.generate_series(0,200)) INTO 'banana' ON CLIENT", "download refused"); + // Exception closes the connection + assertEq("connection is closed", conn.isClosed(), true); + } + + public void test_FileTransferHandlerUploadUtf8() throws IOException, SQLException, Failure { + testFileTransferHandlerUpload(StandardCharsets.UTF_8, "UTF-8"); + } + + public void test_FileTransferHandlerUploadLatin1() throws IOException, SQLException, Failure { + testFileTransferHandlerUpload(Charset.forName("latin1"), "latin1"); + } + + public void test_FileTransferHandlerUploadNull() throws IOException, SQLException, Failure { + testFileTransferHandlerUpload(null, Charset.defaultCharset().name()); + } + + public void testFileTransferHandlerUpload(Charset handlerEncoding, String fileEncoding) throws IOException, SQLException, Failure { + prepare(); + Path d = getTmpDir(currentTestName); + Path f = d.resolve("data.txt"); + OutputStream s = Files.newOutputStream(f, CREATE_NEW); + PrintStream ps = new PrintStream(s, false, fileEncoding); + ps.println("1|one"); + ps.println("2|twø"); + ps.println("3|three"); + ps.close(); + conn.setUploadHandler(new FileTransferHandler(d, handlerEncoding)); + update("COPY INTO foo FROM 'data.txt' ON CLIENT"); + assertQueryInt("SELECT SUM(i) FROM foo", 6); + assertQueryString("SELECT t FROM foo WHERE i = 2", "twø"); + } + + public void test_FileTransferHandlerUploadRefused() throws IOException, SQLException, Failure { + prepare(); + Path d = getTmpDir(currentTestName); + Path f = d.resolve("data.txt"); + OutputStream s = Files.newOutputStream(f, CREATE_NEW); + PrintStream ps = new PrintStream(s, false, "UTF-8"); + ps.println("1|one"); + ps.println("2|two"); + ps.println("3|three"); + ps.close(); + + Path d2 = getTmpDir(currentTestName + "2"); + conn.setUploadHandler(new FileTransferHandler(d2, StandardCharsets.UTF_8)); + String quoted = f.toAbsolutePath().toString().replaceAll("'", "''"); + expectError("COPY INTO foo FROM R'"+ quoted + "' ON CLIENT", "not in upload directory"); + // connection is still alive + assertQueryInt("SELECT SUM(i) FROM foo", 0); + } + + public void test_FileTransferHandlerDownloadUtf8() throws SQLException, Failure, IOException { + testFileTransferHandlerDownload(StandardCharsets.UTF_8, StandardCharsets.UTF_8); + } + + public void test_FileTransferHandlerDownloadLatin1() throws SQLException, Failure, IOException { + Charset latin1 = Charset.forName("latin1"); + testFileTransferHandlerDownload(latin1, latin1); + } + + public void test_FileTransferHandlerDownloadNull() throws SQLException, Failure, IOException { + testFileTransferHandlerDownload(null, Charset.defaultCharset()); + } + + public void testFileTransferHandlerDownload(Charset handlerEncoding, Charset fileEncoding) throws SQLException, Failure, IOException { + prepare(); + update("INSERT INTO foo VALUES (42, 'forty-twø')"); + Path d = getTmpDir(currentTestName); + conn.setDownloadHandler(new FileTransferHandler(d, handlerEncoding)); + update("COPY SELECT * FROM foo INTO 'data.txt' ON CLIENT"); + List<String> lines = Files.readAllLines(d.resolve("data.txt"), fileEncoding); + assertEq("lines written", lines.size(), 1); + assertEq("line content", lines.get(0), "42|\"forty-twø\""); + // connection is still alive + assertQueryInt("SELECT SUM(i) FROM foo", 42); + } + + public void test_FileTransferHandlerDownloadRefused() throws SQLException, Failure, IOException { + prepare(); + BugFixLevel level = getLevel(); + update("INSERT INTO foo VALUES (42, 'forty-two')"); + Path d = getTmpDir(currentTestName); + Path d2 = getTmpDir(currentTestName + "2"); + conn.setDownloadHandler(new FileTransferHandler(d2, StandardCharsets.UTF_8)); + String quoted = d.resolve("data.txt").toAbsolutePath().toString().replaceAll("'", "''"); + expectError("COPY SELECT * FROM foo INTO R'" + quoted + "' ON CLIENT", "not in download directory"); + if (level.compareTo(BugFixLevel.CanRefuseDownload) >= 0) { + // connection is still alive + assertQueryInt("SELECT SUM(i) FROM foo", 42); + } + } + + static class MyUploadHandler implements UploadHandler { + private final long rows; + private final long errorAt; + private final String errorMessage; + private boolean encounteredWriteError = false; + private boolean cancelled = false; + + private int chunkSize = 100; // small number to trigger more bugs + + MyUploadHandler(long rows, long errorAt, String errorMessage) { + this.rows = rows; + this.errorAt = errorAt; + this.errorMessage = errorMessage; + } + + MyUploadHandler(long rows) { + this(rows, -1, null); + } + + MyUploadHandler(String errorMessage) { + this(0, -1, errorMessage); + } + + public void setChunkSize(int chunkSize) { + this.chunkSize = chunkSize; + } + + @Override + public void uploadCancelled() { + cancelled = true; + } + + @Override + public void handleUpload(MonetConnection.Upload handle, String name, boolean textMode, long linesToSkip) throws IOException { + if (errorAt == -1 && errorMessage != null) { + handle.sendError(errorMessage); + return; + } + handle.setChunkSize(chunkSize); + PrintStream stream = handle.getStream(); + for (long i = linesToSkip; i < rows; i++) { + if (i == errorAt) { + throw new IOException(errorMessage); + } + stream.printf("%d|%d%n", i + 1, i + 1); + if (i % 25 == 0 && stream.checkError()) { + encounteredWriteError = true; + break; + } + } + } + + public Object encounteredWriteError() { + return encounteredWriteError; + } + + public boolean isCancelled() { + return cancelled; + } + } + + static class MyDownloadHandler implements DownloadHandler { + private final int errorAtByte; + private final String errorMessage; + private int attempts = 0; + private int bytesSeen = 0; + private int lineEndingsSeen = 0; + private int startOfLine = 0; + + MyDownloadHandler(int errorAtByte, String errorMessage) { + this.errorAtByte = errorAtByte; + this.errorMessage = errorMessage; + } + + MyDownloadHandler(String errorMessage) { + this(-1, errorMessage); + } + + MyDownloadHandler() { + this(-1, null); + } + + @Override + public void handleDownload(MonetConnection.Download handle, String name, boolean textMode) throws IOException { + attempts++; + bytesSeen = 0; + lineEndingsSeen = 0; + startOfLine = 0; + + if (errorMessage != null && errorAtByte < 0) { + handle.sendError(errorMessage); + return; + } + + InputStream stream = handle.getStream(); + byte[] buffer = new byte[1024]; + while (true) { + int toRead = buffer.length; + if (errorMessage != null && errorAtByte >= 0) { + if (bytesSeen == errorAtByte) { + throw new IOException(errorMessage); + } + toRead = Integer.min(toRead, errorAtByte - bytesSeen); + } + int nread = stream.read(buffer, 0, toRead); + if (nread < 0) + break; + for (int i = 0; i < nread; i++) { + if (buffer[i] == '\n') { + lineEndingsSeen += 1; + startOfLine = bytesSeen + i + 1; + } + } + bytesSeen += nread; + } + } + + public int countAttempts() { + return attempts; + } + + public int lineCount() { + int lines = lineEndingsSeen; + if (startOfLine != bytesSeen) + lines++; + return lines; + } + } + +}
new file mode 100644 --- /dev/null +++ b/tests/TestRunner.java @@ -0,0 +1,425 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V. + */ + +import org.monetdb.jdbc.MonetConnection; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.sql.*; +import java.util.ArrayList; + +public class TestRunner { + public static final int VERBOSITY_NONE = 0; + public static final int VERBOSITY_ON = 1; + public static final int VERBOSITY_SHOW_ALL = 2; + protected final String jdbcUrl; + private final int verbosity; + protected String currentTestName; + protected final WatchDog watchDog; + protected MonetConnection conn; + protected Statement stmt; + private StringWriter outBuffer; + protected PrintWriter out; + private Path tmpDir = null; + + public TestRunner(String jdbcUrl, int verbosity, boolean watchDogEnabled) { + this.jdbcUrl = jdbcUrl; + this.verbosity = verbosity; + watchDog = new WatchDog(); + if (watchDogEnabled) + watchDog.enable(); + else + watchDog.disable(); + } + + protected int runTests(String testPrefix) throws SQLException { + int testCount = 0; + int skippedCount = 0; + ArrayList<String> failures = new ArrayList<>(); + + watchDog.stop(); + try { + final String initialPrefix = "test_"; + if (testPrefix == null) + testPrefix = ""; + final String methodPrefix = initialPrefix + testPrefix; + + for (Method method : this.getClass().getDeclaredMethods()) { + if (method.getParameterCount() != 0) { + continue; + } + if (!method.getName().startsWith(initialPrefix)) { + continue; + } + testCount++; + // so user can add $ to force full match + String augmentedMethodName = method.getName() + "$"; + if (!augmentedMethodName.startsWith(methodPrefix)) { + skippedCount++; + continue; + } + String testName = method.getName().substring(initialPrefix.length()); + boolean succeeded = runTest(testName, method); + if (!succeeded) + failures.add(testName); + } + } finally { + watchDog.stop(); + } + + if (testCount > 0 && skippedCount == testCount && !testPrefix.isEmpty()) { + System.err.printf("None of the %d tests matched prefix '%s'%n", testCount, testPrefix); + return 1; + } + + int failureCount = failures.size(); + if (failureCount > 0) { + System.out.println(); + System.out.printf("Ran %d out of %d %s tests, %d failed: %s%n", + testCount - skippedCount, testCount, + this.getClass().getSimpleName(), + failureCount, + String.join(", ", failures) + ); + } else if (verbosity >= VERBOSITY_ON) { + System.out.println(); + System.out.printf("Ran %d out of %d tests, none failed%n", testCount - skippedCount, testCount); + } + + return failureCount; + } + + private synchronized boolean runTest(String testName, Method method) throws SQLException { + currentTestName = testName; + watchDog.setContext("test " + testName); + watchDog.setDuration(3_000); + outBuffer = new StringWriter(); + out = new PrintWriter(outBuffer); + + Connection genericConnection = DriverManager.getConnection(jdbcUrl); + conn = genericConnection.unwrap(MonetConnection.class); + stmt = conn.createStatement(); + + boolean failed = true; + try { + long duration; + try { + long t0 = System.currentTimeMillis(); + method.invoke(this); + long t1 = System.currentTimeMillis(); + duration = t1 - t0; + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof Failure) + throw (Failure) cause; + else if (cause instanceof Exception) { + throw (Exception) cause; + } else { + throw e; + } + } + + failed = false; + + if (verbosity > VERBOSITY_ON) + System.out.println(); + if (verbosity >= VERBOSITY_ON) + System.out.println("Test " + testName + " succeeded in " + duration + "ms"); + if (verbosity >= VERBOSITY_SHOW_ALL) + dumpOutput(testName); + } catch (Failure e) { + System.out.println(); + System.out.println("Test " + testName + " failed"); + dumpOutput(testName); + } catch (Exception e) { + System.out.println(); + System.out.println("Test " + testName + " failed:"); + e.printStackTrace(System.out); + dumpOutput(testName); + // Show the inner bits of the exception again, they may have scrolled off screen + Throwable t = e; + while (t.getCause() != null) { + t = t.getCause(); + } + System.out.println("Innermost cause was " + t); + if (t.getStackTrace().length > 0) { + System.out.println(" at " + t.getStackTrace()[0]); + } + } finally { + watchDog.setContext(null); + if (failed && verbosity == VERBOSITY_ON) { + // next test case will not print separator + System.out.println(); + } + stmt.close(); + conn.close(); + } + + return !failed; + } + + private void dumpOutput(String testName) { + String output = outBuffer.getBuffer().toString(); + if (output.isEmpty()) { + System.out.println("(Test did not produce any output)"); + } else { + System.out.println("------ Accumulated output for test " + testName + ":"); + boolean terminated = output.endsWith(System.lineSeparator()); + if (terminated) { + System.out.print(output); + } else { + System.out.println(output); + } + System.out.println("------ End of accumulated output" + (terminated ? "" : " (no trailing newline)")); + } + } + + private void fail(String message) throws Failure { + out.println("FAILURE: " + message); + throw new Failure(message); + } + + private void checked(String quantity, Object actual) { + out.println(" CHECKED: " + "<" + quantity + "> is " + actual + " as expected"); + } + + protected void assertEq(String quantity, Object expected, Object actual) throws Failure { + if (expected.equals(actual)) { + checked(quantity, actual); + } else { + fail("Expected <" + quantity + "> to be " + expected + " got " + actual); + } + } + + protected boolean execute(String query) throws SQLException { + try { + watchDog.start(); + out.println("EXECUTE: " + query); + boolean result; + result = stmt.execute(query); + if (result) { + out.println(" OK"); + } else { + out.println(" OK, updated " + stmt.getUpdateCount() + " rows"); + } + return result; + } finally { + watchDog.stop(); + } + } + + protected void update(String query) throws SQLException, Failure { + execute(query); + } + + protected void expectError(String query, String expectedError) throws SQLException { + try { + execute(query); + } catch (SQLException e) { + if (e.getMessage().contains(expectedError)) { + out.println(" GOT EXPECTED EXCEPTION: " + e.getMessage()); + } else { + throw e; + } + } + } + + protected void assertQueryInt(String query, int expected) throws SQLException, Failure { + if (execute(query) == false) { + fail("Query does not return a result set"); + } + ResultSet rs = stmt.getResultSet(); + ResultSetMetaData metaData = rs.getMetaData(); + assertEq("column count", 1, metaData.getColumnCount()); + if (!rs.next()) { + fail("Result set is empty"); + } + int result = rs.getInt(1); + if (rs.next()) { + String message = "Result set has more than one row"; + fail(message); + } + rs.close(); + checked("row count", 1); + assertEq("query result", expected, result); + } + + protected void assertQueryString(String query, String expected) throws SQLException, Failure { + if (execute(query) == false) { + fail("Query does not return a result set"); + } + ResultSet rs = stmt.getResultSet(); + ResultSetMetaData metaData = rs.getMetaData(); + assertEq("column count", 1, metaData.getColumnCount()); + if (!rs.next()) { + fail("Result set is empty"); + } + String result = rs.getString(1); + if (rs.next()) { + String message = "Result set has more than one row"; + fail(message); + } + rs.close(); + checked("row count", 1); + assertEq("query result", expected, result); + } + + protected String queryString(String query) throws SQLException, Failure { + if (execute(query) == false) { + fail("Query does not return a result set"); + } + ResultSet rs = stmt.getResultSet(); + ResultSetMetaData metaData = rs.getMetaData(); + assertEq("column count", 1, metaData.getColumnCount()); + if (!rs.next()) { + fail("Result set is empty"); + } + String result = rs.getString(1); + if (rs.next()) { + String message = "Result set has more than one row"; + fail(message); + } + rs.close(); + checked("row count", 1); + return result; + } + + protected synchronized Path getTmpDir(String name) throws IOException { + if (tmpDir == null) { + tmpDir = Files.createTempDirectory("testMonetDB"); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + Files.walkFileTree(tmpDir, new SimpleFileVisitor<Path>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + // we do this on a best effort basis + } + })); + } + Path p = tmpDir.resolve(name); + Files.createDirectory(p); + return p; + } + + static class Failure extends Exception { + + public Failure(String message) { + super(message); + } + + public Failure(String message, Throwable cause) { + super(message, cause); + } + + } + + static class WatchDog { + private boolean enabled; + private long duration = 1000; + private long started = 0; + private String context = "no context"; + + WatchDog() { + Thread watchDog = new Thread(this::work); + watchDog.setName("watchdog_timer"); + watchDog.setDaemon(true); + watchDog.start(); + } + + synchronized void enable() { + this.enabled = true; + this.notifyAll(); + } + + synchronized void disable() { + this.enabled = false; + this.notifyAll(); + } + + synchronized void setContext(String context) { + this.context = context; + } + + synchronized void setDuration(long duration) { + if (duration <= 0) + throw new IllegalArgumentException("duration should be > 0"); + this.duration = duration; + this.notifyAll(); + } + + synchronized void start() { + started = System.currentTimeMillis(); + this.notifyAll(); + } + + synchronized void stop() { + started = 0; + this.notifyAll(); + } + + private synchronized void work() { + long now; + try { + while (true) { + now = System.currentTimeMillis(); + final long sleepTime; + if (started < 0) { + // client asked us to go away + // System.err.println("++ EXIT"); + return; + } else if (!enabled || started == 0) { + // wait for client to enable/start us + sleepTime = 600_000; + } else { + long deadline = started + duration; + sleepTime = deadline - now; + } + // System.err.printf("++ now=%d, started=now%+d, duration=%d, sleep=%d%n", + // now, started - now, duration, sleepTime + // ); + if (sleepTime > 0) { + this.wait(sleepTime); + } else { + trigger(); + return; + } + } + } catch (InterruptedException e) { + System.err.println("WATCHDOG TIMER INTERRUPTED, SHOULDN'T HAPPEN"); + System.exit(4); + } + } + + private void trigger() { + String c = context != null ? context : "no context"; + System.err.println(); + System.err.println(); + System.err.println("WATCHDOG TIMER EXPIRED [" + c + "], KILLING TESTS"); + System.exit(3); + } + } +}
--- a/tests/build.xml +++ b/tests/build.xml @@ -49,7 +49,7 @@ Copyright 1997 - July 2008 CWI, August 2 <javac srcdir="${srcdir}" destdir="${builddir}" - debug="false" + debug="true" optimize="false" includeantruntime="false" source="${jvm.version}" @@ -64,7 +64,7 @@ Copyright 1997 - July 2008 CWI, August 2 </javac> </target> - <!-- a convenience jar collectin all JDBC tests --> + <!-- a convenience jar collecting all JDBC tests --> <target name="jar_jdbctests" depends="compile" unless="uptodate.jdbctests-jar"> @@ -72,6 +72,10 @@ Copyright 1997 - July 2008 CWI, August 2 <jar jarfile="${jdbctests-jar}"> <fileset dir="${builddir}"> <include name="JDBC_API_Tester.class" /> + <include name="TestRunner.class" /> + <include name="TestRunner$*.class" /> + <include name="OnClientTester.class" /> + <include name="OnClientTester$*.class" /> </fileset> </jar> </target> @@ -105,7 +109,8 @@ Copyright 1997 - July 2008 CWI, August 2 <target name="test_class" depends="compile,jdbc"> <echo message="Testing class ${test.class}" /> - <java classname="${test.class}" failonerror="true"> + <!-- fork="true" allows the test program to call System.exit() --> + <java classname="${test.class}" failonerror="true" fork="true"> <classpath> <pathelement path="${builddir}" /> <pathelement path="${jdbc_jar}" /> @@ -122,4 +127,10 @@ Copyright 1997 - July 2008 CWI, August 2 </antcall> </target> + <target name="OnClientTester"> + <antcall target="test_class"> + <param name="test.class" value="OnClientTester" /> + </antcall> + </target> + </project>