Mercurial > hg > monetdb-java
changeset 853:ca7f27763249
Merge 'monetdbs' into 'default'
author | Joeri van Ruth <joeri.van.ruth@monetdbsolutions.com> |
---|---|
date | Fri, 05 Jan 2024 12:59:14 +0100 (15 months ago) |
parents | d7ffef8faf38 (current diff) d9a45743536d (diff) |
children | cd365d70c745 |
files | |
diffstat | 19 files changed, 4456 insertions(+), 680 deletions(-) [+] |
line wrap: on
line diff
--- a/src/main/java/org/monetdb/client/JdbcClient.java +++ b/src/main/java/org/monetdb/client/JdbcClient.java @@ -43,6 +43,7 @@ import java.sql.SQLWarning; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.Properties; /** * This program acts like an extended client program for MonetDB. Its @@ -127,6 +128,7 @@ public final class JdbcClient { * @throws Exception if uncaught exception is thrown */ public final static void main(String[] args) throws Exception { + final Properties props = new Properties(); final CmdLineOpts copts = new CmdLineOpts(); // arguments which take exactly one argument @@ -194,6 +196,10 @@ public final class JdbcClient { "statements read. Batching can greatly speedup the " + "process of restoring a database dump."); + copts.addIgnored("save_history"); + copts.addIgnored("format"); + copts.addIgnored("width"); + // we store user and password in separate variables in order to // be able to properly act on them like forgetting the password // from the user's file if the user supplies a username on the @@ -287,19 +293,24 @@ public final class JdbcClient { user = copts.getOption("user").getArgument(); - // build the hostname + // extract hostname and port String host = copts.getOption("host").getArgument(); - if (host.indexOf(':') == -1) { - host = host + ":" + copts.getOption("port").getArgument(); + String port = copts.getOption("port").getArgument(); + int hostColon = host.indexOf(':'); + if (hostColon > 0) { + port = host.substring(hostColon + 1); + host = host.substring(0, hostColon); } + props.setProperty("host", host); + props.setProperty("port", port); - // build the extra arguments of the JDBC connect string // increase the fetchsize from the default 250 to 10000 - String attr = "?fetchsize=10000&"; + props.setProperty("fetchsize", "10000"); + CmdLineOpts.OptionContainer oc = copts.getOption("language"); final String lang = oc.getArgument(); if (oc.isPresent()) - attr += "language=" + lang + "&"; + props.setProperty("language", lang); /* Xquery is no longer functional or supported // set some behaviour based on the language XQuery @@ -311,13 +322,13 @@ public final class JdbcClient { */ oc = copts.getOption("Xdebug"); if (oc.isPresent()) { - attr += "debug=true&"; + props.setProperty("debug", "true"); if (oc.getArgumentCount() == 1) - attr += "logfile=" + oc.getArgument() + "&"; + props.setProperty("logfile", "logfile=" + oc.getArgument()); } oc = copts.getOption("Xhash"); if (oc.isPresent()) - attr += "hash=" + oc.getArgument() + "&"; + props.setProperty("hash", oc.getArgument()); // request a connection suitable for MonetDB from the driver // manager note that the database specifier is only used when @@ -329,11 +340,18 @@ public final class JdbcClient { // make sure the driver class is loaded (and thus register itself with the DriverManager) Class.forName("org.monetdb.jdbc.MonetDriver"); - con = DriverManager.getConnection( - "jdbc:monetdb://" + host + "/" + database + attr, - user, - pass - ); + // If the database name is a full url, use that. + // Otherwise, construct something. + String url; + if (database.startsWith("jdbc:")) { + url = database; + } else { + url = "jdbc:monetdb:"; // special case + props.setProperty("database", database); + } + props.setProperty("user", user); + props.setProperty("password", pass); + con = DriverManager.getConnection(url, props); SQLWarning warn = con.getWarnings(); while (warn != null) { System.err.println("Connection warning: " + warn.getMessage());
--- a/src/main/java/org/monetdb/jdbc/MonetConnection.java +++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java @@ -35,12 +35,14 @@ import java.util.concurrent.Executor; import org.monetdb.mcl.io.BufferedMCLReader; import org.monetdb.mcl.io.BufferedMCLWriter; import org.monetdb.mcl.io.LineType; -import org.monetdb.mcl.net.HandshakeOption; import org.monetdb.mcl.net.MapiSocket; +import org.monetdb.mcl.net.Target; import org.monetdb.mcl.parser.HeaderLineParser; import org.monetdb.mcl.parser.MCLParseException; import org.monetdb.mcl.parser.StartOfHeaderParser; +import javax.net.ssl.SSLException; + /** *<pre> * A {@link Connection} suitable for the MonetDB database. @@ -73,17 +75,8 @@ public class MonetConnection extends MonetWrapper implements Connection, AutoCloseable { - /** The hostname to connect to */ - private final String hostname; - /** The port to connect on the host to */ - private int port; - /** The database to use (currently not used) */ - private final String database; - /** The username to use when authenticating */ - private final String username; - /** The password to use when authenticating */ - private final String password; - + /* All connection parameters */ + Target target; /** A connection to mserver5 using a TCP socket */ private final MapiSocket server; /** The Reader from the server */ @@ -136,11 +129,6 @@ public class MonetConnection /** The language which is used */ private final int lang; - /** Whether or not BLOB is mapped to Types.VARBINARY instead of Types.BLOB within this connection */ - private boolean treatBlobAsVarBinary = true; // turned on by default for optimal performance (from JDBC Driver release 3.0 onwards) - /** Whether or not CLOB is mapped to Types.VARCHAR instead of Types.CLOB within this connection */ - private boolean treatClobAsVarChar = true; // turned on by default for optimal performance (from JDBC Driver release 3.0 onwards) - /** The last set query timeout on the server as used by Statement, PreparedStatement and CallableStatement */ protected int lastSetQueryTimeout = 0; // 0 means no timeout, which is the default on the server @@ -148,143 +136,27 @@ public class MonetConnection private DatabaseMetaData dbmd; /** - * Constructor of a Connection for MonetDB. At this moment the - * current implementation limits itself to storing the given host, - * database, username and password for later use by the - * createStatement() call. This constructor is only accessible to + * Constructor of a Connection for MonetDB. + * This constructor is only accessible to * classes from the jdbc package. * - * @param props a Property hashtable holding the properties needed for connecting + * @param target a {@link Target} object containing all connection parameters * @throws SQLException if a database error occurs * @throws IllegalArgumentException is one of the arguments is null or empty */ - MonetConnection(final Properties props) + MonetConnection(Target target) throws SQLException, IllegalArgumentException { - HandshakeOption.AutoCommit autoCommitSetting = new HandshakeOption.AutoCommit(true); - HandshakeOption.ReplySize replySizeSetting = new HandshakeOption.ReplySize(DEF_FETCHSIZE); - HandshakeOption.SizeHeader sizeHeaderSetting = new HandshakeOption.SizeHeader(true); - HandshakeOption.TimeZone timeZoneSetting = new HandshakeOption.TimeZone(0); - - // for debug: System.out.println("New connection object. Received properties are: " + props.toString()); - // get supported property values from the props argument. - this.hostname = props.getProperty("host"); - - final String port_prop = props.getProperty("port"); - if (port_prop != null) { - try { - this.port = Integer.parseInt(port_prop); - } catch (NumberFormatException e) { - addWarning("Unable to parse port number from: " + port_prop, "M1M05"); - } - } - - this.database = props.getProperty("database"); - this.username = props.getProperty("user"); - this.password = props.getProperty("password"); - String language = props.getProperty("language"); - - boolean debug = false; - String debug_prop = props.getProperty("debug"); - if (debug_prop != null) { - debug = Boolean.parseBoolean(debug_prop); - } - - final String hash = props.getProperty("hash"); - - String autocommit_prop = props.getProperty("autocommit"); - if (autocommit_prop != null) { - boolean ac = Boolean.parseBoolean(autocommit_prop); - autoCommitSetting.set(ac); - } - - final String fetchsize_prop = props.getProperty("fetchsize"); - if (fetchsize_prop != null) { - try { - int fetchsize = Integer.parseInt(fetchsize_prop); - if (fetchsize > 0 || fetchsize == -1) { - replySizeSetting.set(fetchsize); - } else { - addWarning("Fetch size must either be positive or -1. Value " + fetchsize + " ignored", "M1M05"); - } - } catch (NumberFormatException e) { - addWarning("Unable to parse fetch size number from: " + fetchsize_prop, "M1M05"); - } - } - - final String treatBlobAsVarBinary_prop = props.getProperty("treat_blob_as_binary"); - if (treatBlobAsVarBinary_prop != null) { - treatBlobAsVarBinary = Boolean.parseBoolean(treatBlobAsVarBinary_prop); - if (treatBlobAsVarBinary) - typeMap.put("blob", Byte[].class); - } - - final String treatClobAsVarChar_prop = props.getProperty("treat_clob_as_varchar"); - if (treatClobAsVarChar_prop != null) { - treatClobAsVarChar = Boolean.parseBoolean(treatClobAsVarChar_prop); - if (treatClobAsVarChar) - typeMap.put("clob", String.class); - } - - int sockTimeout = 0; - final String so_timeout_prop = props.getProperty("so_timeout"); - if (so_timeout_prop != null) { - try { - sockTimeout = Integer.parseInt(so_timeout_prop); - if (sockTimeout < 0) { - addWarning("Negative socket timeout not allowed. Value ignored", "M1M05"); - sockTimeout = 0; - } - } catch (NumberFormatException e) { - addWarning("Unable to parse socket timeout number from: " + so_timeout_prop, "M1M05"); - } - } - - // check mandatory input arguments - if (hostname == null || hostname.isEmpty()) - throw new IllegalArgumentException("Missing or empty host name"); - if (port <= 0 || port > 65535) - throw new IllegalArgumentException("Invalid port number: " + port - + ". It should not be " + (port < 0 ? "negative" : (port > 65535 ? "larger than 65535" : "0"))); - if (username == null || username.isEmpty()) - throw new IllegalArgumentException("Missing or empty user name"); - if (password == null || password.isEmpty()) - throw new IllegalArgumentException("Missing or empty password"); - if (language == null || language.isEmpty()) { - // fallback to default language: sql - language = "sql"; - addWarning("No language specified, defaulting to 'sql'", "M1M05"); - } - - // warn about unrecognized property names - for (Entry<Object,Object> e: props.entrySet()) { - checkValidProperty(e.getKey().toString(), "MonetConnection"); - } - + this.target = target; server = new MapiSocket(); - if (hash != null) - server.setHash(hash); - if (database != null) - server.setDatabase(database); - server.setLanguage(language); - - // calculate our time zone offset - final Calendar cal = Calendar.getInstance(); - final int offsetMillis = cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET); - final int offsetSeconds = offsetMillis / 1000; - timeZoneSetting.set(offsetSeconds); - - server.setHandshakeOptions(new HandshakeOption<?>[] { - autoCommitSetting, - replySizeSetting, - sizeHeaderSetting, - timeZoneSetting, - }); // we're debugging here... uhm, should be off in real life - if (debug) { + if (target.isDebug()) { try { - final String fname = props.getProperty("logfile", "monet_" + System.currentTimeMillis() + ".log"); + String fname = target.getLogfile(); + if (fname == null) + fname = "monet_" + System.currentTimeMillis() + ".log"; + File f = new File(fname); int ext = fname.lastIndexOf('.'); @@ -303,26 +175,47 @@ public class MonetConnection } } + SqlOptionsCallback callback = null; + switch (target.getLanguage()) { + case "sql": + lang = LANG_SQL; + queryTempl[0] = "s"; // pre + queryTempl[1] = "\n;"; // post + queryTempl[2] = "\n;\n"; // separator + commandTempl[0] = "X"; // pre + commandTempl[1] = ""; // post + callback = new SqlOptionsCallback(); + break; + case "mal": + lang = LANG_MAL; + queryTempl[0] = ""; // pre + queryTempl[1] = ";\n"; // post + queryTempl[2] = ";\n"; // separator + commandTempl[0] = ""; // pre + commandTempl[1] = ""; // post + break; + default: + lang = LANG_UNKNOWN; + break; + } + try { - final java.util.List<String> warnings = server.connect(hostname, port, username, password); + + final java.util.List<String> warnings = server.connect(target, callback); for (String warning : warnings) { addWarning(warning, "01M02"); } - // apply NetworkTimeout value from legacy (pre 4.1) driver - // so_timeout calls - server.setSoTimeout(sockTimeout); - in = server.getReader(); out = server.getWriter(); final String error = in.discardRemainder(); if (error != null) throw new SQLNonTransientConnectionException((error.length() > 6) ? error.substring(6) : error, "08001"); - } catch (java.net.UnknownHostException e) { - throw new SQLNonTransientConnectionException("Unknown Host (" + hostname + "): " + e.getMessage(), "08006"); + } catch (SSLException e) { + throw new SQLNonTransientConnectionException("Cannot establish secure connection: " + e.getMessage(), e); } catch (IOException e) { - throw new SQLNonTransientConnectionException("Unable to connect (" + hostname + ":" + port + "): " + e.getMessage(), "08006"); + throw new SQLNonTransientConnectionException("Cannot connect: " + e.getMessage(), "08006", e); } catch (MCLParseException e) { throw new SQLNonTransientConnectionException(e.getMessage(), "08001"); } catch (org.monetdb.mcl.MCLException e) { @@ -334,53 +227,17 @@ public class MonetConnection throw sqle; } - // we seem to have managed to log in, let's store the - // language used and language specific query templates - if ("sql".equals(language)) { - lang = LANG_SQL; - - queryTempl[0] = "s"; // pre - queryTempl[1] = "\n;"; // post - queryTempl[2] = "\n;\n"; // separator - - commandTempl[0] = "X"; // pre - commandTempl[1] = ""; // post - //commandTempl[2] = "\nX"; // separator (is not used) - } else if ("mal".equals(language)) { - lang = LANG_MAL; - - queryTempl[0] = ""; // pre - queryTempl[1] = ";\n"; // post - queryTempl[2] = ";\n"; // separator - - commandTempl[0] = ""; // pre - commandTempl[1] = ""; // post - //commandTempl[2] = ""; // separator (is not used) - } else { - lang = LANG_UNKNOWN; - } - // Now take care of any handshake options not handled during the handshake - if (replySizeSetting.isSent()) { - this.curReplySize = replySizeSetting.get(); - } - this.defaultFetchSize = replySizeSetting.get(); + curReplySize = defaultFetchSize; if (lang == LANG_SQL) { - if (autoCommitSetting.mustSend(autoCommit)) { - setAutoCommit(autoCommitSetting.get()); - } else { - // update bookkeeping - autoCommit = autoCommitSetting.get(); + if (autoCommit != target.isAutocommit()) { + setAutoCommit(target.isAutocommit()); } - if (sizeHeaderSetting.mustSend(false)) { + if (!callback.sizeHeaderEnabled) { sendControlCommand("sizeheader 1"); - } else { - // no bookkeeping to update } - if (timeZoneSetting.mustSend(0)) { - setTimezone(timeZoneSetting.get()); - } else { - // no bookkeeping to update + if (!callback.timeZoneSet) { + setTimezone(target.getTimezone()); } } @@ -1387,6 +1244,20 @@ public class MonetConnection } /** + * Construct a Properties object holding all connection parameters such + * as host, port, TLS configuration, autocommit, etc. + * Passing this to {@link DriverManager.getConnection()} together + * with the URL "jdbc:monetdb:" will create a new connection identical to + * the current one. + * + * @return + */ + + public Properties getConnectionProperties() { + return target.getProperties(); + } + + /** * Returns the value of the client info property specified by name. * This method may return null if the specified client info property * has not been set and does not have a default value. @@ -1779,7 +1650,7 @@ public class MonetConnection * @return whether the JDBC BLOB type should be mapped to VARBINARY type. */ boolean mapBlobAsVarBinary() { - return treatBlobAsVarBinary; + return target.isTreatBlobAsBinary(); } /** @@ -1790,7 +1661,7 @@ public class MonetConnection * @return whether the JDBC CLOB type should be mapped to VARCHAR type. */ boolean mapClobAsVarChar() { - return treatClobAsVarChar; + return target.isTreatClobAsVarchar(); } /** @@ -1799,13 +1670,7 @@ public class MonetConnection * @return the MonetDB JDBC Connection URL (without user name and password). */ String getJDBCURL() { - final StringBuilder sb = new StringBuilder(128); - sb.append(MonetDriver.MONETURL).append(hostname) - .append(':').append(port) - .append('/').append(database); - if (lang == LANG_MAL) - sb.append("?language=mal"); - return sb.toString(); + return target.buildUrl(); } /** @@ -3885,4 +3750,53 @@ public class MonetConnection super.close(); } } + + /* encode knowledge of currently available handshake options as an enum. */ + enum SqlOption { + Autocommit(1, "auto_commit"), + ReplySize(2, "reply_size"), + SizeHeader(3, "size_header"), + // NOTE: 4 has been omitted on purpose + TimeZone(5, "time_zone"), + ; + final int level; + final String field; + + SqlOption(int level, String field) { + this.level = level; + this.field = field; + } + } + + + private class SqlOptionsCallback extends MapiSocket.OptionsCallback { + private int level; + boolean sizeHeaderEnabled = false; // used during handshake + boolean timeZoneSet = false; // used during handshake + + + @Override + public void addOptions(String lang, int level) { + if (!lang.equals("sql")) + return; + this.level = level; + + // Try to add options and record that this happened if it succeeds. + if (contribute(SqlOption.Autocommit, target.isAutocommit() ? 1 : 0)) + autoCommit = target.isAutocommit(); + if (contribute(SqlOption.ReplySize, target.getReplySize())) + defaultFetchSize = target.getReplySize(); + if (contribute(SqlOption.SizeHeader, 1)) + sizeHeaderEnabled = true; + if (contribute(SqlOption.TimeZone, target.getTimezone())) + timeZoneSet = true; + } + + private boolean contribute(SqlOption opt, int value) { + if (opt.level >= this.level) + return false; + contribute(opt.field, value); + return true; + } + } }
--- a/src/main/java/org/monetdb/jdbc/MonetDriver.java +++ b/src/main/java/org/monetdb/jdbc/MonetDriver.java @@ -12,14 +12,11 @@ package org.monetdb.jdbc; -import java.net.URI; -import java.sql.Connection; -import java.sql.Driver; -import java.sql.DriverManager; -import java.sql.DriverPropertyInfo; -import java.sql.SQLException; -import java.sql.SQLFeatureNotSupportedException; -import java.sql.Types; +import org.monetdb.mcl.net.Target; +import org.monetdb.mcl.net.ValidationError; + +import java.net.URISyntaxException; +import java.sql.*; import java.util.Map.Entry; import java.util.Properties; @@ -46,8 +43,6 @@ import java.util.Properties; public final class MonetDriver implements Driver { // the url kind will be jdbc:monetdb://<host>[:<port>]/<database> // Chapter 9.2.1 from Sun JDBC 3.0 specification - /** The prefix of a MonetDB url */ - static final String MONETURL = "jdbc:monetdb://"; // initialize this class: register it at the DriverManager // Chapter 9.2 from Sun JDBC 3.0 specification @@ -71,7 +66,9 @@ public final class MonetDriver implement */ @Override public boolean acceptsURL(final String url) { - return url != null && url.startsWith(MONETURL); + if (url == null) + return false; + return url.startsWith("jdbc:monetdb:") || url.startsWith("jdbc:monetdbs:"); } /** @@ -103,56 +100,12 @@ public final class MonetDriver implement if (!acceptsURL(url)) return null; - final Properties props = new Properties(); - // set the optional properties and their defaults here - props.put("port", "50000"); - props.put("debug", "false"); - props.put("language", "sql"); // mal, sql, <future> - props.put("so_timeout", "0"); - - if (info != null) - props.putAll(info); - info = props; - - // remove leading "jdbc:" so the rest is a valid hierarchical URI - final URI uri; try { - uri = new URI(url.substring(5)); - } catch (java.net.URISyntaxException e) { - return null; + Target target = new Target(url, info); + return new MonetConnection(target); + } catch (ValidationError | URISyntaxException e) { + throw new SQLException(e.getMessage()); } - - final String uri_host = uri.getHost(); - if (uri_host == null) - return null; - info.put("host", uri_host); - - int uri_port = uri.getPort(); - if (uri_port > 0) - info.put("port", Integer.toString(uri_port)); - - // check the database - String uri_path = uri.getPath(); - if (uri_path != null && !uri_path.isEmpty()) { - uri_path = uri_path.substring(1).trim(); - if (!uri_path.isEmpty()) - info.put("database", uri_path); - } - - final String uri_query = uri.getQuery(); - if (uri_query != null) { - int pos; - // handle additional connection properties separated by the & character - final String args[] = uri_query.split("&"); - for (int i = 0; i < args.length; i++) { - pos = args[i].indexOf('='); - if (pos > 0) - info.put(args[i].substring(0, pos), args[i].substring(pos + 1)); - } - } - - // finally return the Connection object as requested - return new MonetConnection(info); } /**
--- a/src/main/java/org/monetdb/mcl/MCLException.java +++ b/src/main/java/org/monetdb/mcl/MCLException.java @@ -19,7 +19,11 @@ package org.monetdb.mcl; public final class MCLException extends Exception { private static final long serialVersionUID = 1L; - public MCLException(String e) { - super(e); + public MCLException(String message) { + super(message); + } + + public MCLException(String message, Exception cause) { + super(message, cause); } }
--- a/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java +++ b/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java @@ -99,6 +99,18 @@ public final class BufferedMCLReader { } /** + * Return a substring of the current line, or null if we're at the end or before the beginning. + * + * @return the current line or null + */ + public String getLine(int start) { + String line = getLine(); + if (line != null) + line = line.substring(start); + return line; + } + + /** * getLineType returns the type of the current line. * * @return Linetype representing the kind of line this is, one of the
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java +++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java @@ -21,16 +21,13 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Writer; -import java.net.Socket; -import java.net.SocketException; -import java.net.UnknownHostException; -import java.net.URI; +import java.net.*; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; +import java.util.*; + +import javax.net.ssl.SSLException; import org.monetdb.mcl.MCLException; import org.monetdb.mcl.io.BufferedMCLReader; @@ -89,10 +86,27 @@ import org.monetdb.mcl.parser.MCLParseEx * @see org.monetdb.mcl.io.BufferedMCLWriter */ public final class MapiSocket { + /* an even number of NUL bytes used during the handshake */ + private static final byte[] NUL_BYTES = new byte[]{ 0, 0, 0, 0, 0, 0, 0, 0 }; + + /* A mapping between hash algorithm names as used in the MAPI + * protocol, and the names by which the Java runtime knows them. + */ + private static final String[][] KNOWN_ALGORITHMS = new String[][] { + {"SHA512", "SHA-512"}, + {"SHA384", "SHA-384"}, + {"SHA256", "SHA-256"}, + // should we deprecate this by now? + {"SHA1", "SHA-1"}, + }; + + // MUST be lowercase! + private static final char[] HEXDIGITS = "0123456789abcdef".toCharArray(); + + /** Connection parameters */ + private Target target; /** The TCP Socket to mserver */ private Socket con; - /** The TCP Socket timeout in milliseconds. Default is 0 meaning the timeout is disabled (i.e., timeout of infinity) */ - private int soTimeout = 0; /** Stream from the Socket for reading */ private BlockInputStream fromMonet; /** Stream from the Socket for writing */ @@ -104,36 +118,30 @@ public final class MapiSocket { /** protocol version of the connection */ private int version; - /** The database to connect to */ - private String database = null; - /** The language to connect with */ - private String language = "sql"; - /** The hash methods to use (null = default) */ - private String hash = null; - - /** Whether we should follow redirects */ + /** Whether we should follow redirects. + * Not sure why this needs to be separate + * from 'ttl' but someone someday explicitly documented setTtl + * with 'to disable completely, use followRedirects' so + * apparently there is a use case. + */ private boolean followRedirects = true; /** How many redirections do we follow until we're fed up with it? */ private int ttl = 10; - /** Whether we are debugging or not */ - private boolean debug = false; /** The Writer for the debug log-file */ private Writer log; /** The blocksize (hardcoded in compliance with MonetDB common/stream/stream.h) */ - public final static int BLOCK = 8 * 1024 - 2; + public final static int BLOCK = 8190; /** A short in two bytes for holding the block size in bytes */ private final byte[] blklen = new byte[2]; - /** Options that can be sent during the auth handshake if the server supports it */ - private HandshakeOption<?>[] handshakeOptions; - /** * Constructs a new MapiSocket. */ public MapiSocket() { + target = new Target(); con = null; } @@ -145,7 +153,7 @@ public final class MapiSocket { * @param db the database */ public void setDatabase(final String db) { - this.database = db; + target.setDatabase(db); } /** @@ -154,7 +162,7 @@ public final class MapiSocket { * @param lang the language */ public void setLanguage(final String lang) { - this.language = lang; + target.setLanguage(lang); } /** @@ -167,7 +175,7 @@ public final class MapiSocket { * @param hash the hash method to use */ public void setHash(final String hash) { - this.hash = hash; + target.setHash(hash); } /** @@ -212,7 +220,7 @@ public final class MapiSocket { if (s < 0) { throw new IllegalArgumentException("timeout can't be negative"); } - this.soTimeout = s; + target.setSoTimeout(s); // limit time to wait on blocking operations if (con != null) { con.setSoTimeout(s); @@ -226,10 +234,7 @@ public final class MapiSocket { * @throws SocketException Issue with the socket */ public int getSoTimeout() throws SocketException { - if (con != null) { - this.soTimeout = con.getSoTimeout(); - } - return this.soTimeout; + return target.getSoTimeout(); } /** @@ -238,7 +243,7 @@ public final class MapiSocket { * @param debug Value to set */ public void setDebug(final boolean debug) { - this.debug = debug; + target.setDebug(debug); } /** @@ -261,21 +266,29 @@ public final class MapiSocket { public List<String> connect(final String host, final int port, final String user, final String pass) throws IOException, SocketException, UnknownHostException, MCLParseException, MCLException { - // Wrap around the internal connect that needs to know if it - // should really make a TCP connection or not. - return connect(host, port, user, pass, true); + target.setHost(host); + target.setPort(port); + target.setUser(user); + target.setPassword(pass); + return connect(target, null); + } + + public List<String> connect(String url, Properties props) throws URISyntaxException, ValidationError, MCLException, MCLParseException, IOException { + return connect(new Target(url, props), null); } /** - * Connects to the given host and port, logging in as the given - * user. If followRedirect is false, a RedirectionException is + * Connect according to the settings in the 'target' parameter. + * If followRedirect is false, a RedirectionException is * thrown when a redirect is encountered. * - * @param host the hostname, or null for the loopback address - * @param port the port number (must be between 0 and 65535, inclusive) - * @param user the username - * @param pass the password - * @param makeConnection whether a new socket connection needs to be created + * Some settings, such as the initial reply size, can already be configured + * during the handshake, saving a command round-trip later on. + * To do so, create and pass a subclass of {@link MapiSocket.OptionsCallback}. + * + * @param target the connection settings + * @param callback will be called if the server allows options to be set during the + * initial handshake * @return A List with informational (warning) messages. If this * list is empty; then there are no warnings. * @throws IOException if an I/O error occurs when creating the socket @@ -284,351 +297,344 @@ public final class MapiSocket { * @throws MCLParseException if bogus data is received * @throws MCLException if an MCL related error occurs */ - private List<String> connect(final String host, final int port, final String user, final String pass, final boolean makeConnection) - throws IOException, SocketException, UnknownHostException, MCLParseException, MCLException - { - if (ttl-- <= 0) - throw new MCLException("Maximum number of redirects reached, aborting connection attempt."); + public List<String> connect(Target target, OptionsCallback callback) throws MCLException, MCLParseException, IOException { + // get rid of any earlier connection state, including the existing target + close(); + this.target = target; + + Target.Validated validated; + try { + validated = target.validate(); + } catch (ValidationError e) { + throw new MCLException(e.getMessage()); + } + + if (validated.connectScan()) { + return scanUnixSockets(callback); + } + + ArrayList<String> warnings = new ArrayList<>(); + int attempts = 0; + do { + boolean ok = false; + try { + boolean done = tryConnect(callback, warnings); + ok = true; + if (done) { + return warnings; + } + } finally { + if (!ok) + close(); + } + } while (followRedirects && attempts++ < this.ttl); + throw new MCLException("max redirect count exceeded"); + } - if (makeConnection) { - con = new Socket(host, port); - con.setSoTimeout(this.soTimeout); - // set nodelay, as it greatly speeds up small messages (like we often do) - con.setTcpNoDelay(true); - con.setKeepAlive(true); + private List<String> scanUnixSockets(OptionsCallback callback) throws MCLException, MCLParseException, IOException { + // Because we do not support Unix Domain sockets, we just go back to connect(). + // target.connectScan() will now return false; + target.setHost("localhost"); + return connect(target, callback); + } - fromMonet = new BlockInputStream(con.getInputStream()); - toMonet = new BlockOutputStream(con.getOutputStream()); + private boolean tryConnect(OptionsCallback callback, ArrayList<String> warningBuffer) throws MCLException, IOException { + try { + // We need a valid target + Target.Validated validated = target.validate(); + // con will be non-null if the previous attempt ended in a redirect to mapi:monetdb://proxy + if (con == null) + connectSocket(validated); + return handshake(validated, callback, warningBuffer); + } catch (IOException | MCLException e) { + close(); + throw e; + } catch (ValidationError e) { + close(); + throw new MCLException(e.getMessage()); + } + } + + private void connectSocket(Target.Validated validated) throws MCLException, IOException { + // This method performs steps 2-6 of the procedure outlined in the URL spec + String tcpHost = validated.connectTcp(); + if (tcpHost.isEmpty()) { + throw new MCLException("Unix domain sockets are not supported, only TCP"); + } + int port = validated.connectPort(); + Socket sock = null; + try { + sock = new Socket(tcpHost, port); + sock.setSoTimeout(validated.getSoTimeout()); + sock.setTcpNoDelay(true); + sock.setKeepAlive(true); + + sock = wrapTLS(sock, validated); + + fromMonet = new BlockInputStream(sock.getInputStream()); + toMonet = new BlockOutputStream(sock.getOutputStream()); reader = new BufferedMCLReader(fromMonet, StandardCharsets.UTF_8); writer = new BufferedMCLWriter(toMonet, StandardCharsets.UTF_8); writer.registerReader(reader); - } - - reader.advance(); - final String c = reader.getLine(); - reader.discardRemainder(); - writer.writeLine(getChallengeResponse(c, user, pass, language, database, hash)); - - // read monetdb mserver response till prompt - final ArrayList<String> redirects = new ArrayList<String>(); - final List<String> warns = new ArrayList<String>(); - String err = "", tmp; - do { reader.advance(); - tmp = reader.getLine(); - if (tmp == null) - throw new IOException("Read from " + - con.getInetAddress().getHostName() + ":" + - con.getPort() + ": End of stream reached"); - if (reader.getLineType() == LineType.ERROR) { - err += "\n" + tmp.substring(7); - } else if (reader.getLineType() == LineType.INFO) { - warns.add(tmp.substring(1)); - } else if (reader.getLineType() == LineType.REDIRECT) { - redirects.add(tmp.substring(1)); + + // Only assign to sock when everything went ok so far + con = sock; + sock = null; + } catch (SSLException e) { + throw new MCLException("SSL error: " + e.getMessage(), e); + } catch (IOException e) { + throw new MCLException("Could not connect to " + tcpHost + ":" + port + ": " + e.getMessage(), e); + } finally { + if (sock != null) + try { + sock.close(); + } catch (IOException e) { + // ignore + } + } + } + + private Socket wrapTLS(Socket sock, Target.Validated validated) throws IOException { + if (validated.getTls()) + return SecureSocket.wrap(validated, sock); + else { + // Send an even number of NUL bytes to avoid a deadlock if + // we're accidentally connecting to a TLS-protected server. + // The cause of the deadlock is that we speak MAPI and we wait + // for the server to send a MAPI challenge. + // However, if the server is trying to set up TLS, it will be + // waiting for us to send a TLS 'Client Hello' packet. + // Hence, deadlock. + // NUL NUL is a no-op in MAPI and will hopefully force an error + // in the TLS server. This does not always work, some + // TLS implementations abort on the first NUL, some need more NULs + // than we are prepared to send here. 8 seems to be a good number. + sock.getOutputStream().write(NUL_BYTES); + } + return sock; + } + + private boolean handshake(Target.Validated validated, OptionsCallback callback, ArrayList<String> warnings) throws IOException, MCLException { + String challenge = reader.getLine(); + reader.advance(); + if (reader.getLineType() != LineType.PROMPT) + throw new MCLException("Garbage after server challenge: " + reader.getLine()); + String response = challengeResponse(validated, challenge, callback); + writer.writeLine(response); + reader.advance(); + + // Process the response lines. + String redirect = null; + StringBuilder errors = new StringBuilder(); + while (reader.getLineType() != LineType.PROMPT) { + switch (reader.getLineType()) { + case REDIRECT: + if (redirect == null) + redirect = reader.getLine(1); + break; + case ERROR: + if (errors.length() > 0) + errors.append("\n"); + errors.append(reader.getLine(7)); // 7 not 1! + break; + case INFO: + warnings.add(reader.getLine(1)); + break; + default: + // ignore??!! + break; } - } while (reader.getLineType() != LineType.PROMPT); + reader.advance(); + } + if (errors.length() > 0) + throw new MCLException(errors.toString()); + + if (redirect == null) + return true; // we're happy - if (err.length() > 0) { + // process redirect + try { + MonetUrlParser.parse(target, redirect); + } catch (URISyntaxException | ValidationError e) { + throw new MCLException("While processing redirect " + redirect + ": " + e.getMessage(), e); + } + if (redirect.startsWith("mapi:merovingian://proxy")) { + // The reader is stuck at LineType.PROMPT but actually the + // next challenge is already there. + reader.resetLineType(); + reader.advance(); + } else { close(); - throw new MCLException(err); } - if (!redirects.isEmpty()) { - if (followRedirects) { - // Ok, server wants us to go somewhere else. The list - // might have multiple clues on where to go. For now we - // don't support anything intelligent but trying the - // first one. URI should be in form of: - // "mapi:monetdb://host:port/database?arg=value&..." - // or - // "mapi:merovingian://proxy?arg=value&..." - // note that the extra arguments must be obeyed in both - // cases - final String suri = redirects.get(0).toString(); - if (!suri.startsWith("mapi:")) - throw new MCLException("unsupported redirect: " + suri); + return false; // we need another go + } + + private String challengeResponse(Target.Validated validated, final String challengeLine, OptionsCallback callback) throws MCLException { + // The challengeLine looks like this: + // + // 45IYyVyRnbgEnK92ad:merovingian:9:RIPEMD160,SHA512,SHA384,SHA256,SHA224,SHA1:LIT:SHA512: + // WgHIibSyH:mserver:9:RIPEMD160,SHA512,SHA384,SHA256,SHA224,SHA1:LIT:SHA512:sql=6:BINARY=1: + // 0 1 2 3 4 5 6 7 - final URI u; - try { - u = new URI(suri.substring(5)); - } catch (java.net.URISyntaxException e) { - throw new MCLParseException(e.toString()); - } + String[] parts = challengeLine.split(":"); + if (parts.length < 3) + throw new MCLException("Invalid challenge: expect at least 3 fields"); + String saltPart = parts[0]; + String serverTypePart = parts[1]; + String versionPart = parts[2]; + int version; + if (versionPart.equals("9")) + version = 9; + else + throw new MCLException("Protocol versions other than 9 are note supported: " + versionPart); + if (parts.length < 6) + throw new MCLException("Protocol version " + version + " requires at least 6 fields, found " + parts.length + ": " + challengeLine); + String serverHashesPart = parts[3]; +// String endianPart = parts[4]; + String passwordHashPart = parts[5]; + String optionsPart = parts.length > 6 ? parts[6] : null; +// String binaryPart = parts.length > 7 ? parts[7] : null; - tmp = u.getQuery(); - if (tmp != null) { - final String args[] = tmp.split("&"); - for (int i = 0; i < args.length; i++) { - int pos = args[i].indexOf('='); - if (pos > 0) { - tmp = args[i].substring(0, pos); - switch (tmp) { - case "database": - tmp = args[i].substring(pos + 1); - if (!tmp.equals(database)) { - warns.add("redirect points to different database: " + tmp); - setDatabase(tmp); - } - break; - case "language": - tmp = args[i].substring(pos + 1); - warns.add("redirect specifies use of different language: " + tmp); - setLanguage(tmp); - break; - case "user": - tmp = args[i].substring(pos + 1); - if (!tmp.equals(user)) - warns.add("ignoring different username '" + tmp + "' set by " + - "redirect, what are the security implications?"); - break; - case "password": - warns.add("ignoring different password set by redirect, " + - "what are the security implications?"); - break; - default: - warns.add("ignoring unknown argument '" + tmp + "' from redirect"); - break; - } - } else { - warns.add("ignoring illegal argument from redirect: " + args[i]); - } - } - } + String userResponse; + String password = target.getPassword(); + if (serverTypePart.equals("merovingian") && !target.getLanguage().equals("control")) { + userResponse = "merovingian"; + password = "merovingian"; + } else { + userResponse = target.getUser(); + } + String optionsResponse = handleOptions(callback, optionsPart); - if (u.getScheme().equals("monetdb")) { - // this is a redirect to another (monetdb) server, - // which means a full reconnect - // avoid the debug log being closed - if (debug) { - debug = false; - close(); - debug = true; - } else { - close(); - } - tmp = u.getPath(); - if (tmp != null && tmp.length() > 0) { - tmp = tmp.substring(1).trim(); - if (!tmp.isEmpty() && !tmp.equals(database)) { - warns.add("redirect points to different database: " + tmp); - setDatabase(tmp); - } - } - final int p = u.getPort(); - warns.addAll(connect(u.getHost(), p == -1 ? port : p, user, pass, true)); - warns.add("Redirect by " + host + ":" + port + " to " + suri); - } else if (u.getScheme().equals("merovingian")) { - // reuse this connection to inline connect to the - // right database that Merovingian proxies for us - reader.resetLineType(); - warns.addAll(connect(host, port, user, pass, false)); - } else { - throw new MCLException("unsupported scheme in redirect: " + suri); - } - } else { - final StringBuilder msg = new StringBuilder("The server sent a redirect for this connection:"); - for (String it : redirects) { - msg.append(" [" + it + "]"); - } - throw new MCLException(msg.toString()); + // Response looks like this: + // + // LIT:monetdb:{RIPEMD160}f2236256e5a9b20a5ecab4396e36c14f66c3e3c5:sql:demo + // :FILETRANS:auto_commit=1,reply_size=1000,size_header=0,columnar_protocol=0,time_zone=3600: + StringBuilder response = new StringBuilder(80); + response.append("BIG:"); + response.append(userResponse).append(":"); + hashPassword(response, saltPart, password, passwordHashPart, validated.getHash(), serverHashesPart); + response.append(":"); + response.append(validated.getLanguage()).append(":"); + response.append(validated.getDatabase()).append(":"); + response.append("FILETRANS:"); + response.append(optionsResponse).append(":"); + + return response.toString(); + } + + private String hashPassword(StringBuilder responseBuffer, String salt, String password, String passwordAlgo, String configuredHashes, String serverSupportedAlgos) throws MCLException { + // First determine which hash algorithms we can choose from for the challenge response. + // This defaults to whatever the server offers but may be restricted by the user. + Set<String> algoSet = new HashSet<>(Arrays.asList(serverSupportedAlgos.split(","))); + if (!configuredHashes.isEmpty()) { + String[] allowedList = configuredHashes.toUpperCase().split("[, ]"); + Set<String> allowedSet = new HashSet<>(Arrays.asList(allowedList)); + algoSet.retainAll(allowedSet); + if (algoSet.isEmpty()) { + throw new MCLException("None of the hash algorithms in <" + configuredHashes + "> are supported, server only supports <" + serverSupportedAlgos + ">"); } } - return warns; + + int maxHashDigits = 512 / 4; + + // We'll collect the result in the responseBuffer. + // It will start with '{' HASHNAME '}' followed by hexdigits + + // This is where we accumulate what will eventually be hashed into the hexdigits above. + // It consists of the hexadecimal pre-hash of the password, + // followed by the salt from the server + StringBuilder intermediate = new StringBuilder(maxHashDigits + salt.length()); + + MessageDigest passwordDigest = pickBestAlgorithm(Collections.singleton(passwordAlgo), null); + // Here's the password.. + hexhash(intermediate, passwordDigest, password); + // .. and here's the salt + intermediate.append(salt); + + responseBuffer.append('{'); + MessageDigest responseDigest = pickBestAlgorithm(algoSet, responseBuffer); + // the call above has appended the HASHNAME, now add '}' + responseBuffer.append('}'); + // pickBestAlgorithm has appended HASHNAME, buffer now contains '{' HASHNAME '}' + hexhash(responseBuffer, responseDigest, intermediate.toString()); + // response buffer now contains '{' HASHNAME '}' HEX_DIGITS_OF_INTERMEDIATE_BUFFER + + return responseBuffer.toString(); } /** - * A little helper function that processes a challenge string, and - * returns a response string for the server. If the challenge - * string is null, a challengeless response is returned. + * Pick the most preferred digest algorithm and return a MessageDigest instance for that. * - * @param chalstr the challenge string - * for example: H8sRMhtevGd:mserver:9:PROT10,RIPEMD160,SHA256,SHA1,COMPRESSION_SNAPPY,COMPRESSION_LZ4:LIT:SHA512: - * @param username the username to use - * @param password the password to use - * @param language the language to use - * @param database the database to connect to - * @param hash the hash method(s) to use, or NULL for all supported hashes - * @return the response string for the server - * @throws MCLParseException when parsing failed - * @throws MCLException if an MCL related error occurs - * @throws IOException when IO exception occurred + * @param algos the MAPI names of permitted algorithms + * @param appendMapiName if not null, append MAPI name of chose algorithm to this buffer + * @return instance of the chosen digester + * @throws MCLException if none of the options is supported */ - private String getChallengeResponse( - final String chalstr, - String username, - String password, - final String language, - final String database, - final String hash - ) throws MCLParseException, MCLException, IOException { - // parse the challenge string, split it on ':' - final String[] chaltok = chalstr.split(":"); - if (chaltok.length <= 5) - throw new MCLParseException("Server challenge string unusable! It contains too few (" + chaltok.length + ") tokens: " + chalstr); - - try { - version = Integer.parseInt(chaltok[2]); // protocol version - } catch (NumberFormatException e) { - throw new MCLParseException("Protocol version (" + chaltok[2] + ") unparseable as integer."); + private MessageDigest pickBestAlgorithm(Set<String> algos, StringBuilder appendMapiName) throws MCLException { + for (String[] choice : KNOWN_ALGORITHMS) { + String mapiName = choice[0]; + String algoName = choice[1]; + MessageDigest digest; + if (!algos.contains(mapiName)) + continue; + try { + digest = MessageDigest.getInstance(algoName); + } catch (NoSuchAlgorithmException e) { + continue; + } + // we found a match + if (appendMapiName != null) { + appendMapiName.append(mapiName); + } + return digest; } - - // handle the challenge according to the version it is - switch (version) { - case 9: - // proto 9 is like 8, but uses a hash instead of the plain password - // the server tells us (in 6th token) which hash in the - // challenge after the byte-order token - - String algo; - String pwhash = chaltok[5]; - /* NOTE: Java doesn't support RIPEMD160 :( */ - /* see: https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#MessageDigest */ - switch (pwhash) { - case "SHA512": - algo = "SHA-512"; - break; - case "SHA384": - algo = "SHA-384"; - break; - case "SHA256": - algo = "SHA-256"; - /* NOTE: Java 7 doesn't support SHA-224. Java 8 does but we have not tested it. It is also not requested yet. */ - break; - case "SHA1": - algo = "SHA-1"; - break; - default: - /* Note: MD5 has been deprecated by security experts and support is removed from Oct 2020 release */ - throw new MCLException("Unsupported password hash: " + pwhash); - } - try { - final MessageDigest md = MessageDigest.getInstance(algo); - md.update(password.getBytes(StandardCharsets.UTF_8)); - password = toHex(md.digest()); - } catch (NoSuchAlgorithmException e) { - throw new MCLException("This JVM does not support password hash: " + pwhash + "\n" + e); - } + String algoNames = String.join(",", algos); + throw new MCLException("No supported hash algorithm: " + algoNames); + } - // proto 7 (finally) used the challenge and works with a - // password hash. The supported implementations come - // from the server challenge. We chose the best hash - // we can find, in the order SHA512, SHA1, MD5, plain. - // Also the byte-order is reported in the challenge string, - // which makes sense, since only blockmode is supported. - // proto 8 made this obsolete, but retained the - // byte-order report for future "binary" transports. - // In proto 8, the byte-order of the blocks is always little - // endian because most machines today are. - final String hashes = (hash == null || hash.isEmpty()) ? chaltok[3] : hash; - final HashSet<String> hashesSet = new HashSet<String>(java.util.Arrays.asList(hashes.toUpperCase().split("[, ]"))); // split on comma or space - - // if we deal with merovingian, mask our credentials - if (chaltok[1].equals("merovingian") && !language.equals("control")) { - username = "merovingian"; - password = "merovingian"; - } - - // reuse variables algo and pwhash - algo = null; - pwhash = null; - if (hashesSet.contains("SHA512")) { - algo = "SHA-512"; - pwhash = "{SHA512}"; - } else if (hashesSet.contains("SHA384")) { - algo = "SHA-384"; - pwhash = "{SHA384}"; - } else if (hashesSet.contains("SHA256")) { - algo = "SHA-256"; - pwhash = "{SHA256}"; - } else if (hashesSet.contains("SHA1")) { - algo = "SHA-1"; - pwhash = "{SHA1}"; - } else { - /* Note: MD5 has been deprecated by security experts and support is removed from Oct 2020 release */ - throw new MCLException("no supported hash algorithms found in " + hashes); - } - try { - final MessageDigest md = MessageDigest.getInstance(algo); - md.update(password.getBytes(StandardCharsets.UTF_8)); - md.update(chaltok[0].getBytes(StandardCharsets.UTF_8)); // salt/key - pwhash += toHex(md.digest()); - } catch (NoSuchAlgorithmException e) { - throw new MCLException("This JVM does not support password hash: " + pwhash + "\n" + e); - } - - // TODO: some day when we need this, we should store this - if (chaltok[4].equals("BIG")) { - // byte-order of server is big-endian - } else if (chaltok[4].equals("LIT")) { - // byte-order of server is little-endian - } else { - throw new MCLParseException("Invalid byte-order: " + chaltok[4]); - } - - // compose and return response - String response = "BIG:" // JVM byte-order is big-endian - + username + ":" - + pwhash + ":" - + language + ":" - + (database == null ? "" : database) + ":" - + "FILETRANS:"; // this capability is added in monetdb-jdbc-3.2.jre8.jar - if (chaltok.length > 6) { - // if supported, send handshake options - for (String part : chaltok[6].split(",")) { - if (part.startsWith("sql=") && handshakeOptions != null) { - int level; - try { - level = Integer.parseInt(chaltok[6].substring(4)); - } catch (NumberFormatException e) { - throw new MCLParseException("Invalid handshake level: " + chaltok[6]); - } - boolean first = true; - for (HandshakeOption<?> opt: handshakeOptions) { - if (opt.getLevel() < level) { - // server supports it - if (first) { - first = false; - } else { - response += ","; - } - response += opt.getFieldName() + "=" + opt.numericValue(); - opt.setSent(true); - } - } - break; - } - } - // this ':' delimits the handshake options field. - response += ":"; - } - return response; - default: - throw new MCLException("Unsupported protocol version: " + version); + /** + * Hash the text into the MessageDigest and append the hexadecimal form of the + * resulting digest to buffer. + * + * @param buffer where the hex digits are appended + * @param digest where the hex digits come from after the text has been digested + * @param text text to digest + */ + private void hexhash(StringBuilder buffer, MessageDigest digest, String text) { + byte[] bytes = text.getBytes(StandardCharsets.UTF_8); + digest.update(bytes); + byte[] output = digest.digest(); + for (byte b : output) { + int hi = (b & 0xF0) >> 4; + int lo = b & 0x0F; + buffer.append(HEXDIGITS[hi]); + buffer.append(HEXDIGITS[lo]); } } - /** - * Small helper method to convert a byte string to a hexadecimal - * string representation. - * - * @param digest the byte array to convert - * @return the byte array as hexadecimal string - */ - private final static String toHex(final byte[] digest) { - final char[] result = new char[digest.length * 2]; - int pos = 0; - for (int i = 0; i < digest.length; i++) { - result[pos++] = hexChar((digest[i] & 0xf0) >> 4); - result[pos++] = hexChar(digest[i] & 0x0f); + private String handleOptions(OptionsCallback callback, String optionsPart) throws MCLException { + if (callback == null || optionsPart == null || optionsPart.isEmpty()) + return ""; + + StringBuilder buffer = new StringBuilder(); + callback.setBuffer(buffer); + for (String optlevel : optionsPart.split(",")) { + int eqindex = optlevel.indexOf('='); + if (eqindex < 0) + throw new MCLException("Invalid options part in server challenge: " + optionsPart); + String lang = optlevel.substring(0, eqindex); + int level; + try { + level = Integer.parseInt(optlevel.substring(eqindex + 1)); + } catch (NumberFormatException e) { + throw new MCLException("Invalid option level in server challenge: " + optlevel); + } + callback.addOptions(lang, level); } - return new String(result); - } - private final static char hexChar(final int n) { - return (n > 9) - ? (char) ('a' + (n - 10)) - : (char) ('0' + n); + return buffer.toString(); } /** @@ -720,7 +726,7 @@ public final class MapiSocket { */ public void debug(final Writer out) { log = out; - debug = true; + setDebug(true); } /** @@ -752,15 +758,6 @@ public final class MapiSocket { } /** - * Set the HandshakeOptions - * - * @param handshakeOptions the options array - */ - public void setHandshakeOptions(HandshakeOption<?>[] handshakeOptions) { - this.handshakeOptions = handshakeOptions; - } - - /** * For internal use * * @param b to enable/disable insert 'fake' newline and prompt @@ -770,6 +767,9 @@ public final class MapiSocket { return fromMonet.setInsertFakePrompts(b); } + public boolean isDebug() { + return target.isDebug(); + } /** * Inner class that is used to write data on a normal stream as a @@ -809,7 +809,7 @@ public final class MapiSocket { // it's a bit nasty if an exception is thrown from the log, // but ignoring it can be nasty as well, so it is decided to // let it go so there is feedback about something going wrong - if (debug) { + if (isDebug()) { log.flush(); } } @@ -843,7 +843,7 @@ public final class MapiSocket { // write the actual block out.write(block, 0, writePos); - if (debug) { + if (isDebug()) { if (last) { log("TD ", "write final block: " + writePos + " bytes", false); } else { @@ -968,7 +968,7 @@ public final class MapiSocket { // if we have read something before, we should have been // able to read the whole, so make this fatal if (off > 0) { - if (debug) { + if (isDebug()) { log("RD ", "the following incomplete block was received:", false); log("RX ", new String(b, 0, off, StandardCharsets.UTF_8), true); } @@ -976,7 +976,7 @@ public final class MapiSocket { con.getInetAddress().getHostName() + ":" + con.getPort() + ": Incomplete block read from stream"); } - if (debug) + if (isDebug()) log("RD ", "server closed the connection (EOF)", true); return false; } @@ -1027,7 +1027,7 @@ public final class MapiSocket { readPos = 0; - if (debug) { + if (isDebug()) { if (wasEndBlock) { log("RD ", "read final block: " + blockLen + " bytes", false); } else { @@ -1043,7 +1043,7 @@ public final class MapiSocket { if (!_read(block, blockLen)) return -1; - if (debug) + if (isDebug()) log("RX ", new String(block, 0, blockLen, StandardCharsets.UTF_8), true); // if this is the last block, make it end with a newline and prompt @@ -1058,7 +1058,7 @@ public final class MapiSocket { block[blockLen++] = b; } block[blockLen++] = '\n'; - if (debug) { + if (isDebug()) { log("RD ", "inserting prompt", true); } } @@ -1074,7 +1074,7 @@ public final class MapiSocket { return -1; } - if (debug) + if (isDebug()) log("RX ", new String(block, readPos, 1, StandardCharsets.UTF_8), true); return block[readPos++] & 0xFF; @@ -1213,7 +1213,7 @@ public final class MapiSocket { con = null; } catch (IOException e) { /* ignore it */ } } - if (debug && log != null && log instanceof FileWriter) { + if (isDebug() && log != null && log instanceof FileWriter) { try { log.close(); log = null; @@ -1524,4 +1524,48 @@ public final class MapiSocket { return off - origOff; } } + + /** + * Callback used during the initial MAPI handshake. + * + * Newer MonetDB versions allow setting some options during the handshake. + * The options are language-specific and each has a 'level'. The server + * advertises up to which level options are supported for a given language. + * For each language/option combination, {@link #addOptions} will be invoked + * during the handshake. This method should call {@link #contribute} for each + * option it wants to set. + * + * At the time of writing, only the 'sql' language supports options, + * they are listed in enum mapi_handshake_options_levels in mapi.h. + */ + public static abstract class OptionsCallback { + private StringBuilder buffer; + + /** + * Callback called for each language/level combination supported by the + * server. May call {@link #contribute} for options with a level STRICTLY + * LOWER than the level passed as a parameter. + * @param lang language advertised by the server + * @param level one higher than the maximum supported option + */ + public abstract void addOptions(String lang, int level); + + /** + * Pass option=value during the handshake + * @param field + * @param value + */ + protected void contribute(String field, int value) { + if (buffer.length() > 0) + buffer.append(','); + buffer.append(field); + buffer.append('='); + buffer.append(value); + } + + + void setBuffer(StringBuilder buf) { + buffer = buf; + } + } }
new file mode 100644 --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java @@ -0,0 +1,280 @@ +package org.monetdb.mcl.net; + +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.net.URLEncoder; + +/** + * Helper class to keep the URL parsing code separate from the rest of + * the {@link Target} class. + */ +public class MonetUrlParser { + private final Target target; + private final String urlText; + private final URI url; + + private MonetUrlParser(Target target, String url) throws URISyntaxException { + this.target = target; + this.urlText = url; + // we want to accept monetdb:// but the Java URI parser rejects that. + switch (url) { + case "monetdb:-": + case "monetdbs:-": + throw new URISyntaxException(url, "invalid MonetDB URL"); + case "monetdb://": + case "monetdbs://": + url += "-"; + break; + } + this.url = new URI(url); + } + + public static void parse(Target target, String url) throws URISyntaxException, ValidationError { + if (url.equals("monetdb://")) { + // deal with peculiarity of Java's URI parser + url = "monetdb:///"; + } + + target.barrier(); + if (url.startsWith("mapi:")) { + try { + MonetUrlParser parser = new MonetUrlParser(target, url.substring(5)); + parser.parseClassic(); + } catch (URISyntaxException e) { + URISyntaxException exc = new URISyntaxException(e.getInput(), e.getReason(), -1); + exc.setStackTrace(e.getStackTrace()); + throw exc; + } + } else { + MonetUrlParser parser = new MonetUrlParser(target, url); + parser.parseModern(); + } + target.barrier(); + } + + public static String percentDecode(String context, String text) throws URISyntaxException { + try { + return URLDecoder.decode(text, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("should be unreachable: UTF-8 unknown??", e); + } catch (IllegalArgumentException e) { + throw new URISyntaxException(text, context + ": invalid percent escape"); + } + } + + public static String percentEncode(String text) { + try { + return URLEncoder.encode(text, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + private void parseModern() throws URISyntaxException, ValidationError { + clearBasic(); + + String scheme = url.getScheme(); + if (scheme == null) + throw new URISyntaxException(urlText, "URL scheme must be monetdb:// or monetdbs://"); + switch (scheme) { + case "monetdb": + target.setTls(false); + break; + case "monetdbs": + target.setTls(true); + break; + default: + throw new URISyntaxException(urlText, "URL scheme must be monetdb:// or monetdbs://"); + } + + // The built-in getHost and getPort methods do strange things + // in edge cases such as percent-encoded host names and + // invalid port numbers + String authority = url.getAuthority(); + String host; + String remainder; + int pos; + if (authority == null) { + if (!url.getRawSchemeSpecificPart().startsWith("//")) { + throw new URISyntaxException(urlText, "expected //"); + } + host = ""; + remainder = ""; + } else if (authority.equals("-")) { + host = ""; + remainder = ""; + } else { + if (authority.startsWith("[")) { + // IPv6 + pos = authority.indexOf(']'); + if (pos < 0) + throw new URISyntaxException(urlText, "unmatched '['"); + host = authority.substring(1, pos); + remainder = authority.substring(pos + 1); + } else if ((pos = authority.indexOf(':')) >= 0) { + host = authority.substring(0, pos); + remainder = authority.substring(pos); + } else { + host = authority; + remainder = ""; + } + } + host = Target.unpackHost(host); + target.setHost(host); + + if (remainder.isEmpty()) { + // do nothing + } else if (remainder.startsWith(":")) { + String portStr = remainder.substring(1); + try { + int port = Integer.parseInt(portStr); + if (port <= 0 || port > 65535) + portStr = null; + } catch (NumberFormatException e) { + portStr = null; + } + if (portStr == null) + throw new ValidationError(urlText, "invalid port number"); + target.setString(Parameter.PORT, portStr); + } + + String path = url.getRawPath(); + String[] parts = path.split("/", 4); + // <0: empty before leading slash> / <1: database> / <2: tableschema> / <3: table> / <4: should not exist> + switch (parts.length) { + case 4: + target.setString(Parameter.TABLE, percentDecode(Parameter.TABLE.name, parts[3])); + // fallthrough + case 3: + target.setString(Parameter.TABLESCHEMA, percentDecode(Parameter.TABLESCHEMA.name, parts[2])); + // fallthrough + case 2: + target.setString(Parameter.DATABASE, percentDecode(Parameter.DATABASE.name, parts[1])); + case 1: + case 0: + // fallthrough + break; + } + + final String query = url.getRawQuery(); + if (query != null) { + final String[] args = query.split("&"); + for (int i = 0; i < args.length; i++) { + pos = args[i].indexOf('='); + if (pos <= 0) { + throw new URISyntaxException(args[i], "invalid key=value pair"); + } + String key = args[i].substring(0, pos); + key = percentDecode(key, key); + Parameter parm = Parameter.forName(key); + if (parm != null && parm.isCore) + throw new URISyntaxException(key, key + "= is not allowed as a query parameter"); + + String value = args[i].substring(pos + 1); + target.setString(key, percentDecode(key, value)); + } + } + } + + private void parseClassic() throws URISyntaxException, ValidationError { + if (!url.getRawSchemeSpecificPart().startsWith("//")) { + throw new URISyntaxException(urlText, "expected //"); + } + + String scheme = url.getScheme(); + if (scheme == null) + scheme = ""; + switch (scheme) { + case "monetdb": + parseClassicAuthorityAndPath(); + break; + case "merovingian": + String authority = url.getRawAuthority(); + // authority must be "proxy" ignore authority and path + boolean valid = urlText.startsWith("merovingian://proxy?") || urlText.equals("merovingian://proxy"); + if (!valid) + throw new URISyntaxException(urlText, "with mapi:merovingian:, only //proxy is supported"); + break; + default: + throw new URISyntaxException(urlText, "URL scheme must be mapi:monetdb:// or mapi:merovingian://"); + } + + final String query = url.getRawQuery(); + if (query != null) { + final String[] args = query.split("&"); + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if (arg.startsWith("language=")) { + String language = arg.substring(9); + target.setString(Parameter.LANGUAGE, language); + } else if (arg.startsWith("database=")) { + String database = arg.substring(9); + target.setString(Parameter.DATABASE, database); + } else { + // ignore + } + } + } + } + + private void parseClassicAuthorityAndPath() throws URISyntaxException, ValidationError { + clearBasic(); + String authority = url.getRawAuthority(); + String host; + String portStr; + int pos; + if (authority == null) { + host = ""; + portStr = ""; + } else if (authority.indexOf('@') >= 0) { + throw new URISyntaxException(urlText, "user@host syntax is not allowed"); + } else if ((pos = authority.indexOf(':')) >= 0) { + host = authority.substring(0, pos); + portStr = authority.substring(pos + 1); + } else { + host = authority; + portStr = ""; + } + + if (!portStr.isEmpty()) { + int port; + try { + port = Integer.parseInt(portStr); + } catch (NumberFormatException e) { + port = -1; + } + if (port <= 0) { + throw new ValidationError(urlText, "invalid port number"); + } + target.setString(Parameter.PORT, portStr); + } + + String path = url.getRawPath(); + if (host.isEmpty() && portStr.isEmpty()) { + // socket + target.clear(Parameter.HOST); + target.setString(Parameter.SOCK, path != null ? path : ""); + } else { + // tcp + target.clear(Parameter.SOCK); + target.setString(Parameter.HOST, host); + if (path == null || path.isEmpty()) { + // do nothing + } else if (!path.startsWith("/")) { + throw new URISyntaxException(urlText, "expect path to start with /"); + } else { + String database = path.substring(1); + target.setString(Parameter.DATABASE, database); + } + } + } + + private void clearBasic() { + target.clear(Parameter.TLS); + target.clear(Parameter.HOST); + target.clear(Parameter.PORT); + target.clear(Parameter.DATABASE); + } +}
new file mode 100644 --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/Parameter.java @@ -0,0 +1,146 @@ +package org.monetdb.mcl.net; + + +import java.util.Calendar; + +/** + * Enumerates things that can be configured on a connection to MonetDB. + */ +public enum Parameter { + TLS("tls", ParameterType.Bool, false, "secure the connection using TLS", true), + HOST("host", ParameterType.Str, "", "IP number, domain name or one of the special values `localhost` and `localhost.`", true), + PORT("port", ParameterType.Int, -1, "Port to connect to, 1..65535 or -1 for 'not set'", true), + DATABASE("database", ParameterType.Str, "", "name of database to connect to", true), + TABLESCHEMA("tableschema", ParameterType.Str, "", "only used for REMOTE TABLE, otherwise unused", true), + TABLE("table", ParameterType.Str, "", "only used for REMOTE TABLE, otherwise unused", true), + SOCK("sock", ParameterType.Path, "", "path to Unix domain socket to connect to", false), + SOCKDIR("sockdir", ParameterType.Path, "/tmp", "Directory for implicit Unix domain sockets (.s.monetdb.PORT)", false), + CERT("cert", ParameterType.Path, "", "path to TLS certificate to authenticate server with", false), + CERTHASH("certhash", ParameterType.Str, "", "hash of server TLS certificate must start with these hex digits; overrides cert", false), + CLIENTKEY("clientkey", ParameterType.Path, "", "path to TLS key (+certs) to authenticate with as client", false), + CLIENTCERT("clientcert", ParameterType.Path, "", "path to TLS certs for 'clientkey', if not included there", false), + USER("user", ParameterType.Str, "", "user name to authenticate as", false), + PASSWORD("password", ParameterType.Str, "", "password to authenticate with", false), + LANGUAGE("language", ParameterType.Str, "sql", "for example, \"sql\", \"mal\", \"msql\", \"profiler\"", false), + AUTOCOMMIT("autocommit", ParameterType.Bool, true, "initial value of autocommit", false), + SCHEMA("schema", ParameterType.Str, "", "initial schema", false), + TIMEZONE("timezone", ParameterType.Int, null, "client time zone as minutes east of UTC", false), + BINARY("binary", ParameterType.Str, "on", "whether to use binary result set format (number or bool)", false), + REPLYSIZE("replysize", ParameterType.Int, 250, "rows beyond this limit are retrieved on demand, <1 means unlimited", false), + FETCHSIZE("fetchsize", ParameterType.Int, null, "alias for replysize, specific to jdbc", false), + HASH("hash", ParameterType.Str, "", "specific to jdbc", false), + DEBUG("debug", ParameterType.Bool, false, "specific to jdbc", false), + LOGFILE("logfile", ParameterType.Str, "", "specific to jdbc", false), + + SO_TIMEOUT("so_timeout", ParameterType.Int, 0, "abort if network I/O does not complete in this many milliseconds", false), CLOB_AS_VARCHAR("treat_clob_as_varchar", ParameterType.Bool, true, "return CLOB/TEXT data as type VARCHAR instead of type CLOB", false), BLOB_AS_BINARY("treat_blob_as_binary", ParameterType.Bool, true, "return BLOB data as type BINARY instead of type BLOB", false), + ; + + public final String name; + public final ParameterType type; + public final String description; + public final boolean isCore; + private final Object defaultValue; + + Parameter(String name, ParameterType type, Object defaultValue, String description, boolean isCore) { + this.name = name; + this.type = type; + this.isCore = isCore; + this.defaultValue = defaultValue; + this.description = description; + } + + public static Parameter forName(String name) { + switch (name) { + case "tls": + return TLS; + case "host": + return HOST; + case "port": + return PORT; + case "database": + return DATABASE; + case "tableschema": + return TABLESCHEMA; + case "table": + return TABLE; + case "sock": + return SOCK; + case "sockdir": + return SOCKDIR; + case "cert": + return CERT; + case "certhash": + return CERTHASH; + case "clientkey": + return CLIENTKEY; + case "clientcert": + return CLIENTCERT; + case "user": + return USER; + case "password": + return PASSWORD; + case "language": + return LANGUAGE; + case "autocommit": + return AUTOCOMMIT; + case "schema": + return SCHEMA; + case "timezone": + return TIMEZONE; + case "binary": + return BINARY; + case "replysize": + return REPLYSIZE; + case "fetchsize": + return FETCHSIZE; + case "hash": + return HASH; + case "debug": + return DEBUG; + case "logfile": + return LOGFILE; + case "so_timeout": + return SO_TIMEOUT; + case "treat_clob_as_varchar": + return CLOB_AS_VARCHAR; + case "treat_blob_as_binary": + return BLOB_AS_BINARY; + default: + return null; + } + } + + /** + * Determine if a given setting can safely be ignored. + * The ground rule is that if we encounter an unknown setting + * without an underscore in the name, it is an error. If it has + * an underscore in its name, it can be ignored. + * + * @param name the name of the setting to check + * @return true if it can safely be ignored + */ + public static boolean isIgnored(String name) { + if (Parameter.forName(name) != null) + return false; + return name.contains("_"); + } + + /** + * Return a default value for the given setting, as an Object of the appropriate type. + * Note that the value returned for TIMEZONE may change if the system time zone + * is changed or if Daylight Saving Time starts or ends. + * + * @return + */ + public Object getDefault() { + switch (this) { + case TIMEZONE: + Calendar cal = Calendar.getInstance(); + int offsetMillis = cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET); + int offsetSeconds = offsetMillis / 1000; + return offsetSeconds; + default: + return defaultValue; + } + } +}
new file mode 100644 --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/ParameterType.java @@ -0,0 +1,106 @@ +package org.monetdb.mcl.net; + +/** + * Enumeration of the types a {@link Parameter} may have. + */ +public enum ParameterType { + /** + * The Parameter is an arbitrary string + */ + Str, + /** The Parameter can be interpreted as an {@link Integer} */ + Int, + /** The Parameter is a {@link Boolean} and can be + * written "true", "false", "on", "off", "yes" or "no". + * Uppercase letters are also accepted + */ + Bool, + /** + * Functionally the same as {@link ParameterType.Str } but + * indicates the value is to be interpreted as a path on the + * client's file system. + */ + Path; + + /** + * Convert a string to a boolean, accepting true/false/yes/no/on/off. + * + * Uppercase is also accepted. + * + * @param value text to be parsed + * @return boolean interpretation of the text + */ + public static boolean parseBool(String value) { + boolean lowered = false; + String original = value; + while (true) { + switch (value) { + case "true": + case "yes": + case "on": + return true; + case "false": + case "no": + case "off": + return false; + default: + if (!lowered) { + value = value.toLowerCase(); + lowered = true; + continue; + } + throw new IllegalArgumentException("invalid boolean value: " + original); + } + } + } + + /** + * Convert text into an Object of the appropriate type + * + * @param name name of the setting for use in error messages + * @param value text to be converted + * @return Object representation of the text + * @throws ValidationError if the text cannot be converted + */ + public Object parse(String name, String value) throws ValidationError { + if (value == null) + return null; + + try { + switch (this) { + case Bool: + return parseBool(value); + case Int: + return Integer.parseInt(value); + case Str: + case Path: + return value; + default: + throw new IllegalStateException("unreachable"); + } + } catch (IllegalArgumentException e) { + String message = e.toString(); + throw new ValidationError(name, message); + } + } + + /** + * Represent the object as a string. + * + * @param value, must be of the appropriate type + * @return textual representation + */ + public String format(Object value) { + switch (this) { + case Bool: + return (Boolean) value ? "true" : "false"; + case Int: + return Integer.toString((Integer) value); + case Str: + case Path: + return (String) value; + default: + throw new IllegalStateException("unreachable"); + } + } +}
new file mode 100644 --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/SecureSocket.java @@ -0,0 +1,196 @@ +package org.monetdb.mcl.net; + +import javax.net.ssl.*; +import java.io.FileInputStream; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.Socket; +import java.security.*; +import java.security.cert.CertificateException; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.util.Collections; + +public class SecureSocket { + private static final String[] ENABLED_PROTOCOLS = {"TLSv1.3"}; + private static final String[] APPLICATION_PROTOCOLS = {"mapi/9"}; + + // Cache for the default SSL factory. It must load all trust roots + // so it's worthwhile to cache. + // Only access this through #getDefaultSocketFactory() + private static SSLSocketFactory vanillaFactory = null; + + private static synchronized SSLSocketFactory getDefaultSocketFactory() { + if (vanillaFactory == null) { + vanillaFactory = (SSLSocketFactory) SSLSocketFactory.getDefault(); + } + return vanillaFactory; + } + + public static Socket wrap(Target.Validated validated, Socket inner) throws IOException { + Target.Verify verify = validated.connectVerify(); + SSLSocketFactory socketFactory; + boolean checkName = true; + try { + switch (verify) { + case System: + socketFactory = getDefaultSocketFactory(); + break; + case Cert: + KeyStore keyStore = keyStoreForCert(validated.getCert()); + socketFactory = certBasedSocketFactory(keyStore); + break; + case Hash: + socketFactory = hashBasedSocketFactory(validated.connectCertHashDigits()); + checkName = false; + break; + default: + throw new RuntimeException("unreachable: unexpected verification strategy " + verify.name()); + } + return wrapSocket(inner, validated, socketFactory, checkName); + } catch (CertificateException e) { + throw new SSLException("TLS certificate rejected", e); + } + } + + private static SSLSocket wrapSocket(Socket inner, Target.Validated validated, SSLSocketFactory socketFactory, boolean checkName) throws IOException { + SSLSocket sock = (SSLSocket) socketFactory.createSocket(inner, validated.connectTcp(), validated.connectPort(), true); + sock.setUseClientMode(true); + SSLParameters parameters = sock.getSSLParameters(); + + parameters.setProtocols(ENABLED_PROTOCOLS); + + parameters.setServerNames(Collections.singletonList(new SNIHostName(validated.connectTcp()))); + + if (checkName) { + parameters.setEndpointIdentificationAlgorithm("HTTPS"); + } + + // Unfortunately, SSLParameters.setApplicationProtocols is only available + // since language level 9 and currently we're on 8. + // Still call it if it happens to be available. + try { + Method setApplicationProtocols = SSLParameters.class.getMethod("setApplicationProtocols", String[].class); + setApplicationProtocols.invoke(parameters, (Object) APPLICATION_PROTOCOLS); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException ignored) { + } + + sock.setSSLParameters(parameters); + sock.startHandshake(); + return sock; + } + + private static X509Certificate loadCertificate(String path) throws CertificateException, IOException { + CertificateFactory factory = CertificateFactory.getInstance("X509"); + try (FileInputStream s = new FileInputStream(path)) { + return (X509Certificate) factory.generateCertificate(s); + } + } + + private static SSLSocketFactory certBasedSocketFactory(KeyStore store) throws IOException, CertificateException { + TrustManagerFactory trustManagerFactory; + try { + trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(store); + } catch (NoSuchAlgorithmException | KeyStoreException e) { + throw new RuntimeException("Could not create TrustManagerFactory", e); + } + + SSLContext context; + try { + context = SSLContext.getInstance("TLS"); + context.init(null, trustManagerFactory.getTrustManagers(), null); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new RuntimeException("Could not create SSLContext", e); + } + + return context.getSocketFactory(); + } + + private static KeyStore keyStoreForCert(String path) throws IOException, CertificateException { + try { + X509Certificate cert = loadCertificate(path); + KeyStore store = emptyKeyStore(); + store.setCertificateEntry("root", cert); + return store; + } catch (KeyStoreException e) { + throw new RuntimeException("Could not create KeyStore for certificate", e); + } + } + + private static KeyStore emptyKeyStore() throws IOException, CertificateException { + KeyStore store; + try { + store = KeyStore.getInstance("PKCS12"); + store.load(null, null); + return store; + } catch (KeyStoreException | NoSuchAlgorithmException e) { + throw new RuntimeException("Could not create KeyStore for certificate", e); + } + } + + private static SSLSocketFactory hashBasedSocketFactory(String hashDigits) { + TrustManager trustManager = new HashBasedTrustManager(hashDigits); + try { + SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, new TrustManager[]{trustManager}, null); + return context.getSocketFactory(); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new RuntimeException("Could not create SSLContext", e); + } + + } + + private static class HashBasedTrustManager implements X509TrustManager { + private static final char[] HEXDIGITS = "0123456789abcdef".toCharArray(); + private final String hashDigits; + + public HashBasedTrustManager(String hashDigits) { + this.hashDigits = hashDigits; + } + + + @Override + public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { + throw new RuntimeException("this TrustManager is only suitable for client side connections"); + } + + @Override + public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { + X509Certificate cert = x509Certificates[0]; + byte[] certBytes = cert.getEncoded(); + + // for now it's always SHA256. + byte[] hashBytes; + try { + MessageDigest hasher = MessageDigest.getInstance("SHA-256"); + hasher.update(certBytes); + hashBytes = hasher.digest(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("failed to instantiate hash digest"); + } + + // convert to hex digits + StringBuilder buffer = new StringBuilder(2 * hashBytes.length); + for (byte b : hashBytes) { + int hi = (b & 0xF0) >> 4; + int lo = b & 0x0F; + buffer.append(HEXDIGITS[hi]); + buffer.append(HEXDIGITS[lo]); + } + String certDigits = buffer.toString(); + + if (!certDigits.startsWith(hashDigits)) { + throw new CertificateException("Certificate hash does not start with '" + hashDigits + "': " + certDigits); + } + + + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + } +}
new file mode 100644 --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/Target.java @@ -0,0 +1,824 @@ +package org.monetdb.mcl.net; + +import java.net.URISyntaxException; +import java.util.Properties; +import java.util.regex.Pattern; + +public class Target { + protected static final Target defaults = new Target(); + private static final Pattern namePattern = Pattern.compile("^[a-zzA-Z_][-a-zA-Z0-9_.]*$"); + private static final Pattern hashPattern = Pattern.compile("^sha256:[0-9a-fA-F:]*$"); + private boolean tls = false; + private String host = ""; + private int port = -1; + private String database = ""; + private String tableschema = ""; + private String table = ""; + private String sock = ""; + private String sockdir = "/tmp"; + private String cert = ""; + private String certhash = ""; + private String clientkey = ""; + private String clientcert = ""; + private String user = ""; + private String password = ""; + private String language = "sql"; + private boolean autocommit = true; + private String schema = ""; + private int timezone; + private String binary = "on"; + private int replySize = 250; + private String hash = ""; + private boolean debug = false; + private String logfile = ""; + private int soTimeout = 0; + private boolean treatClobAsVarchar = true; + private boolean treatBlobAsBinary = true; + private boolean userWasSet = false; + private boolean passwordWasSet = false; + private Validated validated = null; + + public Target() { + this.timezone = (int) Parameter.TIMEZONE.getDefault(); + } + + public Target(String url, Properties props) throws URISyntaxException, ValidationError { + this(); + setProperties(props); + parseUrl(url); + } + + public static String packHost(String host) { + switch (host) { + case "localhost": + return "localhost."; + case "": + return "localhost"; + default: + return host; + } + } + + public static String unpackHost(String host) { + switch (host) { + case "localhost.": + return "localhost"; + case "localhost": + return ""; + default: + return host; + } + } + + public void barrier() { + if (userWasSet && !passwordWasSet) + password = ""; + userWasSet = false; + passwordWasSet = false; + } + + public void setString(String key, String value) throws ValidationError { + Parameter parm = Parameter.forName(key); + if (parm != null) + setString(parm, value); + else if (!Parameter.isIgnored(key)) + throw new ValidationError(key, "unknown parameter"); + } + + public void setString(Parameter parm, String value) throws ValidationError { + if (value == null) + throw new NullPointerException("'value' must not be null"); + assign(parm, parm.type.parse(parm.name, value)); + } + + public void clear(Parameter parm) { + assign(parm, parm.getDefault()); + } + + public void parseUrl(String url) throws URISyntaxException, ValidationError { + if (url == null) + return; + if (url.startsWith("jdbc:")) + url = url.substring(5); + if (url.equals("monetdb:")) { + return; + } + MonetUrlParser.parse(this, url); + } + + private void assign(Parameter parm, Object value) { + switch (parm) { + case TLS: + setTls((boolean) value); + break; + case HOST: + setHost((String) value); + break; + case PORT: + setPort((int) value); + break; + case DATABASE: + setDatabase((String) value); + break; + case TABLESCHEMA: + setTableschema((String) value); + break; + case TABLE: + setTable((String) value); + break; + case SOCK: + setSock((String) value); + break; + case SOCKDIR: + setSockdir((String) value); + break; + case CERT: + setCert((String) value); + break; + case CERTHASH: + setCerthash((String) value); + break; + case CLIENTKEY: + setClientkey((String) value); + break; + case CLIENTCERT: + setClientcert((String) value); + break; + case USER: + setUser((String) value); + break; + case PASSWORD: + setPassword((String) value); + break; + case LANGUAGE: + setLanguage((String) value); + break; + case AUTOCOMMIT: + setAutocommit((boolean) value); + break; + case SCHEMA: + setSchema((String) value); + break; + case TIMEZONE: + setTimezone((int) value); + break; + case BINARY: + setBinary((String) value); + break; + case REPLYSIZE: + setReplySize((int) value); + break; + case FETCHSIZE: + setReplySize((int) value); + break; + case HASH: + setHash((String) value); + break; + case DEBUG: + setDebug((boolean) value); + break; + case LOGFILE: + setLogfile((String) value); + break; + + case SO_TIMEOUT: + setSoTimeout((int) value); + break; + case CLOB_AS_VARCHAR: + setTreatClobAsVarchar((boolean) value); + break; + case BLOB_AS_BINARY: + setTreatBlobAsBinary((boolean) value); + break; + + default: + throw new IllegalStateException("unreachable -- missing case: " + parm.name); + } + } + + public String getString(Parameter parm) { + Object value = getObject(parm); + return parm.type.format(value); + } + + public Object getObject(Parameter parm) { + switch (parm) { + case TLS: + return tls; + case HOST: + return host; + case PORT: + return port; + case DATABASE: + return database; + case TABLESCHEMA: + return tableschema; + case TABLE: + return table; + case SOCK: + return sock; + case SOCKDIR: + return sockdir; + case CERT: + return cert; + case CERTHASH: + return certhash; + case CLIENTKEY: + return clientkey; + case CLIENTCERT: + return clientcert; + case USER: + return user; + case PASSWORD: + return password; + case LANGUAGE: + return language; + case AUTOCOMMIT: + return autocommit; + case SCHEMA: + return schema; + case TIMEZONE: + return timezone; + case BINARY: + return binary; + case REPLYSIZE: + return replySize; + case FETCHSIZE: + return replySize; + case HASH: + return hash; + case DEBUG: + return debug; + case LOGFILE: + return logfile; + case SO_TIMEOUT: + return soTimeout; + case CLOB_AS_VARCHAR: + return treatClobAsVarchar; + case BLOB_AS_BINARY: + return treatBlobAsBinary; + default: + throw new IllegalStateException("unreachable -- missing case"); + } + } + + public boolean isTls() { + return tls; + } + + public void setTls(boolean tls) { + this.tls = tls; + validated = null; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + validated = null; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + validated = null; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + validated = null; + } + + public String getTableschema() { + return tableschema; + } + + public void setTableschema(String tableschema) { + this.tableschema = tableschema; + validated = null; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + validated = null; + } + + public String getSock() { + return sock; + } + + public void setSock(String sock) { + this.sock = sock; + validated = null; + } + + public String getSockdir() { + return sockdir; + } + + public void setSockdir(String sockdir) { + this.sockdir = sockdir; + validated = null; + } + + public String getCert() { + return cert; + } + + public void setCert(String cert) { + this.cert = cert; + validated = null; + } + + public String getCerthash() { + return certhash; + } + + public void setCerthash(String certhash) { + this.certhash = certhash; + validated = null; + } + + public String getClientkey() { + return clientkey; + } + + public void setClientkey(String clientkey) { + this.clientkey = clientkey; + validated = null; + } + + public String getClientcert() { + return clientcert; + } + + public void setClientcert(String clientcert) { + this.clientcert = clientcert; + validated = null; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + this.userWasSet = true; + validated = null; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + this.passwordWasSet = true; + validated = null; + } + + public String getLanguage() { + return language; + } + + public void setLanguage(String language) { + this.language = language; + validated = null; + } + + public boolean isAutocommit() { + return autocommit; + } + + public void setAutocommit(boolean autocommit) { + this.autocommit = autocommit; + validated = null; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + validated = null; + } + + public int getTimezone() { + return timezone; + } + + public void setTimezone(int timezone) { + this.timezone = timezone; + validated = null; + } + + public String getBinary() { + return binary; + } + + public void setBinary(String binary) { + this.binary = binary; + validated = null; + } + + public int getReplySize() { + return replySize; + } + + public void setReplySize(int replySize) { + this.replySize = replySize; + validated = null; + } + + public String getHash() { + return hash; + } + + public void setHash(String hash) { + this.hash = hash; + validated = null; + } + + public boolean isDebug() { + return debug; + } + + public void setDebug(boolean debug) { + this.debug = debug; + validated = null; + } + + public String getLogfile() { + return logfile; + } + + public void setLogfile(String logfile) { + this.logfile = logfile; + validated = null; + } + + public int getSoTimeout() { + return soTimeout; + } + + public void setSoTimeout(int soTimeout) { + this.soTimeout = soTimeout; + validated = null; + } + + public boolean isTreatClobAsVarchar() { + return treatClobAsVarchar; + } + + public void setTreatClobAsVarchar(boolean treatClobAsVarchar) { + this.treatClobAsVarchar = treatClobAsVarchar; + validated = null; + } + + public boolean isTreatBlobAsBinary() { + return treatBlobAsBinary; + } + + public void setTreatBlobAsBinary(boolean treatBlobAsBinary) { + this.treatBlobAsBinary = treatBlobAsBinary; + validated = null; + } + + public Validated validate() throws ValidationError { + if (validated == null) + validated = new Validated(); + return validated; + } + + public String buildUrl() { + final StringBuilder sb = new StringBuilder(128); + sb.append("jdbc:"); + sb.append(tls ? "monetdbs" : "monetdb"); + sb.append("://"); + sb.append(packHost(host)); + if (!Parameter.PORT.getDefault().equals(port)) { + sb.append(':'); + sb.append(port); + } + sb.append('/').append(database); + String sep = "?"; + for (Parameter parm : Parameter.values()) { + if (parm.isCore || parm == Parameter.USER || parm == Parameter.PASSWORD) + continue; + Object defaultValue = parm.getDefault(); + if (defaultValue == null) + continue; + Object value = getObject(parm); + if (value.equals(defaultValue)) + continue; + sb.append(sep).append(parm.name).append('='); + String raw = getString(parm); + String encoded = MonetUrlParser.percentEncode(raw); + sb.append(encoded); + sep = "&"; + } + return sb.toString(); + } + + public Properties getProperties() { + Properties props = new Properties(); + for (Parameter parm : Parameter.values()) { + Object defaultValue = parm.getDefault(); + if (defaultValue == null || defaultValue.equals(getObject(parm))) + continue; + String value = getString(parm); + if (parm == Parameter.HOST) + value = packHost(host); + props.setProperty(parm.name, value); + } + + return props; + } + + public void setProperties(Properties props) throws ValidationError { + if (props != null) { + for (String key : props.stringPropertyNames()) { + String value = props.getProperty(key); + if (key.equals(Parameter.HOST.name)) + value = Target.unpackHost(value); + setString(key, value); + } + } + } + + public enum Verify { + None, Cert, Hash, System + } + + public class Validated { + + private final int nbinary; + + Validated() throws ValidationError { + + // 1. The parameters have the types listed in the table in [Section + // Parameters](#parameters). + + String binaryString = binary; + int binaryInt; + try { + binaryInt = (int) ParameterType.Int.parse(Parameter.BINARY.name, binaryString); + } catch (ValidationError e) { + try { + boolean b = (boolean) ParameterType.Bool.parse(Parameter.BINARY.name, binaryString); + binaryInt = b ? 65535 : 0; + } catch (ValidationError ee) { + throw new ValidationError("binary= must be either a number or true/yes/on/false/no/off"); + } + } + if (binaryInt < 0) + throw new ValidationError("binary= cannot be negative"); + nbinary = binaryInt; + + + // 2. At least one of **sock** and **host** must be empty. + if (!sock.isEmpty() && !host.isEmpty()) + throw new ValidationError("sock=" + sock + " cannot be combined with host=" + host); + + // 3. The string parameter **binary** must either parse as a boolean or as a + // non-negative integer. + // + // (checked above) + + // 4. If **sock** is not empty, **tls** must be 'off'. + if (!sock.isEmpty() && tls) + throw new ValidationError("monetdbs:// cannot be combined with sock="); + + // 5. If **certhash** is not empty, it must be of the form `{sha256}hexdigits` + // where hexdigits is a non-empty sequence of 0-9, a-f, A-F and colons. + // TODO + if (!certhash.isEmpty()) { + if (!certhash.toLowerCase().startsWith("sha256:")) + throw new ValidationError("certificate hash must start with 'sha256:'"); + if (!hashPattern.matcher(certhash).matches()) + throw new ValidationError("invalid certificate hash"); + } + + // 6. If **tls** is 'off', **cert** and **certhash** must be 'off' as well. + if (!tls) { + if (!cert.isEmpty() || !certhash.isEmpty()) + throw new ValidationError("cert= and certhash= are only allowed in combination with monetdbs://"); + } + + // 7. Parameters **database**, **tableschema** and **table** must consist only of + // upper- and lowercase letters, digits, periods, dashes and underscores. They must not + // start with a dash. + // If **table** is not empty, **tableschema** must also not be empty. + // If **tableschema** is not empty, **database** must also not be empty. + if (database.isEmpty() && !tableschema.isEmpty()) + throw new ValidationError("table schema cannot be set without database"); + if (tableschema.isEmpty() && !table.isEmpty()) + throw new ValidationError("table cannot be set without schema"); + if (!database.isEmpty() && !namePattern.matcher(database).matches()) + throw new ValidationError("invalid database name"); + if (!tableschema.isEmpty() && !namePattern.matcher(tableschema).matches()) + throw new ValidationError("invalid table schema name"); + if (!table.isEmpty() && !namePattern.matcher(table).matches()) + throw new ValidationError("invalid table name"); + + + // 8. Parameter **port** must be -1 or in the range 1-65535. + if (port < -1 || port == 0 || port > 65535) + throw new ValidationError("invalid port number " + port); + + // 9. If **clientcert** is set, **clientkey** must also be set. + if (!clientcert.isEmpty() && clientkey.isEmpty()) + throw new ValidationError("clientcert= is only valid in combination with clientkey="); + + // JDBC specific + if (soTimeout < 0) + throw new ValidationError("so_timeout= must not be negative"); + } + + public boolean getTls() { + return tls; + } + + // Getter is private because you probably want connectTcp() instead + private String getHost() { + return host; + } + + // Getter is private because you probably want connectPort() instead + private int getPort() { + return port; + } + + public String getDatabase() { + return database; + } + + public String getTableschema() { + return tableschema; + } + + public String getTable() { + return table; + } + + // Getter is private because you probably want connectUnix() instead + private String getSock() { + return sock; + } + + public String getSockdir() { + return sockdir; + } + + public String getCert() { + return cert; + } + + public String getCerthash() { + return certhash; + } + + public String getClientkey() { + return clientkey; + } + + public String getClientcert() { + return clientcert; + } + + public String getUser() { + return user; + } + + public String getPassword() { + return password; + } + + public String getLanguage() { + return language; + } + + public boolean getAutocommit() { + return autocommit; + } + + public String getSchema() { + return schema; + } + + public int getTimezone() { + return timezone; + } + + // Getter is private because you probably want connectBinary() instead + public int getBinary() { + return nbinary; + } + + public int getReplySize() { + return replySize; + } + + public String getHash() { + return hash; + } + + public boolean getDebug() { + return debug; + } + + public String getLogfile() { + return logfile; + } + + public int getSoTimeout() { + return soTimeout; + } + + public boolean isTreatClobAsVarchar() { + return treatClobAsVarchar; + } + + public boolean isTreatBlobAsBinary() { + return treatBlobAsBinary; + } + + public boolean connectScan() { + if (database.isEmpty()) + return false; + if (!sock.isEmpty() || !host.isEmpty() || port != -1) + return false; + return !tls; + } + + public int connectPort() { + return port == -1 ? 50000 : port; + } + + public String connectUnix() { + if (!sock.isEmpty()) + return sock; + if (tls) + return ""; + if (host.isEmpty()) + return sockdir + "/.s.monetdb." + connectPort(); + return ""; + } + + public String connectTcp() { + if (!sock.isEmpty()) + return ""; + if (host.isEmpty()) + return "localhost"; + return host; + } + + public Verify connectVerify() { + if (!tls) + return Verify.None; + if (!certhash.isEmpty()) + return Verify.Hash; + if (!cert.isEmpty()) + return Verify.Cert; + return Verify.System; + } + + public String connectCertHashDigits() { + if (!tls) + return null; + StringBuilder builder = new StringBuilder(certhash.length()); + for (int i = "sha256:".length(); i < certhash.length(); i++) { + char c = certhash.charAt(i); + if (Character.digit(c, 16) >= 0) + builder.append(Character.toLowerCase(c)); + } + return builder.toString(); + } + + public int connectBinary() { + return nbinary; + } + + public String connectClientKey() { + return clientkey; + } + + public String connectClientCert() { + return clientcert.isEmpty() ? clientkey : clientcert; + } + } +}
new file mode 100644 --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/ValidationError.java @@ -0,0 +1,11 @@ +package org.monetdb.mcl.net; + +public class ValidationError extends Exception { + public ValidationError(String parameter, String message) { + super(parameter + ": " + message); + } + + public ValidationError(String message) { + super(message); + } +}
--- a/src/main/java/org/monetdb/util/CmdLineOpts.java +++ b/src/main/java/org/monetdb/util/CmdLineOpts.java @@ -14,11 +14,13 @@ package org.monetdb.util; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Properties; public final class CmdLineOpts { /** the arguments we handle */ private HashMap<String, OptionContainer> opts = new HashMap<String, OptionContainer>(); + private final HashSet<String> ignoredInFile = new HashSet<>(); /** the options themself */ private ArrayList<OptionContainer> options = new ArrayList<OptionContainer>(); @@ -57,6 +59,10 @@ public final class CmdLineOpts { opts.put(longa, oc); } + public void addIgnored(String name) { + ignoredInFile.add(name); + } + public void removeOption(final String name) { final OptionContainer oc = opts.get(name); if (oc != null) { @@ -92,9 +98,10 @@ public final class CmdLineOpts { if (option != null) { option.resetArguments(); option.addArgument(prop.getProperty(key)); - } else + } else if (!ignoredInFile.contains(key)) { // ignore unknown options (it used to throw an OptionsException) System.out.println("Info: Ignoring unknown/unsupported option (in " + file.getAbsolutePath() + "): " + key); + } } } catch (java.io.IOException e) { throw new OptionsException("File IO Exception: " + e);
--- a/tests/JDBC_API_Tester.java +++ b/tests/JDBC_API_Tester.java @@ -16,13 +16,11 @@ import java.io.StringReader; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; +import java.sql.Date; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Iterator; -import java.util.List; -import java.util.TimeZone; - +import java.util.*; + +import org.monetdb.jdbc.MonetConnection; import org.monetdb.jdbc.types.INET; import org.monetdb.jdbc.types.URL; @@ -52,6 +50,9 @@ final public class JDBC_API_Tester { public static void main(String[] args) throws Exception { String con_URL = args[0]; + // Test this before trying to connect + UrlTester.runAllTests(); + JDBC_API_Tester jt = new JDBC_API_Tester(); jt.sb = new StringBuilder(sbInitLen); jt.con = DriverManager.getConnection(con_URL); @@ -2186,12 +2187,10 @@ final public class JDBC_API_Tester { sb.append("1. DatabaseMetadata environment retrieval... "); // retrieve this to simulate a bug report - DatabaseMetaData dbmd = con.getMetaData(); - if (conURL.startsWith(dbmd.getURL())) - sb.append("oke"); - else - sb.append("not oke ").append(dbmd.getURL()); - sb.append("\n"); + con.getMetaData().getURL(); + // There used to be a test "if (conURL.startsWith(dbmdURL))" here + // but with the new URLs that is too simplistic. + sb.append("oke").append("\n"); pstmt = con.prepareStatement("select * from columns"); sb.append("2. empty call..."); @@ -6812,20 +6811,11 @@ final public class JDBC_API_Tester { org.monetdb.mcl.net.MapiSocket server = new org.monetdb.mcl.net.MapiSocket(); try { - server.setLanguage("sql"); - - // extract from conn_URL the used connection properties - String host = extractFromJDBCURL(conn_URL, "host"); - int port = Integer.parseInt(extractFromJDBCURL(conn_URL, "port")); - String login = extractFromJDBCURL(conn_URL, "user"); - String passw = extractFromJDBCURL(conn_URL, "password"); - String database = extractFromJDBCURL(conn_URL, "database"); - // sb.append("conn_URL: " + conn_URL + "\n"); - // sb.append("host: " + host + " port: " + port + " dbname: " + database + " login: " + login + " passwd: " + passw + "\n"); + MonetConnection mcon = (MonetConnection) con; + Properties props = mcon.getConnectionProperties(); sb.append("Before connecting to MonetDB server via MapiSocket\n"); - server.setDatabase(database); - List<String> warning = server.connect(host, port, login, passw); + List<String> warning = server.connect("jdbc:monetdb:", props); if (warning != null) { for (Iterator<String> it = warning.iterator(); it.hasNext(); ) { sb.append("Warning: ").append(it.next().toString()).append("\n");
new file mode 100644 --- /dev/null +++ b/tests/TLSTester.java @@ -0,0 +1,314 @@ +import org.monetdb.mcl.net.Parameter; + +import java.io.*; +import java.net.URL; +import java.net.URLConnection; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Properties; + +public class TLSTester { + final HashMap<String, File> fileCache = new HashMap<>(); + int verbose = 0; + String serverHost = null; + String altHost = null; + int serverPort = -1; + boolean enableTrusted = false; + File tempDir = null; + private final HashSet<String> preparedButNotRun = new HashSet<>(); + + public TLSTester(String[] args) { + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if (arg.equals("-v")) { + verbose = 1; + } else if (arg.equals("-a")) { + altHost = args[++i]; + } else if (arg.equals("-t")) { + enableTrusted = true; + } else if (!arg.startsWith("-") && serverHost == null) { + int idx = arg.indexOf(':'); + if (idx > 0) { + serverHost = arg.substring(0, idx); + try { + serverPort = Integer.parseInt(arg.substring(idx + 1)); + if (serverPort > 0 && serverPort < 65536) + continue; + } catch (NumberFormatException ignored) { + } + } + // if we get here it wasn't very valid + throw new IllegalArgumentException("Invalid argument: " + arg); + } else { + throw new IllegalArgumentException("Unexpected argument: " + arg); + } + } + } + + public static void main(String[] args) throws IOException, SQLException, ClassNotFoundException { + Class.forName("org.monetdb.jdbc.MonetDriver"); + TLSTester main = new TLSTester(args); + main.run(); + } + + private HashMap<String, Integer> loadPortMap(String testName) throws IOException { + HashMap<String, Integer> portMap = new HashMap<>(); + InputStream in = fetchData("/?test=" + testName); + BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); + for (String line = br.readLine(); line != null; line = br.readLine()) { + int idx = line.indexOf(':'); + String service = line.substring(0, idx); + int port; + try { + port = Integer.parseInt(line.substring(idx + 1)); + } catch (NumberFormatException e) { + throw new RuntimeException("Invalid port map line: " + line); + } + portMap.put(service, port); + } + return portMap; + } + + private File resource(String resource) throws IOException { + if (!fileCache.containsKey(resource)) + fetchResource(resource); + return fileCache.get(resource); + } + + private void fetchResource(String resource) throws IOException { + if (!resource.startsWith("/")) { + throw new IllegalArgumentException("Resource must start with slash: " + resource); + } + if (tempDir == null) { + tempDir = Files.createTempDirectory("tlstest").toFile(); + tempDir.deleteOnExit(); + } + File outPath = new File(tempDir, resource.substring(1)); + try (InputStream in = fetchData(resource); FileOutputStream out = new FileOutputStream(outPath)) { + byte[] buffer = new byte[12]; + while (true) { + int n = in.read(buffer); + if (n <= 0) + break; + out.write(buffer, 0, n); + } + } + fileCache.put(resource, outPath); + } + + private byte[] fetchBytes(String resource) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (InputStream in = fetchData(resource)) { + byte[] buffer = new byte[22]; + while (true) { + int nread = in.read(buffer); + if (nread <= 0) + break; + out.write(buffer, 0, nread); + } + return out.toByteArray(); + } + } + + private InputStream fetchData(String resource) throws IOException { + URL url = new URL("http://" + serverHost + ":" + serverPort + resource); + URLConnection conn = url.openConnection(); + conn.connect(); + return conn.getInputStream(); + } + + private void run() throws IOException, SQLException { + test_connect_plain(); + test_connect_tls(); + test_refuse_no_cert(); + test_refuse_wrong_cert(); + test_refuse_wrong_host(); + test_refuse_tlsv12(); + test_refuse_expired(); +// test_connect_client_auth1(); +// test_connect_client_auth2(); + test_fail_tls_to_plain(); + test_fail_plain_to_tls(); + test_connect_server_name(); + test_connect_alpn_mapi9(); + test_connect_trusted(); + test_refuse_trusted_wrong_host(); + + // did we forget to call expectSucceed and expectFailure somewhere? + if (!preparedButNotRun.isEmpty()) { + String names = String.join(", ", preparedButNotRun); + throw new RuntimeException("Not all tests called expectSuccess/expectFailure: " + names); + } + } + + private void test_connect_plain() throws IOException, SQLException { + attempt("connect_plain", "plain").with(Parameter.TLS, false).expectSuccess(); + } + + private void test_connect_tls() throws IOException, SQLException { + Attempt attempt = attempt("connect_tls", "server1"); + attempt.withFile(Parameter.CERT, "/ca1.crt").expectSuccess(); + } + + private void test_refuse_no_cert() throws IOException, SQLException { + attempt("refuse_no_cert", "server1").expectFailure("PKIX path building failed"); + } + + private void test_refuse_wrong_cert() throws IOException, SQLException { + Attempt attempt = attempt("refuse_wrong_cert", "server1"); + attempt.withFile(Parameter.CERT, "/ca2.crt").expectFailure("PKIX path building failed"); + } + + private void test_refuse_wrong_host() throws IOException, SQLException { + Attempt attempt = attempt("refuse_wrong_host", "server1").with(Parameter.HOST, altHost); + attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("No subject alternative DNS name"); + } + + private void test_refuse_tlsv12() throws IOException, SQLException { + Attempt attempt = attempt("refuse_tlsv12", "tls12"); + attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("protocol_version"); + } + + private void test_refuse_expired() throws IOException, SQLException { + Attempt attempt = attempt("refuse_expired", "expiredcert"); + attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("PKIX path validation failed"); + } + + private void test_connect_client_auth1() throws IOException, SQLException { + attempt("connect_client_auth1", "clientauth") + .withFile(Parameter.CERT, "/ca1.crt") + .withFile(Parameter.CLIENTKEY, "/client2.keycrt") + .expectSuccess(); + } + + private void test_connect_client_auth2() throws IOException, SQLException { + attempt("connect_client_auth2", "clientauth") + .withFile(Parameter.CERT, "/ca1.crt") + .withFile(Parameter.CLIENTKEY, "/client2.key") + .withFile(Parameter.CLIENTCERT, "/client2.crt") + .expectSuccess(); + } + + private void test_fail_tls_to_plain() throws IOException, SQLException { + Attempt attempt = attempt("fail_tls_to_plain", "plain"); + attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure(""); + + } + + private void test_fail_plain_to_tls() throws IOException, SQLException { + attempt("fail_plain_to_tls", "server1").with(Parameter.TLS, false).expectFailure("Cannot connect", "Could not connect"); + } + + private void test_connect_server_name() throws IOException, SQLException { + Attempt attempt = attempt("connect_server_name", "sni"); + attempt.withFile(Parameter.CERT, "/ca1.crt").expectSuccess(); + } + + private void test_connect_alpn_mapi9() throws IOException, SQLException { + attempt("connect_alpn_mapi9", "alpn_mapi9").withFile(Parameter.CERT, "/ca1.crt").expectSuccess(); + } + + private void test_connect_trusted() throws IOException, SQLException { + attempt("connect_trusted", null) + .with(Parameter.HOST, "monetdb.ergates.nl") + .with(Parameter.PORT, 50000) + .expectSuccess(); + } + + private void test_refuse_trusted_wrong_host() throws IOException, SQLException { + attempt("test_refuse_trusted_wrong_host", null) + .with(Parameter.HOST, "monetdbxyz.ergates.nl") + .with(Parameter.PORT, 50000) + .expectFailure("No subject alternative DNS name"); + } + + private Attempt attempt(String testName, String portName) throws IOException { + preparedButNotRun.add(testName); + return new Attempt(testName, portName); + } + + private class Attempt { + private final String testName; + private final Properties props = new Properties(); + boolean disabled = false; + + public Attempt(String testName, String portName) throws IOException { + HashMap<String, Integer> portMap = loadPortMap(testName); + + this.testName = testName; + with(Parameter.TLS, true); + with(Parameter.HOST, serverHost); + with(Parameter.SO_TIMEOUT, 3000); + if (portName != null) { + Integer port = portMap.get(portName); + if (port != null) { + with(Parameter.PORT, port); + } else { + throw new RuntimeException("Unknown port name: " + portName); + } + } + } + + private Attempt with(Parameter parm, String value) { + props.setProperty(parm.name, value); + return this; + } + + private Attempt with(Parameter parm, int value) { + props.setProperty(parm.name, Integer.toString(value)); + return this; + } + + private Attempt with(Parameter parm, boolean value) { + props.setProperty(parm.name, value ? "true" : "false"); + return this; + } + + private Attempt withFile(Parameter parm, String certResource) throws IOException { + File certFile = resource(certResource); + String path = certFile.getPath(); + with(parm, path); + return this; + } + + public void expectSuccess() throws SQLException { + preparedButNotRun.remove(testName); + if (disabled) + return; + try { + Connection conn = DriverManager.getConnection("jdbc:monetdb:", props); + conn.close(); + } catch (SQLException e) { + if (e.getMessage().startsWith("Sorry, this is not a real MonetDB instance")) { + // it looks like a failure but this is actually our success scenario + // because this is what the TLS Tester does when the connection succeeds. + return; + } + // other exceptions ARE errors and should be reported. + throw e; + } + } + + public void expectFailure(String... expectedMessages) throws SQLException { + if (disabled) + return; + try { + expectSuccess(); + throw new RuntimeException("Expected test " + testName + " to throw an exception but it didn't"); + } catch (SQLException e) { + for (String expected : expectedMessages) + if (e.getMessage().contains(expected)) + return; + String message = "Test " + testName + " threw the wrong exception: " + e.getMessage() + '\n' + "Expected:\n <" + String.join(">\n <", expectedMessages) + ">"; + throw new RuntimeException(message, e); + + } + } + + } +}
new file mode 100644 --- /dev/null +++ b/tests/UrlTester.java @@ -0,0 +1,414 @@ +import org.monetdb.mcl.net.*; + +import java.io.*; +import java.net.URISyntaxException; +import java.util.ArrayList; + +public class UrlTester { + final String filename; + final int verbose; + final BufferedReader reader; + int lineno = 0; + int testCount = 0; + Target target = null; + Target.Validated validated = null; + + public UrlTester(String filename, BufferedReader reader, int verbose) { + this.filename = filename; + this.verbose = verbose; + this.reader = reader; + } + + public UrlTester(String filename, int verbose) throws IOException { + this.filename = filename; + this.verbose = verbose; + this.reader = new BufferedReader(new FileReader(filename)); + } + + public static void main(String[] args) throws IOException { + ArrayList<String> filenames = new ArrayList<>(); + int verbose = 0; + for (String arg : args) { + switch (arg) { + case "-vvv": + verbose++; + case "-vv": + verbose++; + case "-v": + verbose++; + break; + case "-h": + case "--help": + exitUsage(null); + break; + default: + if (!arg.startsWith("-")) { + filenames.add(arg); + } else { + exitUsage("Unexpected argument: " + arg); + } + break; + } + } + + runUnitTests(); + + try { + if (filenames.isEmpty()) { + runAllTests(); + } else { + for (String filename : filenames) { + new UrlTester(filename, verbose).run(); + } + } + } catch (Failure e) { + System.err.println("Test failed: " + e.getMessage()); + System.exit(1); + } + } + + private static void exitUsage(String message) { + if (message != null) { + System.err.println(message); + } + System.err.println("Usage: UrlTester OPTIONS [FILENAME..]"); + System.err.println("Options:"); + System.err.println(" -v Be more verbose"); + System.err.println(" -h --help Show this help"); + int status = message == null ? 0 : 1; + System.exit(status); + } + + public static UrlTester forResource(String resourceName, int verbose) throws FileNotFoundException { + InputStream stream = UrlTester.class.getResourceAsStream(resourceName); + if (stream == null) { + throw new FileNotFoundException("Resource " + resourceName); + } + BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); + return new UrlTester(resourceName, reader, verbose); + } + + public static void runAllTests() throws IOException, Failure { + runUnitTests(); + UrlTester.forResource("/tests.md", 0).run(); + UrlTester.forResource("/javaspecific.md", 0).run(); + } + + public static void runUnitTests() { + testDefaults(); + testParameters(); + } + + private static void testDefaults() { + Target target = new Target(); + + for (Parameter parm : Parameter.values()) { + Object expected = parm.getDefault(); + if (expected == null) + continue; + Object actual = target.getObject(parm); + if (!expected.equals(actual)) { + throw new RuntimeException("Default for " + parm.name + " expected to be <" + expected + "> but is <" + actual + ">"); + } + } + } + + private static void testParameters() { + for (Parameter parm : Parameter.values()) { + Parameter found = Parameter.forName(parm.name); + if (parm != found) { + String foundStr = found != null ? found.name : "null"; + throw new RuntimeException("Looking up <" + parm.name + ">, found <" + foundStr); + } + } + } + + public void run() throws Failure, IOException { + try { + processFile(); + } catch (Failure e) { + if (e.getFilename() == null) { + e.setFilename(filename); + e.setLineno(lineno); + throw e; + } + } + } + + private void processFile() throws IOException, Failure { + while (true) { + String line = reader.readLine(); + if (line == null) + break; + lineno++; + processLine(line); + } + if (verbose >= 1) { + System.out.println(); + System.out.println("Ran " + testCount + " tests in " + lineno + " lines"); + } + } + + private void processLine(String line) throws Failure { + line = line.replaceFirst("\\s+$", ""); // remove trailing + if (target == null && line.equals("```test")) { + if (verbose >= 2) { + if (testCount > 0) { + System.out.println(); + } + System.out.println("\u25B6 " + filename + ":" + lineno); + } + target = new Target(); + testCount++; + return; + } + if (target != null) { + if (line.equals("```")) { + stopProcessing(); + return; + } + handleCommand(line); + } + } + + private void stopProcessing() { + target = null; + validated = null; + } + + private void handleCommand(String line) throws Failure { + if (verbose >= 3) { + System.out.println(line); + } + if (line.isEmpty()) + return; + + String[] parts = line.split("\\s+", 2); + String command = parts[0]; + switch (command.toUpperCase()) { + case "ONLY": + handleOnly(true, parts[1]); + return; + case "NOT": + handleOnly(false, parts[1]); + return; + case "PARSE": + handleParse(parts[1], null); + return; + case "ACCEPT": + handleParse(parts[1], true); + return; + case "REJECT": + handleParse(parts[1], false); + return; + case "SET": + handleSet(parts[1]); + return; + case "EXPECT": + handleExpect(parts[1]); + return; + default: + throw new Failure("Unexpected command: " + command); + } + + } + + private void handleOnly(boolean mustBePresent, String rest) throws Failure { + boolean found = false; + for (String part : rest.split("\\s+")) { + if (part.equals("jdbc")) { + found = true; + break; + } + } + if (found != mustBePresent) { + // do not further process this block + stopProcessing(); + } + } + + private int findEqualSign(String rest) throws Failure { + int index = rest.indexOf('='); + if (index < -1) + throw new Failure("Expected to find a '='"); + return index; + } + + private String splitKey(String rest) throws Failure { + int index = findEqualSign(rest); + return rest.substring(0, index); + } + + private String splitValue(String rest) throws Failure { + int index = findEqualSign(rest); + return rest.substring(index + 1); + } + + private void handleSet(String rest) throws Failure { + validated = null; + String key = splitKey(rest); + String value = splitValue(rest); + + try { + target.setString(key, value); + } catch (ValidationError e) { + throw new Failure(e.getMessage()); + } + } + + private void handleParse(String rest, Boolean shouldSucceed) throws Failure { + URISyntaxException parseError = null; + ValidationError validationError = null; + + validated = null; + try { + target.barrier(); + MonetUrlParser.parse(target, rest); + } catch (URISyntaxException e) { + parseError = e; + } catch (ValidationError e) { + validationError = e; + } + + if (parseError == null && validationError == null) { + try { + tryValidate(); + } catch (ValidationError e) { + validationError = e; + } + } + + if (shouldSucceed == Boolean.FALSE) { + if (parseError != null || validationError != null) + return; // happy + else + throw new Failure("URL unexpectedly parsed and validated"); + } + + if (parseError != null) + throw new Failure("Parse error: " + parseError); + if (validationError != null && shouldSucceed == Boolean.TRUE) + throw new Failure("Validation error: " + validationError); + } + + private void handleExpect(String rest) throws Failure { + String key = splitKey(rest); + String expectedString = splitValue(rest); + + Object actual = null; + try { + actual = extract(key); + } catch (ValidationError e) { + throw new Failure(e.getMessage()); + } + + Object expected; + try { + if (actual instanceof Boolean) + expected = ParameterType.Bool.parse(key, expectedString); + else if (actual instanceof Integer) + expected = ParameterType.Int.parse(key, expectedString); + else + expected = expectedString; + } catch (ValidationError e) { + String typ = actual.getClass().getName(); + throw new Failure("Cannot convert expected value <" + expectedString + "> to " + typ + ": " + e.getMessage()); + } + + if (actual.equals(expected)) + return; + throw new Failure("Expected " + key + "=<" + expectedString + ">, found <" + actual + ">"); + } + + private Target.Validated tryValidate() throws ValidationError { + if (validated == null) + validated = target.validate(); + return validated; + } + + private Object extract(String key) throws ValidationError, Failure { + switch (key) { + case "valid": + try { + tryValidate(); + } catch (ValidationError e) { + return Boolean.FALSE; + } + return Boolean.TRUE; + + case "connect_scan": + return tryValidate().connectScan(); + case "connect_port": + return tryValidate().connectPort(); + case "connect_unix": + return tryValidate().connectUnix(); + case "connect_tcp": + return tryValidate().connectTcp(); + case "connect_tls_verify": + switch (tryValidate().connectVerify()) { + case None: + return ""; + case Cert: + return "cert"; + case Hash: + return "hash"; + case System: + return "system"; + default: + throw new IllegalStateException("unreachable"); + } + case "connect_certhash_digits": + return tryValidate().connectCertHashDigits(); + case "connect_binary": + return tryValidate().connectBinary(); + case "connect_clientkey": + return tryValidate().connectClientKey(); + case "connect_clientcert": + return tryValidate().connectClientCert(); + + default: + Parameter parm = Parameter.forName(key); + if (parm != null) + return target.getObject(parm); + else + throw new Failure("Unknown attribute: " + key); + } + } + + public static class Failure extends Exception { + private String filename = null; + private int lineno = -1; + + public Failure(String message) { + super(message); + } + + @Override + public String getMessage() { + StringBuilder buffer = new StringBuilder(); + if (filename != null) { + buffer.append(filename).append(":"); + if (lineno >= 0) + buffer.append(lineno).append(":"); + } + buffer.append(super.getMessage()); + return buffer.toString(); + } + + public String getFilename() { + return filename; + } + + public void setFilename(String filename) { + this.filename = filename; + } + + public int getLineno() { + return lineno; + } + + public void setLineno(int lineno) { + this.lineno = lineno; + } + } +}
--- a/tests/build.xml +++ b/tests/build.xml @@ -76,11 +76,17 @@ Copyright 1997 - July 2008 CWI. <jar jarfile="${jdbctests-jar}"> <fileset dir="${builddir}"> <include name="JDBC_API_Tester.class" /> + <include name="UrlTester.class" /> + <include name="UrlTester$*.class" /> <include name="OnClientTester.class" /> <include name="OnClientTester$*.class" /> <include name="ConnectionTests.class" /> <include name="ConnectionTests$*.class" /> </fileset> + <fileset dir="${srcdir}"> + <include name="tests.md" /> + <include name="javaspecific.md" /> + </fileset> </jar> </target>
new file mode 100644 --- /dev/null +++ b/tests/javaspecific.md @@ -0,0 +1,30 @@ +# Java-specific tests + +Test settings that are only in monetdb-java. + +```test +ONLY jdbc +EXPECT so_timeout=0 +SET so_timeout=42 +EXPECT so_timeout=42 +ACCEPT monetdb://?so_timeout=99 +EXPECT so_timeout=99 +``` + +```test +ONLY jdbc +EXPECT treat_clob_as_varchar=true +SET treat_clob_as_varchar=off +EXPECT treat_clob_as_varchar=false +ACCEPT monetdb://?treat_clob_as_varchar=yes +EXPECT treat_clob_as_varchar=on +``` + +```test +ONLY jdbc +EXPECT treat_blob_as_binary=true +SET treat_blob_as_binary=off +EXPECT treat_blob_as_binary=false +ACCEPT monetdb://?treat_blob_as_binary=yes +EXPECT treat_blob_as_binary=on +```
new file mode 100644 --- /dev/null +++ b/tests/tests.md @@ -0,0 +1,1507 @@ +# Tests + +This document contains a large number of test cases. +They are embedded in the Markdown source, in <code>```test</code> +. . .</code>```</code> blocks. + + +The tests are written in a mini language with the following +keywords: + +* `PARSE url`: parse the URL, this should succeed. The validity checks need + not be satisfied. + +* `ACCEPT url`: parse the URL, this should succeed. The validity checks + should pass. + +* `REJECT url`: parse the URL, it should be rejected either in the parsing stage + or by the validity checks. + +* `SET key=value`: modify a parameter, can occur before or after parsing the URL. + Used to model command line parameters, Java Properties objects, etc. + +* `EXPECT key=value`: verify that the given parameter now has the given + value. Fail the test case if the value is different. + +* `ONLY pymonetdb`: only process the rest of the block when testing + pymonetdb, ignore it for other implementations. + +* `NOT pymonetdb`: ignore the rest of the block when testing pymonetdb, + do process it for other implementations. + +At the start of each block the parameters are reset to their default values. + +The EXPECT clause can verify all parameters listen in the Parameters section of +the spec, all 'virtual parameters' and also the special case `valid` which is a +boolean indicating whether all validity rules in section 'Interpreting the +parameters' hold. + +Note: an `EXPECT` of the virtual parameters implies `EXPECT valid=true`. + +TODO before 1.0 does the above explanation make sense? + + +## Tests from the examples + +```test +ACCEPT monetdb:///demo +EXPECT database=demo +EXPECT connect_scan=true +``` + +```test +ACCEPT monetdb://localhost/demo +EXPECT connect_scan=true +EXPECT database=demo +``` + +```test +ACCEPT monetdb://localhost./demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=localhost +EXPECT connect_port=50000 +EXPECT tls=off +EXPECT database=demo +``` + +```test +ACCEPT monetdb://localhost.:12345/demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=localhost +EXPECT connect_port=12345 +EXPECT tls=off +EXPECT database=demo +``` + +```test +ACCEPT monetdb://localhost:12345/demo +EXPECT connect_scan=false +EXPECT connect_unix=/tmp/.s.monetdb.12345 +EXPECT connect_tcp=localhost +EXPECT connect_port=12345 +EXPECT tls=off +EXPECT database=demo +``` + +```test +ACCEPT monetdb:///demo?user=monetdb&password=monetdb +EXPECT connect_scan=true +EXPECT database=demo +EXPECT user=monetdb +EXPECT password=monetdb +``` + +```test +ACCEPT monetdb://mdb.example.com:12345/demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=mdb.example.com +EXPECT connect_port=12345 +EXPECT tls=off +EXPECT database=demo +``` + +```test +ACCEPT monetdb://192.168.13.4:12345/demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=192.168.13.4 +EXPECT connect_port=12345 +EXPECT tls=off +EXPECT database=demo +``` + +```test +ACCEPT monetdb://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:12345/demo +EXPECT host=2001:0db8:85a3:0000:0000:8a2e:0370:7334 +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=2001:0db8:85a3:0000:0000:8a2e:0370:7334 +EXPECT connect_port=12345 +EXPECT tls=off +EXPECT database=demo +``` + +```test +ACCEPT monetdb://localhost/ +EXPECT connect_unix=/tmp/.s.monetdb.50000 +EXPECT connect_scan=false +EXPECT connect_tcp=localhost +EXPECT tls=off +EXPECT database= +``` + +```test +ACCEPT monetdb://localhost +EXPECT connect_scan=false +EXPECT connect_unix=/tmp/.s.monetdb.50000 +EXPECT connect_tcp=localhost +EXPECT tls=off +EXPECT database= +``` + +```test +ACCEPT monetdbs://mdb.example.com/demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=mdb.example.com +EXPECT connect_port=50000 +EXPECT tls=on +EXPECT connect_tls_verify=system +EXPECT database=demo +``` + +```test +ACCEPT monetdbs:///demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=localhost +EXPECT connect_port=50000 +EXPECT tls=on +EXPECT connect_tls_verify=system +EXPECT database=demo +``` + +```test +ACCEPT monetdbs://mdb.example.com/demo?cert=/home/user/server.crt +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=mdb.example.com +EXPECT connect_port=50000 +EXPECT tls=on +EXPECT connect_tls_verify=cert +EXPECT cert=/home/user/server.crt +EXPECT database=demo +``` + +```test +ACCEPT monetdbs://mdb.example.com/demo?certhash=sha256:fb:67:20:aa:00:9f:33:4c +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=mdb.example.com +EXPECT connect_port=50000 +EXPECT tls=on +EXPECT connect_tls_verify=hash +EXPECT certhash=sha256:fb:67:20:aa:00:9f:33:4c +EXPECT connect_certhash_digits=fb6720aa009f334c +EXPECT database=demo +``` + +```test +ACCEPT monetdb:///demo?sock=/var/monetdb/_sock&user=dbuser +EXPECT connect_scan=false +EXPECT connect_unix=/var/monetdb/_sock +EXPECT connect_tcp= +EXPECT tls=off +EXPECT database=demo +EXPECT user=dbuser +EXPECT password= +``` + +```test +ACCEPT monetdb://localhost/demo?sock=/var/monetdb/_sock&user=dbuser +EXPECT connect_scan=false +EXPECT connect_unix=/var/monetdb/_sock +EXPECT connect_tcp= +EXPECT tls=off +EXPECT database=demo +EXPECT user=dbuser +EXPECT password= +``` + + +## Parameter tests + +Tests derived from the parameter section. Test data types and defaults. + +Everything can be SET and EXPECTed + +```test +SET tls=on +EXPECT tls=on +SET host=bananahost +EXPECT host=bananahost +SET port=123 +EXPECT port=123 +SET database=bananadatabase +EXPECT database=bananadatabase +SET tableschema=bananatableschema +EXPECT tableschema=bananatableschema +SET table=bananatable +EXPECT table=bananatable +SET sock=c:\foo.txt +EXPECT sock=c:\foo.txt +SET cert=c:\foo.txt +EXPECT cert=c:\foo.txt +SET certhash=bananacerthash +EXPECT certhash=bananacerthash +SET clientkey=c:\foo.txt +EXPECT clientkey=c:\foo.txt +SET clientcert=c:\foo.txt +EXPECT clientcert=c:\foo.txt +SET user=bananauser +EXPECT user=bananauser +SET password=bananapassword +EXPECT password=bananapassword +SET language=bananalanguage +EXPECT language=bananalanguage +SET autocommit=on +EXPECT autocommit=on +SET schema=bananaschema +EXPECT schema=bananaschema +SET timezone=123 +EXPECT timezone=123 +SET binary=bananabinary +EXPECT binary=bananabinary +SET replysize=123 +EXPECT replysize=123 +SET fetchsize=123 +EXPECT fetchsize=123 +``` + +### core defaults + +```test +EXPECT tls=false +EXPECT host= +EXPECT port=-1 +EXPECT database= +EXPECT tableschema= +EXPECT table= +EXPECT binary=on +``` + +### sock + +Not supported on Windows, but they should still parse. + +```test +EXPECT sock= +ACCEPT monetdb:///?sock=/tmp/sock +EXPECT sock=/tmp/sock +ACCEPT monetdb:///?sock=C:/TEMP/sock +EXPECT sock=C:/TEMP/sock +NOT jdbc +ACCEPT monetdb:///?sock=C:\TEMP\sock +EXPECT sock=C:\TEMP\sock +``` + +### sockdir + +```test +EXPECT sockdir=/tmp +ACCEPT monetdb:///demo?sockdir=/tmp/nonstandard +EXPECT sockdir=/tmp/nonstandard +EXPECT connect_unix=/tmp/nonstandard/.s.monetdb.50000 +``` + +### cert + +```test +EXPECT cert= +ACCEPT monetdbs:///?cert=/tmp/cert.pem +EXPECT cert=/tmp/cert.pem +ACCEPT monetdbs:///?cert=C:/TEMP/cert.pem +EXPECT cert=C:/TEMP/cert.pem +NOT jdbc +ACCEPT monetdbs:///?cert=C:\TEMP\cert.pem +EXPECT cert=C:\TEMP\cert.pem +``` + +### certhash + +```test +EXPECT certhash= +ACCEPT monetdbs:///?certhash=sha256:001122ff +ACCEPT monetdbs:///?certhash=sha256:00:11:22:ff +ACCEPT monetdbs:///?certhash=sha256:::::aa::ff::::: +ACCEPT monetdbs:///?certhash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +``` + +This string of hexdigits is longer than the length of a SHA-256 digest. +It still parses, it will just never match. + +```test +ACCEPT monetdbs:///?certhash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8550 +ACCEPT monetdbs:///?certhash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855000000000000000000000000000000000000000001 +``` + +```test +REJECT monetdbs:///?certhash=001122ff +REJECT monetdbs:///?certhash=Sha256:001122ff +REJECT monetdbs:///?certhash=sha256:001122gg +REJECT monetdbs:///?certhash=sha +REJECT monetdbs:///?certhash=sha1:aabbcc +REJECT monetdbs:///?certhash=sha1: +REJECT monetdbs:///?certhash=sha1:X +REJECT monetdbs:///?certhash=sha99:aabbcc +REJECT monetdbs:///?certhash=sha99: +REJECT monetdbs:///?certhash=sha99:X +``` + +### clientkey, clientcert + +```test +EXPECT clientkey= +EXPECT clientcert= +ACCEPT monetdbs:///?clientkey=/tmp/clientkey.pem +EXPECT clientkey=/tmp/clientkey.pem +ACCEPT monetdbs:///?clientkey=C:/TEMP/clientkey.pem +EXPECT clientkey=C:/TEMP/clientkey.pem +NOT jdbc +ACCEPT monetdbs:///?clientkey=C:\TEMP\clientkey.pem +EXPECT clientkey=C:\TEMP\clientkey.pem +``` + +```test +EXPECT connect_clientkey= +EXPECT connect_clientcert= +``` + +```test +SET clientkey=/tmp/key.pem +SET clientcert=/tmp/cert.pem +EXPECT valid=true +EXPECT connect_clientkey=/tmp/key.pem +EXPECT connect_clientcert=/tmp/cert.pem +``` + +```test +SET clientkey=/tmp/key.pem +EXPECT valid=true +EXPECT connect_clientkey=/tmp/key.pem +EXPECT connect_clientcert=/tmp/key.pem +``` + +```test +SET clientcert=/tmp/cert.pem +EXPECT valid=false +``` + +```test +SET clientkey=dummy +EXPECT clientcert= +ACCEPT monetdbs:///?clientcert=/tmp/clientcert.pem +EXPECT clientcert=/tmp/clientcert.pem +ACCEPT monetdbs:///?clientcert=C:/TEMP/clientcert.pem +EXPECT clientcert=C:/TEMP/clientcert.pem +NOT jdbc +ACCEPT monetdbs:///?clientcert=C:\TEMP\clientcert.pem +EXPECT clientcert=C:\TEMP\clientcert.pem +``` + +### user, password + +Not testing the default because they are (unfortunately) +implementation specific. + +```test +ACCEPT monetdb:///?user=monetdb +EXPECT user=monetdb +ACCEPT monetdb:///?user=me&password=? +EXPECT user=me +EXPECT password=? +``` + +### language + +```test +EXPECT language=sql +ACCEPT monetdb:///?language=msql +EXPECT language=msql +ACCEPT monetdb:///?language=sql +EXPECT language=sql +``` + +### autocommit + +```test +ACCEPT monetdb:///?autocommit=true +EXPECT autocommit=true +ACCEPT monetdb:///?autocommit=on +EXPECT autocommit=true +ACCEPT monetdb:///?autocommit=yes +EXPECT autocommit=true +``` + +```test +ACCEPT monetdb:///?autocommit=false +EXPECT autocommit=false +ACCEPT monetdb:///?autocommit=off +EXPECT autocommit=false +ACCEPT monetdb:///?autocommit=no +EXPECT autocommit=false +``` + +```test +REJECT monetdb:///?autocommit= +REJECT monetdb:///?autocommit=banana +REJECT monetdb:///?autocommit=0 +REJECT monetdb:///?autocommit=1 +``` + +### schema, timezone + +Must be accepted, no constraints on content + +```test +EXPECT schema= +ACCEPT monetdb:///?schema=foo +EXPECT schema=foo +ACCEPT monetdb:///?schema= +EXPECT schema= +ACCEPT monetdb:///?schema=foo +``` + +```test +ACCEPT monetdb:///?timezone=0 +EXPECT timezone=0 +ACCEPT monetdb:///?timezone=120 +EXPECT timezone=120 +ACCEPT monetdb:///?timezone=-120 +EXPECT timezone=-120 +REJECT monetdb:///?timezone=banana +REJECT monetdb:///?timezone= +``` + +### replysize and fetchsize + +Note we never check `EXPECT fetchsize=`, it doesn't exist. + +```test +ACCEPT monetdb:///?replysize=150 +EXPECT replysize=150 +ACCEPT monetdb:///?fetchsize=150 +EXPECT replysize=150 +ACCEPT monetdb:///?fetchsize=100&replysize=200 +EXPECT replysize=200 +ACCEPT monetdb:///?replysize=100&fetchsize=200 +EXPECT replysize=200 +REJECT monetdb:///?replysize= +REJECT monetdb:///?fetchsize= +``` + +### binary + +```test +EXPECT binary=on +EXPECT connect_binary=65535 +``` + +```test +ACCEPT monetdb:///?binary=on +EXPECT connect_binary=65535 + +ACCEPT monetdb:///?binary=yes +EXPECT connect_binary=65535 + +ACCEPT monetdb:///?binary=true +EXPECT connect_binary=65535 + +ACCEPT monetdb:///?binary=yEs +EXPECT connect_binary=65535 +``` + +```test +ACCEPT monetdb:///?binary=off +EXPECT connect_binary=0 + +ACCEPT monetdb:///?binary=no +EXPECT connect_binary=0 + +ACCEPT monetdb:///?binary=false +EXPECT connect_binary=0 +``` + +```test +ACCEPT monetdb:///?binary=0 +EXPECT connect_binary=0 + +ACCEPT monetdb:///?binary=5 +EXPECT connect_binary=5 + +ACCEPT monetdb:///?binary=0100 +EXPECT connect_binary=100 +``` + +```test +REJECT monetdb:///?binary= +REJECT monetdb:///?binary=-1 +REJECT monetdb:///?binary=1.0 +REJECT monetdb:///?binary=banana +``` + +### unknown parameters + +```test +REJECT monetdb:///?banana=bla +``` + +```test +ACCEPT monetdb:///?ban_ana=bla +ACCEPT monetdb:///?hash=sha1 +ACCEPT monetdb:///?debug=true +ACCEPT monetdb:///?logfile=banana +``` + +Unfortunately we can't easily test that it won't allow us +to SET banana. + +```test +SET ban_ana=bla +SET hash=sha1 +SET debug=true +SET logfile=banana +``` + +## Combining sources + +The defaults have been tested in the previous section. + +Rule: If there is overlap, later sources take precedence. + +```test +SET schema=a +ACCEPT monetdb:///db1?schema=b +EXPECT schema=b +EXPECT database=db1 +EXPECT tls=off +ACCEPT monetdbs:///db2?schema=c +EXPECT tls=on +EXPECT database=db2 +EXPECT schema=c +``` + +Rule: a source that sets user must set password or clear. + +```skiptest +ACCEPT monetdb:///?user=foo +EXPECT user=foo +EXPECT password= +SET password=banana +EXPECT user=foo +EXPECT password=banana +SET user=bar +EXPECT password= +``` + +Rule: fetchsize is an alias for replysize, last occurrence counts + +```test +SET replysize=200 +ACCEPT monetdb:///?fetchsize=400 +EXPECT replysize=400 +ACCEPT monetdb:///?replysize=500&fetchsize=600 +EXPECT replysize=600 +``` + +```test +SET replysize=200 +SET fetchsize=300 +EXPECT replysize=300 +``` + +Rule: parsing a URL sets all of tls, host, port and database +even if left out of the URL + +```test +SET tls=on +SET host=banana +SET port=12345 +SET database=foo +SET timezone=120 +ACCEPT monetdb:/// +EXPECT tls=off +EXPECT host= +EXPECT port=-1 +EXPECT database= +``` + +```test +SET tls=on +SET host=banana +SET port=12345 +SET database=foo +SET timezone=120 +ACCEPT monetdb://dbhost/dbdb +EXPECT tls=off +EXPECT host=dbhost +EXPECT port=-1 +EXPECT database=dbdb +``` + +Careful around passwords + +```test +SET user=alan +SET password=turing +ACCEPT monetdbs:/// +EXPECT user=alan +EXPECT password=turing +``` + +```test +SET user=alan +SET password=turing +ACCEPT monetdbs:///?user=mathison +EXPECT user=mathison +EXPECT password= +``` + +The rule is, "if **user** is set", not "if **user** is changed". + +```test +SET user=alan +SET password=turing +ACCEPT monetdbs:///?user=alan +EXPECT user=alan +EXPECT password= +``` + +## URL syntax + +General form + +```test +REJECT monetdb: +REJECT monetdbs: +REJECT monetdb:/ +REJECT monetdbs:/ +ACCEPT monetdb:// +ACCEPT monetdbs:// +ACCEPT monetdb:/// +ACCEPT monetdbs:/// +``` + + +```test +ACCEPT monetdb://host:12345/db1/schema2/table3?user=mr&password=bean +EXPECT tls=off +EXPECT host=host +EXPECT port=12345 +EXPECT database=db1 +EXPECT tableschema=schema2 +EXPECT table=table3 +EXPECT user=mr +EXPECT password=bean +``` + +Also, TLS and percent-escapes + +```test +ACCEPT monetdbs://h%6Fst:12345/db%31/schema%32/table%33?user=%6Dr&p%61ssword=bean +EXPECT tls=on +EXPECT host=host +EXPECT port=12345 +EXPECT database=db1 +EXPECT tableschema=schema2 +EXPECT table=table3 +EXPECT user=mr +EXPECT password=bean +``` + +Port number + +```test +REJECT monetdb://banana:0/ +REJECT monetdb://banana:-1/ +REJECT monetdb://banana:65536/ +REJECT monetdb://banana:100000/ +``` + +Trailing slash can be left off + +```test +ACCEPT monetdb://host?user=claude&password=m%26ms +EXPECT host=host +EXPECT user=claude +EXPECT password=m&ms +``` + +Error to set tls, host, port, database, tableschema and table as query parameters. + +```test +REJECT monetdb://foo:1/bar?tls=off +REJECT monetdb://foo:1/bar?host=localhost +REJECT monetdb://foo:1/bar?port=12345 +REJECT monetdb://foo:1/bar?database=allmydata +REJECT monetdb://foo:1/bar?tableschema=banana +REJECT monetdb://foo:1/bar?table=tabularity +``` + +Last wins, already tested elsewhere but for completeness + +```test +ACCEPT monetdbs:///?timezone=10&timezone=20 +EXPECT timezone=20 +``` + +Interesting case: setting user must clear the password but does +that also happen with repetitions within a URL? +Not sure. For the time being, no. This makes it easier for +situations where for example the query parameters come in +alphabetical order + +```test +ACCEPT monetdb:///?user=foo&password=banana&user=bar +EXPECT user=bar +EXPECT password=banana +``` + +Similar but even simpler: user comes after password but does not +clear it. + +```test +ACCEPT monetdb:///?password=pw&user=foo +EXPECT user=foo +EXPECT password=pw +``` + +Ways of writing booleans and the binary property have already been tested above. + +Ip numbers: + +```test +ACCEPT monetdb://192.168.1.1:12345/foo +EXPECT connect_unix= +EXPECT connect_tcp=192.168.1.1 +EXPECT database=foo +``` + +```test +ACCEPT monetdb://[::1]:12345/foo +EXPECT connect_unix= +EXPECT connect_tcp=::1 +EXPECT database=foo +``` + +```test +REJECT monetdb://[::1]banana/foo +``` + +Bad percent escapes: + +```test +REJECT monetdb:///m%xxbad +``` + + +## Interpreting + +Testing the validity constraints. +They apply both when parsing a URL and with ad-hoc settings. + +Rule 1, the type constraints, has already been tested in [Section Parameter +tests](#parameter-tests). + +Rule 2, interaction between **sock** and **host** is tested below in +[the next subsection](#interaction-between-tls-host-sock-and-database). + +Rule 3, about **binary**, is tested in [Subsection Binary](#binary). + +Rule 4, **sock** vs **tls** is tested below in [the next +subsection](#interaction-between-tls-host-sock-and-database). + +Rule 5, **certhash** syntax, is tested in [Subsection Certhash](#certhash). + +Rule 6, **tls** **cert** **certhash** interaction, is tested +in [Subsection Interaction between tls, cert and certhash](#interaction-between-tls-cert-and-certhash). + +Rule 7, **database**, **tableschema**, **table** is tested in [Subsection +Database, schema, table name +constraints](#database-schema-table-name-constraints). + +Here are some tests for Rule 8, **port**. + +```test +SET port=1 +EXPECT valid=true +SET port=10 +EXPECT valid=true +SET port=000010 +EXPECT valid=true +SET port=65535 +EXPECT valid=true +SET port=-1 +EXPECT valid=true +SET port=0 +EXPECT valid=false +SET port=-2 +EXPECT valid=false +SET port=65536 +EXPECT valid=false +``` + +### Database, schema, table name constraints + +```test +SET database= +EXPECT valid=yes +SET database=banana +EXPECT valid=yes +SET database=UPPERCASE +EXPECT valid=yes +SET database=_under_score_ +EXPECT valid=yes +SET database=with-dashes +EXPECT valid=yes +``` + +```test +SET database=with/slash +EXPECT valid=no +SET database=-flag +EXPECT valid=no +SET database=with space +EXPECT valid=no +SET database=with.period +EXPECT valid=yes +SET database=with%percent +EXPECT valid=no +SET database=with!exclamation +EXPECT valid=no +SET database=with?questionmark +EXPECT valid=no +``` + +```test +SET database=demo +SET tableschema= +EXPECT valid=yes +SET tableschema=banana +EXPECT valid=yes +SET tableschema=UPPERCASE +EXPECT valid=yes +SET tableschema=_under_score_ +EXPECT valid=yes +SET tableschema=with-dashes +EXPECT valid=yes +``` + +```test +SET database=demo +SET tableschema=with/slash +EXPECT valid=no +SET tableschema=-flag +EXPECT valid=no +SET tableschema=with space +EXPECT valid=no +SET tableschema=with.period +EXPECT valid=yes +SET tableschema=with%percent +EXPECT valid=no +SET tableschema=with!exclamation +EXPECT valid=no +SET tableschema=with?questionmark +EXPECT valid=no +``` + +```test +SET database=demo +SET tableschema=sys +SET table= +EXPECT valid=yes +SET table=banana +EXPECT valid=yes +SET table=UPPERCASE +EXPECT valid=yes +SET table=_under_score_ +EXPECT valid=yes +SET table=with-dashes +EXPECT valid=yes +``` + +```test +SET database=demo +SET tableschema=sys +SET table=with/slash +EXPECT valid=no +SET table=-flag +EXPECT valid=no +SET table=with space +EXPECT valid=no +SET table=with.period +EXPECT valid=yes +SET table=with%percent +EXPECT valid=no +SET table=with!exclamation +EXPECT valid=no +SET table=with?questionmark +EXPECT valid=no +``` + +### Interaction between tls, cert and certhash + +```test +ACCEPT monetdbs:///?cert=/a/path +EXPECT connect_tls_verify=cert +ACCEPT monetdbs:///?certhash=sha256:aa +EXPECT connect_tls_verify=hash +ACCEPT monetdbs:///?cert=/a/path&certhash=sha256:aa +EXPECT connect_tls_verify=hash +REJECT monetdb:///?cert=/a/path +REJECT monetdb:///?certhash=sha256:aa +``` + +```test +SET tls=off +SET cert= +SET certhash= +EXPECT valid=yes +EXPECT connect_tls_verify= +``` + +```test +SET tls=off +SET cert= +SET certhash=sha256:abcdef +EXPECT valid=no +``` + +```test +SET tls=off +SET cert=/foo +SET certhash= +EXPECT valid=no +``` + +```test +SET tls=off +SET cert=/foo +SET certhash=sha256:abcdef +EXPECT valid=no +``` + +```test +SET tls=on +SET cert= +SET certhash= +EXPECT valid=yes +EXPECT connect_tls_verify=system +``` + +```test +SET tls=on +SET cert= +SET certhash=sha256:abcdef +EXPECT valid=yes +EXPECT connect_tls_verify=hash +``` + +```test +SET tls=on +SET cert=/foo +SET certhash= +EXPECT valid=yes +EXPECT connect_tls_verify=cert +``` + +```test +SET tls=on +SET cert=/foo +SET certhash=sha256:abcdef +EXPECT valid=yes +EXPECT connect_tls_verify=hash +``` + + +### Interaction between tls, host, sock and database + +The following tests should exhaustively test all variants. + +```test +ACCEPT monetdb:/// +EXPECT connect_scan=off +EXPECT connect_unix=/tmp/.s.monetdb.50000 +EXPECT connect_tcp=localhost +``` + +```test +ACCEPT monetdb:///?sock=/a/path +EXPECT connect_scan=off +EXPECT connect_unix=/a/path +EXPECT connect_tcp= +``` + +```test +ACCEPT monetdb://localhost/ +EXPECT connect_scan=off +EXPECT connect_unix=/tmp/.s.monetdb.50000 +EXPECT connect_tcp=localhost +``` + +```test +ACCEPT monetdb://localhost/?sock=/a/path +EXPECT connect_scan=off +EXPECT connect_unix=/a/path +EXPECT connect_tcp= +``` + +```test +ACCEPT monetdb://localhost./ +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdb://localhost./?sock=/a/path +``` + +```test +ACCEPT monetdb://not.localhost/ +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=not.localhost +``` + +```test +REJECT monetdb://not.localhost/?sock=/a/path +``` + +```test +ACCEPT monetdbs:/// +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdbs:///?sock=/a/path +``` + +```test +ACCEPT monetdbs://localhost/ +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdbs://localhost/?sock=/a/path +``` + +```test +ACCEPT monetdbs://localhost./ +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdbs://localhost./?sock=/a/path +``` + +```test +ACCEPT monetdbs://not.localhost/ +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=not.localhost +``` + +```test +REJECT monetdbs://not.localhost/?sock=/a/path +``` + +```test +ACCEPT monetdb:///demo +EXPECT connect_scan=on +``` + +```test +ACCEPT monetdb:///demo?sock=/a/path +EXPECT connect_scan=off +EXPECT connect_unix=/a/path +EXPECT connect_tcp= +``` + +```test +ACCEPT monetdb://localhost/demo +EXPECT connect_scan=on +``` + +```test +ACCEPT monetdb://localhost/demo?sock=/a/path +EXPECT connect_scan=off +EXPECT connect_unix=/a/path +EXPECT connect_tcp= +``` + +```test +ACCEPT monetdb://localhost./demo +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdb://localhost./?sock=/a/path +``` + +```test +ACCEPT monetdb://not.localhost/demo +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=not.localhost +``` + +```test +REJECT monetdb://not.localhost/?sock=/a/path +``` + +```test +ACCEPT monetdbs:///demo +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdbs:///?sock=/a/path +``` + +```test +ACCEPT monetdbs://localhost/demo +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdbs://localhost/?sock=/a/path +``` + +```test +ACCEPT monetdbs://localhost./demo +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdbs://localhost./?sock=/a/path +``` + +```test +ACCEPT monetdbs://not.localhost/demo +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=not.localhost +``` + +```test +REJECT monetdbs://not.localhost/?sock=/a/path +``` + +### sock and sockdir + +Sockdir only applies to implicit Unix domain sockets, +not to ones that are given explicitly + +```test +EXPECT sockdir=/tmp +EXPECT port=-1 +EXPECT host= +EXPECT connect_unix=/tmp/.s.monetdb.50000 +SET sockdir=/somewhere/else +EXPECT connect_unix=/somewhere/else/.s.monetdb.50000 +SET port=12345 +EXPECT connect_unix=/somewhere/else/.s.monetdb.12345 +``` + +## Legacy URL's + +```test +REJECT mapi: +REJECT mapi:monetdb +REJECT mapi:monetdb: +REJECT mapi:monetdb:/ +``` + +```test +ACCEPT mapi:monetdb://monet.db:12345/demo +EXPECT host=monet.db +EXPECT port=12345 +EXPECT database=demo +EXPECT tls=off +EXPECT language=sql +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=monet.db +``` + +This one is the golden standard: + +```test +ACCEPT mapi:monetdb://localhost:12345/demo +EXPECT host=localhost +EXPECT port=12345 +EXPECT database=demo +EXPECT tls=off +EXPECT language=sql +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +ACCEPT mapi:monetdb://localhost:12345/ +EXPECT host=localhost +EXPECT port=12345 +EXPECT database= +EXPECT tls=off +EXPECT language=sql +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +ACCEPT mapi:monetdb://localhost:12345 +EXPECT host=localhost +EXPECT port=12345 +EXPECT database= +EXPECT tls=off +EXPECT language=sql +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +ACCEPT mapi:monetdb://localhost/demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=localhost +EXPECT connect_port=50000 +``` + +```test +ACCEPT mapi:monetdb://:12345/demo +EXPECT host= +EXPECT port=12345 +EXPECT database=demo +EXPECT tls=off +EXPECT language=sql +EXPECT connect_scan=off +EXPECT connect_unix=/tmp/.s.monetdb.12345 +EXPECT connect_tcp=localhost +``` + +```test +ACCEPT mapi:monetdb://127.0.0.1:12345/demo +EXPECT host=127.0.0.1 +EXPECT port=12345 +EXPECT database=demo +EXPECT tls=off +EXPECT language=sql +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=127.0.0.1 +``` + +Database parameter allowed, overrides path + +```test +ACCEPT mapi:monetdb://localhost:12345/demo?database=foo +EXPECT database=foo +``` + +User, username and password parameters are ignored: + +```test +SET user=alan +SET password=turing +ACCEPT mapi:monetdb://localhost:12345/demo?user=foo +EXPECT user=alan +EXPECT password=turing +ACCEPT mapi:monetdb://localhost:12345/demo?password=foo +EXPECT user=alan +EXPECT password=turing +``` + +Pymonetdb used to accept user name and password before +the host name and should continue to do so. + + +```test +ONLY pymonetdb +SET user=alan +SET password=turing +ACCEPT mapi:monetdb://foo:bar@localhost:12345/demo +EXPECT user=foo +EXPECT password=bar +ACCEPT mapi:monetdb://banana@localhost:12345/demo +EXPECT user=banana +EXPECT password= +``` + +```test +NOT pymonetdb +SET user=alan +SET password=turing +REJECT mapi:monetdb://foo:bar@localhost:12345/demo +REJECT mapi:monetdb://banana@localhost:12345/demo +``` + +Unix domain sockets + +```test +ACCEPT mapi:monetdb:///path/to/sock?database=demo +EXPECT host= +EXPECT sock=/path/to/sock +EXPECT port=-1 +EXPECT database=demo +EXPECT tls=off +EXPECT language=sql +EXPECT connect_unix=/path/to/sock +EXPECT connect_tcp= +``` + +```test +ACCEPT mapi:monetdb:///path/to/sock +EXPECT host= +EXPECT sock=/path/to/sock +EXPECT port=-1 +EXPECT database= +EXPECT tls=off +EXPECT language=sql +EXPECT connect_unix=/path/to/sock +EXPECT connect_tcp= +``` + +Corner case: both libmapi and pymonetdb interpret this as an attempt +to connect to socket '/'. This will fail of course but the URL does parse + +```test +ACCEPT mapi:monetdb:/// +EXPECT host= +EXPECT sock=/ +EXPECT connect_unix=/ +EXPECT connect_tcp= +``` + +```test +NOT pymonetdb +PARSE mapi:monetdb:///foo:bar@path/to/sock +EXPECT sock=/foo:bar@path/to/sock +REJECT mapi:monetdb://foo:bar@/path/to/sock +``` + +```test +ONLY pymonetdb +SET user=alan +SET password=turing +ACCEPT mapi:monetdb://foo:bar@/path/to/sock +EXPECT host= +EXPECT sock=/path/to/sock +EXPECT user=foo +EXPECT password=bar +EXPECT connect_unix=/path/to/sock +EXPECT connect_tcp= +``` + +```test +ONLY pymonetdb +SET user=alan +SET password=turing +ACCEPT mapi:monetdb://foo@/path/to/sock +EXPECT host= +EXPECT sock=/path/to/sock +EXPECT user=foo +EXPECT password= +EXPECT connect_unix=/path/to/sock +EXPECT connect_tcp= +``` + +Language is supported + +```test +SET language=sql +ACCEPT mapi:monetdb://localhost:12345?language=mal +EXPECT host=localhost +EXPECT sock= +EXPECT language=mal +SET language=sql +ACCEPT mapi:monetdb:///path/to/sock?language=mal +EXPECT host= +EXPECT sock=/path/to/sock +EXPECT language=mal +``` + +No percent decoding is performed + +```test +REJECT mapi:monetdb://localhost:1234%35/demo +PARSE mapi:monetdb://loc%61lhost:12345/d%61tabase +EXPECT host=loc%61lhost +EXPECT database=d%61tabase +EXPECT valid=no +``` + +```test +PARSE mapi:monetdb://localhost:12345/db?database=b%61r&language=m%61l +EXPECT database=b%61r +EXPECT language=m%61l +EXPECT valid=no +``` + +l%61nguage is an unknown parameter, thus ignored not rejected + +```test +SET language=sql +ACCEPT mapi:monetdb://localhost:12345/db?l%61nguage=mal +EXPECT language=sql +ACCEPT mapi:monetdb://localhost:12345/db?_l%61nguage=mal +``` + + +## Merovingian URLs + +These occur in redirects. +Leave host, port and database are not cleared the way the are +for `monetdb:`, `monetdbs:` and `mapi:monetdb:` URLs. + +```test +SET host=banana +SET port=123 +SET tls=on +SET sock=/tmp/sock +SET database=dummy +PARSE mapi:merovingian://proxy +EXPECT host=banana +EXPECT port=123 +EXPECT tls=on +EXPECT sock=/tmp/sock +EXPECT database=dummy +``` + +```test +SET host=banana +SET port=123 +SET tls=on +SET sock=/tmp/sock +SET database=dummy +PARSE mapi:merovingian://proxy? +EXPECT host=banana +EXPECT port=123 +EXPECT tls=on +EXPECT sock=/tmp/sock +EXPECT database=dummy +``` + +```test +SET host=banana +SET port=123 +SET tls=on +SET sock=/tmp/sock +SET database=dummy +PARSE mapi:merovingian://proxy?database=yeah&unknown=unknown +EXPECT host=banana +EXPECT port=123 +EXPECT tls=on +EXPECT sock=/tmp/sock +EXPECT database=yeah +```