Re: MonetDB: default - add support for unix sockets

ok, i'll fix it tomorrow. On 09/05/2013 06:56 PM, Stefan Manegold wrote:

Hi Gijs,

in case you did not notice this, yet:

This checkin breaks several tests that do test or use the Python interface, resulting in errors like

" Traceback (most recent call last): File "delete_all.SQL.py", line 9, in <module> dbh = monetdb.sql.Connection(port=port,database=db,autocommit=True) File "PREFIX/lib/python2.7/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python2.7/site-packages/monetdb/mapi.py", line 96, in connect self.socket.connect(unix_socket) File "/usr/lib64/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 2] No such file or directory "

or

" Traceback (most recent call last): File "SOURCE/clients/examples/python/sqlsample.py", line 23, in <module> dbh = monetdb.sql.Connection(port=int(sys.argv[1]),database=sys.argv[2],autocommit=True) File "PREFIX/lib/python3.3/site-packages/monetdb/sql/connections.py", line 49, in __init__ unix_socket=unix_socket) File "PREFIX/lib/python3.3/site-packages/monetdb/mapi.py", line 97, in connect self.socket.connect(unix_socket) FileNotFoundError: [Errno 2] No such file or directory "

See also, e.g., `Mtest.py sql/benchmarks/tpch/fileleak sql/test/concurrent sql/test/mapi`

or our nightly TestWeb @ http://monetdb.cwi.nl/testweb/web/status.php?branch=default as of tomorrow morning.

Best, Stefan

On Thu, Sep 05, 2013 at 02:19:01PM +0200, Gijs Molenaar wrote:

...

Changeset: 0eaa07b061be for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0eaa07b061be Modified Files: clients/python2/monetdb/control.py clients/python2/monetdb/mapi.py clients/python2/monetdb/sql/connections.py clients/python2/test/runtests.py clients/python2/test/test_control.py clients/python3/monetdb/control.py clients/python3/monetdb/mapi.py clients/python3/monetdb/sql/connections.py clients/python3/test/runtests.py clients/python3/test/test_control.py Branch: default Log Message:

add support for unix sockets

diffs (truncated from 649 to 300 lines):

diff --git a/clients/python2/monetdb/control.py b/clients/python2/monetdb/control.py --- a/clients/python2/monetdb/control.py +++ b/clients/python2/monetdb/control.py @@ -8,10 +8,12 @@ def parse_statusline(line): parses a sabdb format status line. Support v1 and v2.

""" - if not line.startswith('=sabdb:'): + if line.startswith("="): + line = line[1:] + if not line.startswith('sabdb:'): raise OperationalError('wrong result recieved')

- prot_version, rest = line.split(":", 2)[1:] + code, prot_version, rest = line.split(":", 2)

if prot_version not in ["1", "2"]: raise InterfaceError("unsupported sabdb protocol") @@ -60,20 +62,26 @@ class Control: Use this module to manage your MonetDB databases. You can create, start, stop, lock, unlock, destroy your databases and request status information. """ - def __init__(self, hostname, port, passphrase): + def __init__(self, hostname=None, port=None, passphrase=None, + unix_socket="/tmp/.s.merovingian.50000"): self.server = mapi.Connection() self.hostname = hostname self.port = port self.passphrase = passphrase + self.unix_socket= unix_socket

# check connection - self.server.connect(hostname, port, 'monetdb', passphrase, - 'merovingian', 'control') + self.server.connect(hostname=hostname, port=port, username='monetdb', + password=passphrase, + database='merovingian', language='control', + unix_socket=unix_socket) self.server.disconnect()

def _send_command(self, database_name, command): - self.server.connect(self.hostname, self.port, 'monetdb', - self.passphrase, 'merovingian', 'control') + self.server.connect(hostname=self.hostname, port=self.port, + username='monetdb', password=self.passphrase, + database='merovingian', language='control', + unix_socket=self.unix_socket) try: return self.server.cmd("%s %s\n" % (database_name, command)) finally: @@ -165,8 +173,9 @@ class Control: """ properties = self._send_command(database_name, "get") values = {} - for dirty_line in properties.split("\n"): - line = dirty_line[1:] + for line in properties.split("\n"): + if line.startswith("="): + line = line[1:] if not line.startswith("#"): if "=" in line: split = line.split("=") diff --git a/clients/python2/monetdb/mapi.py b/clients/python2/monetdb/mapi.py --- a/clients/python2/monetdb/mapi.py +++ b/clients/python2/monetdb/mapi.py @@ -70,8 +70,12 @@ class Connection(object): self.database = "" self.language = ""

- def connect(self, hostname, port, username, password, database, language): - """ setup connection to MAPI server""" + def connect(self, database, username, password, language, hostname=None, + port=None, unix_socket=None): + """ setup connection to MAPI server + + unix_socket is used if hostname is not defined. + """

self.hostname = hostname self.port = port @@ -79,24 +83,34 @@ class Connection(object): self.password = password self.database = database self.language = language + self.unix_socket = unix_socket

- self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if hostname: + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # For performance, mirror MonetDB/src/common/stream.c socket settings. + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0) + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.socket.connect((hostname, port)) + else: + self.socket = socket.socket(socket.AF_UNIX) + self.socket.connect(unix_socket) + if self.language != 'control': + self.socket.send('0') # don't know why, but we need to do this

- # For performance, mirror MonetDB/src/common/stream.c socket settings. - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 0) - self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + if not (self.language == 'control' and not self.hostname): + # control doesn't require authentication over socket + self._login()

- self.socket.connect((hostname, port)) - self.__login() + self.state = STATE_READY

- def __login(self, iteration=0): + def _login(self, iteration=0): """ Reads challenge from line, generate response and check if everything is okay """

- challenge = self.__getblock() - response = self.__challenge_response(challenge) - self.__putblock(response) - prompt = self.__getblock().strip() + challenge = self._getblock() + response = self._challenge_response(challenge) + self._putblock(response) + prompt = self._getblock().strip()

if len(prompt) == 0: # Empty response, server is happy @@ -117,7 +131,7 @@ class Connection(object): if redirect[1] == "merovingian": logger.debug("restarting authentication") if iteration <= 10: - self.__login(iteration=iteration + 1) + self._login(iteration=iteration + 1) else: raise OperationalError("maximal number of redirects " "reached (10)") @@ -138,9 +152,6 @@ class Connection(object): else: raise ProgrammingError("unknown state: %s" % prompt)

- self.state = STATE_READY - return True - def disconnect(self): """ disconnect from the monetdb server """ self.state = STATE_INIT @@ -153,8 +164,8 @@ class Connection(object): if self.state != STATE_READY: raise(ProgrammingError, "Not connected")

- self.__putblock(operation) - response = self.__getblock() + self._putblock(operation) + response = self._getblock() if not len(response): return "" elif response.startswith(MSG_OK): @@ -166,10 +177,16 @@ class Connection(object): return response elif response[0] == MSG_ERROR: raise OperationalError(response[1:]) + elif (self.language == 'control' and not self.hostname): + if response.startswith("OK"): + return response[2:].strip() or "" + else: + return response else: raise ProgrammingError("unknown state: %s" % response)

- def __challenge_response(self, challenge): + + def _challenge_response(self, challenge): """ generate a response to a mapi login challenge """ challenges = challenge.split(':') salt, identity, protocol, hashes, endian = challenges[:5] @@ -204,19 +221,36 @@ class Connection(object): return ":".join(["BIG", self.username, pwhash, self.language, self.database]) + ":"

- def __getblock(self): + def _getblock(self): """ read one mapi encoded block """ + if (self.language == 'control' and not self.hostname): + return self._getblock_socket() # control doesn't do block + # splitting when using a socket + else: + return self._getblock_inet() + + def _getblock_inet(self): result = StringIO() last = 0 while not last: - flag = self.__getbytes(2) + flag = self._getbytes(2) unpacked = struct.unpack('> 1 last = unpacked & 1 - result.write(self.__getbytes(length)) + result.write(self._getbytes(length)) return result.getvalue()

- def __getbytes(self, bytes_): + def _getblock_socket(self): + buffer = StringIO() + while True: + x = self.socket.recv(1) + if len(x): + buffer.write(x) + else: + break + return buffer.getvalue().strip() + + def _getbytes(self, bytes_): """Read an amount of bytes from the socket""" result = StringIO() count = bytes_ @@ -228,8 +262,15 @@ class Connection(object): result.write(recv) return result.getvalue()

- def __putblock(self, block): + def _putblock(self, block): """ wrap the line in mapi format and put it into the socket """ + if (self.language == 'control' and not self.hostname): + return self.socket.send(block) # control doesn't do block + # splitting when using a socket + else: + self._putblock_inet(block) + + def _putblock_inet(self, block): pos = 0 last = 0 while not last: diff --git a/clients/python2/monetdb/sql/connections.py b/clients/python2/monetdb/sql/connections.py --- a/clients/python2/monetdb/sql/connections.py +++ b/clients/python2/monetdb/sql/connections.py @@ -28,25 +28,25 @@ class Connection(object): """A MonetDB SQL database connection""" default_cursor = cursors.Cursor

- def __init__(self, username="monetdb", password="monetdb", - hostname="localhost", port=50000, database="demo", - autocommit=False, user=None, host=None): + def __init__(self, database, hostname=None, port=50000, username="monetdb", + password="monetdb", unix_socket="/tmp/.s.monetdb.50000", + autocommit=False): """ Set up a connection to a MonetDB SQL database.

- username -- username for connection (default: monetdb) - password -- password for connection (default: monetdb) - hostname -- hostname to connect to (default: localhost) - port -- port to connect to (default: 50000) - database -- name of the database (default: demo) - autocommit -- enable/disable auto commit (default: False) + database -- name of the database + hostname -- Hostname where monetDB is running + port -- port to connect to (default: 50000) + username -- username for connection (default: "monetdb") + password -- password for connection (default: "monetdb") + unix_socket -- socket to connect to. used when hostname not set + (default: "/tmp/.s.monetdb.50000") + autocommit -- enable/disable auto commit (default: False) + """ - if user is not None: - username = user - if host is not None: - hostname = host self.mapi = mapi.Connection() self.mapi.connect(hostname=hostname, port=int(port), username=username, - password=password, database=database, language="sql") + password=password, database=database, language="sql", + unix_socket=unix_socket) self.set_autocommit(autocommit) self.set_sizeheader(True) self.set_replysize(100) diff --git a/clients/python2/test/runtests.py b/clients/python2/test/runtests.py --- a/clients/python2/test/runtests.py +++ b/clients/python2/test/runtests.py @@ -45,6 +45,8 @@ TSTDB = os.environ.get('TSTDB', 'demo') TSTHOSTNAME = os.environ.get('TSTHOSTNAME', 'localhost') TSTUSERNAME = os.environ.get('TSTUSERNAME', 'monetdb') TSTPASSWORD = os.environ.get('TSTPASSWORD', 'monetdb') +#TSTHOSTNAME = None # set to none if test a socket +

if os.environ.get("TSTDEBUG", "no") == "yes": logging.basicConfig(level=logging.DEBUG) diff --git a/clients/python2/test/test_control.py b/clients/python2/test/test_control.py --- a/clients/python2/test/test_control.py +++ b/clients/python2/test/test_control.py @@ -15,6 +15,7 @@ # Copyright August 2008-2013 MonetDB B.V. # All Rights Reserved.

+import os import unittest import logging

@@ -33,6 +34,10 @@ from monetdb.exceptions import Operation

_______________________________________________ checkin-list mailing list checkin-list@monetdb.org http://mail.monetdb.org/mailman/listinfo/checkin-list