October 2013 - developers-list

Re: MonetDB: newstorage - pushing changes from laptop
by Lefteris 30 Oct '13

30 Oct '13

what is interesting is not from where you are pushing the changes but what the changes do:)) On Wed, Oct 30, 2013 at 10:30 AM, Rene Tassy <commits(a)monetdb.org> wrote: > Changeset: fb471128db99 for MonetDB > URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fb471128db99 > Modified Files: > sql/storage/bat/bat_storage.c > Branch: newstorage > Log Message: > > pushing changes from laptop > > > diffs (144 lines): > > diff --git a/sql/storage/bat/bat_storage.c b/sql/storage/bat/bat_storage.c > --- a/sql/storage/bat/bat_storage.c > +++ b/sql/storage/bat/bat_storage.c > @@ -41,6 +41,14 @@ timestamp_dbat( sql_dbat *d, int ts) > return d; > } > > +/**static sql_timestamps * > +timestamp_tsbats(sql_timestamps *ts, int timestamp) { > + > + while (ts->next && ts->wtime > timestamp) > + ts = ts->next; > + return ts; > +}*/ > + > > static BAT * > delta_bind_del(sql_dbat *bat, int access) > @@ -1222,16 +1230,6 @@ load_timestamps(sql_timestamps *ts, int > } > > static int > -load_dbat(sql_dbat *bat, int bid) { > - > - BAT *b = quick_descriptor(bid); > - > - bat->dbid = temp_create(b); > - bat->cnt = BATcount(b); > - return LOG_OK; > -} > - > -static int > create_timestamps(sql_trans *tr, sql_table *t) { > > int ok = LOG_OK; > @@ -1275,6 +1273,16 @@ create_timestamps(sql_trans *tr, sql_tab > } > > static int > +load_dbat(sql_dbat *bat, int bid) { > + > + BAT *b = quick_descriptor(bid); > + > + bat->dbid = temp_create(b); > + bat->cnt = BATcount(b); > + return LOG_OK; > +} > + > +static int > create_del(sql_trans *tr, sql_table *t) > { > int ok = LOG_OK; > @@ -1369,8 +1377,7 @@ static int > snapshot_create_del(sql_trans *tr, sql_table *t) > { > sql_dbat *bat = t->data; > - BAT *b = temp_descriptor(bat->dbid); > - //BAT *trdb = temp_descriptor(bat->dtrbid); > + BAT *b = temp_descriptor(bat->dbid);; > > printf("in snapshot_create_del\n"); > > @@ -1383,13 +1390,6 @@ snapshot_create_del(sql_trans *tr, sql_t > BATmmap(b, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP, 0); > bat_destroy(b); > > - /*bat_set_access(trdb, BAT_READ); > - if (BATcount(trdb) > SNAPSHOT_MINSIZE) > - BATmode(trdb, PERSISTENT); > - if (BATcount(trdb) > (BUN) REMAP_PAGE_MAXSIZE) > - BATmmap(trdb, STORE_MMAP, STORE_MMAP, STORE_MMAP, STORE_MMAP, 0); > - bat_destroy(trdb);*/ > - > return LOG_OK; > } > > @@ -1539,16 +1539,12 @@ destroy_dbat(sql_trans *tr, sql_dbat *ba > > if (bat->dname) > _DELETE(bat->dname); > - //if (bat->dtrname) > - //_DELETE(bat->dtrname); > + > if (bat->dbid) > temp_destroy(bat->dbid); > - //if (bat->dtrbid) > - //temp_destroy(bat->dtrbid); > + > bat->dbid = 0; > bat->dname = NULL; > - //bat->dtrbid = 0; > - //bat->dtrname = NULL; > _DELETE(bat); > if (n) > return destroy_dbat(tr, n); > @@ -1798,20 +1794,39 @@ clear_dbat(sql_trans *tr, sql_dbat *bat) > BATcommit(b); > bat_destroy(b); > } > - /*if(bat->dtrbid) { > - BAT *b = temp_descriptor(bat->dtrbid); > - > - sz+= BATcount(b); > - bat_clear(b); > - BATcommit(b); > - bat_destroy(b); > - }*/ > + > bat->cnt = 0; > bat->wtime = tr->wstime; > > return sz; > } > - > +/** > +static BUN > +clear_tsbats(sql_trans *tr, sql_timestamps *ts) { > + > + BUN sz =0; > + > + (void)tr; > + if(ts->insbid) { > + BAT *b = temp_descriptor(ts->insbid); > + > + sz += BATcount(b) > + } > +} > + > +static BUN > +clear_timestamps(sql_trans *tr, sql_table *t) { > + > + if(!t->timestamps || !t->base.allocated) { > + sql_table *ot = tr_find_table(tr->parent, t); > + sql_timestamps *ts = t->timestamps = ZNEW(sql_timestamps), *ots = timestamp_tsbats(ots->timestamps, tr->stime); > + dup_tsbats(tr, ts, ots, isNew(ot), isTempTable(t)); > + t->base.allocated = 1; > + } > + > + return clear_tsbats(tr, t->timestamps); > +} > +*/ > static BUN > clear_del(sql_trans *tr, sql_table *t) > { > _______________________________________________ > checkin-list mailing list > checkin-list(a)monetdb.org > https://www.monetdb.org/mailman/listinfo/checkin-list

1 0

Re: MonetDB: default - (try to) run GSL tests only if we do have GSL...
by Arjen de Rijke 29 Oct '13

29 Oct '13

Hoi Stefan, Dit zou toch niet nodig hoeven zijn. Dit wordt al geregeld via de conditional in de makefile van /sql/backends/monet5/, of vergis ik mij? De andere checkin zal ik even onthouden voor een volgende keer. Arjen ----- Original Message ----- > From: "Stefan Manegold" <commits(a)monetdb.org> > To: checkin-list(a)monetdb.org > Sent: Tuesday, October 29, 2013 9:44:26 AM > Subject: MonetDB: default - (try to) run GSL tests only if we do have GSL... > > Changeset: bb53a64a83fe for MonetDB > URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bb53a64a83fe > Modified Files: > sql/backends/monet5/gsl/Tests/All > Branch: default > Log Message: > > (try to) run GSL tests only if we do have GSL support > > > diffs (8 lines): > > diff --git a/sql/backends/monet5/gsl/Tests/All > b/sql/backends/monet5/gsl/Tests/All > --- a/sql/backends/monet5/gsl/Tests/All > +++ b/sql/backends/monet5/gsl/Tests/All > @@ -1,2 +1,2 @@ > -gsl00 > -gsl01 > +HAVE_GSL?gsl00 > +HAVE_GSL?gsl01 > _______________________________________________ > checkin-list mailing list > checkin-list(a)monetdb.org > https://www.monetdb.org/mailman/listinfo/checkin-list >

2 2

Can an user function return more than one BAT?
by Chi-Young Ku 23 Oct '13

23 Oct '13

Hi, I am looking for a way for a MAL user function to return more than one BAT without success. Could you give me some help? I tried the following way: user.test(X_1:bat[:oid,:oid], X_2:bat[:oid,:int]):void; sql.init(); X_0 := sql.mvc(); X_1 := sql.bind(...); X_2 := sql.bind(...); end test; X1:bat[:oid,:oid] := bat.new(:oid,:oid); X2:bat[:oid,:int] := bat.new(:oid,:int); X_1:str := remote.put(url, X1); X_2:str := remote.put(url, X2); X_3:str := remote.exec(url, "user", "test", X_1, X_2); X3:bat[:oid,:oid] := remote.get(url, X_1); X4:bat[:oid,:int] := remote.get(url, X_2); io.print(X3); io.print(X4);

1 0

Monetdb python error in connection
by Rose Beck 20 Oct '13

20 Oct '13

I am using python-monetdb 11.16.0.7. I created my database farm and database according to instructions given below (source: http://www.monetdb.org/Documentation/monetdbd) % monetdbd start /home/my-dbfarm % monetdb create my-first-db Then I tried to connect to the database using the below mentioned command in python(https://pypi.python.org/pypi/python-monetdb/). Upon doing so I am getting the below mentioned error: >import monetdb.sql >connection=monetdb.sql.connect(username="monetdb",password="monetdb",hostname="localhost",database="my-first-db"); File "/usr/local/lib/python2.7/dist-packages/monetdb/sql/__init__.py", line 28, in connect return Connection(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/monetdb/sql/connections.py", line 58, in __init__ unix_socket=unix_socket) File "/usr/local/lib/python2.7/dist-packages/monetdb/mapi.py", line 93, in connect self.socket.connect((hostname, port)) File "/usr/lib/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) socket.error: [Errno 111] Connection refused Can someone please help me with this? Also my my-first-db resides inside my db-farm? I am not getting as to what should I do?

3 2

Re: MonetDB: newstorage - creating delete-transaction bat
by Martin Kersten 19 Oct '13

19 Oct '13

If intended for debugging only, wrap them in #ifdef DEBUG_STORE ... #endif On 10/19/13 6:26 PM, Rene Tassy wrote: > Changeset: d5436b5b660d for MonetDB > URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d5436b5b660d > Modified Files: > sql/backends/monet5/sql_scenario.c > sql/server/sql_mvc.c > sql/storage/bat/bat_storage.c > sql/storage/bat/bat_storage.h > sql/storage/bat/bat_table.c > sql/storage/store.c > Branch: newstorage > Log Message: > > creating delete-transaction bat > > > diffs (truncated from 1821 to 300 lines): > > diff --git a/sql/backends/monet5/sql_scenario.c b/sql/backends/monet5/sql_scenario.c > --- a/sql/backends/monet5/sql_scenario.c > +++ b/sql/backends/monet5/sql_scenario.c > @@ -1932,6 +1932,9 @@ SQLparser(Client c) > } > } > finalize: > + > + printf("-----finalizing\n"); This printf is not correct, use mstr_printf(c->fdout,.... > + > if (msg) > sqlcleanup(m, 0); > return msg; > diff --git a/sql/server/sql_mvc.c b/sql/server/sql_mvc.c > --- a/sql/server/sql_mvc.c > +++ b/sql/server/sql_mvc.c > @@ -182,6 +182,8 @@ mvc_trans(mvc *m) > int schema_changed = 0, err = m->session->status; > assert(!m->session->active); /* can only start a new transaction */ > > + printf("in mvc_trans\n"); > + > store_lock(); > schema_changed = sql_trans_begin(m->session); > if (m->qc && (schema_changed || m->qc->nr > m->cache || err)){ > @@ -206,6 +208,8 @@ mvc_commit(mvc *m, int chain, char *name > assert(tr); > assert(m->session->active); /* only commit an active transaction */ > > + printf("in mvc commit\n"); > + > if (mvc_debug) > fprintf(stderr, "#mvc_commit %s\n", (name) ? name : ""); > > @@ -288,6 +292,7 @@ build up the hash (not copyied in the tr > mvc_rollback(m, chain, name); > return -1; > } > + > sql_trans_end(m->session); > if (chain) > sql_trans_begin(m->session); > @@ -304,6 +309,8 @@ mvc_rollback(mvc *m, int chain, char *na > int res = 0; > sql_trans *tr = m->session->tr; > > + printf("in mvc_rollback"); > + > if (mvc_debug) > fprintf(stderr, "#mvc_rollback %s\n", (name) ? name : ""); > > @@ -340,6 +347,7 @@ mvc_rollback(mvc *m, int chain, char *na > tr = sql_trans_destroy(tr); > } > m->session-> tr = tr; > + > /* make sure we do not reuse changed data */ > if (tr->wtime) > tr->status = 1; > diff --git a/sql/storage/bat/bat_storage.c b/sql/storage/bat/bat_storage.c > --- a/sql/storage/bat/bat_storage.c > +++ b/sql/storage/bat/bat_storage.c > @@ -12,7 +12,7 @@ > * The Original Code is the MonetDB Database System. > * > * The Initial Developer of the Original Code is CWI. > - * Portions created by CWI are Copyright (C) 1997-July 2008 CWI. > + Portions created by CWI are Copyright (C) 1997-July 2008 CWI. > * Copyright August 2008-2013 MonetDB B.V. > * All Rights Reserved. > */ > @@ -50,6 +50,9 @@ delta_bind_del(sql_dbat *bat, int access > #ifdef NDEBUG > (void) access; /* satisfy compiler */ > #endif > + > + //printf("in delta_bind_del\n"); > + > assert(access == RDONLY || access == RD_INS); > assert(access!=RD_UPD); > > @@ -61,6 +64,9 @@ delta_bind_del(sql_dbat *bat, int access > static BAT * > bind_del(sql_trans *tr, sql_table *t, int access) > { > + > + //printf("in bind_del\n"); > + > if (!t->data) { > sql_table *ot = tr_find_table(tr->parent, t); > t->data = timestamp_dbat(ot->data, tr->stime); > @@ -77,6 +83,8 @@ delta_bind_ubat(sql_delta *bat, int acce > #ifdef NDEBUG > (void) access; /* satisfy compiler */ > #endif > + printf("in delta_bind_ubat\n"); > + > assert(access == RD_UPD); > if (bat->ubid) > b = temp_descriptor(bat->ubid); > @@ -91,6 +99,8 @@ bind_ucol(sql_trans *tr, sql_column *c, > { > BAT *u = NULL, *d, *r; > > + printf("in bind_ucol\n"); > + > if (!c->data) { > sql_column *oc = tr_find_column(tr->parent, c); > c->data = timestamp_delta(oc->data, tr->stime); > @@ -115,6 +125,8 @@ bind_uidx(sql_trans *tr, sql_idx * i, in > { > BAT *u = NULL, *d, *r; > > + printf("in bind_uidx\n"); > + > if (!i->data) { > sql_idx *oi = tr_find_idx(tr->parent, i); > i->data = timestamp_delta(oi->data, tr->stime); > @@ -139,6 +151,8 @@ delta_bind_bat( sql_delta *bat, int acce > { > BAT *b; > > + printf("in delta_bind_bat\n"); > + > assert(access == RDONLY || access == RD_INS); > assert(bat != NULL); > if (temp || access == RD_INS) { > @@ -169,6 +183,8 @@ delta_bind_bat( sql_delta *bat, int acce > static BAT * > bind_col(sql_trans *tr, sql_column *c, int access) > { > + printf("in bind_col\n"); > + > if (!c->data) { > sql_column *oc = tr_find_column(tr->parent, c); > c->data = timestamp_delta(oc->data, tr->stime); > @@ -183,6 +199,9 @@ bind_col(sql_trans *tr, sql_column *c, i > static BAT * > bind_idx(sql_trans *tr, sql_idx * i, int access) > { > + > + printf("in bind_idx\n"); > + > if (!i->data) { > sql_idx *oi = tr_find_idx(tr->parent, i); > i->data = timestamp_delta(oi->data, tr->stime); > @@ -200,6 +219,8 @@ delta_update_bat( sql_delta *bat, BAT *t > BAT *b; > BAT *upd = BATleftfetchjoin(BATmirror(tids), updates, BATcount(tids)); > > + printf("in delta_update_bat\n"); > + > if (bat->cached) { > bat_destroy(bat->cached); > bat->cached = NULL; > @@ -259,6 +280,8 @@ delta_update_val( sql_delta *bat, oid ri > { > BAT *b = NULL; > > + printf("in delta_update_val\n"); > + > assert(rid != oid_nil); > > if (bat->cached) { > @@ -297,6 +320,9 @@ delta_update_val( sql_delta *bat, oid ri > static int > dup_delta(sql_trans *tr, sql_delta *obat, sql_delta *bat, int type, int oc_isnew, int c_isnew, int temp, int sz) > { > + > + printf("in dup_delta\n"); > + > if (!obat) > return LOG_OK; > bat->ibid = obat->ibid; > @@ -359,6 +385,8 @@ dup_delta(sql_trans *tr, sql_delta *obat > int > dup_bat(sql_trans *tr, sql_table *t, sql_delta *obat, sql_delta *bat, int type, int oc_isnew, int c_isnew) > { > + printf("in dup_bat\n"); > + > return dup_delta( tr, obat, bat, type, oc_isnew, c_isnew, isTempTable(t), t->sz); > } > > @@ -368,6 +396,8 @@ update_col(sql_trans *tr, sql_column *c, > BAT *b = tids; > sql_delta *bat; > > + printf("in update_col\n"); > + > if (tpe == TYPE_bat && !BATcount(b)) > return; > > @@ -393,6 +423,8 @@ update_idx(sql_trans *tr, sql_idx * i, v > BAT *b = tids; > sql_delta *bat; > > + printf("in update_idx\n"); > + > if (tpe == TYPE_bat && !BATcount(b)) > return; > > @@ -422,6 +454,8 @@ delta_append_bat( sql_delta *bat, BAT *i > assert(!c || c->htype == TYPE_void); > #endif > > + printf("in delta_append_bat\n"); > + > if (!BATcount(i)) > return ; > b = temp_descriptor(bat->ibid); > @@ -460,12 +494,15 @@ delta_append_bat( sql_delta *bat, BAT *i > static void > delta_append_val( sql_delta *bat, void *i ) > { > + > BAT *b = temp_descriptor(bat->ibid); > #ifndef NDEBUG > BAT *c = BBPquickdesc(bat->bid, 0); > assert(!c || c->htype == TYPE_void); > #endif > > + printf("in delta append val\n"); > + > assert(b->htype == TYPE_void); > if (bat->cached) { > bat_destroy(bat->cached); > @@ -503,6 +540,8 @@ dup_idx(sql_trans *tr, sql_idx *i, sql_i > { > int ok = LOG_OK; > > + printf("in dup_idx\n"); > + > if (i->data) { > int type = (ni->type==join_idx)?TYPE_oid:TYPE_wrd; > sql_delta *bat = ni->data = ZNEW(sql_delta), *obat = i->data; > @@ -515,15 +554,27 @@ dup_idx(sql_trans *tr, sql_idx *i, sql_i > static int > dup_dbat( sql_trans *tr, sql_dbat *obat, sql_dbat *bat, int is_new, int temp) > { > + > + printf("in dup_dbat\n"); > + > bat->dbid = obat->dbid; > + > + bat->dtrbid = obat->dtrbid; > + > bat->cnt = obat->cnt; > + > bat->dname = _STRDUP(obat->dname); > + > + bat->dtrname = _STRDUP(obat->dtrname); > + > bat->wtime = obat->wtime; > if (bat->dbid) { > if (is_new) { > obat->dbid = temp_copy(bat->dbid, temp); > + obat->dtrbid = temp_copy(bat->dtrbid, temp); > } else { > bat->dbid = ebat_copy(bat->dbid, 0, temp); > + bat->dtrbid = ebat_copy(bat->dtrbid, 0, temp); > } > } > (void)tr; > @@ -546,6 +597,8 @@ append_col(sql_trans *tr, sql_column *c, > BAT *b = i; > sql_delta *bat; > > + printf("in append_col\n"); > + > if (tpe == TYPE_bat && !BATcount(b)) > return; > > @@ -574,6 +627,8 @@ append_idx(sql_trans *tr, sql_idx * i, v > BAT *b = ib; > sql_delta *bat; > > + printf("in append_idx\n"); > + > if (tpe == TYPE_bat && !BATcount(b)) > return; > > @@ -598,6 +653,8 @@ delta_delete_bat( sql_dbat *bat, BAT *i > { > BAT *b = temp_descriptor(bat->dbid); > > + printf("in delta_delete_bat\n"); > + > if (isEbat(b)) { > temp_destroy(bat->dbid); > bat->dbid = temp_copy(b->batCacheid, FALSE); > @@ -616,6 +673,8 @@ delta_delete_val( sql_dbat *bat, oid rid > { > BAT *b = temp_descriptor(bat->dbid); > > + printf("in delta_delete_val\n"); > + > if (isEbat(b)) { > temp_destroy(bat->dbid); > bat->dbid = temp_copy(b->batCacheid, FALSE); > @@ -636,6 +695,8 @@ delete_tab(sql_trans *tr, sql_table * t, > _______________________________________________ > checkin-list mailing list > checkin-list(a)monetdb.org > https://www.monetdb.org/mailman/listinfo/checkin-list >

1 0

Announcement: New Feb2013-SP5 Bugfix release of MonetDB suite
by Hannes Muehleisen 16 Oct '13

16 Oct '13

The MonetDB team at CWI/MonetDB BV is pleased to announce the Feb2013-SP5 bugfix release of the MonetDB suite of programs. More information about MonetDB can be found on our website at <http://www.monetdb.org/>. For details on this release, please see the release notes at <http://www.monetdb.org/Downloads/ReleaseNotes>. As usual, the download location is <http://dev.monetdb.org/downloads/>. Feb 2013-SP5 bugfix release Client Package * ODBC: Implemented retrieving variable-length data in parts with SQLGetData. See bug 3381. MonetDB Common * Removed the heap cache. Since the fix for bug 3323 which made that the cache was actually getting used, bug 3376 made clear that it didn't work very well. In addition, on Linux at least, the heap cache slows things down. Java Module * Rearranged order of returned columns of certain metadata functions to comply with the JDBC documentation. See bug 3356. Bug Fixes * 3323: Heapcache bugs/performance issues * 3331: SAMPLE will return same result every time. * 3356: DatabaseMetaData.getColumns() doesn't work correctly when using index-based getters * 3367: Fully qualified order by column gives "relational query without result" * 3368: BAT sortedness info ignored on ORDER BY and TOPN * 3370: SQL environment settings are updated even when the set statement fails * 3373: Setting table to read-only (Bug 3362) fails on big-endian * 3375: LIKE join: BATfetchjoin does not hit always * 3376: COPY INTO fails with HEAPextend: failed to extend: MT_mremap() failed * 3377: Query interfering with next query in same transaction, after SP4 * 3380: Python DBAPI driver throws exception when fetching EXPLAIN results * 3381: Windows ODBC driver hangs or crashes on simple queries

1 0

How to transfer monetdb database to another machine
by Rose Beck 13 Oct '13

13 Oct '13

I created monetdb database using the python library https://pypi.python.org/pypi/python-monetdb/11.16.0.7 on my ubuntu 12.04 lts server. Now I want to transfer this database to another ubuntu 12.04 lts server. However, I am unable to understand as to where(directory name) does monetdb create its database (within ubuntu 12.04) and what is the procedure which I must follow to transfer this database to another machine (so that i may query the database on another machine)?

1 0

Re: MonetDB Feb2013-SP5 Release Candidate available for testing
by Hannes Mühleisen 09 Oct '13

09 Oct '13

Hi Pete and List, On 10/09/2013 09:50 AM, Pete Hollobon wrote: > Thank you for this, we'll give it a try. Great! > Can this version be used with an existing dbfarm, or is a dump / load > necessary? This should work fine with other Feb2013 dbfarms. best, Hannes

1 0

SQLAlchemy-monetdb and change to Python API
by Gijs Molenaar 01 Oct '13

01 Oct '13

Hi monetDB people, The sqlalchemy dialect for monetdb is in quite good shape now, so I made a 0.1 release. It is also available throught Pypi. anybody who wants to give it a test and has some feedback, check it out: https://github.com/gijzelaerr/sqlalchemy-monetdb To add support for the upcomming 0.9 release I had to modify the MonetDB Python API a bit, so it support subtypes of python build-in types. I've added some unittests so the test output will change. Can one of you update the test output, I don't have a the monetdb testing output update script working on this machine. greetings, -- Gijs Molenaar http://www.astro.uva.nl/people/gijs-molenaar/

1 0

TODO statement
by Gijs Molenaar 01 Oct '13

01 Oct '13

Hi MonetDB people, I encountered this TODO message while running a query: sql>SELECT objects.name more> FROM sys.keys, sys.objects, sys.tables, sys.schemas more> WHERE sys.keys.id = objects.id more> AND keys.table_id = tables.id more> AND tables.schema_id = schemas.id more> AND keys.type = 0 more> AND tables.id = 11323; TODO: column names of level >= 3 -- Gijs Molenaar http://www.astro.uva.nl/people/gijs-molenaar/

3 3