- developers-list - monetdb.org

Re: [Monetdb-developers] [Monetdb-checkins] MonetDB5/src/modules/mal tablet_sql.mx, Feb2010, 1.44, 1.44.2.1
by Sjoerd Mullender 25 Apr '10

25 Apr '10

Should I build a new release candidate with this fix included? On 2010-04-25 12:07, Niels Nes wrote: > Update of /cvsroot/monetdb/MonetDB5/src/modules/mal > In directory sfp-cvsdas-4.v30.ch3.sourceforge.com:/tmp/cvs-serv6881/src/modules/mal > > Modified Files: > Tag: Feb2010 > tablet_sql.mx > Log Message: > fixed bug in loader, now (again) we handle embedded seperators properly > > > Index: tablet_sql.mx > =================================================================== > RCS file: /cvsroot/monetdb/MonetDB5/src/modules/mal/tablet_sql.mx,v > retrieving revision 1.44 > retrieving revision 1.44.2.1 > diff -u -d -r1.44 -r1.44.2.1 > --- tablet_sql.mx 10 Jan 2010 14:16:36 -0000 1.44 > +++ tablet_sql.mx 25 Apr 2010 10:07:29 -0000 1.44.2.1 > @@ -448,11 +448,10 @@ > int res = 0; /* < 0: error, > 0: success, == 0: continue processing */ > int j; > BUN i; > - char q=0; > size_t rseplen; > READERtask *task= (READERtask*) GDKzalloc(sizeof(READERtask)); > READERtask ptask[16]; > - int threads= GDKnr_threads < 16? GDKnr_threads: 16; > + int threads= (!maxrow || maxrow > (1<<16)) ? (GDKnr_threads < 16? GDKnr_threads: 16):1; > lng lio =0, tio, t1 = 0, total=0, iototal=0; > int vmtrim = GDK_vm_trim; > > @@ -603,6 +602,7 @@ > rethink the matter. */ > e = s; > while (s < end && (maxrow < 0 || cnt < (BUN) maxrow) ) { > + char q=0; > /* tokenize the record completely > the format of the input should comply to > the following grammar rule > @@ -617,55 +617,50 @@ > */ > if ( quote == 0) { > if (rseplen == 1) > - for ( ; *e ; e++) > - { > + for ( ; *e ; e++) { > if ( *e == '\\' ){ > e++; > continue; > } > if (*e == *rsep ) > - break; > + break; > } > else > - for ( ; *e ; e++) > - { > + for ( ; *e ; e++) { > if ( *e == '\\' ) { > e++; > continue; > } > if (*e == *rsep && strncmp(e, rsep, rseplen) == 0) > - break; > + break; > } > if ( *e == 0) > e = 0; /* nonterminated record, we need more */ > - } else > - if ( rseplen == 1 ) { > - for ( ; *e ; e++) > - { > + } else if ( rseplen == 1 ) { > + for ( ; *e ; e++) { > if (*e == q) > - q = 0; > - else if (*e == quote) > - q = *e; > - else if (*e == '\\') { > - if (e[1]) > - e++; > - } else if (*e == *rsep ) > - break; > + q = 0; > + else if (*e == quote) > + q = *e; > + else if (*e == '\\') { > + if (e[1]) > + e++; > + } else if (!q && *e == *rsep ) > + break; > } > if ( *e == 0) > e = 0; /* nonterminated record, we need more */ > } else { > - for ( ; *e ; e++) > - { > + for ( ; *e ; e++) { > if (*e == q) > - q = 0; > - else if (*e == quote) > - q = *e; > - else if (*e == '\\') { > - if (e[1]) > - e++; > - } else if (*e == *rsep && strncmp(e, rsep, rseplen) == 0 ) > - break; > + q = 0; > + else if (*e == quote) > + q = *e; > + else if (*e == '\\') { > + if (e[1]) > + e++; > + } else if (!q && *e == *rsep && strncmp(e, rsep, rseplen) == 0 ) > + break; > } > if (*e == 0 ) > e = 0; /* nonterminated record, we need more */ > > > ------------------------------------------------------------------------------ > _______________________________________________ > Monetdb-checkins mailing list > Monetdb-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-checkins -- Sjoerd Mullender

1 0

[Monetdb-developers] MonetDB memory management
by Xiao Lei 23 Apr '10

23 Apr '10

Hi, Does anyone has the same memory management problem as me? When I inserted many records like: Insert into tbl1 (id,dt) values(1,blob ‘…………………………………………’); Memory is used up to 1G. That's OK. But when I delete the whole table like: Delete from tbl1; None memory is released, still 1G is used. And when i insert again, memory used keeps increasing. Is this a problem or is it the way MonetDB works? Thx. -- Best Regards, Xiao Lei

3 3

[Monetdb-developers] Nov2009 segmentation fault (invalid read)
by Isidor Zeuner 23 Apr '10

23 Apr '10

Dear MonetDB developers, I am currently experiencing segmentation faults on SQL-based data processing using the MonetDB Nov2009 branch. As the processing is rather complex, I don't know when I will be able to isolate a test case. Based on the clues I gathered (see below), could you give me some pointers on how to investigate further? It always crashes on this query: DELETE FROM progress WHERE EXISTS ( SELECT true FROM finalization WHERE finalization.id=progress.id AND finalization.category=active ); The related tables are defined like this: CREATE TABLE progress ( id VARCHAR(32), attempt INTEGER, clearance INTEGER, PRIMARY KEY (id) ); CREATE TABLE finalization ( category VARCHAR(32), id VARCHAR(32), governing VARCHAR(64), sub_governing VARCHAR(64), domain VARCHAR(64), reason VARCHAR(16), epoch INTEGER, size INTEGER, PRIMARY KEY (category,id,governing,sub_governing,domain,reason,epoch) ); On the crash, the stack backtrace always looks like this: Program received signal SIGSEGV, Segmentation fault. [Switching to Thread 0x42006940 (LWP 17625)] 0x00002b8e691fdad8 in strCmp (l=0x2aaab97c6a20 "06EED104D173", r=0xdbdbdbdbe98848f3 <Address 0xdbdbdbdbe98848f3 out of bounds>) at gdk_atoms.mx:1855 1855 return GDK_STRCMP(l, r); (gdb) bt #0 0x00002b8e691fdad8 in strCmp (l=0x2aaab97c6a20 "06EED104D173", r=0xdbdbdbdbe98848f3 <Address 0xdbdbdbdbe98848f3 out of bounds>) at gdk_atoms.mx:1855 #1 0x00002b8e693d7322 in BAThashjoin (l=0x2aaab9417188, r=0xd614ba0, estimate=9223372036854775807) at gdk_relop.mx:452 #2 0x00002b8e6940e151 in batjoin (l=0x2aaab9417188, r=0xd614ba0, estimate=9223372036854775807, swap=1 '\001') at gdk_relop.mx:1244 #3 0x00002b8e6940e193 in BATjoin (l=0x2aaab9417188, r=0xd614ba0, estimate=9223372036854775807) at gdk_relop.mx:1251 #4 0x00002aaaaafd105e in CMDjoin () from /opt/monetdb-nov2009/lib64/MonetDB5/lib/lib_algebra.so.5 #5 0x00002aaaaaffcef5 in ALGjoinestimate () from /opt/monetdb-nov2009/lib64/MonetDB5/lib/lib_algebra.so.5 #6 0x00002aaaaaffcfc1 in ALGjoin () from /opt/monetdb-nov2009/lib64/MonetDB5/lib/lib_algebra.so.5 #7 0x00002b8e6868b106 in DFLOWstep () from /opt/monetdb-nov2009/lib64/libmonetdb5.so.5 #8 0x00002b8e6868ed7d in runDFLOWworker () from /opt/monetdb-nov2009/lib64/libmonetdb5.so.5 #9 0x00000039d04064a7 in start_thread () from /lib64/libpthread.so.0 #10 0x00000039cf8d3c2d in clone () from /lib64/libc.so.6 The second argument's address to the comparison at the top has always one or more 0xdb memory deletion markers, so it looks like it is at least overlapping with deleted space. Please note that I could not create a backtrace with full symbols as the error does not occur with the MonetDB5 module being compiled with debugging enabled. Best regards, Isidor

1 1

[Monetdb-developers] Fw: BATcacheid
by anshul gangwar 22 Apr '10

22 Apr '10

Does anybody have any idea ............... hello Are BATcacheid's real column id's i.e those BATid's which are given at the time of storage of the document(static id's). If no than from where we can print those static BATid's

1 0

[Monetdb-developers] BATcacheid
by anshul gangwar 21 Apr '10

21 Apr '10

hello Are BATcacheid's real column id's i.e those BATid's which are given at the time of storage of the document(static id's). If no than from where we can print those static BATid's

1 0

[Monetdb-developers] ~~Hi~~
by Erin's Ghost 21 Apr '10

21 Apr '10

http://template09.mydevnet.ca/NepK4RsZ0U.htm

1 0

[Monetdb-developers] flow diagram
by anshul gangwar 21 Apr '10

21 Apr '10

hello Where I can find the full flow diagram of execution of monetdb. Apart from that We are printing the BATcacheid of the columns passed to the BATfetchjoin function ,We are getting the different BATCacheids . So what is the reason behind that.

2 1

[Monetdb-developers] flow diagram
by anshul gangwar 21 Apr '10

21 Apr '10

hello I was talking about flow diagram image, so that i can visualize that.

1 0

Re: [Monetdb-developers] [Monetdb-sql-checkins] sql configure.ag, , 1.166, 1.167
by Milena Ivanova 14 Apr '10

14 Apr '10

Fabian, where should MERO_LOG be defined? Is it an env. variable? A after your last check in, I got /ufs/milena/src/sql/src/backends/monet5/merovingian/merovingian.c: In function ‘main’: /ufs/milena/src/sql/src/backends/monet5/merovingian/merovingian.c:452: error: ‘MERO_LOG’ undeclared (first use in this function) Milena On 14/04/10 11:54, Fabian wrote: > Update of /cvsroot/monetdb/sql > In directory sfp-cvsdas-4.v30.ch3.sourceforge.com:/tmp/cvs-serv6441 > > Modified Files: > configure.ag > Log Message: > Hardwire the default location for the logs in the sources, document them appropriately > > Index: configure.ag > =================================================================== > RCS file: /cvsroot/monetdb/sql/configure.ag,v > retrieving revision 1.166 > retrieving revision 1.167 > diff -u -d -r1.166 -r1.167 > --- configure.ag 13 Apr 2010 16:39:28 -0000 1.166 > +++ configure.ag 14 Apr 2010 09:54:16 -0000 1.167 > @@ -124,6 +124,8 @@ > AC_CHECK_HEADERS([uuid/uuid.h]) > AC_SEARCH_LIBS([uuid_generate], [uuid]) > AC_SUBST([MONETDB5_CONFFILE]) > +AC_SUBST([MERO_LOG], [$MONETDB5_LOGDIR/merovingian.log]) > +AC_DEFINE_UNQUOTED([MERO_LOG], ["$MONETDB5_LOGDIR/merovingian.log"], [where Merovingian will store its logs by default]) > > dnl CFLAGS for our code are stricter than what autoconf can cope with. > CFLAGS="$CFLAGS \$(X_CFLAGS)" > > > ------------------------------------------------------------------------------ > Download Intel® Parallel Studio Eval > Try the new software tools for yourself. Speed compiling, find bugs > proactively, and fine-tune applications for parallel performance. > See why Intel Parallel Studio got high marks during beta. > http://p.sf.net/sfu/intel-sw-dev > _______________________________________________ > Monetdb-sql-checkins mailing list > Monetdb-sql-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-sql-checkins >

1 1

Re: [Monetdb-developers] [Monetdb-checkins] clients/src/mapiclient MapiClient.mx, , 1.201, 1.202
by Stefan Manegold 14 Apr '10

14 Apr '10

On Tue, Apr 13, 2010 at 12:43:06PM +0000, Sjoerd Mullender wrote: > Update of /cvsroot/monetdb/clients/src/mapiclient > In directory sfp-cvsdas-4.v30.ch3.sourceforge.com:/tmp/cvs-serv15694 > > Modified Files: > MapiClient.mx > Log Message: > Once again print timer information to stderr. > If you want to combine timer information with the regular output, you > can do > mclient -t ... 2>&1 > This works since the standard output is flushed before the timer > information is printed. > And if you only want timer information, you can do > mclient -t ... >/dev/null > I briefly tried to do the same for break-down timings given by pathfinder (MonetDB/XQuery), but the changes (see diff below) don't help, mainly since the break-down timings are printed in MIL, and even fprintf(stderr(), ...) in MIL goes to stdout due to MonetDB/src/gdk/gdk.mx:2737:#define GDKerr GDKout which is probably required because MAPI only used one stream. I'm not sure whether it's worth to persue this fruther, e.g., by making mclient aware of the pathfinder/XQuery break-down timings and have re-direct them to (its) stderr ... Stefan ======== Index: pathfinder/compiler/include/mil.h =================================================================== RCS file: /cvsroot/monetdb/pathfinder/compiler/include/mil.h,v retrieving revision 1.89 diff -u -r1.89 mil.h --- pathfinder/compiler/include/mil.h 7 Jan 2010 15:24:27 -0000 1.89 +++ pathfinder/compiler/include/mil.h 14 Apr 2010 10:33:26 -0000 @@ -385,6 +385,7 @@ , m_print /**< MIL print() function */ , m_printf /**< MIL printf() function */ + , m_printerr /**< MIL fprintf(stderr(),...) function */ , m_col_name /**< assign BAT column name (for debugging only) */ @@ -1014,6 +1015,7 @@ PFmil_t * PFmil_print (const PFmil_t *); PFmil_t * PFmil_printf (const PFmil_t *); +PFmil_t * PFmil_printerr (const PFmil_t *); PFmil_t * PFmil_col_name (const PFmil_t *, const PFmil_t *); PFmil_t * PFmil_comment (const char *, ...) __attribute__ ((format (printf, 1, 2))); Index: pathfinder/compiler/compile.c =================================================================== RCS file: /cvsroot/monetdb/pathfinder/compiler/compile.c,v retrieving revision 1.186 diff -u -r1.186 compile.c --- pathfinder/compiler/compile.c 7 Jan 2010 15:24:27 -0000 1.186 +++ pathfinder/compiler/compile.c 14 Apr 2010 10:33:26 -0000 @@ -801,7 +801,7 @@ /* make runtime timing available */ if (status->timing) { mroot = PFmil_seq (mroot, - PFmil_printf ( + PFmil_printerr ( PFmil_arg (PFmil_lit_str ("\nShred % 10.3f msec" "\nQuery % 10.3f msec" "\nPrint % 10.3f msec\n"), @@ -1082,7 +1082,7 @@ if (!strncmp ("timing", genType, 6)) /* add timing information */ mroot = PFmil_seq (mroot, - PFmil_printf ( + PFmil_printerr ( PFmil_arg (PFmil_lit_str ("\nTrans % 10.3f msec" "\nShred % 10.3f msec" "\nQuery % 10.3f msec" Index: pathfinder/compiler/mil/mil.c =================================================================== RCS file: /cvsroot/monetdb/pathfinder/compiler/mil/mil.c,v retrieving revision 1.86 diff -u -r1.86 mil.c --- pathfinder/compiler/mil/mil.c 7 Jan 2010 15:24:28 -0000 1.86 +++ pathfinder/compiler/mil/mil.c 14 Apr 2010 10:33:26 -0000 @@ -1896,6 +1896,12 @@ } PFmil_t * +PFmil_printerr (const PFmil_t *args) +{ + return wire1 (m_printerr, args); +} + +PFmil_t * PFmil_col_name (const PFmil_t *bat, const PFmil_t *name) { return wire2 (m_col_name, bat, name); Index: pathfinder/compiler/mil/mil_dce.c =================================================================== RCS file: /cvsroot/monetdb/pathfinder/compiler/mil/mil_dce.c,v retrieving revision 1.34 diff -u -r1.34 mil_dce.c --- pathfinder/compiler/mil/mil_dce.c 7 Jan 2010 15:24:28 -0000 1.34 +++ pathfinder/compiler/mil/mil_dce.c 14 Apr 2010 10:33:27 -0000 @@ -474,6 +474,7 @@ case m_error: case m_print: case m_printf: + case m_printerr: case m_break: case m_use: case m_update_tape: Index: pathfinder/compiler/mil/milprint.c =================================================================== RCS file: /cvsroot/monetdb/pathfinder/compiler/mil/milprint.c,v retrieving revision 1.101 diff -u -r1.101 milprint.c --- pathfinder/compiler/mil/milprint.c 7 Jan 2010 15:24:28 -0000 1.101 +++ pathfinder/compiler/mil/milprint.c 14 Apr 2010 10:33:27 -0000 @@ -28,6 +28,7 @@ | 'ERROR (' expression ')' <m_error> | 'print (' args ')' <m_print> | 'printf (' args ')' <m_printf> + | 'printerr (' args ')' <m_printerr> | 'col_name (' expr ',' expr ')' <m_col_name> | 'destroy_ws (' expression ')' <m_destroy_ws> | 'end_ws (' expression ',' expression ')' <m_end_ws> @@ -673,6 +674,13 @@ milprintf (")"); break; + /* statement: 'fprintf ( stderr (), ' expression ')' */ + case m_printerr: + milprintf ("fprintf ( stderr (),"); + print_args (n->child[0]); + milprintf (")"); + break; + case m_col_name: print_expression (n->child[0]); milprintf (".%s (", ID[n->kind]); Index: pathfinder/compiler/mil/milprint_summer.c =================================================================== RCS file: /cvsroot/monetdb/pathfinder/compiler/mil/milprint_summer.c,v retrieving revision 1.438 diff -u -r1.438 milprint_summer.c --- pathfinder/compiler/mil/milprint_summer.c 12 Jan 2010 09:22:14 -0000 1.438 +++ pathfinder/compiler/mil/milprint_summer.c 14 Apr 2010 10:33:27 -0000 @@ -11616,7 +11616,7 @@ " ERROR(err); -(try);\n"\ "} else if (genType.startsWith(\"timing\")) {\n"\ " time_print := usec() - time_print;\n"\ - " printf(\"\\nTrans %% 10.3f msec\\nShred %% 10.3f msec\\nQuery %% 10.3f msec\\n" LASTPHASE " %% 10.3f msec\\n\","\ + " fprintf(stderr(),\"\\nTrans %% 10.3f msec\\nShred %% 10.3f msec\\nQuery %% 10.3f msec\\n" LASTPHASE " %% 10.3f msec\\n\","\ " dbl(time_compile)/1000.0, dbl(time_shred)/1000.0, dbl(time_exec - time_shred)/1000.0, time_print/1000.0);\n}" const char* PFstopMIL(int statement_type) { Index: pathfinder/runtime/pathfinder.mx =================================================================== RCS file: /cvsroot/monetdb/pathfinder/runtime/pathfinder.mx,v retrieving revision 1.480 diff -u -r1.480 pathfinder.mx --- pathfinder/runtime/pathfinder.mx 12 Apr 2010 09:20:51 -0000 1.480 +++ pathfinder/runtime/pathfinder.mx 14 Apr 2010 10:33:27 -0000 @@ -6336,7 +6336,7 @@ /* execute query */ xquery = p+1; if (p[1] == '4' && p[2] == '2' && p[3] == '\n' && p[4] == 0 && genType == buf && output[2] == 0) { /* use '42' as latency query test */ - stream_printf(GDKout, "42\n\nTrans 0.000 msec\nShred 0.000 msec\nQuery 0.000 msec\nPrint 0.000 msec\n"); + stream_printf(GDKerr, "42\n\nTrans 0.000 msec\nShred 0.000 msec\nQuery 0.000 msec\nPrint 0.000 msec\n"); } else { err = xquery_change_genType(ctx, genType); if (err == NULL) ======== > > Index: MapiClient.mx > =================================================================== > RCS file: /cvsroot/monetdb/clients/src/mapiclient/MapiClient.mx,v > retrieving revision 1.201 > retrieving revision 1.202 > diff -u -d -r1.201 -r1.202 > --- MapiClient.mx 29 Mar 2010 18:45:15 -0000 1.201 > +++ MapiClient.mx 13 Apr 2010 12:43:04 -0000 1.202 > @@ -332,10 +332,11 @@ > static void > timerEnd(void) > { > + stream_flush(toConsole); > t1 = gettime(); > if (mark && specials == NOmodifier) { > - stream_printf(toConsole, "%s %7ld.%03ld msec %s\n", mark, (long) ((t1 - t0) / 1000), (long) ((t1 - t0) % 1000), mark2 ? mark2 : ""); > - stream_flush(toConsole); > + fprintf(stderr, "%s %7ld.%03ld msec %s\n", mark, (long) ((t1 - t0) / 1000), (long) ((t1 - t0) % 1000), mark2 ? mark2 : ""); > + fflush(stderr); > } > } > > > > ------------------------------------------------------------------------------ > Download Intel® Parallel Studio Eval > Try the new software tools for yourself. Speed compiling, find bugs > proactively, and fine-tune applications for parallel performance. > See why Intel Parallel Studio got high marks during beta. > http://p.sf.net/sfu/intel-sw-dev > _______________________________________________ > Monetdb-checkins mailing list > Monetdb-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-checkins -- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4199 |

2 1