Re: [Monetdb-developers] [Monetdb-checkins] MonetDB/src/gdk gdk.mx, , 1.278, 1.279 gdk_bat.mx, , 1.213, 1.214 gdk_posix.mx, , 1.168, 1.169 gdk_relop.mx, , 1.161, 1.162 gdk_utils.mx, , 1.241, 1.242
Peter, these changes lead to $ Mserver # MonetDB Server v4.29.0 # based on GDK v1.29.0 # Copyright (c) 1993-July 2008, CWI. All rights reserved. # Copyright (c) August 2008-2009, MonetDB B.V.. All rights reserved. # Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs; dynamically linked. # Visit http://monetdb.cwi.nl/ for further information. !ERROR: CMDfind: value not found. !ERROR: CMDfind: operation failed. !ERROR: interpret_params: int(param 1): evaluation error. MonetDB> In fact, Sjoerd fixed this yesterday in =================================================================== 2009/04/02 - sjoerd: MonetDB/src/gdk/gdk_utils.mx,1.241 Reintroduced monet_pid, gdk_mem_pagebits, and gdk_mem_bigsize into the MonetDB environment, but not through advertizing them as options. This should fix bug 2726337. =================================================================== Your checkin apparently removed this fix, again. Why? Stefan ps: page 14 of http://homepages.cwi.nl/~manegold/Presentations/MonetDBdevTTT.pdf says "Do never copy/overwrite files checked-out from CVS!" On Fri, Apr 03, 2009 at 09:14:55AM +0000, Peter Boncz wrote:
Update of /cvsroot/monetdb/MonetDB/src/gdk In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv31767/gdk
Modified Files: gdk.mx gdk_bat.mx gdk_posix.mx gdk_relop.mx gdk_utils.mx Log Message: various tweaks to the memory mangement
- tune the GDK_mmap_minsize to vary between full RAM size and 128MB (lowest when memory pressure is extreme)
- in fetchjoin, when the optimization of string heap copying is applied (instead of creating a new string heap and inserting into it), try to share the heaps in virtual memory (rather than copying it) This is possible with the logical view mechanism.
now fetchjoin, batcopy and remap use this VM heap sharing.
- disable vmalloc on Linux -- well actually on platforms that do not have posix_fadvise. A bit of a hack.
U gdk.mx Index: gdk.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk.mx,v retrieving revision 1.278 retrieving revision 1.279 diff -u -d -r1.278 -r1.279 --- gdk.mx 13 Mar 2009 17:58:12 -0000 1.278 +++ gdk.mx 3 Apr 2009 09:14:51 -0000 1.279 @@ -1048,6 +1048,7 @@ gdk_export int HEAPextend(Heap *h, size_t size); gdk_export int HEAPload(Heap *h, str nme, str ext, int trunc); gdk_export int HEAPsave(Heap *h, str nme, str ext); +gdk_export int HEAPshare(Heap *dst, Heap *src, int copy_on_write); gdk_export int HEAPcopy(Heap *dst, Heap *src); gdk_export int HEAPwarm(Heap *h); gdk_export size_t HEAPsubcopy(Heap *dst, Heap *src, size_t size, size_t offset);
U gdk_posix.mx Index: gdk_posix.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_posix.mx,v retrieving revision 1.168 retrieving revision 1.169 diff -u -d -r1.168 -r1.169 --- gdk_posix.mx 1 Apr 2009 12:37:27 -0000 1.168 +++ gdk_posix.mx 3 Apr 2009 09:14:52 -0000 1.169 @@ -824,6 +824,13 @@ } } } + /* could not save anything -- reset the save_tile to the start to try all heaps again */ + for (i = MT_mmap_next(MT_mmap_cur); i != MT_mmap_cur; i = MT_mmap_next(i)) { + if (MT_mmap_tab[i].fd >= 0 && + MT_mmap_tab[i].writable && + (MT_mmap_tab[i].pincnt == 0 || MT_mmap_tab[i].len > target)) + MT_mmap_tab[i].save_tile = 1; + } MT_mmap_cur = i; } (void) pthread_mutex_unlock(&MT_mmap_lock);
U gdk_relop.mx Index: gdk_relop.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_relop.mx,v retrieving revision 1.161 retrieving revision 1.162 diff -u -d -r1.161 -r1.162 --- gdk_relop.mx 30 Mar 2009 07:30:13 -0000 1.161 +++ gdk_relop.mx 3 Apr 2009 09:14:52 -0000 1.162 @@ -972,7 +972,13 @@ bn->theap->filename = (str) GDKmalloc(strlen(nme) + 12); GDKfilepath(bn->theap->filename, NULL, nme, "theap"); } - if (HEAPcopy(bn->theap, r->theap) < 0) { + if (r->batRestricted == BAT_READ && HEAPshare(bn->theap, r->theap, 0) == 0) { + int parent = r->T->parentid; + if (!parent) parent = -r->batCacheid; + bn->P->lview = TRUE; + bn->T->parentid = parent; + BBPshare(parent); + } else if (HEAPcopy(bn->theap, r->theap) < 0) { BBPreclaim(bn); goto ready; }
U gdk_utils.mx Index: gdk_utils.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_utils.mx,v retrieving revision 1.241 retrieving revision 1.242 diff -u -d -r1.241 -r1.242 --- gdk_utils.mx 2 Apr 2009 08:44:03 -0000 1.241 +++ gdk_utils.mx 3 Apr 2009 09:14:53 -0000 1.242 @@ -1053,6 +1053,7 @@ #endif } size = (size + 7) & ~7; /* round up to a multiple of eight */ +#ifndef HAVE_POSIX_FADVISE if (size > GDK_mem_bigsize) { size_t newsize = size + sizeof(size_t) + sizeof(size_t); size_t newmax = MAX(*maxsize, newsize); @@ -1067,6 +1068,7 @@ *maxsize = newmax - (sizeof(size_t) + sizeof(size_t)); return (void *) s; } +#endif CHKMEM(size, 0); GDKmalloc_prefixsize(s, size); if (s == NULL) { @@ -1203,6 +1205,7 @@ } } #ifndef GDK_MEM_MISALIGN +#ifndef HAVE_POSIX_FADVISE else if (size <= GDK_mem_bigsize) { void *oldblk = blk; size_t newsize = size + 8; @@ -1236,6 +1239,7 @@ return blk; } } +#endif /* alloc© due to failed realloc (may be very big heap that needs vm) */ #else /* alloc© because we cannot trust realloc due to misalignment */ @@ -1446,7 +1450,9 @@ } }
-#ifndef WIN32 +#ifdef HAVE_POSIX_FADVISE +static str highload_name[] = { "idle", "low", "medium", "high", "extreme" }; + static int membits = 0; static void GDKvmtrim(void *limit) @@ -1458,21 +1464,25 @@ /* compute rounded down log2 of memsize */ size_t s = GDK_mem_maxsize_max; while((s >>= 1) > 0) membits++; + membits = (MAX(0,membits-27))/4; } - do { stream *fp = NULL; - MT_sleep_ms(highload ? 500 : 5000); MEMDEBUG { fp = GDKout; - THRprintf(fp, "#GDKvmtrim(" LLFMT ")\n", (long long) *(size_t *) limit); + THRprintf(fp, "#GDKvmtrim(load=%s, rsstarget=" SZFMT ", GDK_mmap_minsize=" SZFMT ")\n", + highload_name[highload], *(size_t *) limit, GDK_mmap_minsize); } highload = MT_mmap_trim(*(size_t *) limit, fp); if (highload) { /* memcrunch distress varies from 1 to 4 */ /* rss is above 0.75mem_maxsize!! start using mmap-on-tempfile */ if (!bak_mmap_minsize) bak_mmap_minsize = GDK_mmap_minsize; - GDK_mmap_minsize = ((size_t) 1) << (membits-(highload+highload)); /* HEAPalloc/extend > [256-16]MB uses it */ + /* minsize = memsize iff highload = 0 + * = ..in between.. iff highload = 1-3 + * = 128MB iff highload = 4 + */ + GDK_mmap_minsize = ((size_t) 1) << (27 + membits*(4-highload)); } else if (bak_mmap_minsize) { GDK_mmap_minsize = bak_mmap_minsize; /* revert to default setting */ bak_mmap_minsize = 0; @@ -1490,7 +1500,6 @@ char *p; opt *n; int i, j, nlen = 0; - char buf[16];
/* some sanity checks (should also find if symbols are not defined) */ assert(sizeof(char) == SIZEOF_CHAR); @@ -1606,20 +1615,8 @@ } if (GDKgetenv_isyes("gdk_embedded") || GDKgetenv_isyes("embedded")) GDKembedded = 1; - if (GDKgetenv("gdk_mem_pagebits") == NULL) { - snprintf(buf, sizeof(buf), "%d", GDK_mem_pagebits); - GDKsetenv("gdk_mem_pagebits", buf); - } - if (GDKgetenv("gdk_mem_bigsize") == NULL) { - snprintf(buf, sizeof(buf), SZFMT, GDK_mem_bigsize); - GDKsetenv("gdk_mem_bigsize", buf); - } - if (GDKgetenv("monet_pid") == NULL) { - snprintf(buf, sizeof(buf), "%d", (int) getpid()); - GDKsetenv("monet_pid", buf); - }
-#ifndef WIN32 +#ifdef HAVE_POSIX_FADVISE if (!GDKembedded && GDK_vm_trim) { MT_Id t;
U gdk_bat.mx Index: gdk_bat.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_bat.mx,v retrieving revision 1.213 retrieving revision 1.214 diff -u -d -r1.213 -r1.214 --- gdk_bat.mx 30 Mar 2009 07:30:07 -0000 1.213 +++ gdk_bat.mx 3 Apr 2009 09:14:52 -0000 1.214 @@ -670,11 +670,9 @@ as rcopy(). @{ @c -static int -heapcopy(Heap *dst, Heap *src, int *remap) -{ - if (*remap && src->storage == STORE_MMAP) { - /* use copy-on-write mmap for isolatable copy */ +int HEAPshare(Heap *dst, Heap *src, int copy_on_write) { + /* use copy-on-write mmap for isolatable copy */ + if (src->storage == STORE_MMAP) { *dst = *src; dst->base = (char *) -1; dst->filename = GDKstrdup(src->filename); @@ -682,16 +680,25 @@ char path[PATHLENGTH];
GDKfilepath(path, BATDIR, dst->filename, NULL); - dst->base = GDKmmap(path, MMAP_READ | MMAP_SEQUENTIAL | MMAP_COPY, 0, dst->maxsize); + dst->base = GDKmmap(path, MMAP_READ | MMAP_SEQUENTIAL | (copy_on_write?MMAP_COPY:0), 0, dst->maxsize); if (dst->base != (char *) -1) { dst->copied = 1; - dst->newstorage = dst->storage = STORE_PRIV; + dst->newstorage = dst->storage = copy_on_write?STORE_PRIV:STORE_MMAP; return 0; } GDKfree(dst->filename); dst->filename = NULL; } } + return -1; +} + +static int +heapcopy(Heap *dst, Heap *src, int *remap) +{ + if (*remap && HEAPshare(dst, src, *remap) == 0) { + return 0; /* use a shared memory heap */ + } *remap = 0; return HEAPcopy(dst, src); } @@ -2766,7 +2773,7 @@ int b0, b1, b2 = 0, b3 = 0;
/* copy-on-write isolated bats that change mode should be made independent */ - if (b->P->lview && batunshare(b, "BATsetaccess") < 0) + if (b->P->lview && newmode != BAT_READ && batunshare(b, "BATsetaccess") < 0) return NULL;
if (b->batSharecnt && newmode != BAT_READ) {
------------------------------------------------------------------------------ _______________________________________________ Monetdb-checkins mailing list Monetdb-checkins@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/monetdb-checkins
-- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4312 |
sorry, I was helping Stefan de Konink, but he had a read-only version of the MonetDB sources.
-----Original Message----- From: Stefan Manegold [mailto:Stefan.Manegold@cwi.nl] Sent: Friday, April 03, 2009 1:18 PM To: monetdb-developers@lists.sourceforge.net; Peter Boncz Subject: Re: [Monetdb-checkins] MonetDB/src/gdk gdk.mx, , 1.278, 1.279 gdk_bat.mx, , 1.213, 1.214 gdk_posix.mx, , 1.168, 1.169 gdk_relop.mx, , 1.161, 1.162 gdk_utils.mx, , 1.241, 1.242
Peter,
these changes lead to
$ Mserver # MonetDB Server v4.29.0 # based on GDK v1.29.0 # Copyright (c) 1993-July 2008, CWI. All rights reserved. # Copyright (c) August 2008-2009, MonetDB B.V.. All rights reserved. # Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs; dynamically linked. # Visit http://monetdb.cwi.nl/ for further information. !ERROR: CMDfind: value not found. !ERROR: CMDfind: operation failed. !ERROR: interpret_params: int(param 1): evaluation error. MonetDB>
In fact, Sjoerd fixed this yesterday in
=================================================================== 2009/04/02 - sjoerd: MonetDB/src/gdk/gdk_utils.mx,1.241 Reintroduced monet_pid, gdk_mem_pagebits, and gdk_mem_bigsize into the MonetDB environment, but not through advertizing them as options. This should fix bug 2726337. ===================================================================
Your checkin apparently removed this fix, again.
Why?
Stefan
ps: page 14 of http://homepages.cwi.nl/~manegold/Presentations/MonetDBdevTTT.pdf says "Do never copy/overwrite files checked-out from CVS!"
On Fri, Apr 03, 2009 at 09:14:55AM +0000, Peter Boncz wrote:
Update of /cvsroot/monetdb/MonetDB/src/gdk In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv31767/gdk
Modified Files: gdk.mx gdk_bat.mx gdk_posix.mx gdk_relop.mx gdk_utils.mx Log Message: various tweaks to the memory mangement
- tune the GDK_mmap_minsize to vary between full RAM size and 128MB (lowest when memory pressure is extreme)
- in fetchjoin, when the optimization of string heap copying is applied (instead of creating a new string heap and inserting into it), try to share the heaps in virtual memory (rather than copying it) This is possible with the logical view mechanism.
now fetchjoin, batcopy and remap use this VM heap sharing.
- disable vmalloc on Linux -- well actually on platforms that do not have posix_fadvise. A bit of a hack.
U gdk.mx Index: gdk.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk.mx,v retrieving revision 1.278 retrieving revision 1.279 diff -u -d -r1.278 -r1.279 --- gdk.mx 13 Mar 2009 17:58:12 -0000 1.278 +++ gdk.mx 3 Apr 2009 09:14:51 -0000 1.279 @@ -1048,6 +1048,7 @@ gdk_export int HEAPextend(Heap *h, size_t size); gdk_export int HEAPload(Heap *h, str nme, str ext, int trunc); gdk_export int HEAPsave(Heap *h, str nme, str ext); +gdk_export int HEAPshare(Heap *dst, Heap *src, int copy_on_write); gdk_export int HEAPcopy(Heap *dst, Heap *src); gdk_export int HEAPwarm(Heap *h); gdk_export size_t HEAPsubcopy(Heap *dst, Heap *src, size_t size, size_t offset);
U gdk_posix.mx Index: gdk_posix.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_posix.mx,v retrieving revision 1.168 retrieving revision 1.169 diff -u -d -r1.168 -r1.169 --- gdk_posix.mx 1 Apr 2009 12:37:27 -0000 1.168 +++ gdk_posix.mx 3 Apr 2009 09:14:52 -0000 1.169 @@ -824,6 +824,13 @@ } } } + /* could not save anything -- reset the save_tile to the start to try all heaps again */ + for (i = MT_mmap_next(MT_mmap_cur); i != MT_mmap_cur; i = MT_mmap_next(i)) { + if (MT_mmap_tab[i].fd >= 0 && + MT_mmap_tab[i].writable && + (MT_mmap_tab[i].pincnt == 0 || MT_mmap_tab[i].len > target)) + MT_mmap_tab[i].save_tile = 1; + } MT_mmap_cur = i; } (void) pthread_mutex_unlock(&MT_mmap_lock);
U gdk_relop.mx Index: gdk_relop.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_relop.mx,v retrieving revision 1.161 retrieving revision 1.162 diff -u -d -r1.161 -r1.162 --- gdk_relop.mx 30 Mar 2009 07:30:13 -0000 1.161 +++ gdk_relop.mx 3 Apr 2009 09:14:52 -0000 1.162 @@ -972,7 +972,13 @@ bn->theap->filename = (str) GDKmalloc(strlen(nme) + 12); GDKfilepath(bn->theap->filename, NULL, nme, "theap"); } - if (HEAPcopy(bn->theap, r->theap) < 0) { + if (r->batRestricted == BAT_READ && HEAPshare(bn->theap, r->theap, 0) == 0) { + int parent = r->T->parentid; + if (!parent) parent = -r->batCacheid; + bn->P->lview = TRUE; + bn->T->parentid = parent; + BBPshare(parent); + } else if (HEAPcopy(bn->theap, r->theap) < 0) { BBPreclaim(bn); goto ready; }
U gdk_utils.mx Index: gdk_utils.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_utils.mx,v retrieving revision 1.241 retrieving revision 1.242 diff -u -d -r1.241 -r1.242 --- gdk_utils.mx 2 Apr 2009 08:44:03 -0000 1.241 +++ gdk_utils.mx 3 Apr 2009 09:14:53 -0000 1.242 @@ -1053,6 +1053,7 @@ #endif } size = (size + 7) & ~7; /* round up to a multiple of eight */ +#ifndef HAVE_POSIX_FADVISE if (size > GDK_mem_bigsize) { size_t newsize = size + sizeof(size_t) + sizeof(size_t); size_t newmax = MAX(*maxsize, newsize); @@ -1067,6 +1068,7 @@ *maxsize = newmax - (sizeof(size_t) + sizeof(size_t)); return (void *) s; } +#endif CHKMEM(size, 0); GDKmalloc_prefixsize(s, size); if (s == NULL) { @@ -1203,6 +1205,7 @@ } } #ifndef GDK_MEM_MISALIGN +#ifndef HAVE_POSIX_FADVISE else if (size <= GDK_mem_bigsize) { void *oldblk = blk; size_t newsize = size + 8; @@ -1236,6 +1239,7 @@ return blk; } } +#endif /* alloc© due to failed realloc (may be very big heap that needs vm) */ #else /* alloc© because we cannot trust realloc due to misalignment */ @@ -1446,7 +1450,9 @@ } }
-#ifndef WIN32 +#ifdef HAVE_POSIX_FADVISE +static str highload_name[] = { "idle", "low", "medium", "high", "extreme" }; + static int membits = 0; static void GDKvmtrim(void *limit) @@ -1458,21 +1464,25 @@ /* compute rounded down log2 of memsize */ size_t s = GDK_mem_maxsize_max; while((s >>= 1) > 0) membits++; + membits = (MAX(0,membits-27))/4; } - do { stream *fp = NULL; - MT_sleep_ms(highload ? 500 : 5000); MEMDEBUG { fp = GDKout; - THRprintf(fp, "#GDKvmtrim(" LLFMT ")\n", (long long) *(size_t *) limit); + THRprintf(fp, "#GDKvmtrim(load=%s, rsstarget=" SZFMT ", GDK_mmap_minsize=" SZFMT ")\n", + highload_name[highload], *(size_t *) limit, GDK_mmap_minsize); } highload = MT_mmap_trim(*(size_t *) limit, fp); if (highload) { /* memcrunch distress varies from 1 to 4 */ /* rss is above 0.75mem_maxsize!! start using mmap-on- tempfile */ if (!bak_mmap_minsize) bak_mmap_minsize = GDK_mmap_minsize; - GDK_mmap_minsize = ((size_t) 1) << (membits- (highload+highload)); /* HEAPalloc/extend > [256-16]MB uses it */ + /* minsize = memsize iff highload = 0 + * = ..in between.. iff highload = 1-3 + * = 128MB iff highload = 4 + */ + GDK_mmap_minsize = ((size_t) 1) << (27 + membits*(4- highload)); } else if (bak_mmap_minsize) { GDK_mmap_minsize = bak_mmap_minsize; /* revert to default setting */ bak_mmap_minsize = 0; @@ -1490,7 +1500,6 @@ char *p; opt *n; int i, j, nlen = 0; - char buf[16];
/* some sanity checks (should also find if symbols are not defined) */ assert(sizeof(char) == SIZEOF_CHAR); @@ -1606,20 +1615,8 @@ } if (GDKgetenv_isyes("gdk_embedded") || GDKgetenv_isyes("embedded")) GDKembedded = 1; - if (GDKgetenv("gdk_mem_pagebits") == NULL) { - snprintf(buf, sizeof(buf), "%d", GDK_mem_pagebits); - GDKsetenv("gdk_mem_pagebits", buf); - } - if (GDKgetenv("gdk_mem_bigsize") == NULL) { - snprintf(buf, sizeof(buf), SZFMT, GDK_mem_bigsize); - GDKsetenv("gdk_mem_bigsize", buf); - } - if (GDKgetenv("monet_pid") == NULL) { - snprintf(buf, sizeof(buf), "%d", (int) getpid()); - GDKsetenv("monet_pid", buf); - }
-#ifndef WIN32 +#ifdef HAVE_POSIX_FADVISE if (!GDKembedded && GDK_vm_trim) { MT_Id t;
U gdk_bat.mx Index: gdk_bat.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_bat.mx,v retrieving revision 1.213 retrieving revision 1.214 diff -u -d -r1.213 -r1.214 --- gdk_bat.mx 30 Mar 2009 07:30:07 -0000 1.213 +++ gdk_bat.mx 3 Apr 2009 09:14:52 -0000 1.214 @@ -670,11 +670,9 @@ as rcopy(). @{ @c -static int -heapcopy(Heap *dst, Heap *src, int *remap) -{ - if (*remap && src->storage == STORE_MMAP) { - /* use copy-on-write mmap for isolatable copy */ +int HEAPshare(Heap *dst, Heap *src, int copy_on_write) { + /* use copy-on-write mmap for isolatable copy */ + if (src->storage == STORE_MMAP) { *dst = *src; dst->base = (char *) -1; dst->filename = GDKstrdup(src->filename); @@ -682,16 +680,25 @@ char path[PATHLENGTH];
GDKfilepath(path, BATDIR, dst->filename, NULL); - dst->base = GDKmmap(path, MMAP_READ | MMAP_SEQUENTIAL | MMAP_COPY, 0, dst->maxsize); + dst->base = GDKmmap(path, MMAP_READ | MMAP_SEQUENTIAL | (copy_on_write?MMAP_COPY:0), 0, dst->maxsize); if (dst->base != (char *) -1) { dst->copied = 1; - dst->newstorage = dst->storage = STORE_PRIV; + dst->newstorage = dst->storage = copy_on_write?STORE_PRIV:STORE_MMAP; return 0; } GDKfree(dst->filename); dst->filename = NULL; } } + return -1; +} + +static int +heapcopy(Heap *dst, Heap *src, int *remap) +{ + if (*remap && HEAPshare(dst, src, *remap) == 0) { + return 0; /* use a shared memory heap */ + } *remap = 0; return HEAPcopy(dst, src); } @@ -2766,7 +2773,7 @@ int b0, b1, b2 = 0, b3 = 0;
/* copy-on-write isolated bats that change mode should be made independent */ - if (b->P->lview && batunshare(b, "BATsetaccess") < 0) + if (b->P->lview && newmode != BAT_READ && batunshare(b, "BATsetaccess") < 0) return NULL;
if (b->batSharecnt && newmode != BAT_READ) {
-------------------------------------------------------------------------
_______________________________________________ Monetdb-checkins mailing list Monetdb-checkins@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/monetdb-checkins
-- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4312 |
On Fri, Apr 03, 2009 at 04:58:45AM -0700, Peter Boncz wrote:
sorry, I was helping Stefan de Konink, but he had a read-only version of the MonetDB sources.
Tip for next time: - use `cvs diff | patch` instead of `cp` - use `cvs diff` to double-check your changes before check-in - check diffs in checkin emails to see whether your check-in went as intended Stefan
-----Original Message----- From: Stefan Manegold [mailto:Stefan.Manegold@cwi.nl] Sent: Friday, April 03, 2009 1:18 PM To: monetdb-developers@lists.sourceforge.net; Peter Boncz Subject: Re: [Monetdb-checkins] MonetDB/src/gdk gdk.mx, , 1.278, 1.279 gdk_bat.mx, , 1.213, 1.214 gdk_posix.mx, , 1.168, 1.169 gdk_relop.mx, , 1.161, 1.162 gdk_utils.mx, , 1.241, 1.242
Peter,
these changes lead to
$ Mserver # MonetDB Server v4.29.0 # based on GDK v1.29.0 # Copyright (c) 1993-July 2008, CWI. All rights reserved. # Copyright (c) August 2008-2009, MonetDB B.V.. All rights reserved. # Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs; dynamically linked. # Visit http://monetdb.cwi.nl/ for further information. !ERROR: CMDfind: value not found. !ERROR: CMDfind: operation failed. !ERROR: interpret_params: int(param 1): evaluation error. MonetDB>
In fact, Sjoerd fixed this yesterday in
=================================================================== 2009/04/02 - sjoerd: MonetDB/src/gdk/gdk_utils.mx,1.241 Reintroduced monet_pid, gdk_mem_pagebits, and gdk_mem_bigsize into the MonetDB environment, but not through advertizing them as options. This should fix bug 2726337. ===================================================================
Your checkin apparently removed this fix, again.
Why?
Stefan
ps: page 14 of http://homepages.cwi.nl/~manegold/Presentations/MonetDBdevTTT.pdf says "Do never copy/overwrite files checked-out from CVS!"
On Fri, Apr 03, 2009 at 09:14:55AM +0000, Peter Boncz wrote:
Update of /cvsroot/monetdb/MonetDB/src/gdk In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv31767/gdk
Modified Files: gdk.mx gdk_bat.mx gdk_posix.mx gdk_relop.mx gdk_utils.mx Log Message: various tweaks to the memory mangement
- tune the GDK_mmap_minsize to vary between full RAM size and 128MB (lowest when memory pressure is extreme)
- in fetchjoin, when the optimization of string heap copying is applied (instead of creating a new string heap and inserting into it), try to share the heaps in virtual memory (rather than copying it) This is possible with the logical view mechanism.
now fetchjoin, batcopy and remap use this VM heap sharing.
- disable vmalloc on Linux -- well actually on platforms that do not have posix_fadvise. A bit of a hack.
U gdk.mx Index: gdk.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk.mx,v retrieving revision 1.278 retrieving revision 1.279 diff -u -d -r1.278 -r1.279 --- gdk.mx 13 Mar 2009 17:58:12 -0000 1.278 +++ gdk.mx 3 Apr 2009 09:14:51 -0000 1.279 @@ -1048,6 +1048,7 @@ gdk_export int HEAPextend(Heap *h, size_t size); gdk_export int HEAPload(Heap *h, str nme, str ext, int trunc); gdk_export int HEAPsave(Heap *h, str nme, str ext); +gdk_export int HEAPshare(Heap *dst, Heap *src, int copy_on_write); gdk_export int HEAPcopy(Heap *dst, Heap *src); gdk_export int HEAPwarm(Heap *h); gdk_export size_t HEAPsubcopy(Heap *dst, Heap *src, size_t size, size_t offset);
U gdk_posix.mx Index: gdk_posix.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_posix.mx,v retrieving revision 1.168 retrieving revision 1.169 diff -u -d -r1.168 -r1.169 --- gdk_posix.mx 1 Apr 2009 12:37:27 -0000 1.168 +++ gdk_posix.mx 3 Apr 2009 09:14:52 -0000 1.169 @@ -824,6 +824,13 @@ } } } + /* could not save anything -- reset the save_tile to the start to try all heaps again */ + for (i = MT_mmap_next(MT_mmap_cur); i != MT_mmap_cur; i = MT_mmap_next(i)) { + if (MT_mmap_tab[i].fd >= 0 && + MT_mmap_tab[i].writable && + (MT_mmap_tab[i].pincnt == 0 || MT_mmap_tab[i].len > target)) + MT_mmap_tab[i].save_tile = 1; + } MT_mmap_cur = i; } (void) pthread_mutex_unlock(&MT_mmap_lock);
U gdk_relop.mx Index: gdk_relop.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_relop.mx,v retrieving revision 1.161 retrieving revision 1.162 diff -u -d -r1.161 -r1.162 --- gdk_relop.mx 30 Mar 2009 07:30:13 -0000 1.161 +++ gdk_relop.mx 3 Apr 2009 09:14:52 -0000 1.162 @@ -972,7 +972,13 @@ bn->theap->filename = (str) GDKmalloc(strlen(nme) + 12); GDKfilepath(bn->theap->filename, NULL, nme, "theap"); } - if (HEAPcopy(bn->theap, r->theap) < 0) { + if (r->batRestricted == BAT_READ && HEAPshare(bn->theap, r->theap, 0) == 0) { + int parent = r->T->parentid; + if (!parent) parent = -r->batCacheid; + bn->P->lview = TRUE; + bn->T->parentid = parent; + BBPshare(parent); + } else if (HEAPcopy(bn->theap, r->theap) < 0) { BBPreclaim(bn); goto ready; }
U gdk_utils.mx Index: gdk_utils.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_utils.mx,v retrieving revision 1.241 retrieving revision 1.242 diff -u -d -r1.241 -r1.242 --- gdk_utils.mx 2 Apr 2009 08:44:03 -0000 1.241 +++ gdk_utils.mx 3 Apr 2009 09:14:53 -0000 1.242 @@ -1053,6 +1053,7 @@ #endif } size = (size + 7) & ~7; /* round up to a multiple of eight */ +#ifndef HAVE_POSIX_FADVISE if (size > GDK_mem_bigsize) { size_t newsize = size + sizeof(size_t) + sizeof(size_t); size_t newmax = MAX(*maxsize, newsize); @@ -1067,6 +1068,7 @@ *maxsize = newmax - (sizeof(size_t) + sizeof(size_t)); return (void *) s; } +#endif CHKMEM(size, 0); GDKmalloc_prefixsize(s, size); if (s == NULL) { @@ -1203,6 +1205,7 @@ } } #ifndef GDK_MEM_MISALIGN +#ifndef HAVE_POSIX_FADVISE else if (size <= GDK_mem_bigsize) { void *oldblk = blk; size_t newsize = size + 8; @@ -1236,6 +1239,7 @@ return blk; } } +#endif /* alloc© due to failed realloc (may be very big heap that needs vm) */ #else /* alloc© because we cannot trust realloc due to misalignment */ @@ -1446,7 +1450,9 @@ } }
-#ifndef WIN32 +#ifdef HAVE_POSIX_FADVISE +static str highload_name[] = { "idle", "low", "medium", "high", "extreme" }; + static int membits = 0; static void GDKvmtrim(void *limit) @@ -1458,21 +1464,25 @@ /* compute rounded down log2 of memsize */ size_t s = GDK_mem_maxsize_max; while((s >>= 1) > 0) membits++; + membits = (MAX(0,membits-27))/4; } - do { stream *fp = NULL; - MT_sleep_ms(highload ? 500 : 5000); MEMDEBUG { fp = GDKout; - THRprintf(fp, "#GDKvmtrim(" LLFMT ")\n", (long long) *(size_t *) limit); + THRprintf(fp, "#GDKvmtrim(load=%s, rsstarget=" SZFMT ", GDK_mmap_minsize=" SZFMT ")\n", + highload_name[highload], *(size_t *) limit, GDK_mmap_minsize); } highload = MT_mmap_trim(*(size_t *) limit, fp); if (highload) { /* memcrunch distress varies from 1 to 4 */ /* rss is above 0.75mem_maxsize!! start using mmap-on- tempfile */ if (!bak_mmap_minsize) bak_mmap_minsize = GDK_mmap_minsize; - GDK_mmap_minsize = ((size_t) 1) << (membits- (highload+highload)); /* HEAPalloc/extend > [256-16]MB uses it */ + /* minsize = memsize iff highload = 0 + * = ..in between.. iff highload = 1-3 + * = 128MB iff highload = 4 + */ + GDK_mmap_minsize = ((size_t) 1) << (27 + membits*(4- highload)); } else if (bak_mmap_minsize) { GDK_mmap_minsize = bak_mmap_minsize; /* revert to default setting */ bak_mmap_minsize = 0; @@ -1490,7 +1500,6 @@ char *p; opt *n; int i, j, nlen = 0; - char buf[16];
/* some sanity checks (should also find if symbols are not defined) */ assert(sizeof(char) == SIZEOF_CHAR); @@ -1606,20 +1615,8 @@ } if (GDKgetenv_isyes("gdk_embedded") || GDKgetenv_isyes("embedded")) GDKembedded = 1; - if (GDKgetenv("gdk_mem_pagebits") == NULL) { - snprintf(buf, sizeof(buf), "%d", GDK_mem_pagebits); - GDKsetenv("gdk_mem_pagebits", buf); - } - if (GDKgetenv("gdk_mem_bigsize") == NULL) { - snprintf(buf, sizeof(buf), SZFMT, GDK_mem_bigsize); - GDKsetenv("gdk_mem_bigsize", buf); - } - if (GDKgetenv("monet_pid") == NULL) { - snprintf(buf, sizeof(buf), "%d", (int) getpid()); - GDKsetenv("monet_pid", buf); - }
-#ifndef WIN32 +#ifdef HAVE_POSIX_FADVISE if (!GDKembedded && GDK_vm_trim) { MT_Id t;
U gdk_bat.mx Index: gdk_bat.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_bat.mx,v retrieving revision 1.213 retrieving revision 1.214 diff -u -d -r1.213 -r1.214 --- gdk_bat.mx 30 Mar 2009 07:30:07 -0000 1.213 +++ gdk_bat.mx 3 Apr 2009 09:14:52 -0000 1.214 @@ -670,11 +670,9 @@ as rcopy(). @{ @c -static int -heapcopy(Heap *dst, Heap *src, int *remap) -{ - if (*remap && src->storage == STORE_MMAP) { - /* use copy-on-write mmap for isolatable copy */ +int HEAPshare(Heap *dst, Heap *src, int copy_on_write) { + /* use copy-on-write mmap for isolatable copy */ + if (src->storage == STORE_MMAP) { *dst = *src; dst->base = (char *) -1; dst->filename = GDKstrdup(src->filename); @@ -682,16 +680,25 @@ char path[PATHLENGTH];
GDKfilepath(path, BATDIR, dst->filename, NULL); - dst->base = GDKmmap(path, MMAP_READ | MMAP_SEQUENTIAL | MMAP_COPY, 0, dst->maxsize); + dst->base = GDKmmap(path, MMAP_READ | MMAP_SEQUENTIAL | (copy_on_write?MMAP_COPY:0), 0, dst->maxsize); if (dst->base != (char *) -1) { dst->copied = 1; - dst->newstorage = dst->storage = STORE_PRIV; + dst->newstorage = dst->storage = copy_on_write?STORE_PRIV:STORE_MMAP; return 0; } GDKfree(dst->filename); dst->filename = NULL; } } + return -1; +} + +static int +heapcopy(Heap *dst, Heap *src, int *remap) +{ + if (*remap && HEAPshare(dst, src, *remap) == 0) { + return 0; /* use a shared memory heap */ + } *remap = 0; return HEAPcopy(dst, src); } @@ -2766,7 +2773,7 @@ int b0, b1, b2 = 0, b3 = 0;
/* copy-on-write isolated bats that change mode should be made independent */ - if (b->P->lview && batunshare(b, "BATsetaccess") < 0) + if (b->P->lview && newmode != BAT_READ && batunshare(b, "BATsetaccess") < 0) return NULL;
if (b->batSharecnt && newmode != BAT_READ) {
-------------------------------------------------------------------------
_______________________________________________ Monetdb-checkins mailing list Monetdb-checkins@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/monetdb-checkins
-- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4312 |
------------------------------------------------------------------------------ _______________________________________________ Monetdb-developers mailing list Monetdb-developers@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/monetdb-developers
-- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4312 |
participants (2)
-
Peter Boncz -
Stefan Manegold