On Sat, Oct 25, 2008 at 07:30:50AM +0000, Martin Kersten wrote:

Update of /cvsroot/monetdb/MonetDB/src/gdk In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv30373

Modified Files: gdk_posix.mx Log Message: The memory map table maintained for the vmtrim should be aligned to the number of mapped files. In more recent applications, such as the SkyServer this may be around 2000 BATs easily. For the time being the limit of 100 is increased to 4096

The remainder are additions of '#' in error messages to prepare for problems in the API.

just an idea (without background knowledge): shouldn't we introduce a new symbol/chracter in the protocol to indicate warnings and informative message? '!' indicates errors (obviously never to be ignored) '#' indicates (usually to be ignored) comments '?' to be added to indicate warnings (and "informative messages)? '?' to be added for ... ?? Stefan

U gdk_posix.mx Index: gdk_posix.mx =================================================================== RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_posix.mx,v retrieving revision 1.157 retrieving revision 1.158 diff -u -d -r1.157 -r1.158 --- gdk_posix.mx 10 Oct 2008 08:50:35 -0000 1.157 +++ gdk_posix.mx 25 Oct 2008 07:30:48 -0000 1.158 @@ -389,7 +389,7 @@ * hope the VM paging algorithm behaves better than Linux which just runs off * the cliff and if MonetDB does not prevent RSS from being too high, enters coma. * - * We will only eb able to sensibly test this on Windows64. On Windows32, mmap sizes + * We will only be able to sensibly test this on Windows64. On Windows32, mmap sizes * do not significantly exceed RAM sizes so MonetDB swapping actually will not happen * (of course, you've got this nasty problem of VM fragemntation and failing mmaps instead). * @@ -488,6 +488,10 @@ * to the cursors (i.e. pointers to b->batBuns->free or the cursors * in radix-cluster), which an enhanced version of this thread might * take into account. + * + * [Kersten] The memory map table should be aligned to the number of + * mapped files. In more recent applications, such as the SkyServer + * this may be around 2000 BATs easily. */

#ifdef HAVE_PTHREAD_H @@ -519,7 +523,7 @@ #endif

#define MT_MMAP_TILE (1<<27) -#define MT_MMAP_BUFSIZE 100 +#define MT_MMAP_BUFSIZE 4096 MT_mmap_t MT_mmap_tab[MT_MMAP_BUFSIZE]; int MT_mmap_cur = -1, MT_mmap_first = -1, MT_mmap_free = 0;

@@ -598,7 +602,7 @@ if (MT_mmap_cur == -1) MT_mmap_cur = i; #ifdef MMAP_DEBUG - stream_printf(GDKerr, "MT_mmap_new: %s fd=%d\n", path, fd); + stream_printf(GDKerr, "#MT_mmap_new: %s fd=%d\n", path, fd); #endif strncpy(MT_mmap_tab[i].path, path, 128); MT_mmap_tab[i].base = base; @@ -642,13 +646,13 @@ /* tell the OS quite clearly that you want to drop this */ ret = posix_fadvise(MT_mmap_tab[victim].fd, 0LL, MT_mmap_tab[victim].len, POSIX_FADV_DONTNEED); #ifdef MMAP_DEBUG - stream_printf(GDKerr, "MT_mmap_del: posix_fadvise(%s,fd=%d,%uMB,POSIX_FADV_DONTNEED) = %d\n", MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, (unsigned int) (MT_mmap_tab[victim].len >> 20), ret); + stream_printf(GDKerr, "#MT_mmap_del: posix_fadvise(%s,fd=%d,%uMB,POSIX_FADV_DONTNEED) = %d\n", MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, (unsigned int) (MT_mmap_tab[victim].len >> 20), ret); #endif } #endif ret = close(MT_mmap_tab[victim].fd); #ifdef MMAP_DEBUG - stream_printf(GDKerr, "MT_mmap_del: close(%s fd=%d) = %d\n", MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, ret); + stream_printf(GDKerr, "#MT_mmap_del: close(%s fd=%d) = %d\n", MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, ret); #endif MT_mmap_tab[victim].next = MT_mmap_free; MT_mmap_empty(victim); @@ -674,7 +678,7 @@ if (MT_mmap_tab[i].fd >= 0) { ret = posix_fadvise(MT_mmap_tab[i].fd, 0, len, advice); #ifdef MMAP_DEBUG - stream_printf(GDKerr, "MT_fadvise: posix_fadvise(%s,fd=%d,%uMB,%d) = %d\n", MT_mmap_tab[i].path, MT_mmap_tab[i].fd, (unsigned int) (len >> 20), advice, ret); + stream_printf(GDKerr, "#MT_fadvise: posix_fadvise(%s,fd=%d,%uMB,%d) = %d\n", MT_mmap_tab[i].path, MT_mmap_tab[i].fd, (unsigned int) (len >> 20), advice, ret); #endif } } @@ -697,7 +701,7 @@ int ret = posix_madvise(MT_mmap_tab[i].base + off, MT_MMAP_TILE, POSIX_MADV_DONTNEED);

if (err) { - stream_printf(err, "MT_mmap_unload_tile: posix_madvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n", + stream_printf(err, "#MT_mmap_unload_tile: posix_madvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n", MT_mmap_tab[i].path, (unsigned int) (off >> 20), (unsigned int) (MT_MMAP_TILE >> 20), MT_mmap_tab[i].fd, ret); } @@ -706,7 +710,7 @@ /* tell the OS quite clearly that you want to drop this */ ret = posix_fadvise(MT_mmap_tab[i].fd, off, MT_MMAP_TILE, POSIX_FADV_DONTNEED); if (err) { - stream_printf(err, "MT_mmap_unload_tile: posix_fadvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n", + stream_printf(err, "#MT_mmap_unload_tile: posix_fadvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n", MT_mmap_tab[i].path, (unsigned int) (off >> 20), (unsigned int) (MT_MMAP_TILE >> 20), MT_mmap_tab[i].fd, ret); } @@ -721,7 +725,7 @@

/* save to disk an 128MB tile, and observe how long this takes */ if (err) { - stream_printf(err, "MT_mmap_save_tile: msync(%s,off=%uM,%u,SYNC)...\n", + stream_printf(err, "#MT_mmap_save_tile: msync(%s,off=%uM,%u,SYNC)...\n", MT_mmap_tab[i].path, (unsigned int) (tile >> 20), (unsigned int) (MT_MMAP_TILE >> 20)); } @@ -729,7 +733,7 @@ ret = MT_msync(MT_mmap_tab[i].base, tile, MT_MMAP_TILE, MMAP_SYNC); t = GDKms() - t; if (err) { - stream_printf(err, "MT_mmap_save_tile: msync(%s,tile=%uM,%uM,SYNC) = %d (%dms)\n", + stream_printf(err, "#MT_mmap_save_tile: msync(%s,tile=%uM,%uM,SYNC) = %d (%dms)\n", MT_mmap_tab[i].path, (unsigned int) (tile >> 20), (unsigned int) (MT_MMAP_TILE >> 20), ret, t); } @@ -768,7 +772,7 @@

(void) pthread_mutex_lock(&MT_mmap_lock); if (err) { - stream_printf(err, "MT_mmap_trim(%u MB): rss = %u MB\n", (unsigned int) ((target) >> 20), (unsigned int) (rss >> 20)); + stream_printf(err, "#MT_mmap_trim(%u MB): rss = %u MB\n", (unsigned int) ((target) >> 20), (unsigned int) (rss >> 20)); } if (rss > target) { /* try to selectively unload pages from the writable regions */ @@ -1085,7 +1089,7 @@ int ret = munmap(p, len);

#ifdef MMAP_DEBUG - stream_printf(GDKerr, "munmap(" LLFMT "," LLFMT ",%d) = %d\n", (long long) p, (long long) len, ret); + stream_printf(GDKerr, "#munmap(" LLFMT "," LLFMT ",%d) = %d\n", (long long) p, (long long) len, ret); #endif MT_mmap_del(p, len); return ret; @@ -1097,7 +1101,7 @@ int ret = msync(((char*) p)+off, len, (mode & MMAP_SYNC) ? MS_SYNC : ((mode & MMAP_ASYNC) ? MS_ASYNC : MS_INVALIDATE));

#ifdef MMAP_DEBUG - stream_printf(GDKerr, "msync(" LLFMT "," LLFMT ",%s) = %d\n", (long long) p, (long long) len, (mode & MMAP_SYNC) ? "MS_SYNC" : ((mode & MMAP_ASYNC) ? "MS_ASYNC" : "MS_INVALIDATE"), ret); + stream_printf(GDKerr, "#msync(" LLFMT "," LLFMT ",%s) = %d\n", (long long) p, (long long) len, (mode & MMAP_SYNC) ? "MS_SYNC" : ((mode & MMAP_ASYNC) ? "MS_ASYNC" : "MS_INVALIDATE"), ret); #endif if (ret < 0) return errno; @@ -1110,7 +1114,7 @@ int ret = posix_madvise(p, len, advise);

#ifdef MMAP_DEBUG - stream_printf(GDKerr, "posix_madvise(" PTRFMT "," SZFMT ",%d) = %d\n", PTRFMTCAST p, len, advise, ret); + stream_printf(GDKerr, "#posix_madvise(" PTRFMT "," SZFMT ",%d) = %d\n", PTRFMTCAST p, len, advise, ret); #endif if (MT_fadvise(p, len, advise)) ret = -1; @@ -1468,7 +1472,7 @@ } if (heapstatus == _HEAPBADPTR || heapstatus == _HEAPBADBEGIN || heapstatus == _HEAPBADNODE) {

- stream_printf(GDKerr, "mallinfo(): heap is corrupt."); + stream_printf(GDKerr, "#mallinfo(): heap is corrupt."); } _heapmin(); return _ret; @@ -1604,7 +1608,7 @@ return;

if (!FindClose(dir->find_file_handle)) - stream_printf(GDKerr, "rewinddir(): FindClose() failed\n"); + stream_printf(GDKerr, "#rewinddir(): FindClose() failed\n");

mask = malloc(strlen(dir->dir_name) + 3); sprintf(mask, "%s\\*", dir->dir_name); @@ -2116,7 +2120,7 @@ p = (void *) VirtualAlloc(a, size, MEM_COMMIT | mode, PAGE_READWRITE); (void) pthread_mutex_unlock(&MT_mmap_lock); if (p == NULL) { - stream_printf(GDKerr, "VirtualAlloc(" PTRFMT "," SZFMT ",MEM_COMMIT,PAGE_READWRITE): failed\n", PTRFMTCAST a, size); + stream_printf(GDKerr, "#VirtualAlloc(" PTRFMT "," SZFMT ",MEM_COMMIT,PAGE_READWRITE): failed\n", PTRFMTCAST a, size); } return p; } @@ -2126,9 +2130,9 @@ MT_vmfree(void *p, size_t size) { if (VirtualFree(p, size, MEM_DECOMMIT) == 0) - stream_printf(GDKerr, "VirtualFree(" PTRFMT "," SZFMT ",MEM_DECOMMIT): failed\n", PTRFMTCAST p, size); + stream_printf(GDKerr, "#VirtualFree(" PTRFMT "," SZFMT ",MEM_DECOMMIT): failed\n", PTRFMTCAST p, size); if (VirtualFree(p, 0, MEM_RELEASE) == 0) - stream_printf(GDKerr, "VirtualFree(" PTRFMT ",0,MEM_RELEASE): failed\n", PTRFMTCAST p); + stream_printf(GDKerr, "#VirtualFree(" PTRFMT ",0,MEM_RELEASE): failed\n", PTRFMTCAST p); }

void * @@ -2149,7 +2153,7 @@ size_t ret = VirtualFree(p + newsize, oldsize - newsize, MEM_DECOMMIT);

if (ret == 0) - stream_printf(GDKerr, "VirtualFree(" PTRFMT "," SSZFMT ",MEM_DECOMMIT): failed\n", PTRFMTCAST(p + newsize), (ssize_t) (oldsize - newsize)); + stream_printf(GDKerr, "#VirtualFree(" PTRFMT "," SSZFMT ",MEM_DECOMMIT): failed\n", PTRFMTCAST(p + newsize), (ssize_t) (oldsize - newsize)); } else if (oldsize < newsize) { (void) pthread_mutex_lock(&MT_mmap_lock); a = (char *) VirtualAlloc(p, newsize, MEM_COMMIT, PAGE_READWRITE);

------------------------------------------------------------------------- This SF.Net email is sponsored by the Moblin Your Move Developer's challenge Build the coolest Linux based applications with Moblin SDK & win great prizes Grand prize is a trip for two to an Open Source event anywhere in the world http://moblin-contest.org/redirect.php?banner_id=100&url=/ _______________________________________________ Monetdb-checkins mailing list Monetdb-checkins@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/monetdb-checkins

-- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4312 |