- developers-list - monetdb.org

Re: [Monetdb-developers] [Monetdb-checkins] MonetDB/src/gdk gdk_posix.mx, Feb2010, 1.176.2.24, 1.176.2.25 gdk_storage.mx, Feb2010, 1.149.2.36, 1.149.2.37
by Stefan Manegold 19 Feb '10

19 Feb '10

On Fri, Feb 19, 2010 at 03:40:10PM +0000, Peter Boncz wrote: > Update of /cvsroot/monetdb/MonetDB/src/gdk > In directory sfp-cvsdas-1.v30.ch3.sourceforge.com:/tmp/cvs-serv10221 > > Modified Files: > Tag: Feb2010 > gdk_posix.mx gdk_storage.mx > Log Message: > - fix newly introduced bug (BUF_SEQUENTIAL passed to posix_madvise iso MMAP_SEQUENTIAL) > most of this was done by Stefan, but the intent of the code was when all users > are gone, to give uniform (sequential) advise to the whole heap > - make sure the length of advise is page aligned (not strictly needed for > Linux, but maybe of other Un*ces) > - in case of shared vheaps (as done by leftfetchjoin into a string bat), assume that > access to the string heap will be random. shouldn't len & ~MT_pagesize() then rather be len & ~(MT_pagesize()-1) (assuming pages size is usually a power of 2) ? Stefan > > > Index: gdk_storage.mx > =================================================================== > RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_storage.mx,v > retrieving revision 1.149.2.36 > retrieving revision 1.149.2.37 > diff -u -d -r1.149.2.36 -r1.149.2.37 > --- gdk_storage.mx 19 Feb 2010 13:13:38 -0000 1.149.2.36 > +++ gdk_storage.mx 19 Feb 2010 15:40:07 -0000 1.149.2.37 > @@ -707,17 +707,19 @@ > * Peter Feb2010: I tried to do prefetches further apart, to trigger multiple readahead > * units in parallel, but it does improve performance visibly > */ > -static size_t access_heap(str id, str hp, Heap *h, char* base, size_t sz, int touch, int preload, int advise) { > +static size_t access_heap(str id, str hp, Heap *h, char* base, size_t sz, int touch, int preload, int adv) { > size_t v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0, v5 =0, v6 = 0, v7 = 0, page = MT_pagesize(); > + str advise = (adv==MMAP_WILLNEED)?"WILLNEED":(adv==MMAP_SEQUENTIAL)?"SEQUENTIAL":(adv==MMAP_RANDOM)?"RANDOM":(adv==MMAP_NORMAL)?"NORMAL":NULL; > int t = GDKms(); > + assert(advise); > if (h->storage != STORE_MEM && h->size > MT_MMAP_TILE) { > - MT_mmap_inform(h->base, h->size, preload, advise, 0); > + MT_mmap_inform(h->base, h->size, preload, adv, 0); > if (preload > 0) { > - void* alignedbase = (void*) (((size_t) base) & ~(page-1)); > - size_t alignedsz = (sz + (page-1)) & ~(page-1); > - int ret = posix_madvise(alignedbase, sz, advise); > - if (ret) THRprintf(GDKerr, "#MT_mmap_inform: posix_madvise(file=%s, base="PTRFMT", len="SZFMT"MB, advice=%d) = %d\n", > - h->filename, PTRFMTCAST alignedbase, alignedsz >> 20, advise, errno); > + size_t alignskip = (page - (((size_t) base) & (page-1))) & (page-1); > + size_t alignedsz = (size_t) (((sz < alignskip)?0:((size_t) (sz-alignskip))) & ~(page-1)); > + int ret = posix_madvise(base + alignskip, alignedsz, adv); > + if (ret) THRprintf(GDKerr, "#MT_mmap_inform: posix_madvise(file=%s, base="PTRFMT", len="SZFMT"MB, advice=%s) = %d\n", > + h->filename, PTRFMTCAST (base + alignskip), alignedsz >> 20, advise, errno); > } > } > if (touch && preload > 0) { > @@ -731,8 +733,7 @@ > } > for (hi += 7*page; lo <= hi; lo +=page) v0 += *lo; > } > - IODEBUG THRprintf(GDKout,"#BATpreload(%s->%s,preload=%d,sz=%dMB,%s) = %dms \n", id, hp, preload, (int) (sz>>20), > - (advise==MMAP_WILLNEED)?"WILLNEED":(advise==MMAP_SEQUENTIAL)?"SEQUENTIAL":"UNKNOWN", GDKms()-t); > + IODEBUG THRprintf(GDKout,"#BATpreload(%s->%s,preload=%d,sz=%dMB,%s) = %dms \n", id, hp, preload, (int) (sz>>20), advise, GDKms()-t); > return v0+v1+v2+v3+v4+v5+v6+v7; > } > > @@ -743,7 +744,6 @@ > str id = BATgetId(b); > BATiter bi = bat_iterator(b); > > - assert(advise==MMAP_NORMAL||advise==MMAP_RANDOM||advise==MMAP_SEQUENTIAL||advise==MMAP_WILLNEED||advise==MMAP_DONTNEED); > if (BATcount(b) == 0) return 0; > > /* HASH indices (inherent random access). handle first as they *will* be access randomly (one can always hope for locality on the other heaps) */ > @@ -760,30 +760,46 @@ > gdk_unset_lock(GDKhashLock(ABS(b->batCacheid) & BBP_BATMASK), "BATaccess"); > } > > - /* we only touch stuff that is going to be read randomly (WILLNEED). Note varheaps are sequential wrt to the references, or small */ > - if ( what&USE_HEAD) { > + /* vheaps next, as shared vheaps are not seq-correlated needing WILLNEED (use prefetch budget for this first) */ > + if ( what&USE_HEAD ) { > + if (b->H->vheap && b->H->vheap->base) { > + char *lo = BUNhead(bi, BUNfirst(b)), *hi = BUNhead(bi, BUNlast(b)-1); > + int heap_advise = advise; > + if (b->H->vheap->copied) { /* shared string heaps are not (likely) to be sequentially correlated */ > + lo = b->H->vheap->base; hi = lo + b->H->vheap->free; > + heap_advise = MADV_WILLNEED; > + } > + budget -= sz = ((hi-lo) > budget)?budget:(hi-lo); > + v += access_heap(id, "hheap", b->H->vheap, lo, sz, (advise == BUF_WILLNEED), preload, heap_advise); > + } > + } > + if ( what&USE_TAIL ) { > + if (b->T->vheap && b->T->vheap->base) { > + char *lo = BUNtail(bi, BUNfirst(b)), *hi = BUNtail(bi, BUNlast(b)-1); > + int heap_advise = advise; > + if (b->T->vheap->copied) { /* shared string heaps are not (likely) to be sequentially correlated */ > + lo = b->T->vheap->base; hi = lo + b->T->vheap->free; > + heap_advise = MADV_WILLNEED; > + } > + budget -= sz = ((hi-lo) > budget)?budget:(hi-lo); > + v += access_heap(id, "theap", b->T->vheap, lo, sz, (advise == BUF_WILLNEED), preload, heap_advise); > + } > + } > + > + /* BUN heaps are last in line for prefetch budget */ > + if ( what&USE_HEAD ) { > if (b->H->heap.base) { > char *lo = BUNhloc(bi, BUNfirst(b)), *hi = BUNhloc(bi, BUNlast(b)-1); > budget -= sz = ((hi-lo) > budget)?budget:(hi-lo); > v += access_heap(id, "hbuns", &b->H->heap, lo, sz, (advise == MMAP_WILLNEED), preload, advise); > } > - if (b->H->vheap && b->H->vheap->base) { > - char *lo = BUNhead(bi, BUNfirst(b)), *hi = BUNhead(bi, BUNlast(b)-1); > - budget -= sz = ((hi-lo) > budget)?budget:(hi-lo); > - v += access_heap(id, "hheap", b->H->vheap, lo, sz, (advise == MMAP_WILLNEED), preload, advise); > - } > } > - if ( what&USE_TAIL) { > + if ( what&USE_TAIL ) { > if (b->T->heap.base) { > char *lo = BUNtloc(bi, BUNfirst(b)), *hi = BUNtloc(bi, BUNlast(b)-1); > budget -= sz = ((hi-lo) > budget)?budget:(hi-lo); > v += access_heap(id, "tbuns", &b->T->heap, lo, sz, (advise == MMAP_WILLNEED), preload, advise); > } > - if (b->T->vheap && b->T->vheap->base) { > - char *lo = BUNtail(bi, BUNfirst(b)), *hi = BUNtail(bi, BUNlast(b)-1); > - budget -= sz = ((hi-lo) > budget)?budget:(hi-lo); > - v += access_heap(id, "theap", b->T->vheap, lo, sz, (advise == MMAP_WILLNEED), preload, advise); > - } > } > return v; > } > > Index: gdk_posix.mx > =================================================================== > RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_posix.mx,v > retrieving revision 1.176.2.24 > retrieving revision 1.176.2.25 > diff -u -d -r1.176.2.24 -r1.176.2.25 > --- gdk_posix.mx 19 Feb 2010 13:13:38 -0000 1.176.2.24 > +++ gdk_posix.mx 19 Feb 2010 15:40:07 -0000 1.176.2.25 > @@ -675,7 +675,7 @@ > #ifdef HAVE_POSIX_FADVISE > if (!do_not_use_posix_fadvise && MT_mmap_tab[victim].fd >= 0) { > /* tell the OS quite clearly that you want to drop this */ > - ret = posix_fadvise(MT_mmap_tab[victim].fd, 0LL, MT_mmap_tab[victim].len, POSIX_FADV_DONTNEED); > + ret = posix_fadvise(MT_mmap_tab[victim].fd, 0LL, MT_mmap_tab[victim].len & ~MT_pagesize(), POSIX_FADV_DONTNEED); > #ifdef MMAP_DEBUG > stream_printf(GDKerr, "#MT_mmap_del: posix_fadvise(%s,fd=%d,%uMB,POSIX_FADV_DONTNEED) = %d\n", MT_mmap_tab[victim].path, MT_mmap_tab[victim].fd, (unsigned int) (MT_mmap_tab[victim].len >> 20), ret); > #endif > @@ -709,7 +709,7 @@ > i = MT_mmap_idx(base, len); > if (i >= 0) { > if (MT_mmap_tab[i].fd >= 0) { > - ret = posix_fadvise(MT_mmap_tab[i].fd, 0, len, advice); > + ret = posix_fadvise(MT_mmap_tab[i].fd, 0, len & ~MT_pagesize(), advice); > #ifdef MMAP_DEBUG > stream_printf(GDKerr, "#MT_fadvise: posix_fadvise(%s,fd=%d,%uMB,%d) = %d\n", MT_mmap_tab[i].path, MT_mmap_tab[i].fd, (unsigned int) (len >> 20), advice, ret); > #endif > @@ -733,7 +733,7 @@ > { > size_t len = MIN((size_t) MT_MMAP_TILE, MT_mmap_tab[i].len - off); > /* tell Linux to please stop caching this stuff */ > - int ret = posix_madvise(MT_mmap_tab[i].base + off, len, POSIX_MADV_DONTNEED); > + int ret = posix_madvise(MT_mmap_tab[i].base + off, len & ~MT_pagesize(), POSIX_MADV_DONTNEED); > > if (err) { > stream_printf(err, "#MT_mmap_unload_tile: posix_madvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n", > @@ -743,7 +743,7 @@ > #ifdef HAVE_POSIX_FADVISE > if (!do_not_use_posix_fadvise) { > /* tell the OS quite clearly that you want to drop this */ > - ret = posix_fadvise(MT_mmap_tab[i].fd, off, len, POSIX_FADV_DONTNEED); > + ret = posix_fadvise(MT_mmap_tab[i].fd, off, len & ~MT_pagesize(), POSIX_FADV_DONTNEED); > if (err) { > stream_printf(err, "#MT_mmap_unload_tile: posix_fadvise(%s,off=%uMB,%uMB,fd=%d,POSIX_MADV_DONTNEED) = %d\n", > MT_mmap_tab[i].path, (unsigned int) (off >> 20), > @@ -908,10 +908,9 @@ > MT_mmap_tab[i].random += preload * (advise == MMAP_WILLNEED); /* done as a counter to keep track of multiple threads */ > MT_mmap_tab[i].usecnt += preload; /* active thread count */ > unload = MT_mmap_tab[i].usecnt == 0; > + if (unload) ret = posix_madvise(MT_mmap_tab[i].base, MT_mmap_tab[i].len & ~MT_pagesize(), MMAP_SEQUENTIAL); > } > (void) pthread_mutex_unlock(&MT_mmap_lock); > - if (unload) > - ret = posix_madvise(base, len, MMAP_SEQUENTIAL); > if (ret) { > stream_printf(GDKerr, "#MT_mmap_inform: posix_madvise(file=%s, fd=%d, base="PTRFMT", len="SZFMT"MB, advice=%d) = %d\n", > (i >= 0 ? MT_mmap_tab[i].path : ""), (i >= 0 ? MT_mmap_tab[i].fd : -1), > @@ -1156,7 +1155,7 @@ > > if (ret != (void *) -1L) { > if (hdl->mode & MMAP_ADVISE) { > - (void) MT_madvise(ret, len, hdl->mode & MMAP_ADVISE); > + (void) MT_madvise(ret, len & ~MT_pagesize(), hdl->mode & MMAP_ADVISE); > } > hdl->fixed = (void *) ((char *) ret + len); > } > @@ -1199,7 +1198,7 @@ > int > MT_madvise(void *p, size_t len, int advise) > { > - int ret = posix_madvise(p, len, advise); > + int ret = posix_madvise(p, len & ~MT_pagesize(), advise); > > #ifdef MMAP_DEBUG > stream_printf(GDKerr, "#posix_madvise(" PTRFMT "," SZFMT ",%d) = %d\n", PTRFMTCAST p, len, advise, ret); > > > ------------------------------------------------------------------------------ > Download Intel® Parallel Studio Eval > Try the new software tools for yourself. Speed compiling, find bugs > proactively, and fine-tune applications for parallel performance. > See why Intel Parallel Studio got high marks during beta. > http://p.sf.net/sfu/intel-sw-dev > _______________________________________________ > Monetdb-checkins mailing list > Monetdb-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-checkins > -- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4199 |

1 0

Re: [Monetdb-developers] [Monetdb-pf-checkins] pathfinder/runtime shredder.mx, Feb2010, 1.162, 1.162.2.1
by Jan Rittinger 19 Feb '10

19 Feb '10

Hi Sjoerd, as far as I can remember we only use the DTD to resolve entities and id references. So if no DTD is available an error is triggered, but the document is still imported. Just my two cents... Jan On Feb 18, 2010, at 10:50, Sjoerd Mullender wrote: > Update of /cvsroot/monetdb/pathfinder/runtime > In directory sfp-cvsdas-1.v30.ch3.sourceforge.com:/tmp/cvs-serv30011 > > Modified Files: > Tag: Feb2010 > shredder.mx > Log Message: > If there is no DTD, I guess there is not much point continuing. > Found by Coverity. > > > Index: shredder.mx > =================================================================== > RCS file: /cvsroot/monetdb/pathfinder/runtime/shredder.mx,v > retrieving revision 1.162 > retrieving revision 1.162.2.1 > diff -u -d -r1.162 -r1.162.2.1 > --- shredder.mx 7 Jan 2010 15:24:29 -0000 1.162 > +++ shredder.mx 18 Feb 2010 09:50:12 -0000 1.162.2.1 > @@ -1362,8 +1362,10 @@ > ctx->myDoc = xmlNewDoc(ctx->version); > } > > - if (!dtd) > + if (!dtd) { > GDKerror("shred_external_subset: problem with reading external DTD (\"%s\")\n", SystemID); > + BAILOUT(shredCtx); > + } > > if (ctx->myDoc->extSubset) { > stream_printf(GDKout, "!WARNING: double external subset (\"%s\") may cause problems.\n", SystemID); > > > ------------------------------------------------------------------------------ > Download Intel® Parallel Studio Eval > Try the new software tools for yourself. Speed compiling, find bugs > proactively, and fine-tune applications for parallel performance. > See why Intel Parallel Studio got high marks during beta. > http://p.sf.net/sfu/intel-sw-dev > _______________________________________________ > Monetdb-pf-checkins mailing list > Monetdb-pf-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-pf-checkins -- Jan Rittinger Lehrstuhl Datenbanken und Informationssysteme Wilhelm-Schickard-Institut für Informatik Eberhard-Karls-Universität Tübingen http://www-db.informatik.uni-tuebingen.de/team/rittinger

3 2

Re: [Monetdb-developers] [Monetdb-pf-checkins] pathfinder/modules/pftijah pftijah.mx, , 1.256, 1.257
by Stefan Manegold 19 Feb '10

19 Feb '10

Jan, out of curiosity: - on nwhich architectures and/or in which cases does semijoin not propagate the 'sorted' properties? - shouldn't this checkin have gone as bug fix to the release branch such that is would be included in the next release? Thanks. Stefan On Fri, Feb 19, 2010 at 08:22:08AM +0000, Jan Flokstra wrote: > Update of /cvsroot/monetdb/pathfinder/modules/pftijah > In directory sfp-cvsdas-1.v30.ch3.sourceforge.com:/tmp/cvs-serv19238 > > Modified Files: > pftijah.mx > Log Message: > - temporary patch to fix a case where on architectures a semijoin does not > propagate the 'sorted' properties. > > > > Index: pftijah.mx > =================================================================== > RCS file: /cvsroot/monetdb/pathfinder/modules/pftijah/pftijah.mx,v > retrieving revision 1.256 > retrieving revision 1.257 > diff -u -d -r1.256 -r1.257 > --- pftijah.mx 12 Feb 2010 10:25:43 -0000 1.256 > +++ pftijah.mx 19 Feb 2010 08:22:06 -0000 1.257 > @@ -4337,6 +4337,7 @@ > # get collection count of term > var t_pre := _getTermPositions($h); > e_pre := e_pre.@5(_getTermDoc_@1_@2(e_pre, e_size, t_pre)); > + e_pre := e_pre.chk_order(); # jf order patch > } > > var res := e_pre; > @@ -4536,6 +4537,7 @@ > # get collection count of term > var t_pre := _getConceptPositions($h); > e_pre := e_pre.@5(_getTermDoc_@1_@2(e_pre, e_size, t_pre)); > + e_pre := e_pre.chk_order(); # jf order patch > } > > var res := e_pre; > > > ------------------------------------------------------------------------------ > Download Intel® Parallel Studio Eval > Try the new software tools for yourself. Speed compiling, find bugs > proactively, and fine-tune applications for parallel performance. > See why Intel Parallel Studio got high marks during beta. > http://p.sf.net/sfu/intel-sw-dev > _______________________________________________ > Monetdb-pf-checkins mailing list > Monetdb-pf-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-pf-checkins > -- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4199 |

1 0

Re: [Monetdb-developers] [Monetdb-checkins] MonetDB/src/gdk gdk_posix.mx, Feb2010, 1.176.2.21, 1.176.2.22 gdk_storage.mx, Feb2010, 1.149.2.32, 1.149.2.33
by Stefan Manegold 19 Feb '10

19 Feb '10

Peter, I have some questions to make sure I understand your new code correctly: 1) I don't see any plance in the hash code (at least not in gdk_search.mx) where the "free" element of a hash heap is set (or used) other than the initialization to 0 in HEAPalloc; thus, I guess, "free" for hash heaps is always 0; hence, shouln't we use "size" instead of "free" for the madvise & preload size of hash heaps (as we did in the original BATpreload/BATaccess code)? 2) Am I right that for string heaps you conclude from a strong order correlation between the off-heap and the string heap (due sequential load/insertion) that also the first and last BUN in the offset point to the "first" and "last" string in the string heap? Well, indeed, since access is to be considered in page size granularity, this might be reasonable ... 3) (This was the same in the previous version of the code) For BUN heaps, in case of views (slices), the base pointer of the view's heap might not be the same as the parent's heap, in fact, it might not be page-aligned. If I understand the MT_mmap_tab[] array correctly, it identifies heap by their page-aligned base pointer of the parent's heap. Hence, BATaccess() on a slice view BAT with non-aligned heap->base pointer calls MT_mmap_inform() (through access_heap()) with a non-aligned heap->base, which is not found in MT_mmap_tab[], and hence MT_mmap_inform() does nothing with that heap. With preload==1 it does hence not resgister the posix_madvise() call that access_heap() does. COnsequently, with preload==-1, MT_mmap_inform() will never reset the advise set via slice views, unless there is (also) access to the original parent's heap (i.e., with page-aligned heap->base pointer. I jjust noticed this, but do not yet understand, whether and if so which consequences this (might) have ... Stefan On Thu, Feb 18, 2010 at 10:39:22PM +0000, Peter Boncz wrote: > Update of /cvsroot/monetdb/MonetDB/src/gdk > In directory sfp-cvsdas-1.v30.ch3.sourceforge.com:/tmp/cvs-serv28734 > > Modified Files: > Tag: Feb2010 > gdk_posix.mx gdk_storage.mx > Log Message: > did experimentation with sequential mmap I/O. > - on very fast subsystems (such as 16xssd) it is three times slower than optimally tuned direct I/O (1GB/s vs 3GB/s) > - with less disks the difference is smaller (e.g. 140 vs 200MB/s) > regrettably, nothing helped to get it higher. > > the below checkin makes the following changes: > - simplified BATaccess code by separating out routine > - made BATaccess more precies in what to preload (ionly BUNfirst-BUNlast) > - observe that large string heaps have a high sequential correletaion > hense always WILLNEED fetching is overkill > - move the madvise() call back to BATaccess at the start of the access but removing > the advise is done in vmtrim, as you need the overview when the last user is away. > - the basic advise is SEQUENTIAL (ie decent I/O) > > > > Index: gdk_storage.mx > =================================================================== > RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_storage.mx,v > retrieving revision 1.149.2.32 > retrieving revision 1.149.2.33 > diff -u -d -r1.149.2.32 -r1.149.2.33 > --- gdk_storage.mx 18 Feb 2010 01:04:11 -0000 1.149.2.32 > +++ gdk_storage.mx 18 Feb 2010 22:39:08 -0000 1.149.2.33 > @@ -697,156 +697,95 @@ > return BATload_intern(i); > } > @- BAT preload > -To avoid random disk access to large (memory-mapped) BATs it may help to issue a preload > -request. > -Of course, it does not make sense to touch more then we can physically accomodate. > +To avoid random disk access to large (memory-mapped) BATs it may help to issue a preload request. > +Of course, it does not make sense to touch more then we can physically accomodate (budget). > @c > -size_t > -BATaccess(BAT *b, int what, int advise, int preload) { > - size_t *i, *limit; > - size_t v1 = 0, v2 = 0, v3 = 0, v4 = 0; > - size_t step = MT_pagesize()/sizeof(size_t); > - size_t pages = (size_t) (0.8 * MT_npages()); > - > - assert(advise==MMAP_NORMAL||advise==MMAP_RANDOM||advise==MMAP_SEQUENTIAL||advise==MMAP_WILLNEED||advise==MMAP_DONTNEED); > - > - /* VAR heaps (inherent random access) */ > - if ( what&USE_HEAD && b->H->vheap && b->H->vheap->base ) { > - if (b->H->vheap->storage != STORE_MEM && b->H->vheap->size > MT_MMAP_TILE) { > - MT_mmap_inform(b->H->vheap->base, b->H->vheap->size, preload, MMAP_WILLNEED, 0); > - } > - if (preload > 0 && pages > 0) { > - IODEBUG THRprintf(GDKout,"#BATaccess(%s,%d): H->vheap\n", BATgetId(b), advise); > - limit = (size_t *) (b->H->vheap->base + b->H->vheap->free) - 4 * step; > - /* we need to ensure alignment, here, as b might be a view and heap.base of views are not necessarily aligned */ > - i = (size_t *) (((size_t)b->H->vheap->base + sizeof(size_t) - 1) & (~(sizeof(size_t) - 1))); > - for (; i <= limit && pages > 3 ; i+= 4*step, pages-= 4) { > - v1 += *i; > - v2 += *(i + step); > - v3 += *(i + 2*step); > - v4 += *(i + 3*step); > - } > - limit += 4 * step; > - for (; i <= limit && pages > 0; i+= step, pages--) { > - v1 += *i; > - } > +/* modern linux tends to use 128K readaround = 64K readahead > + * changes have been going on in 2009, towards true readahead > + * http://tomoyo.sourceforge.jp/cgi-bin/lxr/source/mm/readahead.c > + * > + * Peter Feb2010: I tried to do prefetches further apart, to trigger multiple readahead > + * units in parallel, but it does improve performance visibly > + */ > +static size_t access_heap(str id, str hp, Heap *h, char* base, size_t sz, int touch, int preload, int advise) { > + size_t v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0, v5 =0, v6 = 0, v7 = 0, page = MT_pagesize(); > + int t = GDKms(); > + if (h->storage != STORE_MEM && h->size > MT_MMAP_TILE) { > + MT_mmap_inform(h->base, h->size, preload, advise, 0); > + if (preload > 0) { > + void* alignedbase = (void*) (((size_t) base) & ~(page-1)); > + size_t alignedsz = (sz + (page-1)) & ~(page-1); > + int ret = posix_madvise(alignedbase, sz, advise); > + if (ret) THRprintf(GDKerr, "#MT_mmap_inform: posix_madvise(file=%s, base="PTRFMT", len="SZFMT"MB, advice=%d) = %d\n", > + h->filename, PTRFMTCAST alignedbase, alignedsz >> 20, advise, errno); > } > } > - if ( what&USE_TAIL && b->T->vheap && b->T->vheap->base ) { > - if (b->T->vheap->storage != STORE_MEM && b->T->vheap->size > MT_MMAP_TILE) { > - MT_mmap_inform(b->T->vheap->base, b->T->vheap->size, preload, MMAP_WILLNEED, 0); > - } > - if (preload > 0 && pages > 0) { > - IODEBUG THRprintf(GDKout,"#BATaccess(%s,%d): T->vheap\n", BATgetId(b), advise); > - limit = (size_t *) (b->T->vheap->base + b->T->vheap->free - sizeof(size_t)) - 4 * step; > - /* we need to ensure alignment, here, as b might be a view and heap.base of views are not necessarily aligned */ > - i = (size_t *) (((size_t)b->T->vheap->base + sizeof(size_t) - 1) & (~(sizeof(size_t) - 1))); > - for (; i <= limit && pages > 3; i+= 4*step, pages-= 4) { > - v1 += *i; > - v2 += *(i + step); > - v3 += *(i + 2*step); > - v4 += *(i + 3*step); > - } > - limit += 4 * step; > - for (; i <= limit && pages > 0; i+= step, pages--) { > - v1 += *i; > - } > + if (touch && preload > 0) { > + /* we need to ensure alignment, here, as b might be a view and heap.base of views are not necessarily aligned */ > + size_t *lo = (size_t *) (((size_t) base + sizeof(size_t) - 1) & (~(sizeof(size_t) - 1))); > + size_t *hi = (size_t *) (base + sz); > + for (hi -= 8*page; lo <= hi; lo += 8*page) { > + /* try to trigger loading of multiple pages without blocking */ > + v0 += lo[0*page]; v1 += lo[1*page]; v2 += lo[2*page]; v3 += lo[3*page]; > + v4 += lo[4*page]; v5 += lo[5*page]; v6 += lo[6*page]; v7 += lo[7*page]; > } > + for (hi += 7*page; lo <= hi; lo +=page) v0 += *lo; > } > + IODEBUG THRprintf(GDKout,"#BATpreload(%s->%s,preload=%d,sz=%dMB,%s) = %dms \n", id, hp, preload, (int) (sz>>20), > + (advise==BUF_WILLNEED)?"WILLNEED":(advise==BUF_SEQUENTIAL)?"SEQUENTIAL":"UNKNOWN", GDKms()-t); > + return v0+v1+v2+v3+v4+v5+v6+v7; > +} > > - /* BUN heaps (no need to preload for sequential access) */ > - if ( what&USE_HEAD && b->H->heap.base ) { > - if (b->H->heap.storage != STORE_MEM && b->H->heap.size > MT_MMAP_TILE) { > - MT_mmap_inform(b->H->heap.base, b->H->heap.size, preload, advise, 0); > - } > - if (preload > 0 && pages > 0 && advise != MMAP_SEQUENTIAL) { > - IODEBUG THRprintf(GDKout,"#BATaccess(%s,%d): H->heap\n", BATgetId(b), advise); > - limit = (size_t *) (Hloc(b, BUNlast(b)) - sizeof(size_t)) - 4 * step; > - /* we need to ensure alignment, here, as b might be a view and heap.base of views are not necessarily aligned */ > - i = (size_t *) (((size_t)Hloc(b, BUNfirst(b)) + sizeof(size_t) - 1) & (~(sizeof(size_t) - 1))); > - for (; i <= limit && pages > 3 ; i+= 4*step, pages-= 4) { > - v1 += *i; > - v2 += *(i + step); > - v3 += *(i + 2*step); > - v4 += *(i + 3*step); > - } > - limit += 4 * step; > - for (; i <= limit && pages > 0; i+= step, pages--) { > - v1 += *i; > - } > - } > - } > - if ( what&USE_TAIL && b->T->heap.base ) { > - if (b->T->heap.storage != STORE_MEM && b->T->heap.size > MT_MMAP_TILE) { > - MT_mmap_inform(b->T->heap.base, b->T->heap.size, preload, advise, 0); > +size_t > +BATaccess(BAT *b, int what, int advise, int preload) { > + ssize_t budget = (ssize_t) (0.8 * MT_npages()); > + size_t v = 0, sz; > + str id = BATgetId(b); > + BATiter bi = bat_iterator(b); > + > + assert(advise==MMAP_NORMAL||advise==MMAP_RANDOM||advise==MMAP_SEQUENTIAL||advise==MMAP_WILLNEED||advise==MMAP_DONTNEED); > + if (BATcount(b) == 0) return 0; > + > + /* HASH indices (inherent random access). handle first as they *will* be access randomly (one can always hope for locality on the other heaps) */ > + if ( what&USE_HHASH || what&USE_THASH ) { > + gdk_set_lock(GDKhashLock(ABS(b->batCacheid) & BBP_BATMASK), "BATaccess"); > + if ( what&USE_HHASH && b->H->hash && b->H->hash->heap && b->H->hash->heap->base) { > + budget -= sz = (b->H->hash->heap->free > (size_t) budget)?budget:(ssize_t)b->T->hash->heap->free; > + v += access_heap(id, "hhash", b->H->hash->heap, b->H->hash->heap->base, sz, 1, preload, MMAP_WILLNEED); > } > - if (preload > 0 && pages > 0 && advise != MMAP_SEQUENTIAL) { > - IODEBUG THRprintf(GDKout,"#BATaccess(%s,%d): T->heap\n", BATgetId(b), advise); > - limit = (size_t *) (Tloc(b, BUNlast(b)) - sizeof(size_t)) - 4 * step; > - /* we need to ensure alignment, here, as b might be a view and heap.base of views are not necessarily aligned */ > - i = (size_t *) (((size_t)Tloc(b, BUNfirst(b)) + sizeof(size_t) - 1) & (~(sizeof(size_t) - 1))); > - for (; i <= limit && pages > 3; i+= 4*step, pages-= 4) { > - v1 += *i; > - v2 += *(i + step); > - v3 += *(i + 2*step); > - v4 += *(i + 3*step); > - } > - limit += 4 * step; > - for (; i <= limit && pages > 0; i+= step, pages--) { > - v1 += *i; > - } > + if ( what&USE_THASH && b->T->hash && b->T->hash->heap && b->T->hash->heap->base) { > + budget -= sz = (b->T->hash->heap->free > (size_t) budget)?budget:(ssize_t)b->T->hash->heap->free; > + v += access_heap(id, "thash", b->T->hash->heap, b->T->hash->heap->base, sz, 1, preload, MMAP_WILLNEED); > } > + gdk_unset_lock(GDKhashLock(ABS(b->batCacheid) & BBP_BATMASK), "BATaccess"); > } > > - /* HASH indices (inherent random access) */ > - if ( what&USE_HHASH || what&USE_THASH ) > - gdk_set_lock(GDKhashLock(ABS(b->batCacheid) & BBP_BATMASK), "BATaccess"); > - if ( what&USE_HHASH && b->H->hash && b->H->hash->heap && b->H->hash->heap->base ) { > - if (b->H->hash->heap->storage != STORE_MEM && b->H->hash->heap->size > MT_MMAP_TILE) { > - MT_mmap_inform(b->H->hash->heap->base, b->H->hash->heap->size, preload, MMAP_WILLNEED, 0); > + /* we only touch stuff that is going to be read randomly (WILLNEED). Note varheaps are sequential wrt to the references, or small */ > + if ( what&USE_HEAD) { > + if (b->H->heap.base) { > + char *lo = BUNhloc(bi, BUNfirst(b)), *hi = BUNhloc(bi, BUNlast(b)-1); > + budget -= sz = ((hi-lo) > budget)?budget:(hi-lo); > + v += access_heap(id, "hbuns", &b->H->heap, lo, sz, (advise == BUF_WILLNEED), preload, advise); > } > - if (preload > 0 && pages > 0) { > - IODEBUG THRprintf(GDKout,"#BATaccess(%s,%d): H->hash\n", BATgetId(b), advise); > - limit = (size_t *) (b->H->hash->heap->base + b->H->hash->heap->size - sizeof(size_t)) - 4 * step; > - /* we need to ensure alignment, here, as b might be a view and heap.base of views are not necessarily aligned */ > - i = (size_t *) (((size_t)b->H->hash->heap->base + sizeof(size_t) - 1) & (~(sizeof(size_t) - 1))); > - for (; i <= limit && pages > 3 ; i+= 4*step, pages-= 4) { > - v1 += *i; > - v2 += *(i + step); > - v3 += *(i + 2*step); > - v4 += *(i + 3*step); > - } > - limit += 4 * step; > - for (; i <= limit && pages > 0; i+= step, pages--) { > - v1 += *i; > - } > + if (b->H->vheap && b->H->vheap->base) { > + char *lo = BUNhead(bi, BUNfirst(b)), *hi = BUNhead(bi, BUNlast(b)-1); > + budget -= sz = ((hi-lo) > budget)?budget:(hi-lo); > + v += access_heap(id, "hheap", b->H->vheap, lo, sz, (advise == BUF_WILLNEED), preload, advise); > } > } > - if ( what&USE_THASH && b->T->hash && b->T->hash->heap && b->T->hash->heap->base ) { > - if (b->T->hash->heap->storage != STORE_MEM && b->T->hash->heap->size > MT_MMAP_TILE) { > - MT_mmap_inform(b->T->hash->heap->base, b->T->hash->heap->size, preload, MMAP_WILLNEED, 0); > + if ( what&USE_TAIL) { > + if (b->T->heap.base) { > + char *lo = BUNtloc(bi, BUNfirst(b)), *hi = BUNtloc(bi, BUNlast(b)-1); > + budget -= sz = ((hi-lo) > budget)?budget:(hi-lo); > + v += access_heap(id, "tbuns", &b->T->heap, lo, sz, (advise == BUF_WILLNEED), preload, advise); > } > - if (preload > 0 && pages > 0) { > - IODEBUG THRprintf(GDKout,"#BATaccess(%s,%d): T->hash\n", BATgetId(b), advise); > - limit = (size_t *) (b->T->hash->heap->base + b->T->hash->heap->size - sizeof(size_t)) - 4 * step; > - /* we need to ensure alignment, here, as b might be a view and heap.base of views are not necessarily aligned */ > - i = (size_t *) (((size_t)b->T->hash->heap->base + sizeof(size_t) - 1) & (~(sizeof(size_t) - 1))); > - for (; i <= limit && pages > 3 ; i+= 4*step, pages-= 4) { > - v1 += *i; > - v2 += *(i + step); > - v3 += *(i + 2*step); > - v4 += *(i + 3*step); > - } > - limit += 4 * step; > - for (; i <= limit && pages > 0; i+= step, pages--) { > - v1 += *i; > - } > + if (b->T->vheap && b->T->vheap->base) { > + char *lo = BUNtail(bi, BUNfirst(b)), *hi = BUNtail(bi, BUNlast(b)-1); > + budget -= sz = ((hi-lo) > budget)?budget:(hi-lo); > + v += access_heap(id, "theap", b->T->vheap, lo, sz, (advise == BUF_WILLNEED), preload, advise); > } > } > - if ( what&USE_HHASH || what&USE_THASH ) > - gdk_unset_lock(GDKhashLock(ABS(b->batCacheid) & BBP_BATMASK), "BATaccess"); > - > - return v1 + v2 + v3 + v4; > + return v; > } > @} > > > Index: gdk_posix.mx > =================================================================== > RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_posix.mx,v > retrieving revision 1.176.2.21 > retrieving revision 1.176.2.22 > diff -u -d -r1.176.2.21 -r1.176.2.22 > --- gdk_posix.mx 18 Feb 2010 01:03:55 -0000 1.176.2.21 > +++ gdk_posix.mx 18 Feb 2010 22:38:53 -0000 1.176.2.22 > @@ -909,10 +909,8 @@ > unload = MT_mmap_tab[i].usecnt == 0; > } > (void) pthread_mutex_unlock(&MT_mmap_lock); > - if (i >= 0 && preload > 0) > - ret = posix_madvise(base, len, advise); > - else if (unload) > - ret = posix_madvise(base, len, MMAP_NORMAL); > + if (unload) > + ret = posix_madvise(base, len, BUF_SEQUENTIAL); > if (ret) { > stream_printf(GDKerr, "#MT_mmap_inform: posix_madvise(file=%s, fd=%d, base="PTRFMT", len="SZFMT"MB, advice=%d) = %d\n", > (i >= 0 ? MT_mmap_tab[i].path : ""), (i >= 0 ? MT_mmap_tab[i].fd : -1), > > > ------------------------------------------------------------------------------ > Download Intel® Parallel Studio Eval > Try the new software tools for yourself. Speed compiling, find bugs > proactively, and fine-tune applications for parallel performance. > See why Intel Parallel Studio got high marks during beta. > http://p.sf.net/sfu/intel-sw-dev > _______________________________________________ > Monetdb-checkins mailing list > Monetdb-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-checkins > > -- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4199 |

3 4

[Monetdb-developers] Single column select vs aggregation
by Stefan de Konink 18 Feb '10

18 Feb '10

Attached is a trace of a single column select operation, with a max, avg and equality. The equality takes about 4 times more time than the avg operation. From the plan I see that the operation is executed paralel, but I do wonder why suddenly the uselect takes such a high amount of time. Stefan

3 7

[Monetdb-developers] Maximum length of /any/ entry
by Stefan de Konink 16 Feb '10

16 Feb '10

Hi, Is the maximum length of a str (intmax) equal to maximum length of any other object we can currently store? Stefan

1 0

[Monetdb-developers] ODBC call-failed
by filippo.zonta＠it.abb.com 16 Feb '10

16 Feb '10

Hello, I installed the ODBC driver for windows32 (MonetDB-ODBC-Installer-i386-20100112.msi) and configured an ODBC data source linked to 'demo' database, but using the connection in MS Access I obtain an error ODBC CONFIGURATION DB SERVER WINDOWS RUNNING MS ACCESS LINK TABLE : ONLY 1 TABLE IS LISTED Using the same connection in Excel instead, the odbc driver works: Let me know if the ODBC driver doesn't support MS Access table links. Thanks Best Regards Filippo Il Gruppo ABB in Italia ha adottato il Modello Organizzativo ai sensi del D.Lgs. 231/2001, in forza del quale l'assunzione di obbligazioni da parte di societa' ABB avviene a firma congiunta di due procuratori muniti di idonei poteri, con la sola eccezione delle persone che rivestono la carica di Amministratore Delegato o di Direttore Generale.Le informazioni contenute in questo messaggio di posta elettronica sono riservate e confidenziali e ne e' vietata la diffusione in qualunque modo eseguita.Qualora Lei non fosse la persona cui il presente messaggio e' destinato, e' invitata a non diffonderlo, e ad eliminarlo, dandone gentilmente comunicazione al mittente.ABB Group in Italy adopts a Compliance Programme under the Italian Law (D.Lgs.231/2001). According to this ABB Compliance Programme, any commitment of ABB Italian Companies is taken by the double signature of ABB Representatives granted by a proper Power of Attorney with the only exception of Managing Director or General Manager.The information included in this e-mail and any attachments are confidential and may also be privileged. If you are not the correct recipient, you are kindly requested to notify the sender immediately, to cancel it and not to disclose the contents to any other person.

2 1

[Monetdb-developers] BATpreload_ and _BATpreload
by Sjoerd Mullender 15 Feb '10

15 Feb '10

I am quite unhappy with the names BATpreload, BATpreload_, and _BATpreload. I really don't like names that differ only by an initial or final _. I would much rather see more descriptive names. BATpreload_ is now more an interface to give advice to the underlying system about how the BAT is going to be accessed in the near future, and _BATpreload is telling the underlying system that you're done with accessing the BAT according to the advice given. Names such as BATaccessadvice/BATendaccessadvice, though longer, are more descriptive. I have seen some checkins fixing the incorrect use of BATpreload_ and _BATpreload, proofing my point. Now we still have a chance to fix this. -- Sjoerd Mullender

4 7

Re: [Monetdb-developers] [Monetdb-checkins] MonetDB/src/gdk gdk_posix.mx, Feb2010, 1.176.2.14, 1.176.2.15
by Stefan Manegold 15 Feb '10

15 Feb '10

On Sun, Feb 14, 2010 at 09:23:10PM +0000, Martin Kersten wrote: > Update of /cvsroot/monetdb/MonetDB/src/gdk > In directory sfp-cvsdas-1.v30.ch3.sourceforge.com:/tmp/cvs-serv28118 > > Modified Files: > Tag: Feb2010 > gdk_posix.mx > Log Message: > Move the load advice outside the lock. FYI: This checkin also changes the semantics of MT_mmap_inform() such that, now, (1) with preload > 0, posix_madvise(base, len, advise) is also called on areas that are not maintained in MT_mmap_tab[] (2) with preload < 0, posix_madvise(base, len, BUF_NORMAL) is never called on areas that are not maintained in MT_mmap_tab[] Was this intended? Stefan > > Index: gdk_posix.mx > =================================================================== > RCS file: /cvsroot/monetdb/MonetDB/src/gdk/gdk_posix.mx,v > retrieving revision 1.176.2.14 > retrieving revision 1.176.2.15 > diff -u -d -r1.176.2.14 -r1.176.2.15 > --- gdk_posix.mx 14 Feb 2010 13:17:50 -0000 1.176.2.14 > +++ gdk_posix.mx 14 Feb 2010 21:22:29 -0000 1.176.2.15 > @@ -884,7 +884,7 @@ > void > MT_mmap_inform(void *base, size_t len, int preload, int advise, int writable) > { > - int i; > + int i, unload =0; > > (void) pthread_mutex_lock(&MT_mmap_lock); > i = MT_mmap_idx(base, len); > @@ -892,12 +892,13 @@ > if (writable) MT_mmap_tab[i].writable = (writable > 0); > MT_mmap_tab[i].random += preload * (advise == BUF_WILLNEED); /* done as a counter to keep track of multiple threads */ > MT_mmap_tab[i].usecnt += preload; /* active thread count */ > - if (preload > 0) > - (void) posix_madvise(base, len, advise); > - else if (MT_mmap_tab[i].usecnt == 0) > - (void) posix_madvise(base, len, BUF_NORMAL); > + unload = MT_mmap_tab[i].usecnt == 0; > } > (void) pthread_mutex_unlock(&MT_mmap_lock); > + if (preload > 0) > + (void) posix_madvise(base, len, advise); > + else if (unload) > + (void) posix_madvise(base, len, BUF_NORMAL); > > } > > > > ------------------------------------------------------------------------------ > SOLARIS 10 is the OS for Data Centers - provides features such as DTrace, > Predictive Self Healing and Award Winning ZFS. Get Solaris 10 NOW > http://p.sf.net/sfu/solaris-dev2dev > _______________________________________________ > Monetdb-checkins mailing list > Monetdb-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-checkins > -- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4199 |

2 1

Re: [Monetdb-developers] [Monetdb-checkins] MonetDB5/src/modules/kernel group.mx, Feb2010, 1.146.2.14, 1.146.2.15
by Martin Kersten 14 Feb '10

14 Feb '10

Stefan Manegold wrote: > when BATpreloading the sequentially accessed outer BAT, > make sure we also BATpreload the (often) randomly > accessed "inner" BAT We probably also have to inject advice for the target variable(s), because it is likely to be used in the future a BUF_WILLNEED might be appropriate to avoid early eviction. This advice should be given when we create them, e.g. in BATnew?

1 0