February 2014 - developers-list

status of default branch
by Fabian Groffen 05 Mar '14

05 Mar '14

Hi all, Since I'm no longer really in the loop with MonetDB, I was wondering what the state of the default branch is supposed to be. I just fixed a configure error and now hit a redeclaration problem regarding gdk.h and gdk_private.h. gdk_batop.c includes both headers so it seems like a valid complaint to me. Is OSX still a supported platform, these days? I guess there is little to no point in supporting it. Fabian

5 20

Re: MonetDB: rdf - Generate sample data for all tables
by Lefteris 26 Feb '14

26 Feb '14

Hi Duc, I did not read in detail your function, but you can also use BATsample_ (which is the void headed version of BATsample) lefteris On Wed, Feb 26, 2014 at 2:10 PM, Minh-Duc Pham <commits(a)monetdb.org> wrote: > Changeset: e7109fc24610 for MonetDB > URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e7109fc24610 > Modified Files: > monetdb5/extras/rdf/rdfschema.c > monetdb5/extras/rdf/rdfschema.h > Branch: rdf > Log Message: > > Generate sample data for all tables > > > diffs (truncated from 721 to 300 lines): > > diff --git a/monetdb5/extras/rdf/rdfschema.c b/monetdb5/extras/rdf/rdfschema.c > --- a/monetdb5/extras/rdf/rdfschema.c > +++ b/monetdb5/extras/rdf/rdfschema.c > @@ -695,6 +695,11 @@ char isInfrequentSampleProp(CS freqCS, i > if (freqCS.lstPropSupport[propIdx] * 100 < freqCS.support * SAMPLE_FILTER_THRESHOLD) return 1; > else return 0; > } > +static > +char isInfrequentSampleCol(CS freqCS, PropTypes pt){ > + if (pt.propFreq * 100 < freqCS.support * SAMPLE_FILTER_THRESHOLD) return 1; > + else return 0; > +} > > static > void genCSPropTypesColIdx(CSPropTypes* csPropTypes, int numMergedCS, CSset* freqCSset){ > @@ -4830,6 +4835,211 @@ void initSampleData(CSSample *csSample,B > > } > } > + > + > +static > +void getSubjIdFromTablePosition(int tblIdx, int pos, oid *sOid){ > + oid id; > + id = pos; > + id |= (BUN)(tblIdx + 1) << (sizeof(BUN)*8 - NBITS_FOR_CSID); > + *sOid = id; > +} > + > +static > +str getOrigSbt(oid *sbt, oid *origSbt, BAT *lmap, BAT *rmap){ > + BUN pos; > + oid *tmp; > + pos = BUNfnd(BATmirror(rmap),sbt); > + if (pos == BUN_NONE){ > + throw(RDF, "rdf.RDFdistTriplesToCSs", "This encoded subject must be in rmap"); > + } > + tmp = (oid *) Tloc(lmap, pos); > + if (*tmp == BUN_NONE){ > + throw(RDF, "rdf.RDFdistTriplesToCSs", "The encoded subject must be in lmap"); > + } > + > + *origSbt = *tmp; > + > + return MAL_SUCCEED; > +} > + > +static > +str getOrigObt(oid *obt, oid *origObt, BAT *lmap, BAT *rmap){ > + BUN pos; > + oid *tmp; > + oid tmporigOid = BUN_NONE; > + char objType; > + BUN maxObjectURIOid = ((oid)1 << (sizeof(BUN)*8 - NBITS_FOR_CSID - 1)) - 1; //Base on getTblIdxFromS > + > + objType = getObjType(*obt); > + > + if (objType == URI || objType == BLANKNODE){ > + tmporigOid = (*obt) - ((oid)objType << (sizeof(BUN)*8 - 4)); > + } > + > + if (tmporigOid > maxObjectURIOid){ > + pos = BUNfnd(BATmirror(rmap),&tmporigOid); > + if (pos == BUN_NONE){ > + throw(RDF, "rdf.RDFdistTriplesToCSs", "This encoded object must be in rmap"); > + } > + tmp = (oid *) Tloc(lmap, pos); > + if (*tmp == BUN_NONE){ > + throw(RDF, "rdf.RDFdistTriplesToCSs", "The encoded object must be in lmap"); > + } > + > + *origObt = *tmp; > + } > + else{ > + *origObt = tmporigOid; > + } > + > + return MAL_SUCCEED; > +} > + > +static > +str initFullSampleData(CSSampleExtend *csSampleEx, int *mTblIdxFreqIdxMapping, CSlabel *label, CStableStat* cstablestat, CSPropTypes *csPropTypes, CSset *freqCSset, int numTables, bat *lmapbatid, bat *rmapbatid){ > + int i, j, k; > + int freqId; > + int tmpNumcand; > + oid tmpCandidate; > + int randValue = 0; > + int ranPosition = 0; //random position of the instance in a table > + int tmpNumCols; > + int colIdx; > + BAT *tmpbat = NULL; > + BATiter tmpi; > + BAT *cursamplebat = NULL; > + int tmpNumRows = 0; > + oid tmpSoid = BUN_NONE, origSoid = BUN_NONE; > + oid origOid = BUN_NONE; > + BAT *lmap = NULL, *rmap = NULL; > + > + if ((lmap = BATdescriptor(*lmapbatid)) == NULL) { > + throw(MAL, "rdf.RDFdistTriplesToCSs", RUNTIME_OBJECT_MISSING); > + } > + > + if ((rmap = BATdescriptor(*rmapbatid)) == NULL) { > + BBPreleaseref(lmap->batCacheid); > + throw(MAL, "rdf.RDFdistTriplesToCSs", RUNTIME_OBJECT_MISSING); > + } > + srand(123456); > + for (i = 0; i < numTables; i++){ > + freqId = mTblIdxFreqIdxMapping[i]; > + csSampleEx[i].freqIdx = freqId; > + tmpNumcand = (NUM_SAMPLE_CANDIDATE > label[freqId].candidatesCount)?label[freqId].candidatesCount:NUM_SAMPLE_CANDIDATE; > + csSampleEx[i].name = cstablestat->lstcstable[i].tblname; > + csSampleEx[i].candidateCount = tmpNumcand; > + csSampleEx[i].candidates = (oid*)malloc(sizeof(oid) * tmpNumcand); > + for (k = 0; k < tmpNumcand; k++){ > + csSampleEx[i].candidates[k] = label[freqId].candidates[k]; > + } > + //Randomly exchange the value, change the position k with a random pos > + for (k = 0; k < tmpNumcand; k++){ > + randValue = rand() % tmpNumcand; > + tmpCandidate = csSampleEx[i].candidates[k]; > + csSampleEx[i].candidates[k] = csSampleEx[i].candidates[randValue]; > + csSampleEx[i].candidates[randValue] = tmpCandidate; > + } > + > + csSampleEx[i].lstSubjOid = (oid*)malloc(sizeof(oid) * NUM_SAMPLE_INSTANCE); > + for (k = 0; k < NUM_SAMPLE_INSTANCE; k++) > + csSampleEx[i].lstSubjOid[k] = BUN_NONE; > + > + tmpNumCols = csPropTypes[i].numProp - csPropTypes[i].numInfreqProp; //already remove infrequent column; > + csSampleEx[i].numProp = tmpNumCols; > + > + assert(tmpNumCols > 0); > + > + csSampleEx[i].lstProp = (oid*)malloc(sizeof(oid) * tmpNumCols); > + csSampleEx[i].lstIsInfrequentProp = (char*)malloc(sizeof(char) * tmpNumCols); > + csSampleEx[i].lstIsMVCol = (char*)malloc(sizeof(char) * tmpNumCols); > + csSampleEx[i].colBats = (BAT**)malloc(sizeof(BAT*) * tmpNumCols); > + colIdx = -1; > + csSampleEx[i].numInstances = 0; > + for(j = 0; j < csPropTypes[i].numProp; j++){ > + #if REMOVE_INFREQ_PROP > + if (csPropTypes[i].lstPropTypes[j].defColIdx == -1) continue; //Infrequent prop > + #endif > + colIdx++; > + csSampleEx[i].lstProp[colIdx] = csPropTypes[i].lstPropTypes[j].prop; > + > + csSampleEx[i].colBats[colIdx] = BATnew(TYPE_void, cstablestat->lstcstable[i].colBats[colIdx]->ttype , NUM_SAMPLE_INSTANCE + 1); > + > + //Mark whether this col is infrequent sample cols > + if ( isInfrequentSampleCol(freqCSset->items[freqId], csPropTypes[i].lstPropTypes[j])){ > + csSampleEx[i].lstIsInfrequentProp[colIdx] = 1; > + } > + else > + csSampleEx[i].lstIsInfrequentProp[colIdx] = 0; > + > + //Mark whther this col is a MV col > + csSampleEx[i].lstIsMVCol[colIdx] = csPropTypes[i].lstPropTypes[j].isMVProp; > + > + //if this is a multivalue column, get the data type of the first column > + > + } > + assert(colIdx == (tmpNumCols - 1)); > + > + > + // Inserting instances to csSampleEx > + > + tmpNumRows = BATcount(cstablestat->lstcstable[i].colBats[0]); > + > + for (k = 0; k < NUM_SAMPLE_INSTANCE; k++){ > + ranPosition = rand() % tmpNumRows; > + > + getSubjIdFromTablePosition(i, ranPosition, &tmpSoid); > + > + if (getOrigSbt(&tmpSoid, &origSoid, lmap, rmap) != MAL_SUCCEED){ > + throw(RDF, "rdf.RDFdistTriplesToCSs","Problem in getting the orignal sbt "); > + } > + > + csSampleEx[i].lstSubjOid[k] = origSoid; > + > + for (j = 0; j < tmpNumCols; j++){ > + cursamplebat = csSampleEx[i].colBats[j]; > + > + tmpbat = cstablestat->lstcstable[i].colBats[j]; > + tmpi = bat_iterator(tmpbat); > + > + if (tmpbat->ttype == TYPE_oid && csSampleEx[i].lstIsMVCol[j] == 0){ > + //Get the original object oid > + oid *tmpOid = (oid *) BUNtail(tmpi, ranPosition); > + if(*tmpOid != oid_nil){ > + if (getOrigObt(tmpOid, &origOid, lmap, rmap) != MAL_SUCCEED){ > + throw(RDF, "rdf.RDFdistTriplesToCSs","Problem in getting the orignal obt "); > + } > + BUNappend(cursamplebat, &origOid, TRUE); > + } > + else{ > + BUNappend(cursamplebat, ATOMnilptr(TYPE_oid), TRUE); > + } > + > + } > + else > + BUNappend(cursamplebat, BUNtail(tmpi, ranPosition), TRUE); > + > + > + > + } > + csSampleEx[i].numInstances++; > + } > + > + if (i == 0) > + for (j = 0; j < tmpNumCols; j++){ > + //BATprint(cstablestat->lstcstable[i].colBats[j]); > + BATprint(csSampleEx[i].colBats[j]); > + } > + > + } > + > + BBPunfix(lmap->batCacheid); > + BBPunfix(rmap->batCacheid); > + > + return MAL_SUCCEED; > + > +} > + > static > void freeSampleData(CSSample *csSample, int numCand){ > int i, j; > @@ -4846,6 +5056,25 @@ void freeSampleData(CSSample *csSample, > free(csSample); > } > > + > +static > +void freeSampleExData(CSSampleExtend *csSampleEx, int numCand){ > + int i, j; > + for (i = 0; i < numCand; i++){ > + free(csSampleEx[i].lstProp); > + free(csSampleEx[i].lstIsInfrequentProp); > + free(csSampleEx[i].lstIsMVCol); > + free(csSampleEx[i].candidates); > + free(csSampleEx[i].lstSubjOid); > + for (j = 0; j < csSampleEx[i].numProp; j++){ > + BBPunfix(csSampleEx[i].colBats[j]->batCacheid); > + } > + free(csSampleEx[i].colBats); > + } > + > + free(csSampleEx); > +} > + > static > void addSampleInstance(oid subj, oid *buffO, oid* buffP, int numP, int sampleIdx, CSSample *csSample){ > int i,j; > @@ -5217,6 +5446,295 @@ str printSampleData(CSSample *csSample, > return MAL_SUCCEED; > } > > +#if 0 > +static > +str printFullSampleData(CSSampleExtend *csSampleEx, CSset *freqCSset, BAT *mbat, int num, int sampleVersion){ > + > + int i,j, k; > + FILE *fout, *fouttb, *foutis; > + char filename[100], filename2[100], filename3[100]; > + char tmpStr[20], tmpStr2[20], tmpStr3[20]; > + int ret; > + > + str propStr; > + str subjStr; > + char* schema = "rdf"; > + CSSample sample; > + CS freqCS; > + char objType = 0; > + str objStr; > + oid objOid = BUN_NONE; > + BATiter mapi; > + str canStr; > + char isTitle = 0; > + char isUrl = 0; > + char isType = 0; > + char isDescription = 0; > + char isImage = 0; > + char isSite = 0; > + char isEmail = 0; > + char isCountry = 0; > + char isLocality = 0; > + BAT *lmap = NULL, *rmap = NULL > +#if USE_SHORT_NAMES > + str propStrShort = NULL; > + char *pch; > +#endif > + > + > + > + mapi = bat_iterator(mbat); > + > + if (TKNZRopen (NULL, &schema) != MAL_SUCCEED) { > + throw(RDF, "rdf.rdfschema", > + "could not open the tokenizer\n"); > + } > _______________________________________________ > checkin-list mailing list > checkin-list(a)monetdb.org > https://www.monetdb.org/mailman/listinfo/checkin-list

2 1

Re: MonetDB: default - Move tokenizer to the attic
by Stefan Manegold 24 Feb '14

24 Feb '14

Martin, et al., please be aware that this means that also monetdb5/extras/rdf (aka. --enable-rdf) does no longer compile, as it depends on the (now gone) tokenizer. Best, Stefan ----- Original Message ----- > Changeset: 155c3a3fcfdb for MonetDB > URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=155c3a3fcfdb > Removed Files: > monetdb5/modules/mal/tokenizer.c > monetdb5/modules/mal/tokenizer.h > monetdb5/modules/mal/tokenizer.mal > Modified Files: > monetdb5/modules/mal/Makefile.ag > Branch: default > Log Message: > > Move tokenizer to the attic > The experiment to organise urls using a variation of graph-based > tokenization requires a major overhaul to support the void-headed approach. > > > diffs (truncated from 737 to 300 lines): > > diff --git a/monetdb5/modules/mal/Makefile.ag > b/monetdb5/modules/mal/Makefile.ag > --- a/monetdb5/modules/mal/Makefile.ag > +++ b/monetdb5/modules/mal/Makefile.ag > @@ -54,7 +54,6 @@ lib_mal = { > sabaoth.c sabaoth.h \ > sysmon.c sysmon.h \ > tablet.c tablet.h \ > - tokenizer.c tokenizer.h \ > trader.c trader.h \ > transaction.c \ > txtsim.c txtsim.h \ > @@ -76,7 +75,7 @@ headers_mal = { > mal_mapi.mal sabaoth.mal remote.mal \ > txtsim.mal recycle.mal \ > cluster.mal trader.mal \ > - tokenizer.mal zorder.mal sample.mal json_util.mal \ > + zorder.mal sample.mal json_util.mal \ > calc.mal batcalc.mal batmtime.mal querylog.mal sysmon.mal > } > > @@ -84,7 +83,7 @@ EXTRA_DIST = batExtensions.mal iterator. > groupby.mal mal_init.mal manual.mal mkey.mal manifold.mal pcre.mal \ > profiler.mal recycle.mal remote.mal sabaoth.mal trader.mal \ > transaction.mal txtsim.mal tablet.mal tablet.h sample.mal json_util.mal \ > - mal_mapi.mal mat.mal tokenizer.mal pqueue.mal calc.mal \ > + mal_mapi.mal mat.mal pqueue.mal calc.mal \ > batcalc.mal batmtime.mal querylog.mal sysmon.mal > > EXTRA_DIST_DIR = Tests > diff --git a/monetdb5/modules/mal/tokenizer.c > b/monetdb5/modules/mal/tokenizer.c > deleted file mode 100644 > --- a/monetdb5/modules/mal/tokenizer.c > +++ /dev/null > @@ -1,585 +0,0 @@ > -/* > - * The contents of this file are subject to the MonetDB Public License > - * Version 1.1 (the "License"); you may not use this file except in > - * compliance with the License. You may obtain a copy of the License at > - * http://www.monetdb.org/Legal/MonetDBLicense > - * > - * Software distributed under the License is distributed on an "AS IS" > - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the > - * License for the specific language governing rights and limitations > - * under the License. > - * > - * The Original Code is the MonetDB Database System. > - * > - * The Initial Developer of the Original Code is CWI. > - * Portions created by CWI are Copyright (C) 1997-July 2008 CWI. > - * Copyright August 2008-2014 MonetDB B.V. > - * All Rights Reserved. > - */ > - > -/* > - * author Lefteris Sidirourgos > - * Tokenizer > - * This module implements a vertical fragmented tokenizer for strings. > - * It is based on the ideas of the urlbox module by mk. > - * > - * The input string is tokenized according to a separator character. > - * Each token is inserted to the next BAT with the same order of > - * appearance in the string. We currently support 255 tokens in each > - * string as this module is intended for use with short and similar > - * strings such as URLs. In addition we maintain a 2-dimensional index > - * that points to the depth and height of the last token of each string. > - * The 2-dimensional index is combined to one BAT where the 8 least > - * significant bits represent the depth, and the rest bits the height. > - * > - * The tokenizer can be accessed in two ways. Given the oid retrieve the > - * re-constructed string, or given a string return its oid if present, > - * otherwise nil. > - * > - * Strings can be added either in batch (from a file or a bat of > - * strings) and by appending a single string. Duplicate elimination is > - * always performed. > - * > - * There can be only one tokenizer open at the same time. This is > - * achieved by setting a TRANSaction bat. This might change in the > - * future. However there can be more than one tokenizers stored in the > - * disk, each of which is identified by its name (usually the name of > - * the active schema of the db). These administrative issues and > - * security aspects (e.g., opening a tokenizer of a different schema) > - * should be addressed more thoroughly. > - */ > -#include "monetdb_config.h" > -#include "bat5.h" > -#include "tokenizer.h" > -#include "mal_linker.h" > - > -#define MAX_TKNZR_DEPTH 256 > -#define INDEX MAX_TKNZR_DEPTH > -static int tokenDepth = 0; > -static BAT *tokenBAT[MAX_TKNZR_DEPTH + 1]; > -static BAT *TRANS = NULL; /* the catalog of tokenizers */ > -static char name[128]; > - > -#if SIZEOF_OID == 4 /* 32-bit oid */ > -#define MAX_h ((((oid) 1) << 23) - 1) > -#else /* 64-bit oid */ > -#define MAX_h ((((oid) 1) << 55) - 1) > -#endif > - > -#define COMP(h, d) ((h << 8) | (d & 255)) > -#define GET_d(x) ((sht) ((x) & 255)) > -#define GET_h(x) ((x) >> 8) > - > -static int prvlocate(BAT* b, oid *prv, str part) > -{ > - BAT *m = BATmirror(b); > - BATiter mi = bat_iterator(m); > - BUN p; > - if (m->H->hash == NULL) > - BAThash(m, 2 * BATcount(m)); > - HASHloop_str(mi, m->H->hash, p, part) > - { > - if (*((oid *) BUNtail(mi, p)) == *prv) { > - *prv = (oid) p; > - return TRUE; > - } > - } > - return FALSE; > -} > - > -str > -TKNZRopen(int *ret, str *in) > -{ > - int depth, r; > - bat idx; > - str batname = NULL; > - BAT *b; > - > - (void) ret; > - if (strlen(*in) > 127) > - throw(MAL, "tokenizer.open", > - ILLEGAL_ARGUMENT " tokenizer name too long"); > - > - MT_lock_set(&mal_contextLock, "tokenizer"); > - if (TRANS != NULL) { > - MT_lock_unset(&mal_contextLock, "tokenizer"); > - throw(MAL, "tokenizer.open", "Another tokenizer is already open"); > - } > - > - for (depth = 0; depth < MAX_TKNZR_DEPTH; depth++) { > - tokenBAT[depth] = 0; > - } > - tokenDepth = 0; > - > - TRANS = BATnew(TYPE_void, TYPE_str, MAX_TKNZR_DEPTH + 1); > - if (TRANS == NULL) { > - MT_lock_unset(&mal_contextLock, "tokenizer"); > - throw(MAL, "tokenizer.open", MAL_MALLOC_FAIL); > - } > - /* now we are sure that none overwrites the tokenizer table*/ > - MT_lock_unset(&mal_contextLock, "tokenizer"); > - BATseqbase(TRANS, 0); > - > - snprintf(name, 128, "%s", *in); > - batname = (str) GDKmalloc(134 * sizeof(char)); > - if( batname == NULL) > - throw(MAL, "tokenizer.open", MAL_MALLOC_FAIL); > - > - snprintf(batname, 134, "%s_index", name); > - idx = BBPindex(batname); > - > - if (idx == 0) { /* new tokenizer */ > - b = BATnew(TYPE_void, TYPE_oid, 1024); > - if (b == NULL) > - throw(MAL, "tokenizer.open", MAL_MALLOC_FAIL); > - BATkey(b, FALSE); > - BATseqbase(b, 0); > - tokenBAT[INDEX] = b; > - if (BKCsetName(&r, (int *) &(b->batCacheid), (str *) &batname) != > MAL_SUCCEED) > - throw(MAL, "tokenizer.open", OPERATION_FAILED); > - if (BKCsetPersistent(&r, (int *) &(b->batCacheid)) != MAL_SUCCEED) > - throw(MAL, "tokenizer.open", OPERATION_FAILED); > - BUNappend(TRANS, batname, FALSE); > - } else { /* existing tokenizer */ > - tokenBAT[INDEX] = BATdescriptor(idx); > - BUNappend(TRANS, batname, FALSE); > - > - for (depth = 0; depth < MAX_TKNZR_DEPTH; depth++) { > - snprintf(batname, 128, "%s_%d", name, depth); > - idx = BBPindex(batname); > - if (idx == 0) > - break; > - tokenBAT[depth] = BATdescriptor(idx); > - BUNappend(TRANS, batname, FALSE); > - } > - tokenDepth = depth; > - } > - > - GDKfree(batname); > - return MAL_SUCCEED; > -} > - > -str > -TKNZRclose(int *r) > -{ > - int i; > - (void) r; > - > - if (TRANS == NULL) > - throw(MAL, "tokenizer", "no tokenizer store open"); > - > - TMsubcommit(TRANS); > - > - for (i = 0; i < tokenDepth; i++) { > - BBPunfix(tokenBAT[i]->batCacheid); > - } > - BBPunfix(tokenBAT[INDEX]->batCacheid); > - > - tokenDepth = 0; > - > - BBPreclaim(TRANS); > - TRANS = NULL; > - return MAL_SUCCEED; > -} > - > -/* > - * Tokenize operations > - * The tokenizer operation assumes a private copy to mark the end of the > - * token separators with a zero byte. Tokens are separated by a single > - * character for simplicity. Might be a good scheme to assume that > - * strings to be broken are properly ended with either 0 or nl, not > - * both. It seems 0 can be assumed. > - */ > -static int > -TKNZRtokenize(str in, str *parts, char tkn) > -{ > - char *s, *t; > - int depth = 0; > - > - s = in; > - while (*s && *s != '\n') { > - t = s; > - while (*t != tkn && *t != '\n' && *t) > - t++; > - parts[depth++] = s; > - s = t + (*t != 0); > - *t = 0; > - if (depth > MAX_TKNZR_DEPTH) > - break; > - } > - return depth; > -} > - > -str > -TKNZRappend(oid *pos, str *s) > -{ > - str url; > - str batname; > - str parts[MAX_TKNZR_DEPTH]; > - int i, new, r, depth; > - BAT *b; > - BUN p; > - BUN idx = 0; > - oid prv = 0; > - oid comp; > - > - if (TRANS == NULL) > - throw(MAL, "tokenizer", "no tokenizer store open"); > - > - if ((url = GDKstrdup(*s)) == NULL) { > - throw(MAL, "tokenizer.append", OPERATION_FAILED MAL_MALLOC_FAIL); > - } > - > - depth = TKNZRtokenize(url, parts, '/'); > - new = depth; > - > - if (depth == 0) { > - GDKfree(url); > - return MAL_SUCCEED; > - } > - if (depth > MAX_TKNZR_DEPTH) { > - GDKfree(url); > - throw(MAL, "tokenizer", > - ILLEGAL_ARGUMENT "input string breaks to too many parts"); > - } > - if (depth > tokenDepth || tokenBAT[0] == NULL) { > - new = tokenDepth; > - for (i = tokenDepth; i < depth; i++) { > - /* make new bat */ > - batname = (str) GDKmalloc(128 * sizeof(char)); > - snprintf(batname, 128, "%s_%d", name, i); > - b = BATnew(TYPE_oid, TYPE_str, 1024); > - if (b == NULL) { > - GDKfree(batname); > - GDKfree(url); > - throw(MAL, "tokenizer.append", MAL_MALLOC_FAIL); > - } > - BATkey(b, FALSE); > - tokenBAT[i] = b; > - > - if (BKCsetName(&r, (int *) &(b->batCacheid), (str *) &batname) > - != MAL_SUCCEED) { > - GDKfree(batname); > - GDKfree(url); > - throw(MAL, "tokenizer.open", OPERATION_FAILED); > - } > - if (BKCsetPersistent(&r, (int *) &(b->batCacheid)) > _______________________________________________ > checkin-list mailing list > checkin-list(a)monetdb.org > https://www.monetdb.org/mailman/listinfo/checkin-list > -- | Stefan.Manegold(a)CWI.nl | DB Architectures (DA) | | www.CWI.nl/~manegold/ | Science Park 123 (L321) | | +31 (0)20 592-4212 | 1098 XG Amsterdam (NL) |

1 0

Announcement: New Jan2014 Feature release of MonetDB suite
by Sjoerd Mullender 20 Feb '14

20 Feb '14

The MonetDB team at CWI/MonetDB BV is pleased to announce the Jan2014 feature release of the MonetDB suite of programs. More information about MonetDB can be found on our website at <http://www.monetdb.org/>. For details on this release, please see the release notes at <http://www.monetdb.org/Downloads/ReleaseNotes>. As usual, the download location is <http://dev.monetdb.org/downloads/>. Jan 2014 feature release Build Environment * Created development packages for RPM based systems and Debian/Ubuntu containing the files needed to create extensions to the SQL front end. * Removed Mx, the literate programming tool. All code for the server is now pure C. * Created packages for RPM based systems and Debian/Ubunty containing the MonetDB interface to the GNU Scientific Library (gsl). * We no longer install the .la files in our Fedora/Debian/Ubuntu packages. Client Package * ODBC: Implemented {fn scalar()} and {interval ...} escapes. python2 * Changed defaults for connecting (defaults to unix socket now). * Unix sockets partially working for control protocol. * Add support for unix socket. python3 * Changed defaults for connecting (defaults to unix socket now). * Unix sockets partially working for control protocol. * Add support for unix socket. R * The R connector is now distributed in the source code packages. MonetDB Common * The join code has been completely rewritten. It now optionally uses candidate lists, like the select code that was introduced in the previous release. * A new indexing structure for range selections on unsorted data has been added. * The vmtrim thread is no longer started by default on 64 bit architectures. The vmtrim thread monitors memory usage and drops BATs from memory when memory gets tight. However, in the age of large address spaces and virtual memory, the kernel does a good enough job. And in addition to dropping BATs, the thread also destroyed indexing structures which would have to be recreated the next time they were needed. * Cleaned up some of the parameters dealing with memory usage. * If available on the system, we now use atomic instructions for latching. * Removed some unused fields in the atomDesc structure. This change requires a complete recompilation of the whole suite. * Replaced the mutex implementation for both GNU C and Visual Studio with a home-grown implementation that uses atomic instructions (__sync_*() in gcc, _Interlocked*() in VS). SQL * Added support for quantiles (generalization of median). Usage: SELECT quantile(column_name,0.25) FROM table_name; The value should be in the range 0..1. Bug Fixes * 3040: Wrong NULL behavior in EXCEPT and INTERSECT * 3092: ODBC client doesn't support scalar function escape * 3198: SIGSEGV insert_string_bat (b=0x7fffe419d0a0, n=0x7fffc4006010, append=0) at gdk_batop.c:196 * 3210: Unexpected concurrency conflict when inserting to 2 tables simultaneously and querying one of them * 3273: Add support to Python DBAPI package for timetz, inet and url types * 3285: no such table 'queryHistory' * 3298: GDKmmap messages and monetdb start db takes very long * 3354: Introduce query time-out * 3371: (i)like generates batloop instead of algebra.likesubselect * 3372: Large group by queries never complete - server at 100% cpu(all cores) until MonetDB stopped * 3383: Bad performance with DISTINCT GROUP BY * 3391: Bad performance with GROUP BY and FK with out aggregate function * 3393: "COPY .. INTO ..." - escape of string quotes * 3399: server crashed on simple (malformed) query * 3401: inconsistent/strange handling of invalid dates (e.g. 2013-02-29) in where clause * 3403: NOT NULL constraint can't be applied after deleting rows with null values * 3404: Assertion `h->storage == STORE_MMAP' failed. * 3408: nested concat query crashed server * 3411: (disguised) BETWEEN clause not recognised. Hence no rangejoin. * 3412: Boolean expressions in WHERE clause, result in incorrect resulsts * 3417: Nested Common Table Expressions Crash * 3418: Segmentation fault on a query from table expression * 3419: Database does not start after upgrade * 3420: Database does not start after upgrade * 3423: Group by alias with distinct count doesn't work * 3425: Temporal extraction glitches * 3427: Consistent use of current_timestamp and now() * 3428: Aggregation over two columns is broken * 3429: SAMPLE on JOIN result crashes server * 3430: Wrong temporary handling * 3431: SQLGetInfo returns incorrect value for SQL_FN_NUM_TRUNCATE * 3432: MonetDB SQL syntax incompatible with SQL-92 <delimited identifier> syntax * 3435: INDEX prevents JOIN from discovering matches * 3436: COPY INTO from file containing leading Byte Order Mark (BOM) causes corruption

1 0

Re: MonetDB: Jan2014 - monetdb: allow databases to be created with a...
by Stefan Manegold 17 Feb '14

17 Feb '14

Fabian, when configured with --enable-strict, this fail to compile on my Fedora 20 desktop: .../MonetDB/tools/merovingian/daemon/controlrunner.c: In function ‘ctl_handle_client’: .../MonetDB/tools/merovingian/daemon/controlrunner.c:387:13: error: ignoring return value of ‘chdir’, declared with attribute warn_unused_result [-Werror=unused-result] chdir(q); ^ cc1: all warnings being treated as errors I could not fix it quickly (i.e., checking and handling the return value of chdir()), as I could not quickly grasp how to do proper error/exception handling in this case ... Would be great, if you could have a look at it. Thanks! Stefan ----- Original Message ----- > Changeset: 6177dd32ac5d for MonetDB > URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6177dd32ac5d > Modified Files: > tools/merovingian/ChangeLog.Jan2014 > tools/merovingian/client/monetdb.1 > tools/merovingian/client/monetdb.c > tools/merovingian/daemon/Makefile.ag > tools/merovingian/daemon/controlrunner.c > Branch: Jan2014 > Log Message: > > monetdb: allow databases to be created with a password for monetdb user > > Creating a database with a custom password for the monetdb user makes it > safer than when created with default/well-known credentials, hence it is > not necessary to lock the database after creation. > > > diffs (190 lines): > > diff --git a/tools/merovingian/ChangeLog.Jan2014 > b/tools/merovingian/ChangeLog.Jan2014 > --- a/tools/merovingian/ChangeLog.Jan2014 > +++ b/tools/merovingian/ChangeLog.Jan2014 > @@ -2,5 +2,7 @@ > # This file is updated with mchangelog > > * Sun Feb 16 2014 Fabian Groffen <fabian(a)monetdb.org> > +- monetdb create: add -p flag to set monetdb user password on creation, > + and therefore allow creating the database in unlocked state > - monetdb destroy -f now also works on running databases > > diff --git a/tools/merovingian/client/monetdb.1 > b/tools/merovingian/client/monetdb.1 > --- a/tools/merovingian/client/monetdb.1 > +++ b/tools/merovingian/client/monetdb.1 > @@ -52,12 +52,14 @@ stopping a database inside the MonetDB D > For all commands, database arguments can be glob-like expressions. > This allows to do wildcard matches. For details on the syntax, see > .IR EXPRESSIONS . > -.IP "create [\-m pattern] database [database ...]" > +.IP "create [\-m pattern] [\-p password] database [database ...]" > Initialises a new database in the MonetDB Database Server. A database > created with this command makes it available under its database name, > but not yet for use by clients, as the database is put into maintenance > mode. This allows the database administrator to perform initialisation > -steps before releasing it to users. See also > +steps before releasing it to users, unless the > +.I \-p > +argument is supplied. See also > .BR "monetdb lock" . > The name of the database must match the expression [A\-Za\-z0\-9\-_]+. > .IP " \-m pattern" > @@ -74,6 +76,14 @@ or discovery. Each parallel target for > sequence, separated by commas. Here the pattern is an ordinary pattern > as would be used for connecting to a database, and can hence also be > just the name of a database. > +.IP " \-p password" > +The > +.I \-p > +flag allows to create a database with the given password for the monetdb > +user. Since this protects the database from being accessed via > +well-known credentials, the created database is not locked after > +creation. This way, a new database can be created and used right away > +using the password supplied. > .IP "destroy [\-f] database [database ...]" > Removes the given database, including all its data and logfiles. Once > destroy has completed, all data is lost. Be careful when using this > diff --git a/tools/merovingian/client/monetdb.c > b/tools/merovingian/client/monetdb.c > --- a/tools/merovingian/client/monetdb.c > +++ b/tools/merovingian/client/monetdb.c > @@ -1435,6 +1435,7 @@ command_create(int argc, char *argv[]) > { > int i; > char *mfunnel = NULL; > + char *password = NULL; > sabdb *orig = NULL; > sabdb *stats = NULL; > > @@ -1464,6 +1465,19 @@ command_create(int argc, char *argv[]) > command_help(2, &argv[-1]); > exit(1); > } > + } else if (argv[i][1] == 'p') { > + if (argv[i][2] != '\0') { > + password = &argv[i][2]; > + argv[i] = NULL; > + } else if (i + 1 < argc && argv[i + 1][0] != '-') { > + argv[i] = NULL; > + password = argv[++i]; > + argv[i] = NULL; > + } else { > + fprintf(stderr, "create: -p needs an argument\n"); > + command_help(2, &argv[-1]); > + exit(1); > + } > } else { > fprintf(stderr, "create: unknown option: %s\n", argv[i]); > command_help(argc + 1, &argv[-1]); > @@ -1492,6 +1506,13 @@ command_create(int argc, char *argv[]) > simple_argv_cmd(argv[0], orig, cmd, > "created multiplex-funnel in maintenance mode", NULL); > free(cmd); > + } else if (password != NULL) { > + size_t len = strlen("create password=") + strlen(password) + 1; > + char *cmd = malloc(len); > + snprintf(cmd, len, "create password=%s", password); > + simple_argv_cmd(argv[0], orig, cmd, > + "created database with password for monetdb user", NULL); > + free(cmd); > } else { > simple_argv_cmd(argv[0], orig, "create", > "created database in maintenance mode", NULL); > diff --git a/tools/merovingian/daemon/Makefile.ag > b/tools/merovingian/daemon/Makefile.ag > --- a/tools/merovingian/daemon/Makefile.ag > +++ b/tools/merovingian/daemon/Makefile.ag > @@ -27,7 +27,10 @@ monetdb_var_dbfarm_DATA = .merovingian_p > INCLUDES = .. \ > ../../../common/stream \ > ../../../common/utils \ > + ../../../common/options \ > ../../../clients/mapilib \ > + ../../../gdk \ > + ../../../monetdb5/mal \ > $(MSGCONTROL_FLAGS) > > EXTRA_DIST = $(man_MANS) .merovingian_properties > @@ -54,6 +57,8 @@ bin_monetdbd = { > ../../../common/utils/libmsabaoth \ > ../../../common/utils/libmuuid \ > ../../../common/utils/libmutils \ > + ../../../monetdb5/tools/libmonetdb5 \ > + ../../../gdk/libbat \ > $(UUID_LIBS) \ > $(curl_LIBS) \ > $(SOCKET_LIBS) \ > diff --git a/tools/merovingian/daemon/controlrunner.c > b/tools/merovingian/daemon/controlrunner.c > --- a/tools/merovingian/daemon/controlrunner.c > +++ b/tools/merovingian/daemon/controlrunner.c > @@ -40,6 +40,10 @@ > #include <utils/database.h> > #include <utils/control.h> > > +#include "gdk.h" /* these three for creation of dbs with password */ > +#include "gdk_private.h" > +#include "mal_authorize.h" > + > #include "merovingian.h" > #include "discoveryrunner.h" /* broadcast, remotedb */ > #include "forkmserver.h" > @@ -353,8 +357,15 @@ static void ctl_handle_client( > "database is not running: %s\n", q); > send_client("!"); > } > - } else if (strcmp(p, "create") == 0) { > - err e = db_create(q); > + } else if (strcmp(p, "create") == 0 || > + strncmp(p, "create password=", strlen("create password=")) == 0) { > + err e; > + > + p += strlen("create"); > + if (*p == ' ') > + p += strlen(" password="); > + > + e = db_create(q); > if (e != NO_ERR) { > Mfprintf(_mero_ctlerr, "%s: failed to create " > "database '%s': %s\n", origin, q, getErrMsg(e)); > @@ -363,6 +374,43 @@ static void ctl_handle_client( > send_client("!"); > free(e); > } else { > + if (*p != '\0') { > + pid_t child; > + if ((child = fork()) == 0) { > + FILE *secretf; > + size_t len; > + char *err; > + char *vaultkey; > + > + /* the child, pollute scope by loading BBP */ > + chdir(q); > + > + buf2[0] = '\0'; > + if ((secretf = fopen(".vaultkey", "r")) != NULL) { > + len = fread(buf2, 1, sizeof(buf2), secretf); > + buf2[len] = '\0'; > + len = strlen(buf2); /* secret can contain null-bytes */ > + fclose(secretf); > + } > + BBPinit(); > + vaultkey = buf2; > + AUTHunlockVault(&vaultkey); > + err = AUTHinitTables(&p); > + if (err != NULL) { > + Mfprintf(_mero_ctlerr, "%s: could not setup " > + "database '%s': %s\n", origin, q, err); > + } else { > + /* don't start locked */ > + unlink(".maintenance"); > + } > + > + exit(0); /* return to the parent */ > + } else { > + /* wait for the child to finish */ > + waitpid(child, NULL, 0); > + } > + } > + > Mfprintf(_mero_ctlout, "%s: created database '%s'\n", > origin, q); > len = snprintf(buf2, sizeof(buf2), "OK\n"); > _______________________________________________ > checkin-list mailing list > checkin-list(a)monetdb.org > https://www.monetdb.org/mailman/listinfo/checkin-list > -- | Stefan.Manegold(a)CWI.nl | DB Architectures (DA) | | www.CWI.nl/~manegold/ | Science Park 123 (L321) | | +31 (0)20 592-4212 | 1098 XG Amsterdam (NL) |

2 1

Re: MonetDB: Jan2014 - AUTHinitTables: allow passing a password for ...
by Sjoerd Mullender 17 Feb '14

17 Feb '14

Fabian, this reeks of a forbidden ABI change. On 2014-02-16 22:36, Fabian Groffen wrote: > Changeset: 8f6aa798f4c5 for MonetDB > URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=8f6aa798f4c5 > Modified Files: > clients/Tests/exports.stable.out > monetdb5/mal/mal_authorize.c > monetdb5/mal/mal_authorize.h > Branch: Jan2014 > Log Message: > > AUTHinitTables: allow passing a password for monetdb user > > This allows to securely setup a new database. > > > diffs (62 lines): > > diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out > --- a/clients/Tests/exports.stable.out > +++ b/clients/Tests/exports.stable.out > @@ -881,7 +881,7 @@ str AUTHcheckCredentials(oid *ret, Clien > str AUTHgetPasswordHash(str *ret, Client *c, str *username); > str AUTHgetUsername(str *ret, Client *c); > str AUTHgetUsers(BAT **ret, Client *c); > -str AUTHinitTables(void); > +str AUTHinitTables(str *passwd); > str AUTHremoveUser(Client *c, str *username); > str AUTHrequireAdmin(Client *c); > str AUTHresolveUser(str *ret, oid *uid); > diff --git a/monetdb5/mal/mal_authorize.c b/monetdb5/mal/mal_authorize.c > --- a/monetdb5/mal/mal_authorize.c > +++ b/monetdb5/mal/mal_authorize.c > @@ -112,12 +112,13 @@ AUTHcommit(void) > * Localize the authorization tables in the database. The authorization > * tables are a set of aligned BATs that store username, password (hashed) > * and scenario permissions. > - * If the BATs do not exist, they are created, and the monetdb/monetdb > - * administrator account is added. Initialising the authorization tables > - * can only be done after the GDK kernel has been initialized. > + * If the BATs do not exist, they are created, and the monetdb > + * administrator account is added with the given password (or 'monetdb' > + * if NULL). Initialising the authorization tables can only be done > + * after the GDK kernel has been initialized. > */ > str > -AUTHinitTables(void) { > +AUTHinitTables(str *passwd) { > bat bid; > BAT *b; > int isNew = 1; > @@ -168,12 +169,13 @@ AUTHinitTables(void) { > /* insert the monetdb/monetdb administrator account on a > * complete fresh and new auth tables system */ > str user = "monetdb"; > - str pw; /* will become the right hash for "monetdb" */ > - int len = (int) strlen(user); > + str pw = "monetdb"; > oid uid; > Client c = &mal_clients[0]; > > - pw = mcrypt_BackendSum(user /* because user == pass */, len); > + if (passwd != NULL && *passwd != NULL) > + pw = *passwd; > + pw = mcrypt_BackendSum(pw, strlen(pw)); > msg = AUTHaddUser(&uid, &c, &user, &pw); > free(pw); > if (msg) > diff --git a/monetdb5/mal/mal_authorize.h b/monetdb5/mal/mal_authorize.h > --- a/monetdb5/mal/mal_authorize.h > +++ b/monetdb5/mal/mal_authorize.h > @@ -38,7 +38,7 @@ mal_export str AUTHgetUsers(BAT **ret, C > mal_export str AUTHgetPasswordHash(str *ret, Client *c, str *username); > > mal_export str AUTHrequireAdmin(Client *c); > -mal_export str AUTHinitTables(void); > +mal_export str AUTHinitTables(str *passwd); > > > /* > _______________________________________________ > checkin-list mailing list > checkin-list(a)monetdb.org > https://www.monetdb.org/mailman/listinfo/checkin-list > -- Sjoerd Mullender

2 1

Announcement: (Hopefully) Final Release Candidate for Jan2014 Release
by Sjoerd Mullender 10 Feb '14

10 Feb '14

-----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 The MonetDB team at CWI/MonetDB BV is pleased to announce a new release candidate for the upcoming Jan2014 feature release. Many things have changed in this release compared to the previous (Feb2013) release (that's why it took so long). For a smooth upgrade I would urge you to try out this release candidate and to provide feedback on our bug tracker (http://bugs.monetdb.org) A number of bugs in the previous release candidate have been fixed. We hope that this release candidate is good enough to be the new release, but we will give it a few more days. The release candidate can be found at http://dev.monetdb.org/downloads/testing. - -- Sjoerd Mullender -----BEGIN PGP SIGNATURE----- Version: GnuPG v1 Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/ iQCVAwUBUviUtz7g04AjvIQpAQJy5wQArulRMgV7YWX1NoOJMoSSKINjEQLeneZC vj4NqMJ0igxYMRxqBLd7EcSfLKvgDR3ElmP7HhDkXbmCDk6POD4PjxPRmh66AEwY uYBhT3iIEe01PFzqjMhWm2HnDnfDOTzvn8Ai4n1/qCIVmW0p14kIYOkTRXWiNX5o tcGfy384cx8= =2un1 -----END PGP SIGNATURE-----

1 0

Re: MonetDB: Jan2014 - make sure we don't call BATappend(a,b) where ...
by Sjoerd Mullender 05 Feb '14

05 Feb '14

What happens to i after this? It still has a heap base pointer that is pointing into freed memory. On 2014-02-05 20:51, Niels Nes wrote: > Changeset: 262ad3ce2b2e for MonetDB > URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=262ad3ce2b2e > Modified Files: > sql/storage/bat/bat_storage.c > sql/test/BugTracker-2014/Tests/sample-crash.Bug-3429.stable.err > Branch: Jan2014 > Log Message: > > make sure we don't call BATappend(a,b) where b is a VIEW on a. > > > diffs (29 lines): > > diff --git a/sql/storage/bat/bat_storage.c b/sql/storage/bat/bat_storage.c > --- a/sql/storage/bat/bat_storage.c > +++ b/sql/storage/bat/bat_storage.c > @@ -448,7 +448,12 @@ delta_append_bat( sql_delta *bat, BAT *i > bat_destroy(b); > b = temp_descriptor(bat->ibid); > } > - BATappend(b, i, TRUE); > + if (isVIEW(i) && b->batCacheid == ABS(VIEWtparent(i))) { > + BAT *ic = BATcopy(i, TYPE_void, i->ttype, TRUE); > + BATappend(b, ic, TRUE); > + bat_destroy(ic); > + } else > + BATappend(b, i, TRUE); > assert(BUNlast(b) > b->batInserted); > bat_destroy(b); > } > diff --git a/sql/test/BugTracker-2014/Tests/sample-crash.Bug-3429.stable.err b/sql/test/BugTracker-2014/Tests/sample-crash.Bug-3429.stable.err > --- a/sql/test/BugTracker-2014/Tests/sample-crash.Bug-3429.stable.err > +++ b/sql/test/BugTracker-2014/Tests/sample-crash.Bug-3429.stable.err > @@ -28,8 +28,6 @@ stderr of test 'sample-crash.Bug-3429` i > # 20:54:49 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-31548" "--port=33290" > # 20:54:49 > > > -MAPI = (monetdb) /var/tmp/mtest-31984/.s.monetdb.34383 > -QUERY = rollback; > > > > _______________________________________________ > checkin-list mailing list > checkin-list(a)monetdb.org > https://www.monetdb.org/mailman/listinfo/checkin-list > -- Sjoerd Mullender

1 0