Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : /*
14 : * M.L. Kersten
15 : */
16 : #include "monetdb_config.h"
17 : #include "manifold.h"
18 : #include "mal_resolve.h"
19 : #include "mal_builder.h"
20 :
21 : /* The default iterator over known scalar commands.
22 : * It can be less efficient then the vector based implementations,
23 : * but saves quite some hacking in non-essential cases or
24 : * expensive user defined functions.
25 : *
26 : * To keep things simple and reasonably performant we limit the
27 : * implementation to those cases where a single BAT is returned.
28 : * Arguments may be of any type. The MAL signature should be a COMMAND.
29 : *
30 : * The functionality has been extended to also perform the manifold
31 : * over aligned BATs, provided the underlying scalar function carries
32 : * the 'manifold' property.
33 : */
34 :
35 : typedef struct {
36 : BAT *b;
37 : void *first;
38 : void *last;
39 : int size;
40 : int type;
41 : BUN cnt;
42 : BATiter bi;
43 : BUN o;
44 : BUN q;
45 : str *s;
46 : } MULTIarg;
47 :
48 : typedef struct {
49 : Client cntxt;
50 : MalBlkPtr mb;
51 : MalStkPtr stk;
52 : InstrPtr pci;
53 : int fvar, lvar;
54 : MULTIarg *args;
55 : } MULTItask;
56 :
57 :
58 : // Loop through the first BAT
59 : // keep the last error message received
60 : #define MALfcn1(Type) (str (*) (Type *, void *))
61 : #define MALfcn2(Type) (str (*) (Type *, void *, void *))
62 : #define MALfcn3(Type) (str (*) (Type *, void *, void *, void *))
63 : #define MALfcn4(Type) (str (*) (Type *, void *, void *, void *, void *))
64 : #define MALfcn5(Type) (str (*) (Type *, void *, void *, void *, void *, void *))
65 : #define ManifoldLoop(N, Type, ...) \
66 : do { \
67 : Type *v = (Type *) mut->args[0].first; \
68 : for (;;) { \
69 : msg = (*(MALfcn##N(Type) mut->pci->fcn))(v, __VA_ARGS__); \
70 : if (msg) \
71 : break; \
72 : if (++oo == olimit) \
73 : break; \
74 : for (i = mut->fvar; i <= mut->lvar; i++) { \
75 : if (ATOMstorage(mut->args[i].type) == TYPE_void) { \
76 : args[i] = (void *) &mut->args[i].o; \
77 : mut->args[i].o++; \
78 : } else if (mut->args[i].size == 0) { \
79 : ; \
80 : } else if (ATOMstorage(mut->args[i].type) < TYPE_str) { \
81 : args[i] += mut->args[i].size; \
82 : } else if (ATOMvarsized(mut->args[i].type)) { \
83 : mut->args[i].o++; \
84 : mut->args[i].s = (str *) BUNtvar(mut->args[i].bi, mut->args[i].o); \
85 : args[i] = (void *) &mut->args[i].s; \
86 : } else { \
87 : mut->args[i].o++; \
88 : mut->args[i].s = (str *) BUNtloc(mut->args[i].bi, mut->args[i].o); \
89 : args[i] = (void *) &mut->args[i].s; \
90 : } \
91 : } \
92 : v++; \
93 : } \
94 : } while (0)
95 :
96 : // The target BAT tail type determines the result variable
97 : #ifdef HAVE_HGE
98 : #define Manifoldbody_hge(N,...) \
99 : case TYPE_hge: ManifoldLoop(N,hge,__VA_ARGS__); break
100 : #else
101 : #define Manifoldbody_hge(N,...)
102 : #endif
103 : #define Manifoldbody(N,...) \
104 : do { \
105 : switch (ATOMstorage(mut->args[0].b->ttype)) { \
106 : case TYPE_bte: ManifoldLoop(N,bte,__VA_ARGS__); break; \
107 : case TYPE_sht: ManifoldLoop(N,sht,__VA_ARGS__); break; \
108 : case TYPE_int: ManifoldLoop(N,int,__VA_ARGS__); break; \
109 : case TYPE_lng: ManifoldLoop(N,lng,__VA_ARGS__); break; \
110 : Manifoldbody_hge(N,__VA_ARGS__); \
111 : case TYPE_oid: ManifoldLoop(N,oid,__VA_ARGS__); break; \
112 : case TYPE_flt: ManifoldLoop(N,flt,__VA_ARGS__); break; \
113 : case TYPE_dbl: ManifoldLoop(N,dbl,__VA_ARGS__); break; \
114 : case TYPE_uuid: ManifoldLoop(N,uuid,__VA_ARGS__); break; \
115 : case TYPE_str: \
116 : default: { \
117 : for (;;) { \
118 : msg = (*(MALfcn##N(str) mut->pci->fcn))(&y, __VA_ARGS__); \
119 : if (msg) \
120 : break; \
121 : if (bunfastapp(mut->args[0].b, (void*) y) != GDK_SUCCEED) \
122 : goto bunins_failed; \
123 : GDKfree(y); \
124 : y = NULL; \
125 : if (++oo == olimit) \
126 : break; \
127 : for (i = mut->fvar; i <= mut->lvar; i++) { \
128 : if (ATOMstorage(mut->args[i].type) == TYPE_void) { \
129 : args[i] = (void *) &mut->args[i].o; \
130 : mut->args[i].o++; \
131 : } else if(mut->args[i].size == 0) { \
132 : ; \
133 : } else if (ATOMstorage(mut->args[i].type) < TYPE_str) { \
134 : args[i] += mut->args[i].size; \
135 : } else if (ATOMvarsized(mut->args[i].type)) { \
136 : mut->args[i].o++; \
137 : mut->args[i].s = (str *) BUNtvar(mut->args[i].bi, mut->args[i].o); \
138 : args[i] = (void *) &mut->args[i].s; \
139 : } else { \
140 : mut->args[i].o++; \
141 : mut->args[i].s = (str *) BUNtloc(mut->args[i].bi, mut->args[i].o); \
142 : args[i] = (void*) &mut->args[i].s; \
143 : } \
144 : } \
145 : } \
146 : break; \
147 : } \
148 : } \
149 : mut->args[0].b->theap->dirty = true; \
150 : } while (0)
151 :
152 : // single argument is preparatory step for GDK_mapreduce
153 : // Only the last error message is returned, the value of
154 : // an erroneous call depends on the operator itself.
155 : static str
156 802 : MANIFOLDjob(MULTItask *mut)
157 : {
158 802 : int i;
159 802 : char **args;
160 802 : str y = NULL, msg = MAL_SUCCEED;
161 802 : oid oo = 0, olimit = mut->args[mut->fvar].cnt;
162 :
163 802 : if (olimit == 0)
164 : return msg; /* nothing to do */
165 :
166 720 : args = (char **) GDKzalloc(sizeof(char *) * mut->pci->argc);
167 725 : if (args == NULL)
168 0 : throw(MAL, "mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
169 :
170 : // the mod.fcn arguments are ignored from the call
171 2141 : for (i = mut->pci->retc + 2; i < mut->pci->argc; i++) {
172 1429 : if (mut->args[i].b) {
173 952 : if (ATOMstorage(mut->args[i].type) < TYPE_str) {
174 67 : args[i] = (char *) mut->args[i].first;
175 885 : } else if (ATOMvarsized(mut->args[i].type)) {
176 867 : mut->args[i].s = BUNtvar(mut->args[i].bi, mut->args[i].o);
177 867 : args[i] = (void *) &mut->args[i].s;
178 : } else {
179 18 : mut->args[i].s = BUNtloc(mut->args[i].bi, mut->args[i].o);
180 18 : args[i] = (void *) &mut->args[i].s;
181 : }
182 : } else {
183 477 : args[i] = (char *) getArgReference(mut->stk, mut->pci, i);
184 : }
185 : }
186 :
187 : /* TRC_DEBUG(MAL_SERVER, "fvar %d lvar %d type %d\n", mut->fvar,mut->lvar, ATOMstorage(mut->args[mut->fvar].b->ttype)); */
188 :
189 : // use limited argument list expansion.
190 712 : switch (mut->pci->argc) {
191 172 : case 4:
192 220640 : Manifoldbody(1, args[3]);
193 174 : break;
194 419 : case 5:
195 1720 : Manifoldbody(2, args[3], args[4]);
196 425 : break;
197 91 : case 6:
198 57621 : Manifoldbody(3, args[3], args[4], args[5]);
199 92 : break;
200 27 : case 7:
201 474 : Manifoldbody(4, args[3], args[4], args[5], args[6]);
202 27 : break;
203 3 : case 8:
204 45 : Manifoldbody(5, args[3], args[4], args[5], args[6], args[7]);
205 3 : break;
206 0 : default:
207 0 : msg = createException(MAL, "mal.manifold", "manifold call limitation ");
208 : }
209 721 : if (ATOMextern(mut->args[0].type) && y)
210 0 : GDKfree(y);
211 721 : bunins_failed:
212 721 : GDKfree(args);
213 721 : return msg;
214 : }
215 :
216 : /* The manifold optimizer should check for the possibility
217 : * to use this implementation instead of the MAL loop.
218 : */
219 : MALfcn
220 12959 : MANIFOLDtypecheck(Client cntxt, MalBlkPtr mb, InstrPtr pci, int checkprops)
221 : {
222 12959 : int i, k, tpe = 0;
223 12959 : InstrPtr q = 0;
224 12959 : MalBlkPtr nmb;
225 12959 : MALfcn fcn;
226 :
227 12959 : if (mb->errors)
228 : return NULL;
229 12959 : if (getArgType(mb, pci, pci->retc) == TYPE_lng) {
230 : // TODO: trivial case
231 : return NULL;
232 : }
233 :
234 12959 : if (pci->retc > 1 || pci->argc > 8 || getModuleId(pci) == NULL) // limitation on MANIFOLDjob
235 : return NULL;
236 : // We need a private MAL context to resolve the function call
237 12917 : nmb = newMalBlk(2);
238 12928 : if (nmb == NULL) {
239 0 : mb->errors = createException(MAL, "mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
240 0 : return NULL;
241 : }
242 : // the scalar function
243 25851 : q = newStmt(nmb,
244 12928 : getVarConstant(mb, getArg(pci, pci->retc)).val.sval,
245 12928 : getVarConstant(mb, getArg(pci, pci->retc + 1)).val.sval);
246 12923 : if (q == NULL) {
247 0 : goto bailout;
248 : }
249 : // Prepare the single result variable
250 12923 : tpe = getBatType(getArgType(mb, pci, 0));
251 12923 : k = getArg(q, 0);
252 12923 : setVarType(nmb, k, tpe);
253 12923 : if (isVarFixed(nmb, k))
254 0 : setVarFixed(nmb, k);
255 :
256 : // extract their scalar argument type
257 38199 : for (i = pci->retc + 2; i < pci->argc; i++) {
258 25273 : tpe = getBatType(getArgType(mb, pci, i));
259 25273 : k = newTmpVariable(nmb, tpe);
260 25274 : if (k < 0) {
261 0 : freeInstruction(q);
262 0 : goto bailout;
263 : }
264 25274 : q = pushArgument(nmb, q, k);
265 25276 : setVarFixed(nmb, k);
266 : }
267 12926 : pushInstruction(nmb, q);
268 :
269 12923 : if (nmb->errors)
270 0 : goto bailout;
271 :
272 : /*
273 : TRC_DEBUG(MAL_SERVER, "Manifold operation\n");
274 : traceInstruction(MAL_SERVER, mb, 0, pci, LIST_MAL_ALL);
275 : traceInstruction(MAL_SERVER, nmb, 0, q, LIST_MAL_ALL);
276 : */
277 : // Localize the underlying scalar operator
278 12923 : typeChecker(cntxt->usermodule, nmb, q, getPC(nmb, q), TRUE);
279 12914 : if (nmb->errors)
280 0 : goto bailout;
281 12914 : if (q->fcn == NULL || q->token != CMDcall ||
282 1526 : (checkprops && q->blk && q->blk->unsafeProp))
283 : fcn = NULL;
284 : else {
285 3855 : fcn = q->fcn;
286 : // retain the type detected
287 3855 : if (!isVarFixed(mb, getArg(pci, 0)))
288 0 : setVarType(mb, getArg(pci, 0), newBatType(getArgType(nmb, q, 0)));
289 : }
290 :
291 : /*
292 : TRC_DEBUG(MAL_SERVER, "Success? %s\n", (fcn == NULL? "no":"yes"));
293 : traceInstruction(MAL_SERVER, nmb, 0, q, LIST_MAL_ALL);
294 : */
295 :
296 12914 : freeMalBlk(nmb);
297 12914 : return fcn;
298 :
299 0 : bailout:
300 : /* there was an error, perhaps it's in nmb->errors */
301 0 : assert(mb->errors == NULL);
302 0 : if ((mb->errors = nmb->errors) == NULL)
303 0 : mb->errors = createException(MAL, "mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
304 0 : nmb->errors = NULL;
305 0 : freeMalBlk(nmb);
306 0 : return NULL;
307 : }
308 :
309 : /*
310 : * The manifold should support aligned BATs as well
311 : */
312 : static str
313 799 : MANIFOLDevaluate(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
314 : {
315 799 : MULTItask mut;
316 799 : MULTIarg *mat;
317 799 : int i, tpe = 0;
318 799 : BUN cnt = 0;
319 799 : oid o = 0;
320 799 : str msg = MAL_SUCCEED;
321 799 : MALfcn fcn;
322 :
323 799 : fcn = MANIFOLDtypecheck(cntxt, mb, pci, 0);
324 808 : if (fcn == NULL) {
325 0 : if (mb->errors) {
326 0 : msg = mb->errors;
327 0 : mb->errors = NULL;
328 0 : return msg;
329 : }
330 0 : throw(MAL, "mal.manifold", "Illegal manifold function call");
331 : }
332 :
333 808 : mat = (MULTIarg *) GDKzalloc(sizeof(MULTIarg) * pci->argc);
334 808 : if (mat == NULL)
335 0 : throw(MAL, "mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
336 :
337 : // mr-job structure preparation
338 808 : mut = (MULTItask) {
339 : .cntxt = cntxt,
340 : .mb = mb,
341 : .stk = stk,
342 : .args = mat,
343 : .pci = pci,
344 : };
345 :
346 : // prepare iterators
347 2360 : for (i = pci->retc + 2; i < pci->argc; i++) {
348 1557 : if (isaBatType(getArgType(mb, pci, i))) {
349 1058 : mat[i].b = BATdescriptor(*getArgReference_bat(stk, pci, i));
350 1057 : if (mat[i].b == NULL) {
351 0 : msg = createException(MAL, "mal.manifold",
352 : SQLSTATE(HY013) MAL_MALLOC_FAIL);
353 0 : goto wrapup;
354 : }
355 1057 : mat[i].bi = bat_iterator(mat[i].b);
356 1054 : mat[i].type = tpe = getBatType(getArgType(mb, pci, i));
357 1054 : if (mut.fvar == 0) {
358 808 : mut.fvar = i;
359 808 : cnt = BATcount(mat[i].b);
360 246 : } else if (BATcount(mat[i].b) != cnt) {
361 0 : msg = createException(MAL, "mal.manifold",
362 : "Columns must be of same length");
363 0 : goto wrapup;
364 : }
365 1054 : mut.lvar = i;
366 1054 : mat[i].size = mat[i].bi.width;
367 1054 : mat[i].cnt = cnt;
368 1054 : if (mat[i].b->ttype == TYPE_void) {
369 0 : o = mat[i].b->tseqbase;
370 0 : mat[i].first = mat[i].last = (void *) &o;
371 : } else {
372 1054 : mat[i].first = (void *) mat[i].bi.base;
373 1054 : mat[i].last = (void *) ((char *) mat[i].bi.base + (BATcount(mat[i].b) << mat[i].bi.shift));
374 : }
375 1054 : mat[i].o = 0;
376 1054 : mat[i].q = BATcount(mat[i].b);
377 : } else {
378 499 : mat[i].last = mat[i].first = (void *) getArgReference(stk, pci, i);
379 498 : mat[i].type = getArgType(mb, pci, i);
380 : }
381 : }
382 :
383 : // Then iterator over all BATs
384 803 : if (mut.fvar == 0) {
385 0 : msg = createException(MAL, "mal.manifold",
386 : "At least one column required");
387 0 : goto wrapup;
388 : }
389 : // prepare result variable
390 1604 : mat[0].b = COLnew(mat[mut.fvar].b->hseqbase,
391 803 : getBatType(getArgType(mb, pci, 0)), cnt, TRANSIENT);
392 801 : if (mat[0].b == NULL) {
393 0 : msg = createException(MAL, "mal.manifold",
394 : SQLSTATE(HY013) MAL_MALLOC_FAIL);
395 0 : goto wrapup;
396 : }
397 801 : mat[0].b->tnonil = false;
398 801 : mat[0].b->tsorted = false;
399 801 : mat[0].b->trevsorted = false;
400 801 : mat[0].b->tkey = false;
401 801 : mat[0].bi = (BATiter) {.b = NULL, };
402 801 : mat[0].first = (void *) Tloc(mat[0].b, 0);
403 801 : mat[0].last = (void *) Tloc(mat[0].b, BATcount(mat[0].b));
404 :
405 801 : mut.pci = copyInstruction(pci);
406 807 : if (mut.pci == NULL) {
407 0 : msg = createException(MAL, "mal.manifold",
408 : SQLSTATE(HY013) MAL_MALLOC_FAIL);
409 0 : goto wrapup;
410 : }
411 807 : mut.pci->fcn = fcn;
412 807 : msg = MANIFOLDjob(&mut);
413 801 : freeInstruction(mut.pci);
414 :
415 806 : wrapup:
416 : // restore the argument types
417 3969 : for (i = pci->retc; i < pci->argc; i++) {
418 3160 : if (mat[i].b) {
419 1055 : bat_iterator_end(&mat[i].bi);
420 1058 : BBPunfix(mat[i].b->batCacheid);
421 : }
422 : }
423 809 : if (msg) {
424 6 : BBPreclaim(mat[0].b);
425 803 : } else if (!msg) {
426 : // consolidate the properties
427 803 : if (ATOMstorage(mat[0].b->ttype) < TYPE_str)
428 507 : BATsetcount(mat[0].b, cnt);
429 804 : BATsettrivprop(mat[0].b);
430 787 : *getArgReference_bat(stk, pci, 0) = mat[0].b->batCacheid;
431 787 : BBPkeepref(mat[0].b);
432 : }
433 793 : GDKfree(mat);
434 809 : return msg;
435 : }
436 :
437 : // The old code
438 : static str
439 1 : MANIFOLDremapMultiplex(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
440 : {
441 1 : (void) mb;
442 1 : (void) cntxt;
443 4 : throw(MAL, "mal.multiplex", "Function '%s.%s' not defined",
444 1 : *getArgReference_str(stk, p, p->retc),
445 1 : *getArgReference_str(stk, p, p->retc + 1));
446 : }
447 :
448 : #include "mel.h"
449 : mel_func manifold_init_funcs[] = {
450 : pattern("mal", "multiplex", MANIFOLDremapMultiplex, false, "", args(1,4, varargany("",0),arg("mod",str),arg("fcn",str),varargany("a",0))),
451 : pattern("mal", "multiplex", MANIFOLDremapMultiplex, false, "", args(1,4, varargany("",0),arg("card", lng), arg("mod",str),arg("fcn",str))),
452 : pattern("mal", "multiplex", MANIFOLDremapMultiplex, false, "", args(1,5, varargany("",0),arg("card", lng), arg("mod",str),arg("fcn",str),varargany("a",0))),
453 : pattern("batmal", "multiplex", MANIFOLDremapMultiplex, false, "", args(1,4, varargany("",0),arg("mod",str),arg("fcn",str),varargany("a",0))),
454 : pattern("mal", "manifold", MANIFOLDevaluate, false, "", args(1,4, batargany("",0),arg("mod",str),arg("fcn",str),varargany("a",0))),
455 : { .imp=NULL }
456 : };
457 : #include "mal_import.h"
458 : #ifdef _MSC_VER
459 : #undef read
460 : #pragma section(".CRT$XCU",read)
461 : #endif
462 345 : LIB_STARTUP_FUNC(init_manifold_mal)
463 345 : { mal_module("manifold", NULL, manifold_init_funcs); }
|
