Line data    Source code

       1             : /*
       2             :  * SPDX-License-Identifier: MPL-2.0
       3             :  *
       4             :  * This Source Code Form is subject to the terms of the Mozilla Public
       5             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       6             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       7             :  *
       8             :  * Copyright 2024, 2025 MonetDB Foundation;
       9             :  * Copyright August 2008 - 2023 MonetDB B.V.;
      10             :  * Copyright 1997 - July 2008 CWI.
      11             :  */
      12             : 
      13             : /*
      14             :  * M.L. Kersten
      15             :  */
      16             : #include "monetdb_config.h"
      17             : #include "manifold.h"
      18             : #include "mal_resolve.h"
      19             : #include "mal_builder.h"
      20             : 
      21             : /* The default iterator over known scalar commands.
      22             :  * It can be less efficient then the vector based implementations,
      23             :  * but saves quite some hacking in non-essential cases or
      24             :  * expensive user defined functions.
      25             :  *
      26             :  * To keep things simple and reasonably performant we limit the
      27             :  * implementation to those cases where a single BAT is returned.
      28             :  * Arguments may be of any type. The MAL signature should be a COMMAND.
      29             :  *
      30             :  * The functionality has been extended to also perform the manifold
      31             :  * over aligned BATs, provided the underlying scalar function carries
      32             :  * the 'manifold' property.
      33             :  */
      34             : 
      35             : typedef struct {
      36             :         BAT *b;
      37             :         void *first;
      38             :         void *last;
      39             :         int size;
      40             :         int type;
      41             :         BUN cnt;
      42             :         BATiter bi;
      43             :         BUN o;
      44             :         BUN q;
      45             :         str *s;
      46             : } MULTIarg;
      47             : 
      48             : typedef struct {
      49             :         Client cntxt;
      50             :         MalBlkPtr mb;
      51             :         MalStkPtr stk;
      52             :         InstrPtr pci;
      53             :         int fvar, lvar;
      54             :         MULTIarg *args;
      55             : } MULTItask;
      56             : 
      57             : 
      58             : // Loop through the first BAT
      59             : // keep the last error message received
      60             : #define MALfcn1(Type) (str (*) (Type *, void *))
      61             : #define MALfcn2(Type) (str (*) (Type *, void *, void *))
      62             : #define MALfcn3(Type) (str (*) (Type *, void *, void *, void *))
      63             : #define MALfcn4(Type) (str (*) (Type *, void *, void *, void *, void *))
      64             : #define MALfcn5(Type) (str (*) (Type *, void *, void *, void *, void *, void *))
      65             : #define ManifoldLoop(N, Type, ...)                                                                              \
      66             :         do {                                                                                                                            \
      67             :                 Type *v = (Type *) mut->args[0].first;                                                       \
      68             :                 for (;;) {                                                                                                              \
      69             :                         msg = (*(MALfcn##N(Type) mut->pci->fcn))(v, __VA_ARGS__); \
      70             :                         if (msg)                                                                                                        \
      71             :                                 break;                                                                                                  \
      72             :                         if (++oo == olimit)                                                                                     \
      73             :                                 break;                                                                                                  \
      74             :                         for (i = mut->fvar; i <= mut->lvar; i++) {                                     \
      75             :                                 if (ATOMstorage(mut->args[i].type) == TYPE_void) {           \
      76             :                                         args[i] = (void *) &mut->args[i].o;                                      \
      77             :                                         mut->args[i].o++;                                                                    \
      78             :                                 } else if (mut->args[i].size == 0) {                                 \
      79             :                                         ;                                                                                                       \
      80             :                                 } else if (ATOMstorage(mut->args[i].type) < TYPE_str) {   \
      81             :                                         args[i] += mut->args[i].size;                                                \
      82             :                                 } else if (ATOMvarsized(mut->args[i].type)) {                        \
      83             :                                         mut->args[i].o++;                                                                    \
      84             :                                         mut->args[i].s = (str *) BUNtvar(mut->args[i].bi, mut->args[i].o); \
      85             :                                         args[i] = (void *) &mut->args[i].s;                                      \
      86             :                                 } else {                                                                                                \
      87             :                                         mut->args[i].o++;                                                                    \
      88             :                                         mut->args[i].s = (str *) BUNtloc(mut->args[i].bi, mut->args[i].o); \
      89             :                                         args[i] = (void *) &mut->args[i].s;                                      \
      90             :                                 }                                                                                                               \
      91             :                         }                                                                                                                       \
      92             :                         v++;                                                                                                            \
      93             :                 }                                                                                                                               \
      94             :         } while (0)
      95             : 
      96             : // The target BAT tail type determines the result variable
      97             : #ifdef HAVE_HGE
      98             : #define Manifoldbody_hge(N,...)                                                         \
      99             :         case TYPE_hge: ManifoldLoop(N,hge,__VA_ARGS__); break
     100             : #else
     101             : #define Manifoldbody_hge(N,...)
     102             : #endif
     103             : #define Manifoldbody(N,...)                                                                                             \
     104             :         do {                                                                                                                            \
     105             :                 switch (ATOMstorage(mut->args[0].b->ttype)) {                                     \
     106             :                 case TYPE_bte: ManifoldLoop(N,bte,__VA_ARGS__); break;                  \
     107             :                 case TYPE_sht: ManifoldLoop(N,sht,__VA_ARGS__); break;                  \
     108             :                 case TYPE_int: ManifoldLoop(N,int,__VA_ARGS__); break;                  \
     109             :                 case TYPE_lng: ManifoldLoop(N,lng,__VA_ARGS__); break;                  \
     110             :                 Manifoldbody_hge(N,__VA_ARGS__);                                                                \
     111             :                 case TYPE_oid: ManifoldLoop(N,oid,__VA_ARGS__); break;                  \
     112             :                 case TYPE_flt: ManifoldLoop(N,flt,__VA_ARGS__); break;                  \
     113             :                 case TYPE_dbl: ManifoldLoop(N,dbl,__VA_ARGS__); break;                  \
     114             :                 case TYPE_uuid: ManifoldLoop(N,uuid,__VA_ARGS__); break;                \
     115             :                 case TYPE_str:                                                                                                  \
     116             :                 default: {                                                                                                              \
     117             :                         for (;;) {                                                                                                      \
     118             :                             msg = (*(MALfcn##N(str) mut->pci->fcn))(&y, __VA_ARGS__); \
     119             :                                 if (msg)                                                                                                \
     120             :                                         break;                                                                                          \
     121             :                                 if (bunfastapp(mut->args[0].b, (void*) y) != GDK_SUCCEED) \
     122             :                                         goto bunins_failed;                                                                     \
     123             :                                 GDKfree(y);                                                                                             \
     124             :                                 y = NULL;                                                                                               \
     125             :                                 if (++oo == olimit)                                                                             \
     126             :                                         break;                                                                                          \
     127             :                                 for (i = mut->fvar; i <= mut->lvar; i++) {                             \
     128             :                                         if (ATOMstorage(mut->args[i].type) == TYPE_void) {   \
     129             :                                                 args[i] = (void *) &mut->args[i].o;                              \
     130             :                                                 mut->args[i].o++;                                                            \
     131             :                                         } else if(mut->args[i].size == 0) {                                  \
     132             :                                                 ;                                                                                               \
     133             :                                         } else if (ATOMstorage(mut->args[i].type) < TYPE_str) { \
     134             :                                                 args[i] += mut->args[i].size;                                        \
     135             :                                         } else if (ATOMvarsized(mut->args[i].type)) {                \
     136             :                                                 mut->args[i].o++;                                                            \
     137             :                                                 mut->args[i].s = (str *) BUNtvar(mut->args[i].bi, mut->args[i].o); \
     138             :                                                 args[i] = (void *) &mut->args[i].s;                              \
     139             :                                         } else {                                                                                        \
     140             :                                                 mut->args[i].o++;                                                            \
     141             :                                                 mut->args[i].s = (str *) BUNtloc(mut->args[i].bi, mut->args[i].o); \
     142             :                                                 args[i] = (void*) &mut->args[i].s;                               \
     143             :                                         }                                                                                                       \
     144             :                                 }                                                                                                               \
     145             :                         }                                                                                                                       \
     146             :                         break;                                                                                                          \
     147             :                 }                                                                                                                               \
     148             :                 }                                                                                                                               \
     149             :                 mut->args[0].b->theap->dirty = true;                                                   \
     150             :         } while (0)
     151             : 
     152             : // single argument is preparatory step for GDK_mapreduce
     153             : // Only the last error message is returned, the value of
     154             : // an erroneous call depends on the operator itself.
     155             : static str
     156         674 : MANIFOLDjob(MULTItask *mut)
     157             : {
     158         674 :         int i;
     159         674 :         char **args;
     160         674 :         str y = NULL, msg = MAL_SUCCEED;
     161         674 :         oid oo = 0, olimit = mut->args[mut->fvar].cnt;
     162             : 
     163         674 :         if (olimit == 0)
     164             :                 return msg;                             /* nothing to do */
     165             : 
     166         615 :         args = (char **) GDKzalloc(sizeof(char *) * mut->pci->argc);
     167         615 :         if (args == NULL)
     168           0 :                 throw(MAL, "mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     169             : 
     170             :         // the mod.fcn arguments are ignored from the call
     171        1850 :         for (i = mut->pci->retc + 2; i < mut->pci->argc; i++) {
     172        1235 :                 if (mut->args[i].b) {
     173         818 :                         if (ATOMstorage(mut->args[i].type) < TYPE_str) {
     174          60 :                                 args[i] = (char *) mut->args[i].first;
     175         758 :                         } else if (ATOMvarsized(mut->args[i].type)) {
     176         740 :                                 mut->args[i].s = BUNtvar(mut->args[i].bi, mut->args[i].o);
     177         740 :                                 args[i] = (void *) &mut->args[i].s;
     178             :                         } else {
     179          18 :                                 mut->args[i].s = BUNtloc(mut->args[i].bi, mut->args[i].o);
     180          18 :                                 args[i] = (void *) &mut->args[i].s;
     181             :                         }
     182             :                 } else {
     183         417 :                         args[i] = (char *) getArgReference(mut->stk, mut->pci, i);
     184             :                 }
     185             :         }
     186             : 
     187             :         /* TRC_DEBUG(MAL_SERVER, "fvar %d lvar %d type %d\n", mut->fvar,mut->lvar, ATOMstorage(mut->args[mut->fvar].b->ttype)); */
     188             : 
     189             :         // use limited argument list expansion.
     190         615 :         switch (mut->pci->argc) {
     191         145 :         case 4:
     192      220673 :                 Manifoldbody(1, args[3]);
     193         145 :                 break;
     194         352 :         case 5:
     195        1864 :                 Manifoldbody(2, args[3], args[4]);
     196         352 :                 break;
     197          89 :         case 6:
     198       59837 :                 Manifoldbody(3, args[3], args[4], args[5]);
     199          89 :                 break;
     200          26 :         case 7:
     201         476 :                 Manifoldbody(4, args[3], args[4], args[5], args[6]);
     202          26 :                 break;
     203           3 :         case 8:
     204          45 :                 Manifoldbody(5, args[3], args[4], args[5], args[6], args[7]);
     205           3 :                 break;
     206           0 :         default:
     207           0 :                 msg = createException(MAL, "mal.manifold", "manifold call limitation ");
     208             :         }
     209         615 :         if (ATOMextern(mut->args[0].type) && y)
     210           0 :                 GDKfree(y);
     211         615 :   bunins_failed:
     212         615 :         GDKfree(args);
     213         615 :         return msg;
     214             : }
     215             : 
     216             : /* The manifold optimizer should check for the possibility
     217             :  * to use this implementation instead of the MAL loop.
     218             :  */
     219             : MALfcn
     220        8694 : MANIFOLDtypecheck(Client cntxt, MalBlkPtr mb, InstrPtr pci, int checkprops)
     221             : {
     222        8694 :         int i, k, tpe = 0;
     223        8694 :         InstrPtr q = 0;
     224        8694 :         MalBlkPtr nmb;
     225        8694 :         MALfcn fcn;
     226             : 
     227        8694 :         if (mb->errors)
     228             :                 return NULL;
     229        8694 :         if (getArgType(mb, pci, pci->retc) == TYPE_lng) {
     230             :                 // TODO: trivial case
     231             :                 return NULL;
     232             :         }
     233             : 
     234        8694 :         if (pci->retc > 1 || pci->argc > 8 || getModuleId(pci) == NULL)     // limitation on MANIFOLDjob
     235             :                 return NULL;
     236             :         // We need a private MAL context to resolve the function call
     237        8652 :         nmb = newMalBlk(2);
     238        8652 :         if (nmb == NULL) {
     239           0 :                 mb->errors = createException(MAL, "mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     240           0 :                 return NULL;
     241             :         }
     242             :         // the scalar function
     243       17304 :         q = newStmt(nmb,
     244        8652 :                                 getVarConstant(mb, getArg(pci, pci->retc)).val.sval,
     245        8652 :                                 getVarConstant(mb, getArg(pci, pci->retc + 1)).val.sval);
     246        8652 :         if (q == NULL) {
     247           0 :                 goto bailout;
     248             :         }
     249             :         // Prepare the single result variable
     250        8652 :         tpe = getBatType(getArgType(mb, pci, 0));
     251        8652 :         k = getArg(q, 0);
     252        8652 :         setVarType(nmb, k, tpe);
     253        8652 :         if (isVarFixed(nmb, k))
     254           0 :                 setVarFixed(nmb, k);
     255             : 
     256             :         // extract their scalar argument type
     257       25808 :         for (i = pci->retc + 2; i < pci->argc; i++) {
     258       17156 :                 tpe = getBatType(getArgType(mb, pci, i));
     259       17156 :                 k = newTmpVariable(nmb, tpe);
     260       17156 :                 if (k < 0) {
     261           0 :                         freeInstruction(q);
     262           0 :                         goto bailout;
     263             :                 }
     264       17156 :                 q = pushArgument(nmb, q, k);
     265       17156 :                 setVarFixed(nmb, k);
     266             :         }
     267        8652 :         pushInstruction(nmb, q);
     268             : 
     269        8652 :         if (nmb->errors)
     270           0 :                 goto bailout;
     271             : 
     272             :         // Localize the underlying scalar operator
     273        8652 :         typeChecker(cntxt->usermodule, nmb, q, getPC(nmb, q), TRUE);
     274        8652 :         if (nmb->errors)
     275           0 :                 goto bailout;
     276        8652 :         if (q->fcn == NULL || q->token != CMDcall ||
     277        1256 :                 (checkprops && q->blk && q->blk->unsafeProp))
     278             :                 fcn = NULL;
     279             :         else {
     280        3190 :                 fcn = q->fcn;
     281             :                 // retain the type detected
     282        3190 :                 if (!isVarFixed(mb, getArg(pci, 0)))
     283           0 :                         setVarType(mb, getArg(pci, 0), newBatType(getArgType(nmb, q, 0)));
     284             :         }
     285             : 
     286        8652 :         freeMalBlk(nmb);
     287        8652 :         return fcn;
     288             : 
     289           0 :   bailout:
     290             :         /* there was an error, perhaps it's in nmb->errors */
     291           0 :         assert(mb->errors == NULL);
     292           0 :         if ((mb->errors = nmb->errors) == NULL)
     293           0 :                 mb->errors = createException(MAL, "mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     294           0 :         nmb->errors = NULL;
     295           0 :         freeMalBlk(nmb);
     296           0 :         return NULL;
     297             : }
     298             : 
     299             : /*
     300             :  * The manifold should support aligned BATs as well
     301             :  */
     302             : static str
     303         674 : MANIFOLDevaluate(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
     304             : {
     305         674 :         MULTItask mut;
     306         674 :         MULTIarg *mat;
     307         674 :         int i, tpe = 0;
     308         674 :         BUN cnt = 0;
     309         674 :         oid o = 0;
     310         674 :         str msg = MAL_SUCCEED;
     311         674 :         MALfcn fcn;
     312             : 
     313         674 :         fcn = MANIFOLDtypecheck(cntxt, mb, pci, 0);
     314         674 :         if (fcn == NULL) {
     315           0 :                 if (mb->errors) {
     316           0 :                         msg = mb->errors;
     317           0 :                         mb->errors = NULL;
     318           0 :                         return msg;
     319             :                 }
     320           0 :                 throw(MAL, "mal.manifold", "Illegal manifold function call");
     321             :         }
     322             : 
     323         674 :         mat = (MULTIarg *) GDKzalloc(sizeof(MULTIarg) * pci->argc);
     324         674 :         if (mat == NULL)
     325           0 :                 throw(MAL, "mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     326             : 
     327             :         // mr-job structure preparation
     328         674 :         mut = (MULTItask) {
     329             :                 .cntxt = cntxt,
     330             :                 .mb = mb,
     331             :                 .stk = stk,
     332             :                 .args = mat,
     333             :                 .pci = pci,
     334             :         };
     335             : 
     336             :         // prepare iterators
     337        2002 :         for (i = pci->retc + 2; i < pci->argc; i++) {
     338        1328 :                 if (isaBatType(getArgType(mb, pci, i))) {
     339         894 :                         mat[i].b = BATdescriptor(*getArgReference_bat(stk, pci, i));
     340         894 :                         if (mat[i].b == NULL) {
     341           0 :                                 msg = createException(MAL, "mal.manifold",
     342             :                                                                           SQLSTATE(HY013) MAL_MALLOC_FAIL);
     343           0 :                                 goto wrapup;
     344             :                         }
     345         894 :                         mat[i].bi = bat_iterator(mat[i].b);
     346         894 :                         mat[i].type = tpe = getBatType(getArgType(mb, pci, i));
     347         894 :                         if (mut.fvar == 0) {
     348         674 :                                 mut.fvar = i;
     349         674 :                                 cnt = BATcount(mat[i].b);
     350         220 :                         } else if (BATcount(mat[i].b) != cnt) {
     351           0 :                                 msg = createException(MAL, "mal.manifold",
     352             :                                                                           "Columns must be of same length");
     353           0 :                                 goto wrapup;
     354             :                         }
     355         894 :                         mut.lvar = i;
     356         894 :                         mat[i].size = mat[i].bi.width;
     357         894 :                         mat[i].cnt = cnt;
     358         894 :                         if (mat[i].b->ttype == TYPE_void) {
     359           0 :                                 o = mat[i].b->tseqbase;
     360           0 :                                 mat[i].first = mat[i].last = (void *) &o;
     361             :                         } else {
     362         894 :                                 mat[i].first = (void *) mat[i].bi.base;
     363         894 :                                 mat[i].last = (void *) ((char *) mat[i].bi.base + (BATcount(mat[i].b) << mat[i].bi.shift));
     364             :                         }
     365         894 :                         mat[i].o = 0;
     366         894 :                         mat[i].q = BATcount(mat[i].b);
     367             :                 } else {
     368         434 :                         mat[i].last = mat[i].first = (void *) getArgReference(stk, pci, i);
     369         434 :                         mat[i].type = getArgType(mb, pci, i);
     370             :                 }
     371             :         }
     372             : 
     373             :         // Then iterator over all BATs
     374         674 :         if (mut.fvar == 0) {
     375           0 :                 msg = createException(MAL, "mal.manifold",
     376             :                                                           "At least one column required");
     377           0 :                 goto wrapup;
     378             :         }
     379             :         // prepare result variable
     380        1348 :         mat[0].b = COLnew(mat[mut.fvar].b->hseqbase,
     381         674 :                                           getBatType(getArgType(mb, pci, 0)), cnt, TRANSIENT);
     382         674 :         if (mat[0].b == NULL) {
     383           0 :                 msg = createException(MAL, "mal.manifold",
     384             :                                                           SQLSTATE(HY013) MAL_MALLOC_FAIL);
     385           0 :                 goto wrapup;
     386             :         }
     387         674 :         mat[0].b->tnonil = false;
     388         674 :         mat[0].b->tsorted = false;
     389         674 :         mat[0].b->trevsorted = false;
     390         674 :         mat[0].b->tkey = false;
     391         674 :         mat[0].bi = (BATiter) {.b = NULL, };
     392         674 :         mat[0].first = (void *) Tloc(mat[0].b, 0);
     393         674 :         mat[0].last = (void *) Tloc(mat[0].b, BATcount(mat[0].b));
     394             : 
     395         674 :         mut.pci = copyInstruction(pci);
     396         674 :         if (mut.pci == NULL) {
     397           0 :                 msg = createException(MAL, "mal.manifold",
     398             :                                                           SQLSTATE(HY013) MAL_MALLOC_FAIL);
     399           0 :                 goto wrapup;
     400             :         }
     401         674 :         mut.pci->fcn = fcn;
     402         674 :         msg = MANIFOLDjob(&mut);
     403         674 :         freeInstruction(mut.pci);
     404             : 
     405         674 :   wrapup:
     406             :         // restore the argument types
     407        3350 :         for (i = pci->retc; i < pci->argc; i++) {
     408        2676 :                 if (mat[i].b) {
     409         894 :                         bat_iterator_end(&mat[i].bi);
     410         894 :                         BBPunfix(mat[i].b->batCacheid);
     411             :                 }
     412             :         }
     413         674 :         if (msg) {
     414           6 :                 BBPreclaim(mat[0].b);
     415         668 :         } else if (!msg) {
     416             :                 // consolidate the properties
     417         668 :                 if (ATOMstorage(mat[0].b->ttype) < TYPE_str)
     418         422 :                         BATsetcount(mat[0].b, cnt);
     419         668 :                 BATsettrivprop(mat[0].b);
     420         668 :                 *getArgReference_bat(stk, pci, 0) = mat[0].b->batCacheid;
     421         668 :                 BBPkeepref(mat[0].b);
     422             :         }
     423         674 :         GDKfree(mat);
     424         674 :         return msg;
     425             : }
     426             : 
     427             : // The old code
     428             : static str
     429           1 : MANIFOLDremapMultiplex(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
     430             : {
     431           1 :         (void) mb;
     432           1 :         (void) cntxt;
     433           4 :         throw(MAL, "mal.multiplex", "Function '%s.%s' not defined",
     434           1 :                   *getArgReference_str(stk, p, p->retc),
     435           1 :                   *getArgReference_str(stk, p, p->retc + 1));
     436             : }
     437             : 
     438             : #include "mel.h"
     439             : mel_func manifold_init_funcs[] = {
     440             :  pattern("mal", "multiplex", MANIFOLDremapMultiplex, false, "", args(1,4, varargany("",0),arg("mod",str),arg("fcn",str),varargany("a",0))),
     441             :  pattern("mal", "multiplex", MANIFOLDremapMultiplex, false, "", args(1,4, varargany("",0),arg("card", lng), arg("mod",str),arg("fcn",str))),
     442             :  pattern("mal", "multiplex", MANIFOLDremapMultiplex, false, "", args(1,5, varargany("",0),arg("card", lng), arg("mod",str),arg("fcn",str),varargany("a",0))),
     443             :  pattern("batmal", "multiplex", MANIFOLDremapMultiplex, false, "", args(1,4, varargany("",0),arg("mod",str),arg("fcn",str),varargany("a",0))),
     444             :  pattern("mal", "manifold", MANIFOLDevaluate, false, "", args(1,4, batargany("",0),arg("mod",str),arg("fcn",str),varargany("a",0))),
     445             :  { .imp=NULL }
     446             : };
     447             : #include "mal_import.h"
     448             : #ifdef _MSC_VER
     449             : #undef read
     450             : #pragma section(".CRT$XCU",read)
     451             : #endif
     452         350 : LIB_STARTUP_FUNC(init_manifold_mal)
     453         350 : { mal_module("manifold", NULL, manifold_init_funcs); }