Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : #include "monetdb_config.h"
14 : #include "conversion.h"
15 : #include "convert_loops.h"
16 : #include "pytypes.h"
17 : #include "type_conversion.h"
18 : #include "unicode.h"
19 :
20 : //! Wrapper to get eclass of SQL type
21 : int GetSQLType(sql_subtype *sql_subtype);
22 :
23 : static bool IsBlobType(int type)
24 : {
25 : return type == TYPE_blob;
26 : }
27 :
28 : static bool IsVoidType(int type)
29 : {
30 : return type == TYPE_void;
31 : }
32 :
33 85 : PyObject *PyArrayObject_FromScalar(PyInput *inp, char **return_message)
34 : {
35 85 : PyObject *vararray = NULL;
36 85 : char *msg = NULL;
37 85 : assert(inp->scalar); // input has to be a scalar
38 :
39 85 : switch (inp->bat_type) {
40 0 : case TYPE_void:
41 : #if SIZEOF_OID == SIZEOF_INT
42 : vararray = PyArray_Arange(0, 1, 1, NPY_UINT);
43 : #else
44 0 : vararray = PyArray_Arange(0, 1, 1, NPY_ULONGLONG);
45 : #endif
46 0 : break;
47 0 : case TYPE_oid:
48 0 : vararray = PyLong_FromLong((long)(*(oid *)inp->dataptr));
49 0 : break;
50 0 : case TYPE_bit:
51 0 : vararray = PyLong_FromLong((long)(*(bit *)inp->dataptr));
52 0 : break;
53 0 : case TYPE_bte:
54 0 : vararray = PyLong_FromLong((long)(*(bte *)inp->dataptr));
55 0 : break;
56 0 : case TYPE_sht:
57 0 : vararray = PyLong_FromLong((long)(*(sht *)inp->dataptr));
58 0 : break;
59 63 : case TYPE_int:
60 63 : vararray = PyLong_FromLong((long)(*(int *)inp->dataptr));
61 63 : break;
62 0 : case TYPE_lng:
63 0 : vararray = PyLong_FromLongLong((*(lng *)inp->dataptr));
64 0 : break;
65 0 : case TYPE_flt:
66 0 : vararray = PyFloat_FromDouble((double)(*(flt *)inp->dataptr));
67 0 : break;
68 1 : case TYPE_dbl:
69 1 : vararray = PyFloat_FromDouble((double)(*(dbl *)inp->dataptr));
70 1 : break;
71 : #ifdef HAVE_HGE
72 0 : case TYPE_hge:
73 0 : vararray = PyLong_FromHge(*((hge *)inp->dataptr));
74 0 : break;
75 : #endif
76 1 : case TYPE_date: {
77 1 : USE_DATETIME_API;
78 1 : date dt = *(date *)inp->dataptr;
79 1 : vararray = PyDate_FromDate(date_year(dt), date_month(dt), date_day(dt));
80 : /* error checking */
81 1 : break;
82 : }
83 1 : case TYPE_daytime: {
84 1 : USE_DATETIME_API;
85 1 : daytime dt = *(daytime *)inp->dataptr;
86 1 : vararray = PyTime_FromTime(daytime_hour(dt), daytime_min(dt), daytime_sec(dt), daytime_usec(dt));
87 : /* error checking */
88 1 : break;
89 : }
90 1 : case TYPE_timestamp: {
91 1 : USE_DATETIME_API;
92 1 : timestamp ts = *(timestamp *)inp->dataptr;
93 1 : date dt = timestamp_date(ts);
94 1 : daytime dtm = timestamp_daytime(ts);
95 1 : vararray = PyDateTime_FromDateAndTime(date_year(dt), date_month(dt), date_day(dt), daytime_hour(dtm), daytime_min(dtm), daytime_sec(dtm), daytime_usec(dtm));
96 : /* error checking */
97 1 : break;
98 : }
99 18 : case TYPE_str:
100 18 : vararray = PyUnicode_FromString(*((char **)inp->dataptr));
101 18 : break;
102 0 : default:
103 0 : msg = createException(MAL, "pyapi3.eval",
104 : SQLSTATE(PY000) "Unsupported scalar type %i.", inp->bat_type);
105 0 : goto wrapup;
106 : }
107 85 : if (vararray == NULL) {
108 0 : msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Something went wrong "
109 : "converting the MonetDB "
110 : "scalar to a Python scalar.");
111 0 : goto wrapup;
112 : }
113 85 : wrapup:
114 85 : *return_message = msg;
115 85 : return vararray;
116 : }
117 :
118 : PyObject *
119 339 : PyMaskedArray_FromBAT(PyInput *inp, size_t t_start, size_t t_end, char **return_message, bool copy)
120 : {
121 339 : BAT *b;
122 339 : char *msg;
123 339 : PyObject *vararray = PyArrayObject_FromBAT(inp, t_start, t_end, return_message, copy);
124 :
125 339 : if (vararray == NULL)
126 : return NULL;
127 339 : b = inp->bat;
128 : // To deal with null values, we use the numpy masked array structure
129 : // The masked array structure is an object with two arrays of equal size, a
130 : // data array and a mask array
131 : // The mask array is a boolean array that has the value 'True' when the
132 : // element is NULL, and 'False' otherwise
133 : // if we know for sure that the BAT has no NULL values, we can skip the construction
134 : // of this masked array. Otherwise, we create it.
135 339 : MT_lock_set(&b->theaplock);
136 339 : bool bnonil = b->tnonil;
137 339 : MT_lock_unset(&b->theaplock);
138 339 : if (!bnonil) {
139 39 : PyObject *mod = PyImport_ImportModule("numpy.ma");
140 39 : PyObject *mafunc = PyObject_GetAttrString( mod, "masked_array");
141 39 : PyObject *nullmask = PyNullMask_FromBAT(b, t_start, t_end);
142 :
143 39 : if (!nullmask) {
144 0 : Py_DECREF(vararray);
145 0 : Py_DECREF(mafunc);
146 0 : Py_DECREF(mod);
147 0 : msg = createException(MAL, "pyapi3.eval", "Failed to create mask for some reason");
148 0 : goto wrapup;
149 39 : } else if (nullmask == Py_None) {
150 15 : Py_DECREF(nullmask);
151 : } else {
152 24 : PyObject *maargs = PyTuple_New(2);
153 24 : PyTuple_SetItem(maargs, 0, vararray);
154 24 : PyTuple_SetItem(maargs, 1, nullmask);
155 :
156 : // Now we will actually construct the mask by calling the masked
157 : // array constructor
158 24 : PyObject *mask = PyObject_CallObject(mafunc, maargs);
159 24 : Py_DECREF(maargs);
160 24 : if (!mask) {
161 0 : Py_DECREF(mafunc);
162 0 : Py_DECREF(mod);
163 0 : msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Failed to create mask");
164 0 : goto wrapup;
165 : }
166 : vararray = mask;
167 : }
168 39 : Py_DECREF(mafunc);
169 39 : Py_DECREF(mod);
170 : }
171 : return vararray;
172 0 : wrapup:
173 0 : *return_message = msg;
174 0 : return NULL;
175 : }
176 :
177 : PyObject *
178 339 : PyArrayObject_FromBAT(PyInput *inp, size_t t_start, size_t t_end, char **return_message, bool copy)
179 : {
180 : // This variable will hold the converted Python object
181 339 : PyObject *vararray = NULL;
182 339 : char *msg;
183 339 : size_t j = 0;
184 339 : BUN p = 0, q = 0;
185 339 : BATiter li;
186 339 : BAT *b = inp->bat;
187 339 : npy_intp elements[1] = {t_end - t_start};
188 :
189 339 : assert(!inp->scalar); // input has to be a BAT
190 :
191 339 : if (!b) {
192 : // No BAT was found, we can't do anything in this case
193 0 : msg = createException(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL " bat missing");
194 0 : goto wrapup;
195 : }
196 :
197 665 : if (!IsVoidType(inp->bat_type) && !IsBlobType(inp->bat_type) &&
198 651 : (!IsStandardBATType(inp->bat_type) ||
199 325 : ConvertableSQLType(inp->sql_subtype))) { // if the sql type is set, we
200 : // have to do some conversion
201 4 : if (inp->scalar) {
202 : // FIXME: scalar SQL types
203 0 : msg = createException(
204 : MAL, "pyapi3.eval",
205 : SQLSTATE(0A000) "Scalar SQL types haven't been implemented yet... sorry");
206 0 : goto wrapup;
207 : } else {
208 4 : BAT *ret_bat = NULL;
209 4 : msg = ConvertFromSQLType(b, inp->sql_subtype, &ret_bat, &inp->bat_type);
210 4 : if (msg != MAL_SUCCEED) {
211 0 : freeException(msg);
212 0 : msg = createException(MAL, "pyapi3.eval",
213 : SQLSTATE(PY000) "Failed to convert BAT.");
214 0 : goto wrapup;
215 : }
216 4 : b = ret_bat;
217 : }
218 : }
219 :
220 339 : if (IsBlobType(inp->bat_type)) {
221 2 : PyObject **data;
222 2 : li = bat_iterator(b);
223 2 : vararray = PyArray_EMPTY(1, elements, NPY_OBJECT, 0);
224 2 : data = PyArray_DATA((PyArrayObject *)vararray);
225 8 : BATloop(b, p, q)
226 : {
227 6 : const blob *t = (const blob *)BUNtvar(li, p);
228 6 : if (t->nitems == ~(size_t)0) {
229 0 : data[p] = Py_None;
230 0 : Py_INCREF(Py_None);
231 : } else {
232 6 : data[p] = PyByteArray_FromStringAndSize((char *) t->data, t->nitems);
233 : }
234 : }
235 2 : bat_iterator_end(&li);
236 : } else {
237 337 : switch (inp->bat_type) {
238 11 : case TYPE_void:
239 : #if SIZEOF_OID == SIZEOF_INT
240 : BAT_TO_NP_CREATE_ALWAYS(b, NPY_UINT);
241 : #else
242 11 : BAT_TO_NP_CREATE_ALWAYS(b, NPY_ULONGLONG);
243 : #endif
244 11 : break;
245 19 : case TYPE_oid:
246 : #if SIZEOF_OID == SIZEOF_INT
247 : BAT_TO_NP(b, oid, NPY_UINT32);
248 : #else
249 19 : BAT_TO_NP(b, oid, NPY_UINT64);
250 : #endif
251 19 : break;
252 1 : case TYPE_bit:
253 1 : BAT_TO_NP(b, bit, NPY_INT8);
254 1 : break;
255 3 : case TYPE_bte:
256 3 : BAT_TO_NP(b, bte, NPY_INT8);
257 3 : break;
258 1 : case TYPE_sht:
259 1 : BAT_TO_NP(b, sht, NPY_INT16);
260 1 : break;
261 271 : case TYPE_int:
262 271 : BAT_TO_NP(b, int, NPY_INT32);
263 271 : break;
264 1 : case TYPE_lng:
265 1 : BAT_TO_NP(b, lng, NPY_INT64);
266 1 : break;
267 0 : case TYPE_flt:
268 0 : BAT_TO_NP(b, flt, NPY_FLOAT32);
269 0 : break;
270 16 : case TYPE_dbl:
271 16 : BAT_TO_NP(b, dbl, NPY_FLOAT64);
272 16 : break;
273 2 : case TYPE_date: {
274 2 : li = bat_iterator(b);
275 :
276 2 : USE_DATETIME_API;
277 2 : vararray = PyArray_EMPTY(1, elements, NPY_OBJECT, 0);
278 : {
279 2 : PyObject **data = ((PyObject **)PyArray_DATA((PyArrayObject *)vararray));
280 : // PyObject *obj;
281 2 : j = 0;
282 6 : BATloop(b, p, q)
283 : {
284 4 : date dt = *(const date*)BUNtail(li, p);
285 4 : if (is_date_nil(dt))
286 0 : dt = date_create(1, 1, 1);
287 4 : data[j++] = PyDate_FromDate(date_year(dt), date_month(dt), date_day(dt));
288 : }
289 : }
290 2 : bat_iterator_end(&li);
291 2 : break;
292 : }
293 1 : case TYPE_daytime: {
294 1 : li = bat_iterator(b);
295 :
296 1 : USE_DATETIME_API;
297 1 : vararray = PyArray_EMPTY(1, elements, NPY_OBJECT, 0);
298 : {
299 1 : PyObject **data = ((PyObject **)PyArray_DATA((PyArrayObject *)vararray));
300 : // PyObject *obj;
301 1 : j = 0;
302 2 : BATloop(b, p, q)
303 : {
304 1 : daytime dt = *(const daytime*)BUNtail(li, p);
305 1 : if (is_daytime_nil(dt))
306 0 : dt = daytime_create(0, 0, 0, 0);
307 1 : data[j++] = PyTime_FromTime(daytime_hour(dt),
308 : daytime_min(dt),
309 : daytime_sec(dt),
310 : daytime_usec(dt));
311 : }
312 : }
313 1 : bat_iterator_end(&li);
314 1 : break;
315 : }
316 2 : case TYPE_timestamp: {
317 2 : li = bat_iterator(b);
318 :
319 2 : USE_DATETIME_API;
320 2 : vararray = PyArray_EMPTY(1, elements, NPY_OBJECT, 0);
321 : {
322 2 : PyObject **data = ((PyObject **)PyArray_DATA((PyArrayObject *)vararray));
323 : // PyObject *obj;
324 2 : j = 0;
325 5 : BATloop(b, p, q)
326 : {
327 3 : const timestamp ts = *(const timestamp*)BUNtail(li, p);
328 3 : const date dt = is_timestamp_nil(ts) ? date_create(1, 1, 1) : timestamp_date(ts);
329 3 : const daytime dtm = is_timestamp_nil(ts) ? daytime_create(0, 0, 0, 0) : timestamp_daytime(ts);
330 :
331 3 : data[j++] = PyDateTime_FromDateAndTime(date_year(dt), date_month(dt), date_day(dt), daytime_hour(dtm), daytime_min(dtm), daytime_sec(dtm), daytime_usec(dtm));
332 : }
333 : }
334 2 : bat_iterator_end(&li);
335 2 : break;
336 : }
337 :
338 9 : case TYPE_str: {
339 9 : bool unicode = false;
340 9 : li = bat_iterator(b);
341 : // create a NPY_OBJECT array object
342 9 : vararray = PyArray_New(&PyArray_Type, 1, elements, NPY_OBJECT,
343 : NULL, NULL, 0, 0, NULL);
344 :
345 1100037 : BATloop(b, p, q)
346 : {
347 1100028 : const char *t = (const char *)BUNtvar(li, p);
348 6589029 : for (; *t != 0; t++) {
349 5489001 : if (*t & 0x80) {
350 : unicode = true;
351 : break;
352 : }
353 : }
354 1100028 : if (unicode) {
355 : break;
356 : }
357 : }
358 :
359 : {
360 9 : PyObject **data = ((PyObject **)PyArray_DATA((PyArrayObject *)vararray));
361 9 : PyObject *obj;
362 9 : j = 0;
363 9 : if (unicode) {
364 0 : if (GDK_ELIMDOUBLES(b->tvheap)) {
365 0 : PyObject **pyptrs = GDKzalloc(b->tvheap->free * sizeof(PyObject *));
366 0 : if (!pyptrs) {
367 0 : bat_iterator_end(&li);
368 0 : msg = createException(MAL, "pyapi3.eval",
369 : SQLSTATE(HY013) MAL_MALLOC_FAIL
370 : " PyObject strings.");
371 0 : goto wrapup;
372 : }
373 0 : BATloop(b, p, q)
374 : {
375 0 : const char *t = (const char *)BUNtvar(li, p);
376 0 : ptrdiff_t offset = t - b->tvheap->base;
377 0 : if (!pyptrs[offset]) {
378 0 : if (strNil(t)) {
379 : // str_nil isn't a valid UTF-8 character
380 : // (it's 0x80), so we can't decode it as
381 : // UTF-8 (it will throw an error)
382 0 : pyptrs[offset] = PyUnicode_FromString("-");
383 : } else {
384 : // otherwise we can just decode the
385 : // string as UTF-8
386 0 : pyptrs[offset] = PyUnicode_FromString(t);
387 : }
388 0 : if (!pyptrs[offset]) {
389 0 : bat_iterator_end(&li);
390 0 : msg = createException(
391 : MAL, "pyapi3.eval",
392 : SQLSTATE(PY000) "Failed to create string.");
393 0 : goto wrapup;
394 : }
395 : } else {
396 0 : Py_INCREF(pyptrs[offset]);
397 : }
398 0 : data[j++] = pyptrs[offset];
399 : }
400 0 : GDKfree(pyptrs);
401 : } else {
402 0 : BATloop(b, p, q)
403 : {
404 0 : const char *t = (const char *)BUNtvar(li, p);
405 0 : if (strNil(t)) {
406 : // str_nil isn't a valid UTF-8 character
407 : // (it's 0x80), so we can't decode it as
408 : // UTF-8 (it will throw an error)
409 0 : obj = PyUnicode_FromString("-");
410 : } else {
411 : // otherwise we can just decode the string
412 : // as UTF-8
413 0 : obj = PyUnicode_FromString(t);
414 : }
415 :
416 0 : if (obj == NULL) {
417 0 : bat_iterator_end(&li);
418 0 : msg = createException(
419 : MAL, "pyapi3.eval",
420 : SQLSTATE(PY000) "Failed to create string.");
421 0 : goto wrapup;
422 : }
423 0 : data[j++] = obj;
424 : }
425 : }
426 : } else {
427 : /* special case where we exploit the
428 : * duplicate-eliminated string heap */
429 9 : if (GDK_ELIMDOUBLES(b->tvheap)) {
430 8 : PyObject **pyptrs = GDKzalloc(b->tvheap->free * sizeof(PyObject *));
431 8 : if (!pyptrs) {
432 0 : bat_iterator_end(&li);
433 0 : msg = createException(MAL, "pyapi3.eval",
434 : SQLSTATE(HY013) MAL_MALLOC_FAIL
435 : " PyObject strings.");
436 0 : goto wrapup;
437 : }
438 1000036 : BATloop(b, p, q)
439 : {
440 1000028 : const char *t = (const char *)BUNtvar(li, p);
441 1000028 : ptrdiff_t offset = t - b->tvheap->base;
442 1000028 : if (!pyptrs[offset]) {
443 21 : pyptrs[offset] = PyUnicode_FromString(t);
444 : } else {
445 1000007 : Py_INCREF(pyptrs[offset]);
446 : }
447 1000028 : data[j++] = pyptrs[offset];
448 : }
449 8 : GDKfree(pyptrs);
450 : } else {
451 100001 : BATloop(b, p, q)
452 : {
453 100000 : const char *t = (const char *)BUNtvar(li, p);
454 100000 : obj = PyUnicode_FromString(t);
455 100000 : if (obj == NULL) {
456 0 : bat_iterator_end(&li);
457 0 : msg = createException(
458 : MAL, "pyapi3.eval",
459 : SQLSTATE(PY000) "Failed to create string.");
460 0 : goto wrapup;
461 : }
462 100000 : data[j++] = obj;
463 : }
464 : }
465 : }
466 : }
467 9 : bat_iterator_end(&li);
468 9 : break;
469 : }
470 : #ifdef HAVE_HGE
471 0 : case TYPE_hge: {
472 : // create a NPY_FLOAT64 array to hold the huge type
473 0 : vararray = PyArray_New(&PyArray_Type, 1,
474 0 : (npy_intp[1]){t_end - t_start},
475 : NPY_FLOAT64, NULL, NULL, 0, 0, NULL);
476 :
477 0 : j = 0;
478 0 : npy_float64 *data = (npy_float64 *)PyArray_DATA((PyArrayObject *)vararray);
479 0 : BATiter bi = bat_iterator(b);
480 0 : const hge *vals = (const hge *) bi.base;
481 0 : BATloop(b, p, q) {
482 0 : data[j++] = (npy_float64)vals[p];
483 : }
484 0 : bat_iterator_end(&bi);
485 0 : break;
486 : }
487 : #endif
488 0 : default:
489 0 : if (!inp->sql_subtype || !inp->sql_subtype->type) {
490 0 : msg = createException(MAL, "pyapi3.eval",
491 : SQLSTATE(PY000) "unknown argument type");
492 : } else {
493 0 : msg = createException(MAL, "pyapi3.eval",
494 : SQLSTATE(PY000) "Unsupported SQL Type: %s",
495 : inp->sql_subtype->type->base.name);
496 : }
497 0 : goto wrapup;
498 : }
499 : }
500 :
501 339 : if (vararray == NULL) {
502 0 : msg = createException(MAL, "pyapi3.eval",
503 : SQLSTATE(PY000) "Failed to convert BAT to Numpy array.");
504 0 : goto wrapup;
505 : }
506 339 : if (b != inp->bat)
507 4 : inp->conv_bat = b; /* still in use, free later */
508 : return vararray;
509 0 : wrapup:
510 0 : *return_message = msg;
511 0 : if (b != inp->bat)
512 0 : BBPunfix(b->batCacheid);
513 : return NULL;
514 : }
515 :
516 : #define CreateNullMask(tpe) \
517 : { \
518 : tpe *bat_ptr = (tpe *)b->theap->base + b->tbaseoff; \
519 : for (j = 0; j < count; j++) { \
520 : mask_data[j] = is_##tpe##_nil(bat_ptr[j]); \
521 : found_nil |= mask_data[j]; \
522 : } \
523 : }
524 :
525 : PyObject *
526 39 : PyNullMask_FromBAT(BAT *b, size_t t_start, size_t t_end)
527 : {
528 : // We will now construct the Masked array, we start by setting everything to
529 : // False
530 39 : size_t count = t_end - t_start;
531 39 : npy_intp elements[1] = {count};
532 39 : PyArrayObject *nullmask = (PyArrayObject *)PyArray_EMPTY(1, elements, NPY_BOOL, 0);
533 39 : const void *nil = ATOMnilptr(b->ttype);
534 39 : size_t j;
535 39 : bool found_nil = false;
536 39 : BATiter bi = bat_iterator(b);
537 39 : bool *mask_data = (bool *)PyArray_DATA(nullmask);
538 :
539 77 : switch (ATOMbasetype(getBatType(b->ttype))) {
540 0 : case TYPE_bit:
541 0 : CreateNullMask(bit);
542 : break;
543 0 : case TYPE_bte:
544 0 : CreateNullMask(bte);
545 : break;
546 1 : case TYPE_sht:
547 100001 : CreateNullMask(sht);
548 : break;
549 32 : case TYPE_int:
550 435079 : CreateNullMask(int);
551 : break;
552 2 : case TYPE_lng:
553 100004 : CreateNullMask(lng);
554 : break;
555 0 : case TYPE_flt:
556 0 : CreateNullMask(flt);
557 : break;
558 4 : case TYPE_dbl:
559 200018 : CreateNullMask(dbl);
560 : break;
561 : #ifdef HAVE_HGE
562 0 : case TYPE_hge:
563 0 : CreateNullMask(hge);
564 : break;
565 : #endif
566 0 : default: {
567 0 : int (*atomcmp)(const void *, const void *) = ATOMcompare(b->ttype);
568 0 : for (j = 0; j < count; j++) {
569 0 : mask_data[j] = (*atomcmp)(BUNtail(bi, (BUN)(j)), nil) == 0;
570 0 : found_nil |= mask_data[j];
571 : }
572 : }
573 : }
574 39 : bat_iterator_end(&bi);
575 :
576 39 : if (!found_nil) {
577 15 : Py_DECREF(nullmask);
578 15 : Py_RETURN_NONE;
579 : }
580 :
581 : return (PyObject *)nullmask;
582 : }
583 :
584 : PyObject *
585 41 : PyDict_CheckForConversion(PyObject *pResult, int expected_columns, char **retcol_names, char **return_message)
586 : {
587 41 : char *msg = MAL_SUCCEED;
588 41 : PyObject *result = PyList_New(expected_columns);
589 41 : int i;
590 :
591 181 : for (i = 0; i < expected_columns; i++) {
592 102 : PyObject *object = PyDict_GetItemString(pResult, retcol_names[i]);
593 102 : if (object == NULL) {
594 2 : msg = createException(MAL, "pyapi3.eval",
595 : SQLSTATE(PY000) "Expected a return value with name \"%s\", but "
596 : "this key was not present in the dictionary.",
597 : retcol_names[i]);
598 2 : goto wrapup;
599 : }
600 100 : Py_INCREF(object);
601 100 : object = PyObject_CheckForConversion(object, 1, NULL, return_message);
602 100 : if (object == NULL) {
603 1 : msg = createException(
604 : MAL, "pyapi3.eval",
605 : SQLSTATE(PY000) "Error converting dict return value \"%s\": %s",
606 : retcol_names[i], getExceptionMessage(*return_message));
607 1 : freeException(*return_message);
608 1 : goto wrapup;
609 : }
610 99 : if (PyList_CheckExact(object)) {
611 99 : PyObject *item = PyList_GetItem(object, 0);
612 99 : PyList_SetItem(result, i, item);
613 99 : Py_INCREF(item);
614 99 : Py_DECREF(object);
615 : } else {
616 0 : if (object)
617 0 : Py_DECREF(object);
618 0 : msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Why is this not a list?");
619 0 : goto wrapup;
620 : }
621 : }
622 38 : Py_DECREF(pResult);
623 38 : return result;
624 3 : wrapup:
625 3 : *return_message = msg;
626 3 : Py_DECREF(result);
627 3 : Py_DECREF(pResult);
628 3 : return NULL;
629 : }
630 :
631 : PyObject *
632 212 : PyObject_CheckForConversion(PyObject *pResult, int expected_columns, int *actual_columns, char **return_message)
633 : {
634 212 : char *msg;
635 212 : int columns = 0;
636 :
637 212 : if (pResult) {
638 212 : PyObject *pColO = NULL;
639 212 : if (PyType_IsPandasDataFrame(pResult)) {
640 : // the result object is a Pandas data frame
641 : // we can convert the pandas data frame to a numpy array by simply
642 : // accessing the "values" field (as pandas dataframes are numpy
643 : // arrays internally)
644 0 : PyObject *nResult = PyObject_GetAttrString(pResult, "values");
645 0 : Py_DECREF(pResult);
646 0 : if (nResult == NULL) {
647 0 : msg = createException(MAL, "pyapi3.eval",
648 : SQLSTATE(PY000) "Invalid Pandas data frame.");
649 0 : goto wrapup;
650 : }
651 : // we transpose the values field so it's aligned correctly for our
652 : // purposes
653 0 : pResult = PyObject_GetAttrString(nResult, "T");
654 0 : Py_DECREF(nResult);
655 0 : if (pResult == NULL) {
656 0 : msg = createException(MAL, "pyapi3.eval",
657 : SQLSTATE(PY000) "Invalid Pandas data frame.");
658 0 : goto wrapup;
659 : }
660 : }
661 :
662 212 : if (PyType_IsPyScalar(pResult)) { // check if the return object is a scalar
663 68 : if (expected_columns == 1 || expected_columns <= 0) {
664 : // if we only expect a single return value, we can accept
665 : // scalars by converting it into an array holding an array
666 : // holding the element (i.e. [[pResult]])
667 68 : PyObject *list = PyList_New(1);
668 68 : PyList_SetItem(list, 0, pResult);
669 68 : pResult = list;
670 :
671 68 : list = PyList_New(1);
672 68 : PyList_SetItem(list, 0, pResult);
673 68 : pResult = list;
674 :
675 68 : columns = 1;
676 : } else {
677 : // the result object is a scalar, yet we expect more than one
678 : // return value. We can only convert the result into a list with
679 : // a single element, so the output is necessarily wrong.
680 0 : msg = createException(
681 : MAL, "pyapi3.eval",
682 : SQLSTATE(PY000) "A single scalar was returned, yet we expect a list of %d "
683 : "columns. We can only convert a single scalar into a "
684 : "single column, thus the result is invalid.",
685 : expected_columns);
686 0 : goto wrapup;
687 : }
688 : } else {
689 : // if it is not a scalar, we check if it is a single array
690 144 : bool IsSingleArray = true;
691 144 : PyObject *data = pResult, *ndata = NULL;
692 144 : if (PyType_IsNumpyMaskedArray(data)) {
693 30 : ndata = PyObject_GetAttrString(pResult, "data");
694 30 : if (ndata == NULL) {
695 0 : msg = createException(MAL, "pyapi3.eval",
696 : SQLSTATE(PY000) "Invalid masked array.");
697 0 : goto wrapup;
698 : }
699 : data = ndata;
700 : }
701 144 : if (PyType_IsNumpyArray(data)) {
702 105 : if (PyArray_NDIM((PyArrayObject *)data) != 1) {
703 : IsSingleArray = false;
704 105 : } else if (PyArray_SIZE((PyArrayObject *)data) == 0) {
705 : IsSingleArray = true;
706 : } else {
707 103 : pColO = PyArray_GETITEM( (PyArrayObject *)data, PyArray_GETPTR1((PyArrayObject *)data, 0));
708 103 : IsSingleArray = PyType_IsPyScalar(pColO);
709 103 : Py_DECREF(pColO);
710 : }
711 39 : } else if (PyList_Check(data)) {
712 39 : if (PyList_Size(data) == 0) {
713 : IsSingleArray = true;
714 : } else {
715 39 : pColO = PyList_GetItem(data, 0);
716 39 : IsSingleArray = PyType_IsPyScalar(pColO);
717 : }
718 0 : } else if (!PyType_IsNumpyMaskedArray(data)) {
719 : // it is neither a python array, numpy array or numpy masked
720 : // array, thus the result is unsupported! Throw an exception!
721 0 : msg = createException(
722 : MAL, "pyapi3.eval",
723 : SQLSTATE(PY000) "Unsupported result object. Expected either a list, "
724 : "dictionary, a numpy array, a numpy masked array or a "
725 : "pandas data frame, but received an object of type \"%s\"",
726 : /* leaks */
727 : PyUnicode_AsUTF8(PyObject_Str(PyObject_Type(data))));
728 0 : goto wrapup;
729 : }
730 :
731 142 : if (IsSingleArray) {
732 136 : if (expected_columns == 1 || expected_columns <= 0) {
733 : // if we only expect a single return value, we can accept a
734 : // single array by converting it into an array holding an
735 : // array holding the element (i.e. [pResult])
736 136 : PyObject *list = PyList_New(1);
737 136 : PyList_SetItem(list, 0, pResult);
738 136 : pResult = list;
739 :
740 136 : columns = 1;
741 : } else {
742 : // the result object is a single array, yet we expect more
743 : // than one return value. We can only convert the result
744 : // into a list with a single array, so the output is
745 : // necessarily wrong.
746 0 : msg = createException(MAL, "pyapi3.eval",
747 : SQLSTATE(PY000) "A single array was returned, yet we "
748 : "expect a list of %d columns. The "
749 : "result is invalid.",
750 : expected_columns);
751 0 : goto wrapup;
752 : }
753 : } else {
754 : // the return value is an array of arrays, all we need to do is
755 : // check if it is the correct size
756 8 : int results = 0;
757 8 : if (PyList_Check(data))
758 8 : results = (int)PyList_Size(data);
759 : else
760 0 : results = (int)PyArray_DIMS((PyArrayObject *)data)[0];
761 8 : columns = results;
762 8 : if (results != expected_columns && expected_columns > 0) {
763 : // wrong return size, we expect pci->retc arrays
764 1 : msg = createException(MAL, "pyapi3.eval",
765 : SQLSTATE(PY000) "An array of size %d was returned, "
766 : "yet we expect a list of %d columns. "
767 : "The result is invalid.",
768 : results, expected_columns);
769 1 : goto wrapup;
770 : }
771 : }
772 143 : if (ndata)
773 30 : Py_DECREF(ndata);
774 : }
775 : } else {
776 0 : msg = createException(
777 : MAL, "pyapi3.eval",
778 : SQLSTATE(PY000) "Invalid result object. No result object could be generated.");
779 0 : goto wrapup;
780 : }
781 :
782 211 : if (actual_columns != NULL)
783 0 : *actual_columns = columns;
784 : return pResult;
785 1 : wrapup:
786 1 : if (actual_columns != NULL)
787 0 : *actual_columns = columns;
788 1 : *return_message = msg;
789 1 : return NULL;
790 : }
791 :
792 : #ifndef NUMPY_CORE_INCLUDE_NUMPY_NPY_2_COMPAT_H_
793 : #define PyDataType_ELSIZE(x) (x)->elsize
794 : #endif
795 :
796 : str
797 254 : PyObject_GetReturnValues(PyObject *obj, PyReturn *ret)
798 : {
799 254 : PyObject *pMask = NULL;
800 254 : str msg = MAL_SUCCEED;
801 : // If it isn't we need to convert pColO to the expected Numpy Array type
802 254 : ret->numpy_array = PyArray_FromAny( obj, NULL, 1, 1, NPY_ARRAY_CARRAY | NPY_ARRAY_FORCECAST, NULL);
803 254 : if (ret->numpy_array == NULL) {
804 0 : msg = createException(
805 : MAL, "pyapi3.eval",
806 : SQLSTATE(PY000) "Could not create a Numpy array from the return type.\n");
807 0 : goto wrapup;
808 : }
809 :
810 254 : ret->result_type = PyArray_DESCR((PyArrayObject *)ret->numpy_array)->type_num; // We read the result type from the resulting array
811 254 : ret->memory_size = PyDataType_ELSIZE(PyArray_DESCR((PyArrayObject *)ret->numpy_array));
812 254 : ret->count = PyArray_DIMS((PyArrayObject *)ret->numpy_array)[0];
813 254 : ret->array_data = PyArray_DATA((PyArrayObject *)ret->numpy_array);
814 254 : ret->mask_data = NULL;
815 254 : ret->numpy_mask = NULL;
816 : // If pColO is a Masked array, we convert the mask to a NPY_BOOL numpy array
817 254 : if (PyObject_HasAttrString(obj, "mask")) {
818 30 : pMask = PyObject_GetAttrString(obj, "mask");
819 30 : if (pMask != NULL) {
820 30 : ret->numpy_mask = PyArray_FromAny(pMask, PyArray_DescrFromType(NPY_BOOL), 1, 1, NPY_ARRAY_CARRAY, NULL);
821 30 : Py_DECREF(pMask);
822 30 : if (ret->numpy_mask == NULL || PyArray_DIMS((PyArrayObject *)ret->numpy_mask)[0] != (int)ret->count) {
823 0 : PyErr_Clear();
824 0 : if (ret->numpy_mask)
825 0 : Py_DECREF(ret->numpy_mask);
826 0 : ret->numpy_mask = NULL;
827 : }
828 : }
829 : }
830 254 : if (ret->numpy_mask != NULL)
831 30 : ret->mask_data = PyArray_DATA((PyArrayObject *)ret->numpy_mask);
832 224 : wrapup:
833 254 : return msg;
834 : }
835 :
836 : bool
837 150 : PyObject_PreprocessObject(PyObject *pResult, PyReturn *pyreturn_values, int column_count, char **return_message)
838 : {
839 150 : int i;
840 150 : char *msg;
841 363 : for (i = 0; i < column_count; i++) {
842 : // Refers to the current Numpy mask (if it exists)
843 213 : PyObject *pMask = NULL;
844 : // Refers to the current Numpy array
845 213 : PyObject *pColO = NULL;
846 : // This is the PyReturn header information for the current return value,
847 : // we will fill this now
848 213 : PyReturn *ret = &pyreturn_values[i];
849 :
850 213 : ret->multidimensional = false;
851 : // There are three possibilities (we have ensured this right after
852 : // executing the Python call by calling PyObject_CheckForConversion)
853 : // 1: The top level result object is a PyList or Numpy Array containing
854 : // pci->retc Numpy Arrays
855 : // 2: The top level result object is a (pci->retc x N) dimensional Numpy
856 : // Array [Multidimensional]
857 : // 3: The top level result object is a (pci->retc x N) dimensional Numpy
858 : // Masked Array [Multidimensional]
859 213 : if (PyList_Check(pResult)) {
860 : // If it is a PyList, we simply get the i'th Numpy array from the
861 : // PyList
862 213 : pColO = PyList_GetItem(pResult, i);
863 213 : Py_INCREF(pColO);
864 : } else {
865 : // If it isn't, the result object is either a Nump Masked Array or a
866 : // Numpy Array
867 0 : PyObject *data = pResult;
868 0 : if (PyType_IsNumpyMaskedArray(data)) {
869 0 : assert(0);
870 : data = PyObject_GetAttrString( pResult, "data");
871 : // If it is a Masked array, the data is stored in the masked_array.data attribute
872 : pMask = PyObject_GetAttrString(pResult, "mask");
873 : }
874 :
875 : // We can either have a multidimensional numpy array, or a single
876 : // dimensional numpy array
877 0 : if (PyArray_NDIM((PyArrayObject *)data) != 1) {
878 : // If it is a multidimensional numpy array, we have to convert
879 : // the i'th dimension to a NUMPY array object
880 0 : ret->multidimensional = true;
881 0 : ret->result_type = PyArray_DESCR((PyArrayObject *)data)->type_num;
882 : } else {
883 : // If it is a single dimensional Numpy array, we get the i'th
884 : // Numpy array from the Numpy Array
885 0 : pColO = PyArray_GETITEM((PyArrayObject *)data, PyArray_GETPTR1((PyArrayObject *)data, i));
886 : }
887 : }
888 :
889 : // Now we have to do some preprocessing on the data
890 213 : if (ret->multidimensional) {
891 0 : assert(pColO==NULL);
892 : // If it is a multidimensional Numpy array, we don't need to do any
893 : // conversion, we can just do some pointers
894 0 : ret->count = PyArray_DIMS((PyArrayObject *)pResult)[1];
895 0 : ret->numpy_array = pResult;
896 0 : ret->numpy_mask = pMask;
897 0 : ret->array_data = PyArray_DATA((PyArrayObject *)ret->numpy_array);
898 0 : if (ret->numpy_mask != NULL)
899 : ret->mask_data = PyArray_DATA((PyArrayObject *)ret->numpy_mask);
900 0 : ret->memory_size = PyDataType_ELSIZE(PyArray_DESCR((PyArrayObject *)ret->numpy_array));
901 : } else {
902 213 : msg = PyObject_GetReturnValues(pColO, ret);
903 213 : Py_DECREF(pColO);
904 213 : if (msg != MAL_SUCCEED) {
905 0 : goto wrapup;
906 : }
907 : }
908 : }
909 : return true;
910 0 : wrapup:
911 0 : *return_message = msg;
912 0 : return false;
913 : }
914 :
915 : BAT *
916 213 : PyObject_ConvertToBAT(PyReturn *ret, sql_subtype *type, int bat_type, int i, oid seqbase, char **return_message, bool copy)
917 : {
918 213 : BAT *b = NULL;
919 213 : size_t index_offset = 0;
920 213 : char *msg;
921 213 : size_t iu;
922 :
923 213 : if (ret->multidimensional)
924 0 : index_offset = i;
925 :
926 213 : switch (GetSQLType(type)) {
927 3 : case EC_TIMESTAMP:
928 3 : bat_type = TYPE_timestamp;
929 3 : break;
930 3 : case EC_TIME:
931 3 : bat_type = TYPE_daytime;
932 3 : break;
933 4 : case EC_DATE:
934 4 : bat_type = TYPE_date;
935 4 : break;
936 4 : case EC_DEC:
937 4 : bat_type = TYPE_dbl;
938 4 : break;
939 : default:
940 : break;
941 : }
942 :
943 213 : if (IsBlobType(bat_type)) {
944 2 : bool *mask = NULL;
945 2 : char *data = NULL;
946 2 : blob *ele_blob;
947 2 : size_t blob_fixed_size = ret->memory_size;
948 :
949 2 : PyObject *pickle_module = NULL, *pickle = NULL;
950 2 : bool gstate = 0;
951 :
952 2 : if (ret->result_type == NPY_OBJECT) {
953 : // Python objects, we may need to pickle them, so we
954 : // may execute Python code, we have to obtain the GIL
955 0 : gstate = Python_ObtainGIL();
956 0 : pickle_module = PyImport_ImportModule("pickle");
957 0 : if (pickle_module == NULL) {
958 0 : msg = createException(MAL, "pyapi3.eval",
959 : SQLSTATE(PY000) "Can't load pickle module to pickle python object to blob");
960 0 : Python_ReleaseGIL(gstate);
961 0 : goto wrapup;
962 : }
963 : blob_fixed_size = 0; // Size depends on the objects
964 : }
965 :
966 2 : if (ret->mask_data != NULL) {
967 : mask = (bool *)ret->mask_data;
968 : }
969 2 : if (ret->array_data == NULL) {
970 0 : msg = createException(MAL, "pyapi3.eval",
971 : SQLSTATE(PY000) "No return value stored in the structure.");
972 0 : if (ret->result_type == NPY_OBJECT) {
973 0 : Py_XDECREF(pickle_module);
974 0 : Python_ReleaseGIL(gstate);
975 : }
976 0 : goto wrapup;
977 : }
978 2 : data = (char *)ret->array_data;
979 2 : data += (index_offset * ret->count) * ret->memory_size;
980 2 : b = COLnew(seqbase, TYPE_blob, (BUN)ret->count, TRANSIENT);
981 2 : if (b == NULL) {
982 0 : if (ret->result_type == NPY_OBJECT) {
983 0 : Py_XDECREF(pickle_module);
984 0 : Python_ReleaseGIL(gstate);
985 : }
986 0 : msg = createException(MAL, "pyapi3.eval", GDK_EXCEPTION);
987 0 : goto wrapup;
988 : }
989 2 : b->tnil = false;
990 2 : b->tnonil = true;
991 2 : b->tkey = false;
992 2 : b->tsorted = false;
993 2 : b->trevsorted = false;
994 8 : for (iu = 0; iu < ret->count; iu++) {
995 :
996 6 : char* memcpy_data;
997 6 : size_t blob_len = 0;
998 :
999 6 : if (ret->result_type == NPY_OBJECT) {
1000 0 : PyObject *object = *((PyObject **)&data[0]);
1001 0 : if (PyByteArray_Check(object)) {
1002 0 : memcpy_data = PyByteArray_AsString(object);
1003 0 : blob_len = pyobject_get_size(object);
1004 : } else {
1005 0 : pickle = PyObject_CallMethod(pickle_module, "dumps", "O", object);
1006 0 : if (pickle == NULL) {
1007 0 : msg = createException(MAL, "pyapi3.eval",
1008 : SQLSTATE(PY000) "Can't pickle object to blob");
1009 0 : Py_XDECREF(pickle_module);
1010 0 : Python_ReleaseGIL(gstate);
1011 0 : goto wrapup;
1012 : }
1013 0 : memcpy_data = PyBytes_AsString(pickle);
1014 0 : blob_len = pyobject_get_size(pickle);
1015 0 : Py_XDECREF(pickle);
1016 : }
1017 0 : if (memcpy_data == NULL) {
1018 0 : msg = createException(MAL, "pyapi3.eval",
1019 : SQLSTATE(PY000) "Can't get blob pickled object as char*");
1020 0 : Py_XDECREF(pickle_module);
1021 0 : Python_ReleaseGIL(gstate);
1022 0 : goto wrapup;
1023 : }
1024 : } else {
1025 : memcpy_data = data;
1026 : }
1027 :
1028 6 : if (mask && mask[iu]) {
1029 0 : ele_blob = (blob *)GDKmalloc(offsetof(blob, data));
1030 0 : ele_blob->nitems = ~(size_t)0;
1031 : } else {
1032 6 : if (blob_fixed_size > 0) {
1033 6 : blob_len = blob_fixed_size;
1034 : }
1035 6 : ele_blob = GDKmalloc(blobsize(blob_len));
1036 6 : ele_blob->nitems = blob_len;
1037 6 : memcpy(ele_blob->data, memcpy_data, blob_len);
1038 : }
1039 6 : if (BUNappend(b, ele_blob, false) != GDK_SUCCEED) {
1040 0 : if (ret->result_type == NPY_OBJECT) {
1041 0 : Py_XDECREF(pickle_module);
1042 0 : Python_ReleaseGIL(gstate);
1043 : }
1044 0 : BBPunfix(b->batCacheid);
1045 0 : msg = createException(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL);
1046 0 : goto wrapup;
1047 : }
1048 6 : GDKfree(ele_blob);
1049 6 : data += ret->memory_size;
1050 :
1051 : }
1052 :
1053 : // We are done, we can release the GIL
1054 2 : if (ret->result_type == NPY_OBJECT) {
1055 0 : Py_XDECREF(pickle_module);
1056 0 : Python_ReleaseGIL(gstate);
1057 : }
1058 :
1059 2 : BATsetcount(b, (BUN)ret->count);
1060 : } else {
1061 211 : switch (bat_type) {
1062 0 : case TYPE_void:
1063 0 : NP_CREATE_EMPTY_BAT(b, oid);
1064 0 : break;
1065 9 : case TYPE_bit:
1066 18 : NP_CREATE_BAT(b, bit);
1067 : break;
1068 1 : case TYPE_bte:
1069 101 : NP_CREATE_BAT(b, bte);
1070 : break;
1071 2 : case TYPE_sht:
1072 100003 : NP_CREATE_BAT(b, sht);
1073 : break;
1074 136 : case TYPE_int:
1075 20470561 : NP_CREATE_BAT(b, int);
1076 : break;
1077 0 : case TYPE_oid:
1078 0 : NP_CREATE_BAT(b, oid);
1079 : break;
1080 2 : case TYPE_lng:
1081 4 : NP_CREATE_BAT(b, lng);
1082 : break;
1083 0 : case TYPE_flt:
1084 0 : NP_CREATE_BAT(b, flt);
1085 : break;
1086 30 : case TYPE_dbl:
1087 1700171 : NP_CREATE_BAT(b, dbl);
1088 : break;
1089 : #ifdef HAVE_HGE
1090 0 : case TYPE_hge:
1091 0 : NP_CREATE_BAT(b, hge);
1092 : break;
1093 : #endif
1094 4 : case TYPE_date:
1095 10 : NP_CREATE_BAT(b, date);
1096 : break;
1097 3 : case TYPE_daytime:
1098 6 : NP_CREATE_BAT(b, daytime);
1099 : break;
1100 3 : case TYPE_timestamp:
1101 6 : NP_CREATE_BAT(b, timestamp);
1102 : break;
1103 21 : case TYPE_str: {
1104 21 : bool *mask = NULL;
1105 21 : char *data = NULL;
1106 21 : char *utf8_string = NULL;
1107 21 : if (ret->mask_data != NULL) {
1108 : mask = (bool *)ret->mask_data;
1109 : }
1110 21 : if (ret->array_data == NULL) {
1111 0 : msg = createException(
1112 : MAL, "pyapi3.eval",
1113 : SQLSTATE(PY000) "No return value stored in the structure. n");
1114 0 : goto wrapup;
1115 : }
1116 21 : data = (char *)ret->array_data;
1117 :
1118 21 : if (ret->result_type != NPY_OBJECT) {
1119 15 : utf8_string = GDKzalloc(utf8string_minlength + ret->memory_size + 1);
1120 15 : utf8_string[utf8string_minlength + ret->memory_size] = '\0';
1121 : }
1122 :
1123 21 : b = COLnew(seqbase, TYPE_str, (BUN)ret->count, TRANSIENT);
1124 21 : if (b == NULL) {
1125 0 : GDKfree(utf8_string);
1126 0 : msg = createException(MAL, "pyapi3.eval", GDK_EXCEPTION);
1127 0 : goto wrapup;
1128 : }
1129 21 : b->tnil = false;
1130 21 : b->tnonil = true;
1131 21 : b->tkey = false;
1132 21 : b->tsorted = false;
1133 21 : b->trevsorted = false;
1134 1300085 : NP_INSERT_STRING_BAT(b);
1135 21 : GDKfree(utf8_string);
1136 21 : BATsetcount(b, (BUN)ret->count);
1137 21 : break;
1138 : }
1139 0 : default:
1140 0 : msg = createException(MAL, "pyapi3.eval",
1141 : SQLSTATE(PY000) "Unrecognized BAT type %s.\n",
1142 : BatType_Format(bat_type));
1143 0 : goto wrapup;
1144 : }
1145 : }
1146 213 : if (ConvertableSQLType(type)) {
1147 4 : BAT *result;
1148 4 : msg = ConvertToSQLType(NULL, b, type, &result, &bat_type);
1149 4 : BBPunfix(b->batCacheid);
1150 4 : if (msg != MAL_SUCCEED) {
1151 0 : goto wrapup;
1152 : }
1153 4 : b = result;
1154 : }
1155 :
1156 : return b;
1157 :
1158 0 : wrapup:
1159 0 : *return_message = msg;
1160 0 : return NULL;
1161 : }
1162 :
1163 538 : bit ConvertableSQLType(sql_subtype *sql_subtype)
1164 : {
1165 538 : switch (GetSQLType(sql_subtype)) {
1166 : case EC_DEC:
1167 : return 1;
1168 : }
1169 531 : return 0;
1170 : }
1171 :
1172 751 : int GetSQLType(sql_subtype *sql_subtype)
1173 : {
1174 751 : if (!sql_subtype)
1175 : return -1;
1176 535 : if (!sql_subtype->type)
1177 : return -1;
1178 535 : return (int) sql_subtype->type->eclass;
1179 : }
1180 :
1181 : str
1182 4 : ConvertFromSQLType(BAT *b, sql_subtype *sql_subtype, BAT **ret_bat, int *ret_type)
1183 : {
1184 4 : str res = MAL_SUCCEED;
1185 4 : int conv_type;
1186 :
1187 4 : assert(sql_subtype);
1188 4 : assert(sql_subtype->type);
1189 :
1190 4 : switch (sql_subtype->type->eclass) {
1191 : case EC_DEC:
1192 : conv_type = TYPE_dbl;
1193 : break;
1194 : default:
1195 1 : conv_type = TYPE_str;
1196 : }
1197 :
1198 4 : if (conv_type == TYPE_str) {
1199 1 : BUN p = 0, q = 0;
1200 1 : char *result = NULL;
1201 1 : size_t length = 0;
1202 1 : ssize_t (*strConversion)(str *, size_t *, const void *, bool) =
1203 1 : BATatoms[b->ttype].atomToStr;
1204 1 : *ret_bat = COLnew(0, TYPE_str, 0, TRANSIENT);
1205 1 : *ret_type = conv_type;
1206 1 : if (!(*ret_bat)) {
1207 0 : return createException(MAL, "pyapi3.eval",
1208 : SQLSTATE(HY013) MAL_MALLOC_FAIL " string conversion BAT.");
1209 : }
1210 1 : BATiter li = bat_iterator(b);
1211 2 : BATloop(b, p, q)
1212 : {
1213 1 : const void *element = (const void*)BUNtail(li, p);
1214 1 : if (strConversion(&result, &length, element, false) < 0) {
1215 0 : bat_iterator_end(&li);
1216 0 : BBPunfix((*ret_bat)->batCacheid);
1217 0 : return createException(MAL, "pyapi3.eval",
1218 : SQLSTATE(PY000) "Failed to convert element to string.");
1219 : }
1220 1 : if (BUNappend(*ret_bat, result, false) != GDK_SUCCEED) {
1221 0 : bat_iterator_end(&li);
1222 0 : BBPunfix((*ret_bat)->batCacheid);
1223 0 : throw(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL);
1224 : }
1225 : }
1226 1 : bat_iterator_end(&li);
1227 1 : if (result) {
1228 1 : GDKfree(result);
1229 : }
1230 1 : return res;
1231 3 : } else if (conv_type == TYPE_dbl) {
1232 3 : int bat_type = ATOMstorage(b->ttype);
1233 3 : int hpos = sql_subtype->scale;
1234 3 : bat result = 0;
1235 : // decimal values can be stored in various numeric fields, so check the
1236 : // numeric field and convert the one it's actually stored in
1237 3 : switch (bat_type) {
1238 0 : case TYPE_bte:
1239 0 : res = batbte_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
1240 0 : break;
1241 0 : case TYPE_sht:
1242 0 : res = batsht_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
1243 0 : break;
1244 0 : case TYPE_int:
1245 0 : res = batint_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
1246 0 : break;
1247 3 : case TYPE_lng:
1248 3 : res = batlng_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
1249 3 : break;
1250 : #ifdef HAVE_HGE
1251 0 : case TYPE_hge:
1252 0 : res = bathge_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
1253 0 : break;
1254 : #endif
1255 0 : default:
1256 0 : return createException(MAL, "pyapi3.eval",
1257 : "Unsupported decimal storage type.");
1258 : }
1259 3 : if (res == MAL_SUCCEED) {
1260 3 : *ret_bat = BATdescriptor(result);
1261 3 : BBPrelease(result);
1262 3 : *ret_type = TYPE_dbl;
1263 : } else {
1264 0 : *ret_bat = NULL;
1265 : }
1266 3 : return res;
1267 : }
1268 : return createException(MAL, "pyapi3.eval", "Unrecognized conv type.");
1269 : }
1270 :
1271 4 : str ConvertToSQLType(Client cntxt, BAT *b, sql_subtype *sql_subtype,
1272 : BAT **ret_bat, int *ret_type)
1273 : {
1274 4 : str res = MAL_SUCCEED;
1275 4 : bat result_bat = 0;
1276 4 : int digits = sql_subtype->digits;
1277 4 : int scale = sql_subtype->scale;
1278 4 : (void)cntxt;
1279 :
1280 4 : assert(sql_subtype);
1281 4 : assert(sql_subtype->type);
1282 :
1283 4 : switch (sql_subtype->type->eclass) {
1284 0 : case EC_TIMESTAMP:
1285 0 : res = batstr_2time_timestamp(&result_bat, &b->batCacheid, NULL, &digits);
1286 0 : break;
1287 0 : case EC_TIME:
1288 0 : res = batstr_2time_daytime(&result_bat, &b->batCacheid, NULL, &digits);
1289 0 : break;
1290 0 : case EC_DATE:
1291 0 : if ((*ret_bat = BATconvert(b, NULL, TYPE_date, 0, 0, 0)) == NULL)
1292 0 : throw(MAL, "pyapi3.eval", GDK_EXCEPTION);
1293 0 : *ret_type = TYPE_date;
1294 0 : return MAL_SUCCEED;
1295 4 : case EC_DEC:
1296 4 : res = batdbl_num2dec_lng(&result_bat, &b->batCacheid, NULL,
1297 : &digits, &scale);
1298 4 : break;
1299 0 : default:
1300 0 : return createException(
1301 : MAL, "pyapi3.eval",
1302 : "Convert To SQL Type: Unrecognized SQL type %s (%d).",
1303 : sql_subtype->type->base.name, (int) sql_subtype->type->eclass);
1304 : }
1305 4 : if (res == MAL_SUCCEED) {
1306 4 : *ret_bat = BATdescriptor(result_bat);
1307 4 : BBPrelease(result_bat);
1308 4 : *ret_type = (*ret_bat)->ttype;
1309 : }
1310 :
1311 : return res;
1312 : }
1313 :
1314 224 : ssize_t PyType_Size(PyObject *obj)
1315 : {
1316 224 : if (PyType_IsPyScalar(obj)) {
1317 : return 1;
1318 : }
1319 62 : if (PyArray_Check(obj)) {
1320 57 : return PyArray_Size(obj);
1321 : }
1322 5 : if (PyList_Check(obj)) {
1323 5 : return Py_SIZE(obj);
1324 : }
1325 : return -1;
1326 : }
1327 :
1328 326 : bit IsStandardBATType(int type)
1329 : {
1330 326 : switch (type) {
1331 : case TYPE_bit:
1332 : case TYPE_bte:
1333 : case TYPE_sht:
1334 : case TYPE_int:
1335 : case TYPE_oid:
1336 : case TYPE_lng:
1337 : case TYPE_flt:
1338 : case TYPE_dbl:
1339 : #ifdef HAVE_HGE
1340 : case TYPE_hge:
1341 : #endif
1342 : case TYPE_date:
1343 : case TYPE_daytime:
1344 : case TYPE_timestamp:
1345 : case TYPE_str:
1346 : return 1;
1347 1 : default:
1348 1 : return 0;
1349 : }
1350 : }
1351 :
1352 10 : static void conversion_import_array(void) { _import_array(); }
1353 :
1354 10 : str _conversion_init(void)
1355 : {
1356 10 : str msg = MAL_SUCCEED;
1357 10 : conversion_import_array();
1358 :
1359 10 : return msg;
1360 : }
|
