Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : #include "monetdb_config.h"
14 : #include "conversion.h"
15 : #include "convert_loops.h"
16 : #include "pytypes.h"
17 : #include "type_conversion.h"
18 : #include "unicode.h"
19 :
20 : //! Wrapper to get eclass of SQL type
21 : int GetSQLType(sql_subtype *sql_subtype);
22 :
23 : static bool IsBlobType(int type)
24 : {
25 : return type == TYPE_blob;
26 : }
27 :
28 : static bool IsVoidType(int type)
29 : {
30 : return type == TYPE_void;
31 : }
32 :
33 85 : PyObject *PyArrayObject_FromScalar(PyInput *inp, char **return_message)
34 : {
35 85 : PyObject *vararray = NULL;
36 85 : char *msg = NULL;
37 85 : assert(inp->scalar); // input has to be a scalar
38 :
39 85 : switch (inp->bat_type) {
40 0 : case TYPE_void:
41 : #if SIZEOF_OID == SIZEOF_INT
42 : vararray = PyArray_Arange(0, 1, 1, NPY_UINT);
43 : #else
44 0 : vararray = PyArray_Arange(0, 1, 1, NPY_ULONGLONG);
45 : #endif
46 0 : break;
47 0 : case TYPE_oid:
48 0 : vararray = PyLong_FromLong((long)(*(oid *)inp->dataptr));
49 0 : break;
50 0 : case TYPE_bit:
51 0 : vararray = PyLong_FromLong((long)(*(bit *)inp->dataptr));
52 0 : break;
53 0 : case TYPE_bte:
54 0 : vararray = PyLong_FromLong((long)(*(bte *)inp->dataptr));
55 0 : break;
56 0 : case TYPE_sht:
57 0 : vararray = PyLong_FromLong((long)(*(sht *)inp->dataptr));
58 0 : break;
59 63 : case TYPE_int:
60 63 : vararray = PyLong_FromLong((long)(*(int *)inp->dataptr));
61 63 : break;
62 0 : case TYPE_lng:
63 0 : vararray = PyLong_FromLongLong((*(lng *)inp->dataptr));
64 0 : break;
65 0 : case TYPE_flt:
66 0 : vararray = PyFloat_FromDouble((double)(*(flt *)inp->dataptr));
67 0 : break;
68 1 : case TYPE_dbl:
69 1 : vararray = PyFloat_FromDouble((double)(*(dbl *)inp->dataptr));
70 1 : break;
71 : #ifdef HAVE_HGE
72 0 : case TYPE_hge:
73 0 : vararray = PyLong_FromHge(*((hge *)inp->dataptr));
74 0 : break;
75 : #endif
76 1 : case TYPE_date: {
77 1 : USE_DATETIME_API;
78 1 : date dt = *(date *)inp->dataptr;
79 1 : vararray = PyDate_FromDate(date_year(dt), date_month(dt), date_day(dt));
80 : /* error checking */
81 1 : break;
82 : }
83 1 : case TYPE_daytime: {
84 1 : USE_DATETIME_API;
85 1 : daytime dt = *(daytime *)inp->dataptr;
86 1 : vararray = PyTime_FromTime(daytime_hour(dt), daytime_min(dt), daytime_sec(dt), daytime_usec(dt));
87 : /* error checking */
88 1 : break;
89 : }
90 1 : case TYPE_timestamp: {
91 1 : USE_DATETIME_API;
92 1 : timestamp ts = *(timestamp *)inp->dataptr;
93 1 : date dt = timestamp_date(ts);
94 1 : daytime dtm = timestamp_daytime(ts);
95 1 : vararray = PyDateTime_FromDateAndTime(date_year(dt), date_month(dt), date_day(dt), daytime_hour(dtm), daytime_min(dtm), daytime_sec(dtm), daytime_usec(dtm));
96 : /* error checking */
97 1 : break;
98 : }
99 18 : case TYPE_str:
100 18 : vararray = PyUnicode_FromString(*((char **)inp->dataptr));
101 18 : break;
102 0 : default:
103 0 : msg = createException(MAL, "pyapi3.eval",
104 : SQLSTATE(PY000) "Unsupported scalar type %i.", inp->bat_type);
105 0 : goto wrapup;
106 : }
107 85 : if (vararray == NULL) {
108 0 : msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Something went wrong "
109 : "converting the MonetDB "
110 : "scalar to a Python scalar.");
111 0 : goto wrapup;
112 : }
113 85 : wrapup:
114 85 : *return_message = msg;
115 85 : return vararray;
116 : }
117 :
118 : PyObject *
119 339 : PyMaskedArray_FromBAT(PyInput *inp, size_t t_start, size_t t_end, char **return_message, bool copy)
120 : {
121 339 : BAT *b;
122 339 : char *msg;
123 339 : PyObject *vararray = PyArrayObject_FromBAT(inp, t_start, t_end, return_message, copy);
124 :
125 339 : if (vararray == NULL)
126 : return NULL;
127 339 : b = inp->bat;
128 : // To deal with null values, we use the numpy masked array structure
129 : // The masked array structure is an object with two arrays of equal size, a
130 : // data array and a mask array
131 : // The mask array is a boolean array that has the value 'True' when the
132 : // element is NULL, and 'False' otherwise
133 : // if we know for sure that the BAT has no NULL values, we can skip the construction
134 : // of this masked array. Otherwise, we create it.
135 339 : MT_lock_set(&b->theaplock);
136 339 : bool bnonil = b->tnonil;
137 339 : MT_lock_unset(&b->theaplock);
138 339 : if (!bnonil) {
139 291 : PyObject *nme = PyUnicode_FromString("numpy.ma");
140 291 : PyObject *mod = PyImport_Import(nme);
141 291 : PyObject *mafunc = PyObject_GetAttrString( mod, "masked_array");
142 291 : PyObject *nullmask = PyNullMask_FromBAT(b, t_start, t_end);
143 291 : Py_DECREF(nme);
144 :
145 291 : if (!nullmask) {
146 0 : Py_DECREF(vararray);
147 0 : Py_DECREF(mafunc);
148 0 : Py_DECREF(mod);
149 0 : msg = createException(MAL, "pyapi3.eval", "Failed to create mask for some reason");
150 0 : goto wrapup;
151 291 : } else if (nullmask == Py_None) {
152 267 : Py_DECREF(nullmask);
153 : } else {
154 24 : PyObject *maargs = PyTuple_New(2);
155 24 : PyTuple_SetItem(maargs, 0, vararray);
156 24 : PyTuple_SetItem(maargs, 1, nullmask);
157 :
158 : // Now we will actually construct the mask by calling the masked
159 : // array constructor
160 24 : PyObject *mask = PyObject_CallObject(mafunc, maargs);
161 24 : Py_DECREF(maargs);
162 24 : if (!mask) {
163 0 : Py_DECREF(mafunc);
164 0 : Py_DECREF(mod);
165 0 : msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Failed to create mask");
166 0 : goto wrapup;
167 : }
168 : vararray = mask;
169 : }
170 291 : Py_DECREF(mafunc);
171 291 : Py_DECREF(mod);
172 : }
173 : return vararray;
174 0 : wrapup:
175 0 : *return_message = msg;
176 0 : return NULL;
177 : }
178 :
179 : PyObject *
180 339 : PyArrayObject_FromBAT(PyInput *inp, size_t t_start, size_t t_end, char **return_message, bool copy)
181 : {
182 : // This variable will hold the converted Python object
183 339 : PyObject *vararray = NULL;
184 339 : char *msg;
185 339 : size_t j = 0;
186 339 : BUN p = 0, q = 0;
187 339 : BATiter li;
188 339 : BAT *b = inp->bat;
189 339 : npy_intp elements[1] = {t_end - t_start};
190 :
191 339 : assert(!inp->scalar); // input has to be a BAT
192 :
193 339 : if (!b) {
194 : // No BAT was found, we can't do anything in this case
195 0 : msg = createException(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL " bat missing");
196 0 : goto wrapup;
197 : }
198 :
199 665 : if (!IsVoidType(inp->bat_type) && !IsBlobType(inp->bat_type) &&
200 651 : (!IsStandardBATType(inp->bat_type) ||
201 325 : ConvertableSQLType(inp->sql_subtype))) { // if the sql type is set, we
202 : // have to do some conversion
203 4 : if (inp->scalar) {
204 : // FIXME: scalar SQL types
205 0 : msg = createException(
206 : MAL, "pyapi3.eval",
207 : SQLSTATE(0A000) "Scalar SQL types haven't been implemented yet... sorry");
208 0 : goto wrapup;
209 : } else {
210 4 : BAT *ret_bat = NULL;
211 4 : msg = ConvertFromSQLType(b, inp->sql_subtype, &ret_bat, &inp->bat_type);
212 4 : if (msg != MAL_SUCCEED) {
213 0 : freeException(msg);
214 0 : msg = createException(MAL, "pyapi3.eval",
215 : SQLSTATE(PY000) "Failed to convert BAT.");
216 0 : goto wrapup;
217 : }
218 4 : b = ret_bat;
219 : }
220 : }
221 :
222 339 : if (IsBlobType(inp->bat_type)) {
223 2 : PyObject **data;
224 2 : li = bat_iterator(b);
225 2 : vararray = PyArray_EMPTY(1, elements, NPY_OBJECT, 0);
226 2 : data = PyArray_DATA((PyArrayObject *)vararray);
227 8 : BATloop(b, p, q)
228 : {
229 6 : const blob *t = (const blob *)BUNtvar(li, p);
230 6 : if (t->nitems == ~(size_t)0) {
231 0 : data[p] = Py_None;
232 0 : Py_INCREF(Py_None);
233 : } else {
234 6 : data[p] = PyByteArray_FromStringAndSize(t->data, t->nitems);
235 : }
236 : }
237 2 : bat_iterator_end(&li);
238 : } else {
239 337 : switch (inp->bat_type) {
240 11 : case TYPE_void:
241 : #if SIZEOF_OID == SIZEOF_INT
242 : BAT_TO_NP_CREATE_ALWAYS(b, NPY_UINT);
243 : #else
244 11 : BAT_TO_NP_CREATE_ALWAYS(b, NPY_ULONGLONG);
245 : #endif
246 11 : break;
247 19 : case TYPE_oid:
248 : #if SIZEOF_OID == SIZEOF_INT
249 : BAT_TO_NP(b, oid, NPY_UINT32);
250 : #else
251 19 : BAT_TO_NP(b, oid, NPY_UINT64);
252 : #endif
253 19 : break;
254 1 : case TYPE_bit:
255 1 : BAT_TO_NP(b, bit, NPY_INT8);
256 1 : break;
257 1 : case TYPE_bte:
258 1 : BAT_TO_NP(b, bte, NPY_INT8);
259 1 : break;
260 1 : case TYPE_sht:
261 1 : BAT_TO_NP(b, sht, NPY_INT16);
262 1 : break;
263 271 : case TYPE_int:
264 271 : BAT_TO_NP(b, int, NPY_INT32);
265 271 : break;
266 1 : case TYPE_lng:
267 1 : BAT_TO_NP(b, lng, NPY_INT64);
268 1 : break;
269 0 : case TYPE_flt:
270 0 : BAT_TO_NP(b, flt, NPY_FLOAT32);
271 0 : break;
272 16 : case TYPE_dbl:
273 16 : BAT_TO_NP(b, dbl, NPY_FLOAT64);
274 16 : break;
275 2 : case TYPE_date: {
276 2 : li = bat_iterator(b);
277 :
278 2 : USE_DATETIME_API;
279 2 : vararray = PyArray_EMPTY(1, elements, NPY_OBJECT, 0);
280 : {
281 2 : PyObject **data = ((PyObject **)PyArray_DATA((PyArrayObject *)vararray));
282 : // PyObject *obj;
283 2 : j = 0;
284 6 : BATloop(b, p, q)
285 : {
286 4 : const date* dt = (const date*)BUNtail(li, p);
287 4 : data[j++] = PyDate_FromDate(date_year(*dt), date_month(*dt), date_day(*dt));
288 : }
289 : }
290 2 : bat_iterator_end(&li);
291 2 : break;
292 : }
293 1 : case TYPE_daytime: {
294 1 : li = bat_iterator(b);
295 :
296 1 : USE_DATETIME_API;
297 1 : vararray = PyArray_EMPTY(1, elements, NPY_OBJECT, 0);
298 : {
299 1 : PyObject **data = ((PyObject **)PyArray_DATA((PyArrayObject *)vararray));
300 : // PyObject *obj;
301 1 : j = 0;
302 2 : BATloop(b, p, q)
303 : {
304 1 : const daytime* dt = (const daytime*)BUNtail(li, p);
305 1 : data[j++] = PyTime_FromTime(daytime_hour(*dt),
306 : daytime_min(*dt),
307 : daytime_sec(*dt),
308 : daytime_usec(*dt));
309 : }
310 : }
311 1 : bat_iterator_end(&li);
312 1 : break;
313 : }
314 2 : case TYPE_timestamp: {
315 2 : li = bat_iterator(b);
316 :
317 2 : USE_DATETIME_API;
318 2 : vararray = PyArray_EMPTY(1, elements, NPY_OBJECT, 0);
319 : {
320 2 : PyObject **data = ((PyObject **)PyArray_DATA((PyArrayObject *)vararray));
321 : // PyObject *obj;
322 2 : j = 0;
323 5 : BATloop(b, p, q)
324 : {
325 3 : const timestamp* ts = (const timestamp*)BUNtail(li, p);
326 3 : const date dt = timestamp_date(*ts);
327 3 : const daytime dtm = timestamp_daytime(*ts);
328 :
329 3 : data[j++] = PyDateTime_FromDateAndTime(date_year(dt), date_month(dt), date_day(dt), daytime_hour(dtm), daytime_min(dtm), daytime_sec(dtm), daytime_usec(dtm));
330 : }
331 : }
332 2 : bat_iterator_end(&li);
333 2 : break;
334 : }
335 :
336 9 : case TYPE_str: {
337 9 : bool unicode = false;
338 9 : li = bat_iterator(b);
339 : // create a NPY_OBJECT array object
340 9 : vararray = PyArray_New(&PyArray_Type, 1, elements, NPY_OBJECT,
341 : NULL, NULL, 0, 0, NULL);
342 :
343 1100037 : BATloop(b, p, q)
344 : {
345 1100028 : const char *t = (const char *)BUNtvar(li, p);
346 6589029 : for (; *t != 0; t++) {
347 5489001 : if (*t & 0x80) {
348 : unicode = true;
349 : break;
350 : }
351 : }
352 1100028 : if (unicode) {
353 : break;
354 : }
355 : }
356 :
357 : {
358 9 : PyObject **data = ((PyObject **)PyArray_DATA((PyArrayObject *)vararray));
359 9 : PyObject *obj;
360 9 : j = 0;
361 9 : if (unicode) {
362 0 : if (GDK_ELIMDOUBLES(b->tvheap)) {
363 0 : PyObject **pyptrs = GDKzalloc(b->tvheap->free * sizeof(PyObject *));
364 0 : if (!pyptrs) {
365 0 : bat_iterator_end(&li);
366 0 : msg = createException(MAL, "pyapi3.eval",
367 : SQLSTATE(HY013) MAL_MALLOC_FAIL
368 : " PyObject strings.");
369 0 : goto wrapup;
370 : }
371 0 : BATloop(b, p, q)
372 : {
373 0 : const char *t = (const char *)BUNtvar(li, p);
374 0 : ptrdiff_t offset = t - b->tvheap->base;
375 0 : if (!pyptrs[offset]) {
376 0 : if (strNil(t)) {
377 : // str_nil isn't a valid UTF-8 character
378 : // (it's 0x80), so we can't decode it as
379 : // UTF-8 (it will throw an error)
380 0 : pyptrs[offset] = PyUnicode_FromString("-");
381 : } else {
382 : // otherwise we can just decode the
383 : // string as UTF-8
384 0 : pyptrs[offset] = PyUnicode_FromString(t);
385 : }
386 0 : if (!pyptrs[offset]) {
387 0 : bat_iterator_end(&li);
388 0 : msg = createException(
389 : MAL, "pyapi3.eval",
390 : SQLSTATE(PY000) "Failed to create string.");
391 0 : goto wrapup;
392 : }
393 : } else {
394 0 : Py_INCREF(pyptrs[offset]);
395 : }
396 0 : data[j++] = pyptrs[offset];
397 : }
398 0 : GDKfree(pyptrs);
399 : } else {
400 0 : BATloop(b, p, q)
401 : {
402 0 : const char *t = (const char *)BUNtvar(li, p);
403 0 : if (strNil(t)) {
404 : // str_nil isn't a valid UTF-8 character
405 : // (it's 0x80), so we can't decode it as
406 : // UTF-8 (it will throw an error)
407 0 : obj = PyUnicode_FromString("-");
408 : } else {
409 : // otherwise we can just decode the string
410 : // as UTF-8
411 0 : obj = PyUnicode_FromString(t);
412 : }
413 :
414 0 : if (obj == NULL) {
415 0 : bat_iterator_end(&li);
416 0 : msg = createException(
417 : MAL, "pyapi3.eval",
418 : SQLSTATE(PY000) "Failed to create string.");
419 0 : goto wrapup;
420 : }
421 0 : data[j++] = obj;
422 : }
423 : }
424 : } else {
425 : /* special case where we exploit the
426 : * duplicate-eliminated string heap */
427 9 : if (GDK_ELIMDOUBLES(b->tvheap)) {
428 8 : PyObject **pyptrs = GDKzalloc(b->tvheap->free * sizeof(PyObject *));
429 8 : if (!pyptrs) {
430 0 : bat_iterator_end(&li);
431 0 : msg = createException(MAL, "pyapi3.eval",
432 : SQLSTATE(HY013) MAL_MALLOC_FAIL
433 : " PyObject strings.");
434 0 : goto wrapup;
435 : }
436 1000036 : BATloop(b, p, q)
437 : {
438 1000028 : const char *t = (const char *)BUNtvar(li, p);
439 1000028 : ptrdiff_t offset = t - b->tvheap->base;
440 1000028 : if (!pyptrs[offset]) {
441 21 : pyptrs[offset] = PyUnicode_FromString(t);
442 : } else {
443 1000007 : Py_INCREF(pyptrs[offset]);
444 : }
445 1000028 : data[j++] = pyptrs[offset];
446 : }
447 8 : GDKfree(pyptrs);
448 : } else {
449 100001 : BATloop(b, p, q)
450 : {
451 100000 : const char *t = (const char *)BUNtvar(li, p);
452 100000 : obj = PyUnicode_FromString(t);
453 100000 : if (obj == NULL) {
454 0 : bat_iterator_end(&li);
455 0 : msg = createException(
456 : MAL, "pyapi3.eval",
457 : SQLSTATE(PY000) "Failed to create string.");
458 0 : goto wrapup;
459 : }
460 100000 : data[j++] = obj;
461 : }
462 : }
463 : }
464 : }
465 9 : bat_iterator_end(&li);
466 9 : break;
467 : }
468 : #ifdef HAVE_HGE
469 2 : case TYPE_hge: {
470 : // create a NPY_FLOAT64 array to hold the huge type
471 4 : vararray = PyArray_New(&PyArray_Type, 1,
472 2 : (npy_intp[1]){t_end - t_start},
473 : NPY_FLOAT64, NULL, NULL, 0, 0, NULL);
474 :
475 2 : j = 0;
476 2 : npy_float64 *data = (npy_float64 *)PyArray_DATA((PyArrayObject *)vararray);
477 2 : BATiter bi = bat_iterator(b);
478 2 : const hge *vals = (const hge *) bi.base;
479 4 : BATloop(b, p, q) {
480 2 : data[j++] = (npy_float64)vals[p];
481 : }
482 2 : bat_iterator_end(&bi);
483 2 : break;
484 : }
485 : #endif
486 0 : default:
487 0 : if (!inp->sql_subtype || !inp->sql_subtype->type) {
488 0 : msg = createException(MAL, "pyapi3.eval",
489 : SQLSTATE(PY000) "unknown argument type");
490 : } else {
491 0 : msg = createException(MAL, "pyapi3.eval",
492 : SQLSTATE(PY000) "Unsupported SQL Type: %s",
493 : inp->sql_subtype->type->base.name);
494 : }
495 0 : goto wrapup;
496 : }
497 : }
498 :
499 339 : if (vararray == NULL) {
500 0 : msg = createException(MAL, "pyapi3.eval",
501 : SQLSTATE(PY000) "Failed to convert BAT to Numpy array.");
502 0 : goto wrapup;
503 : }
504 339 : if (b != inp->bat)
505 4 : inp->conv_bat = b; /* still in use, free later */
506 : return vararray;
507 0 : wrapup:
508 0 : *return_message = msg;
509 0 : if (b != inp->bat)
510 0 : BBPunfix(b->batCacheid);
511 : return NULL;
512 : }
513 :
514 : #define CreateNullMask(tpe) \
515 : { \
516 : tpe *bat_ptr = (tpe *)b->theap->base + b->tbaseoff; \
517 : for (j = 0; j < count; j++) { \
518 : mask_data[j] = is_##tpe##_nil(bat_ptr[j]); \
519 : found_nil |= mask_data[j]; \
520 : } \
521 : }
522 :
523 : PyObject *
524 291 : PyNullMask_FromBAT(BAT *b, size_t t_start, size_t t_end)
525 : {
526 : // We will now construct the Masked array, we start by setting everything to
527 : // False
528 291 : size_t count = t_end - t_start;
529 291 : npy_intp elements[1] = {count};
530 291 : PyArrayObject *nullmask = (PyArrayObject *)PyArray_EMPTY(1, elements, NPY_BOOL, 0);
531 291 : const void *nil = ATOMnilptr(b->ttype);
532 291 : size_t j;
533 291 : bool found_nil = false;
534 291 : BATiter bi = bat_iterator(b);
535 291 : bool *mask_data = (bool *)PyArray_DATA(nullmask);
536 :
537 576 : switch (ATOMbasetype(getBatType(b->ttype))) {
538 0 : case TYPE_bit:
539 0 : CreateNullMask(bit);
540 : break;
541 2 : case TYPE_bte:
542 12 : CreateNullMask(bte);
543 : break;
544 1 : case TYPE_sht:
545 100001 : CreateNullMask(sht);
546 : break;
547 265 : case TYPE_int:
548 436450 : CreateNullMask(int);
549 : break;
550 5 : case TYPE_lng:
551 100010 : CreateNullMask(lng);
552 : break;
553 0 : case TYPE_flt:
554 0 : CreateNullMask(flt);
555 : break;
556 10 : case TYPE_dbl:
557 202121 : CreateNullMask(dbl);
558 : break;
559 : #ifdef HAVE_HGE
560 0 : case TYPE_hge:
561 0 : CreateNullMask(hge);
562 : break;
563 : #endif
564 8 : default: {
565 8 : int (*atomcmp)(const void *, const void *) = ATOMcompare(b->ttype);
566 35 : for (j = 0; j < count; j++) {
567 27 : mask_data[j] = (*atomcmp)(BUNtail(bi, (BUN)(j)), nil) == 0;
568 27 : found_nil |= mask_data[j];
569 : }
570 : }
571 : }
572 291 : bat_iterator_end(&bi);
573 :
574 291 : if (!found_nil) {
575 267 : Py_DECREF(nullmask);
576 267 : Py_RETURN_NONE;
577 : }
578 :
579 : return (PyObject *)nullmask;
580 : }
581 :
582 : PyObject *
583 41 : PyDict_CheckForConversion(PyObject *pResult, int expected_columns, char **retcol_names, char **return_message)
584 : {
585 41 : char *msg = MAL_SUCCEED;
586 41 : PyObject *result = PyList_New(expected_columns);
587 41 : int i;
588 :
589 181 : for (i = 0; i < expected_columns; i++) {
590 102 : PyObject *object = PyDict_GetItemString(pResult, retcol_names[i]);
591 102 : if (object == NULL) {
592 2 : msg = createException(MAL, "pyapi3.eval",
593 : SQLSTATE(PY000) "Expected a return value with name \"%s\", but "
594 : "this key was not present in the dictionary.",
595 : retcol_names[i]);
596 2 : goto wrapup;
597 : }
598 100 : Py_INCREF(object);
599 100 : object = PyObject_CheckForConversion(object, 1, NULL, return_message);
600 100 : if (object == NULL) {
601 1 : msg = createException(
602 : MAL, "pyapi3.eval",
603 : SQLSTATE(PY000) "Error converting dict return value \"%s\": %s",
604 : retcol_names[i], getExceptionMessage(*return_message));
605 1 : freeException(*return_message);
606 1 : goto wrapup;
607 : }
608 99 : if (PyList_CheckExact(object)) {
609 99 : PyObject *item = PyList_GetItem(object, 0);
610 99 : PyList_SetItem(result, i, item);
611 99 : Py_INCREF(item);
612 99 : Py_DECREF(object);
613 : } else {
614 0 : if (object)
615 0 : Py_DECREF(object);
616 0 : msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Why is this not a list?");
617 0 : goto wrapup;
618 : }
619 : }
620 38 : Py_DECREF(pResult);
621 38 : return result;
622 3 : wrapup:
623 3 : *return_message = msg;
624 3 : Py_DECREF(result);
625 3 : Py_DECREF(pResult);
626 3 : return NULL;
627 : }
628 :
629 : PyObject *
630 212 : PyObject_CheckForConversion(PyObject *pResult, int expected_columns, int *actual_columns, char **return_message)
631 : {
632 212 : char *msg;
633 212 : int columns = 0;
634 :
635 212 : if (pResult) {
636 212 : PyObject *pColO = NULL;
637 212 : if (PyType_IsPandasDataFrame(pResult)) {
638 : // the result object is a Pandas data frame
639 : // we can convert the pandas data frame to a numpy array by simply
640 : // accessing the "values" field (as pandas dataframes are numpy
641 : // arrays internally)
642 0 : PyObject *nResult = PyObject_GetAttrString(pResult, "values");
643 0 : Py_DECREF(pResult);
644 0 : if (nResult == NULL) {
645 0 : msg = createException(MAL, "pyapi3.eval",
646 : SQLSTATE(PY000) "Invalid Pandas data frame.");
647 0 : goto wrapup;
648 : }
649 : // we transpose the values field so it's aligned correctly for our
650 : // purposes
651 0 : pResult = PyObject_GetAttrString(nResult, "T");
652 0 : Py_DECREF(nResult);
653 0 : if (pResult == NULL) {
654 0 : msg = createException(MAL, "pyapi3.eval",
655 : SQLSTATE(PY000) "Invalid Pandas data frame.");
656 0 : goto wrapup;
657 : }
658 : }
659 :
660 212 : if (PyType_IsPyScalar(pResult)) { // check if the return object is a scalar
661 68 : if (expected_columns == 1 || expected_columns <= 0) {
662 : // if we only expect a single return value, we can accept
663 : // scalars by converting it into an array holding an array
664 : // holding the element (i.e. [[pResult]])
665 68 : PyObject *list = PyList_New(1);
666 68 : PyList_SetItem(list, 0, pResult);
667 68 : pResult = list;
668 :
669 68 : list = PyList_New(1);
670 68 : PyList_SetItem(list, 0, pResult);
671 68 : pResult = list;
672 :
673 68 : columns = 1;
674 : } else {
675 : // the result object is a scalar, yet we expect more than one
676 : // return value. We can only convert the result into a list with
677 : // a single element, so the output is necessarily wrong.
678 0 : msg = createException(
679 : MAL, "pyapi3.eval",
680 : SQLSTATE(PY000) "A single scalar was returned, yet we expect a list of %d "
681 : "columns. We can only convert a single scalar into a "
682 : "single column, thus the result is invalid.",
683 : expected_columns);
684 0 : goto wrapup;
685 : }
686 : } else {
687 : // if it is not a scalar, we check if it is a single array
688 144 : bool IsSingleArray = true;
689 144 : PyObject *data = pResult, *ndata = NULL;
690 144 : if (PyType_IsNumpyMaskedArray(data)) {
691 30 : ndata = PyObject_GetAttrString(pResult, "data");
692 30 : if (ndata == NULL) {
693 0 : msg = createException(MAL, "pyapi3.eval",
694 : SQLSTATE(PY000) "Invalid masked array.");
695 0 : goto wrapup;
696 : }
697 : data = ndata;
698 : }
699 144 : if (PyType_IsNumpyArray(data)) {
700 105 : if (PyArray_NDIM((PyArrayObject *)data) != 1) {
701 : IsSingleArray = false;
702 105 : } else if (PyArray_SIZE((PyArrayObject *)data) == 0) {
703 : IsSingleArray = true;
704 : } else {
705 103 : pColO = PyArray_GETITEM( (PyArrayObject *)data, PyArray_GETPTR1((PyArrayObject *)data, 0));
706 103 : IsSingleArray = PyType_IsPyScalar(pColO);
707 103 : Py_DECREF(pColO);
708 : }
709 39 : } else if (PyList_Check(data)) {
710 39 : if (PyList_Size(data) == 0) {
711 : IsSingleArray = true;
712 : } else {
713 39 : pColO = PyList_GetItem(data, 0);
714 39 : IsSingleArray = PyType_IsPyScalar(pColO);
715 : }
716 0 : } else if (!PyType_IsNumpyMaskedArray(data)) {
717 : // it is neither a python array, numpy array or numpy masked
718 : // array, thus the result is unsupported! Throw an exception!
719 0 : msg = createException(
720 : MAL, "pyapi3.eval",
721 : SQLSTATE(PY000) "Unsupported result object. Expected either a list, "
722 : "dictionary, a numpy array, a numpy masked array or a "
723 : "pandas data frame, but received an object of type \"%s\"",
724 : /* leaks */
725 : PyUnicode_AsUTF8(PyObject_Str(PyObject_Type(data))));
726 0 : goto wrapup;
727 : }
728 :
729 142 : if (IsSingleArray) {
730 136 : if (expected_columns == 1 || expected_columns <= 0) {
731 : // if we only expect a single return value, we can accept a
732 : // single array by converting it into an array holding an
733 : // array holding the element (i.e. [pResult])
734 136 : PyObject *list = PyList_New(1);
735 136 : PyList_SetItem(list, 0, pResult);
736 136 : pResult = list;
737 :
738 136 : columns = 1;
739 : } else {
740 : // the result object is a single array, yet we expect more
741 : // than one return value. We can only convert the result
742 : // into a list with a single array, so the output is
743 : // necessarily wrong.
744 0 : msg = createException(MAL, "pyapi3.eval",
745 : SQLSTATE(PY000) "A single array was returned, yet we "
746 : "expect a list of %d columns. The "
747 : "result is invalid.",
748 : expected_columns);
749 0 : goto wrapup;
750 : }
751 : } else {
752 : // the return value is an array of arrays, all we need to do is
753 : // check if it is the correct size
754 8 : int results = 0;
755 8 : if (PyList_Check(data))
756 8 : results = (int)PyList_Size(data);
757 : else
758 0 : results = (int)PyArray_DIMS((PyArrayObject *)data)[0];
759 8 : columns = results;
760 8 : if (results != expected_columns && expected_columns > 0) {
761 : // wrong return size, we expect pci->retc arrays
762 1 : msg = createException(MAL, "pyapi3.eval",
763 : SQLSTATE(PY000) "An array of size %d was returned, "
764 : "yet we expect a list of %d columns. "
765 : "The result is invalid.",
766 : results, expected_columns);
767 1 : goto wrapup;
768 : }
769 : }
770 143 : if (ndata)
771 30 : Py_DECREF(ndata);
772 : }
773 : } else {
774 0 : msg = createException(
775 : MAL, "pyapi3.eval",
776 : SQLSTATE(PY000) "Invalid result object. No result object could be generated.");
777 0 : goto wrapup;
778 : }
779 :
780 211 : if (actual_columns != NULL)
781 0 : *actual_columns = columns;
782 : return pResult;
783 1 : wrapup:
784 1 : if (actual_columns != NULL)
785 0 : *actual_columns = columns;
786 1 : *return_message = msg;
787 1 : return NULL;
788 : }
789 :
790 : str
791 254 : PyObject_GetReturnValues(PyObject *obj, PyReturn *ret)
792 : {
793 254 : PyObject *pMask = NULL;
794 254 : str msg = MAL_SUCCEED;
795 : // If it isn't we need to convert pColO to the expected Numpy Array type
796 254 : ret->numpy_array = PyArray_FromAny( obj, NULL, 1, 1, NPY_ARRAY_CARRAY | NPY_ARRAY_FORCECAST, NULL);
797 254 : if (ret->numpy_array == NULL) {
798 0 : msg = createException(
799 : MAL, "pyapi3.eval",
800 : SQLSTATE(PY000) "Could not create a Numpy array from the return type.\n");
801 0 : goto wrapup;
802 : }
803 :
804 254 : ret->result_type = PyArray_DESCR((PyArrayObject *)ret->numpy_array)->type_num; // We read the result type from the resulting array
805 254 : ret->memory_size = PyArray_DESCR((PyArrayObject *)ret->numpy_array)->elsize;
806 254 : ret->count = PyArray_DIMS((PyArrayObject *)ret->numpy_array)[0];
807 254 : ret->array_data = PyArray_DATA((PyArrayObject *)ret->numpy_array);
808 254 : ret->mask_data = NULL;
809 254 : ret->numpy_mask = NULL;
810 : // If pColO is a Masked array, we convert the mask to a NPY_BOOL numpy array
811 254 : if (PyObject_HasAttrString(obj, "mask")) {
812 30 : pMask = PyObject_GetAttrString(obj, "mask");
813 30 : if (pMask != NULL) {
814 30 : ret->numpy_mask = PyArray_FromAny(pMask, PyArray_DescrFromType(NPY_BOOL), 1, 1, NPY_ARRAY_CARRAY, NULL);
815 30 : Py_DECREF(pMask);
816 30 : if (ret->numpy_mask == NULL || PyArray_DIMS((PyArrayObject *)ret->numpy_mask)[0] != (int)ret->count) {
817 0 : PyErr_Clear();
818 0 : if (ret->numpy_mask)
819 0 : Py_DECREF(ret->numpy_mask);
820 0 : ret->numpy_mask = NULL;
821 : }
822 : }
823 : }
824 254 : if (ret->numpy_mask != NULL)
825 30 : ret->mask_data = PyArray_DATA((PyArrayObject *)ret->numpy_mask);
826 224 : wrapup:
827 254 : return msg;
828 : }
829 :
830 : bool
831 150 : PyObject_PreprocessObject(PyObject *pResult, PyReturn *pyreturn_values, int column_count, char **return_message)
832 : {
833 150 : int i;
834 150 : char *msg;
835 363 : for (i = 0; i < column_count; i++) {
836 : // Refers to the current Numpy mask (if it exists)
837 213 : PyObject *pMask = NULL;
838 : // Refers to the current Numpy array
839 213 : PyObject *pColO = NULL;
840 : // This is the PyReturn header information for the current return value,
841 : // we will fill this now
842 213 : PyReturn *ret = &pyreturn_values[i];
843 :
844 213 : ret->multidimensional = false;
845 : // There are three possibilities (we have ensured this right after
846 : // executing the Python call by calling PyObject_CheckForConversion)
847 : // 1: The top level result object is a PyList or Numpy Array containing
848 : // pci->retc Numpy Arrays
849 : // 2: The top level result object is a (pci->retc x N) dimensional Numpy
850 : // Array [Multidimensional]
851 : // 3: The top level result object is a (pci->retc x N) dimensional Numpy
852 : // Masked Array [Multidimensional]
853 213 : if (PyList_Check(pResult)) {
854 : // If it is a PyList, we simply get the i'th Numpy array from the
855 : // PyList
856 213 : pColO = PyList_GetItem(pResult, i);
857 213 : Py_INCREF(pColO);
858 : } else {
859 : // If it isn't, the result object is either a Nump Masked Array or a
860 : // Numpy Array
861 0 : PyObject *data = pResult;
862 0 : if (PyType_IsNumpyMaskedArray(data)) {
863 0 : assert(0);
864 : data = PyObject_GetAttrString( pResult, "data");
865 : // If it is a Masked array, the data is stored in the masked_array.data attribute
866 : pMask = PyObject_GetAttrString(pResult, "mask");
867 : }
868 :
869 : // We can either have a multidimensional numpy array, or a single
870 : // dimensional numpy array
871 0 : if (PyArray_NDIM((PyArrayObject *)data) != 1) {
872 : // If it is a multidimensional numpy array, we have to convert
873 : // the i'th dimension to a NUMPY array object
874 0 : ret->multidimensional = true;
875 0 : ret->result_type = PyArray_DESCR((PyArrayObject *)data)->type_num;
876 : } else {
877 : // If it is a single dimensional Numpy array, we get the i'th
878 : // Numpy array from the Numpy Array
879 0 : pColO = PyArray_GETITEM((PyArrayObject *)data, PyArray_GETPTR1((PyArrayObject *)data, i));
880 : }
881 : }
882 :
883 : // Now we have to do some preprocessing on the data
884 213 : if (ret->multidimensional) {
885 0 : assert(pColO==NULL);
886 : // If it is a multidimensional Numpy array, we don't need to do any
887 : // conversion, we can just do some pointers
888 0 : ret->count = PyArray_DIMS((PyArrayObject *)pResult)[1];
889 0 : ret->numpy_array = pResult;
890 0 : ret->numpy_mask = pMask;
891 0 : ret->array_data = PyArray_DATA((PyArrayObject *)ret->numpy_array);
892 0 : if (ret->numpy_mask != NULL)
893 : ret->mask_data = PyArray_DATA((PyArrayObject *)ret->numpy_mask);
894 0 : ret->memory_size = PyArray_DESCR((PyArrayObject *)ret->numpy_array)->elsize;
895 : } else {
896 213 : msg = PyObject_GetReturnValues(pColO, ret);
897 213 : Py_DECREF(pColO);
898 213 : if (msg != MAL_SUCCEED) {
899 0 : goto wrapup;
900 : }
901 : }
902 : }
903 : return true;
904 0 : wrapup:
905 0 : *return_message = msg;
906 0 : return false;
907 : }
908 :
909 : BAT *
910 213 : PyObject_ConvertToBAT(PyReturn *ret, sql_subtype *type, int bat_type, int i, oid seqbase, char **return_message, bool copy)
911 : {
912 213 : BAT *b = NULL;
913 213 : size_t index_offset = 0;
914 213 : char *msg;
915 213 : size_t iu;
916 :
917 213 : if (ret->multidimensional)
918 0 : index_offset = i;
919 :
920 213 : switch (GetSQLType(type)) {
921 3 : case EC_TIMESTAMP:
922 3 : bat_type = TYPE_timestamp;
923 3 : break;
924 3 : case EC_TIME:
925 3 : bat_type = TYPE_daytime;
926 3 : break;
927 4 : case EC_DATE:
928 4 : bat_type = TYPE_date;
929 4 : break;
930 4 : case EC_DEC:
931 4 : bat_type = TYPE_dbl;
932 4 : break;
933 : default:
934 : break;
935 : }
936 :
937 213 : if (IsBlobType(bat_type)) {
938 2 : bool *mask = NULL;
939 2 : char *data = NULL;
940 2 : blob *ele_blob;
941 2 : size_t blob_fixed_size = ret->memory_size;
942 :
943 2 : PyObject *pickle_module = NULL, *pickle = NULL;
944 2 : bool gstate = 0;
945 :
946 2 : if (ret->result_type == NPY_OBJECT) {
947 : // Python objects, we may need to pickle them, so we
948 : // may execute Python code, we have to obtain the GIL
949 0 : gstate = Python_ObtainGIL();
950 0 : pickle_module = PyImport_ImportModule("pickle");
951 0 : if (pickle_module == NULL) {
952 0 : msg = createException(MAL, "pyapi3.eval",
953 : SQLSTATE(PY000) "Can't load pickle module to pickle python object to blob");
954 0 : Python_ReleaseGIL(gstate);
955 0 : goto wrapup;
956 : }
957 : blob_fixed_size = 0; // Size depends on the objects
958 : }
959 :
960 2 : if (ret->mask_data != NULL) {
961 : mask = (bool *)ret->mask_data;
962 : }
963 2 : if (ret->array_data == NULL) {
964 0 : msg = createException(MAL, "pyapi3.eval",
965 : SQLSTATE(PY000) "No return value stored in the structure.");
966 0 : if (ret->result_type == NPY_OBJECT) {
967 0 : Py_XDECREF(pickle_module);
968 0 : Python_ReleaseGIL(gstate);
969 : }
970 0 : goto wrapup;
971 : }
972 2 : data = (char *)ret->array_data;
973 2 : data += (index_offset * ret->count) * ret->memory_size;
974 2 : b = COLnew(seqbase, TYPE_blob, (BUN)ret->count, TRANSIENT);
975 2 : if (b == NULL) {
976 0 : if (ret->result_type == NPY_OBJECT) {
977 0 : Py_XDECREF(pickle_module);
978 0 : Python_ReleaseGIL(gstate);
979 : }
980 0 : msg = createException(MAL, "pyapi3.eval", GDK_EXCEPTION);
981 0 : goto wrapup;
982 : }
983 2 : b->tnil = false;
984 2 : b->tnonil = true;
985 2 : b->tkey = false;
986 2 : b->tsorted = false;
987 2 : b->trevsorted = false;
988 8 : for (iu = 0; iu < ret->count; iu++) {
989 :
990 6 : char* memcpy_data;
991 6 : size_t blob_len = 0;
992 :
993 6 : if (ret->result_type == NPY_OBJECT) {
994 0 : PyObject *object = *((PyObject **)&data[0]);
995 0 : if (PyByteArray_Check(object)) {
996 0 : memcpy_data = PyByteArray_AsString(object);
997 0 : blob_len = pyobject_get_size(object);
998 : } else {
999 0 : pickle = PyObject_CallMethod(pickle_module, "dumps", "O", object);
1000 0 : if (pickle == NULL) {
1001 0 : msg = createException(MAL, "pyapi3.eval",
1002 : SQLSTATE(PY000) "Can't pickle object to blob");
1003 0 : Py_XDECREF(pickle_module);
1004 0 : Python_ReleaseGIL(gstate);
1005 0 : goto wrapup;
1006 : }
1007 0 : memcpy_data = PyBytes_AsString(pickle);
1008 0 : blob_len = pyobject_get_size(pickle);
1009 0 : Py_XDECREF(pickle);
1010 : }
1011 0 : if (memcpy_data == NULL) {
1012 0 : msg = createException(MAL, "pyapi3.eval",
1013 : SQLSTATE(PY000) "Can't get blob pickled object as char*");
1014 0 : Py_XDECREF(pickle_module);
1015 0 : Python_ReleaseGIL(gstate);
1016 0 : goto wrapup;
1017 : }
1018 : } else {
1019 : memcpy_data = data;
1020 : }
1021 :
1022 6 : if (mask && mask[iu]) {
1023 0 : ele_blob = (blob *)GDKmalloc(offsetof(blob, data));
1024 0 : ele_blob->nitems = ~(size_t)0;
1025 : } else {
1026 6 : if (blob_fixed_size > 0) {
1027 6 : blob_len = blob_fixed_size;
1028 : }
1029 6 : ele_blob = GDKmalloc(blobsize(blob_len));
1030 6 : ele_blob->nitems = blob_len;
1031 6 : memcpy(ele_blob->data, memcpy_data, blob_len);
1032 : }
1033 6 : if (BUNappend(b, ele_blob, false) != GDK_SUCCEED) {
1034 0 : if (ret->result_type == NPY_OBJECT) {
1035 0 : Py_XDECREF(pickle_module);
1036 0 : Python_ReleaseGIL(gstate);
1037 : }
1038 0 : BBPunfix(b->batCacheid);
1039 0 : msg = createException(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL);
1040 0 : goto wrapup;
1041 : }
1042 6 : GDKfree(ele_blob);
1043 6 : data += ret->memory_size;
1044 :
1045 : }
1046 :
1047 : // We are done, we can release the GIL
1048 2 : if (ret->result_type == NPY_OBJECT) {
1049 0 : Py_XDECREF(pickle_module);
1050 0 : Python_ReleaseGIL(gstate);
1051 : }
1052 :
1053 2 : BATsetcount(b, (BUN)ret->count);
1054 : } else {
1055 211 : switch (bat_type) {
1056 0 : case TYPE_void:
1057 0 : NP_CREATE_EMPTY_BAT(b, oid);
1058 0 : break;
1059 9 : case TYPE_bit:
1060 18 : NP_CREATE_BAT(b, bit);
1061 : break;
1062 1 : case TYPE_bte:
1063 101 : NP_CREATE_BAT(b, bte);
1064 : break;
1065 2 : case TYPE_sht:
1066 100003 : NP_CREATE_BAT(b, sht);
1067 : break;
1068 136 : case TYPE_int:
1069 20470561 : NP_CREATE_BAT(b, int);
1070 : break;
1071 0 : case TYPE_oid:
1072 0 : NP_CREATE_BAT(b, oid);
1073 : break;
1074 2 : case TYPE_lng:
1075 4 : NP_CREATE_BAT(b, lng);
1076 : break;
1077 0 : case TYPE_flt:
1078 0 : NP_CREATE_BAT(b, flt);
1079 : break;
1080 30 : case TYPE_dbl:
1081 1700171 : NP_CREATE_BAT(b, dbl);
1082 : break;
1083 : #ifdef HAVE_HGE
1084 0 : case TYPE_hge:
1085 0 : NP_CREATE_BAT(b, hge);
1086 : break;
1087 : #endif
1088 4 : case TYPE_date:
1089 10 : NP_CREATE_BAT(b, date);
1090 : break;
1091 3 : case TYPE_daytime:
1092 6 : NP_CREATE_BAT(b, daytime);
1093 : break;
1094 3 : case TYPE_timestamp:
1095 6 : NP_CREATE_BAT(b, timestamp);
1096 : break;
1097 21 : case TYPE_str: {
1098 21 : bool *mask = NULL;
1099 21 : char *data = NULL;
1100 21 : char *utf8_string = NULL;
1101 21 : if (ret->mask_data != NULL) {
1102 : mask = (bool *)ret->mask_data;
1103 : }
1104 21 : if (ret->array_data == NULL) {
1105 0 : msg = createException(
1106 : MAL, "pyapi3.eval",
1107 : SQLSTATE(PY000) "No return value stored in the structure. n");
1108 0 : goto wrapup;
1109 : }
1110 21 : data = (char *)ret->array_data;
1111 :
1112 21 : if (ret->result_type != NPY_OBJECT) {
1113 15 : utf8_string = GDKzalloc(utf8string_minlength + ret->memory_size + 1);
1114 15 : utf8_string[utf8string_minlength + ret->memory_size] = '\0';
1115 : }
1116 :
1117 21 : b = COLnew(seqbase, TYPE_str, (BUN)ret->count, TRANSIENT);
1118 21 : if (b == NULL) {
1119 0 : GDKfree(utf8_string);
1120 0 : msg = createException(MAL, "pyapi3.eval", GDK_EXCEPTION);
1121 0 : goto wrapup;
1122 : }
1123 21 : b->tnil = false;
1124 21 : b->tnonil = true;
1125 21 : b->tkey = false;
1126 21 : b->tsorted = false;
1127 21 : b->trevsorted = false;
1128 1300085 : NP_INSERT_STRING_BAT(b);
1129 21 : GDKfree(utf8_string);
1130 21 : BATsetcount(b, (BUN)ret->count);
1131 21 : break;
1132 : }
1133 0 : default:
1134 0 : msg = createException(MAL, "pyapi3.eval",
1135 : SQLSTATE(PY000) "Unrecognized BAT type %s.\n",
1136 : BatType_Format(bat_type));
1137 0 : goto wrapup;
1138 : }
1139 : }
1140 213 : if (ConvertableSQLType(type)) {
1141 4 : BAT *result;
1142 4 : msg = ConvertToSQLType(NULL, b, type, &result, &bat_type);
1143 4 : BBPunfix(b->batCacheid);
1144 4 : if (msg != MAL_SUCCEED) {
1145 0 : goto wrapup;
1146 : }
1147 4 : b = result;
1148 : }
1149 :
1150 : return b;
1151 :
1152 0 : wrapup:
1153 0 : *return_message = msg;
1154 0 : return NULL;
1155 : }
1156 :
1157 538 : bit ConvertableSQLType(sql_subtype *sql_subtype)
1158 : {
1159 538 : switch (GetSQLType(sql_subtype)) {
1160 : case EC_DEC:
1161 : return 1;
1162 : }
1163 531 : return 0;
1164 : }
1165 :
1166 751 : int GetSQLType(sql_subtype *sql_subtype)
1167 : {
1168 751 : if (!sql_subtype)
1169 : return -1;
1170 535 : if (!sql_subtype->type)
1171 : return -1;
1172 535 : return (int) sql_subtype->type->eclass;
1173 : }
1174 :
1175 : str
1176 4 : ConvertFromSQLType(BAT *b, sql_subtype *sql_subtype, BAT **ret_bat, int *ret_type)
1177 : {
1178 4 : str res = MAL_SUCCEED;
1179 4 : int conv_type;
1180 :
1181 4 : assert(sql_subtype);
1182 4 : assert(sql_subtype->type);
1183 :
1184 4 : switch (sql_subtype->type->eclass) {
1185 : case EC_DEC:
1186 : conv_type = TYPE_dbl;
1187 : break;
1188 : default:
1189 1 : conv_type = TYPE_str;
1190 : }
1191 :
1192 4 : if (conv_type == TYPE_str) {
1193 1 : BUN p = 0, q = 0;
1194 1 : char *result = NULL;
1195 1 : size_t length = 0;
1196 1 : ssize_t (*strConversion)(str *, size_t *, const void *, bool) =
1197 1 : BATatoms[b->ttype].atomToStr;
1198 1 : *ret_bat = COLnew(0, TYPE_str, 0, TRANSIENT);
1199 1 : *ret_type = conv_type;
1200 1 : if (!(*ret_bat)) {
1201 0 : return createException(MAL, "pyapi3.eval",
1202 : SQLSTATE(HY013) MAL_MALLOC_FAIL " string conversion BAT.");
1203 : }
1204 1 : BATiter li = bat_iterator(b);
1205 2 : BATloop(b, p, q)
1206 : {
1207 1 : const void *element = (const void*)BUNtail(li, p);
1208 1 : if (strConversion(&result, &length, element, false) < 0) {
1209 0 : bat_iterator_end(&li);
1210 0 : BBPunfix((*ret_bat)->batCacheid);
1211 0 : return createException(MAL, "pyapi3.eval",
1212 : SQLSTATE(PY000) "Failed to convert element to string.");
1213 : }
1214 1 : if (BUNappend(*ret_bat, result, false) != GDK_SUCCEED) {
1215 0 : bat_iterator_end(&li);
1216 0 : BBPunfix((*ret_bat)->batCacheid);
1217 0 : throw(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL);
1218 : }
1219 : }
1220 1 : bat_iterator_end(&li);
1221 1 : if (result) {
1222 1 : GDKfree(result);
1223 : }
1224 1 : return res;
1225 3 : } else if (conv_type == TYPE_dbl) {
1226 3 : int bat_type = ATOMstorage(b->ttype);
1227 3 : int hpos = sql_subtype->scale;
1228 3 : bat result = 0;
1229 : // decimal values can be stored in various numeric fields, so check the
1230 : // numeric field and convert the one it's actually stored in
1231 3 : switch (bat_type) {
1232 0 : case TYPE_bte:
1233 0 : res = batbte_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
1234 0 : break;
1235 0 : case TYPE_sht:
1236 0 : res = batsht_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
1237 0 : break;
1238 0 : case TYPE_int:
1239 0 : res = batint_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
1240 0 : break;
1241 3 : case TYPE_lng:
1242 3 : res = batlng_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
1243 3 : break;
1244 : #ifdef HAVE_HGE
1245 0 : case TYPE_hge:
1246 0 : res = bathge_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
1247 0 : break;
1248 : #endif
1249 0 : default:
1250 0 : return createException(MAL, "pyapi3.eval",
1251 : "Unsupported decimal storage type.");
1252 : }
1253 3 : if (res == MAL_SUCCEED) {
1254 3 : *ret_bat = BATdescriptor(result);
1255 3 : BBPrelease(result);
1256 3 : *ret_type = TYPE_dbl;
1257 : } else {
1258 0 : *ret_bat = NULL;
1259 : }
1260 3 : return res;
1261 : }
1262 : return createException(MAL, "pyapi3.eval", "Unrecognized conv type.");
1263 : }
1264 :
1265 4 : str ConvertToSQLType(Client cntxt, BAT *b, sql_subtype *sql_subtype,
1266 : BAT **ret_bat, int *ret_type)
1267 : {
1268 4 : str res = MAL_SUCCEED;
1269 4 : bat result_bat = 0;
1270 4 : int digits = sql_subtype->digits;
1271 4 : int scale = sql_subtype->scale;
1272 4 : (void)cntxt;
1273 :
1274 4 : assert(sql_subtype);
1275 4 : assert(sql_subtype->type);
1276 :
1277 4 : switch (sql_subtype->type->eclass) {
1278 0 : case EC_TIMESTAMP:
1279 0 : res = batstr_2time_timestamp(&result_bat, &b->batCacheid, NULL, &digits);
1280 0 : break;
1281 0 : case EC_TIME:
1282 0 : res = batstr_2time_daytime(&result_bat, &b->batCacheid, NULL, &digits);
1283 0 : break;
1284 0 : case EC_DATE:
1285 0 : if ((*ret_bat = BATconvert(b, NULL, TYPE_date, 0, 0, 0)) == NULL)
1286 0 : throw(MAL, "pyapi3.eval", GDK_EXCEPTION);
1287 0 : *ret_type = TYPE_date;
1288 0 : return MAL_SUCCEED;
1289 4 : case EC_DEC:
1290 4 : res = batdbl_num2dec_lng(&result_bat, &b->batCacheid, NULL,
1291 : &digits, &scale);
1292 4 : break;
1293 0 : default:
1294 0 : return createException(
1295 : MAL, "pyapi3.eval",
1296 : "Convert To SQL Type: Unrecognized SQL type %s (%d).",
1297 : sql_subtype->type->base.name, (int) sql_subtype->type->eclass);
1298 : }
1299 4 : if (res == MAL_SUCCEED) {
1300 4 : *ret_bat = BATdescriptor(result_bat);
1301 4 : BBPrelease(result_bat);
1302 4 : *ret_type = (*ret_bat)->ttype;
1303 : }
1304 :
1305 : return res;
1306 : }
1307 :
1308 224 : ssize_t PyType_Size(PyObject *obj)
1309 : {
1310 224 : if (PyType_IsPyScalar(obj)) {
1311 : return 1;
1312 : }
1313 62 : if (PyArray_Check(obj)) {
1314 57 : return PyArray_Size(obj);
1315 : }
1316 5 : if (PyList_Check(obj)) {
1317 5 : return Py_SIZE(obj);
1318 : }
1319 : return -1;
1320 : }
1321 :
1322 326 : bit IsStandardBATType(int type)
1323 : {
1324 326 : switch (type) {
1325 : case TYPE_bit:
1326 : case TYPE_bte:
1327 : case TYPE_sht:
1328 : case TYPE_int:
1329 : case TYPE_oid:
1330 : case TYPE_lng:
1331 : case TYPE_flt:
1332 : case TYPE_dbl:
1333 : #ifdef HAVE_HGE
1334 : case TYPE_hge:
1335 : #endif
1336 : case TYPE_date:
1337 : case TYPE_daytime:
1338 : case TYPE_timestamp:
1339 : case TYPE_str:
1340 : return 1;
1341 1 : default:
1342 1 : return 0;
1343 : }
1344 : }
1345 :
1346 10 : static void conversion_import_array(void) { _import_array(); }
1347 :
1348 10 : str _conversion_init(void)
1349 : {
1350 10 : str msg = MAL_SUCCEED;
1351 10 : conversion_import_array();
1352 :
1353 10 : return msg;
1354 : }
|
