Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024, 2025 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : #include "monetdb_config.h"
14 : #include "bat_logger.h"
15 : #include "bat_utils.h"
16 : #include "sql_types.h" /* EC_POS */
17 : #include "gdk_logger_internals.h"
18 : #include "mutils.h"
19 :
20 : #define CATALOG_JUL2021 52300 /* first in Jul2021 */
21 : #define CATALOG_JAN2022 52301 /* first in Jan2022 */
22 : #define CATALOG_SEP2022 52302 /* first in Sep2022 */
23 : #define CATALOG_AUG2024 52303 /* first in Aug2024 */
24 :
25 : /* Note, CATALOG version 52300 is the first one where the basic system
26 : * tables (the ones created in store.c) have fixed and unchangeable
27 : * ids. */
28 :
29 : /* return GDK_SUCCEED if we can handle the upgrade from oldversion to
30 : * newversion */
31 : static gdk_return
32 16 : bl_preversion(sqlstore *store, int oldversion, int newversion)
33 : {
34 16 : (void)newversion;
35 :
36 : #ifdef CATALOG_JUL2021
37 16 : if (oldversion == CATALOG_JUL2021) {
38 : /* upgrade to default releases */
39 0 : store->catalog_version = oldversion;
40 0 : return GDK_SUCCEED;
41 : }
42 : #endif
43 :
44 : #ifdef CATALOG_JAN2022
45 16 : if (oldversion == CATALOG_JAN2022) {
46 : /* upgrade to default releases */
47 0 : store->catalog_version = oldversion;
48 0 : return GDK_SUCCEED;
49 : }
50 : #endif
51 :
52 : #ifdef CATALOG_SEP2022
53 16 : if (oldversion == CATALOG_SEP2022) {
54 : /* upgrade to default releases */
55 8 : store->catalog_version = oldversion;
56 8 : return GDK_SUCCEED;
57 : }
58 : #endif
59 :
60 : #ifdef CATALOG_AUG2024
61 8 : if (oldversion == CATALOG_AUG2024) {
62 : /* upgrade to default releases */
63 8 : store->catalog_version = oldversion;
64 8 : return GDK_SUCCEED;
65 : }
66 : #endif
67 :
68 : return GDK_FAIL;
69 : }
70 :
71 : #if defined CATALOG_JUL2021 || defined CATALOG_JAN2022
72 : /* replace a column in a system table with a new column
73 : * colid is the SQL id for the column, newcol is the new BAT */
74 : static gdk_return
75 32 : replace_bat(logger *lg, int colid, BAT *newcol)
76 : {
77 32 : gdk_return rc;
78 32 : newcol = BATsetaccess(newcol, BAT_READ);
79 32 : if (newcol == NULL)
80 : return GDK_FAIL;
81 32 : if ((rc = BAThash(lg->catalog_id)) == GDK_SUCCEED) {
82 32 : BATiter cii = bat_iterator_nolock(lg->catalog_id);
83 32 : BUN p;
84 32 : MT_rwlock_rdlock(&cii.b->thashlock);
85 86 : HASHloop_int(cii, cii.b->thash, p, &colid) {
86 32 : if (BUNfnd(lg->dcatalog, &(oid){(oid)p}) == BUN_NONE) {
87 64 : if (BUNappend(lg->dcatalog, &(oid){(oid)p}, true) != GDK_SUCCEED ||
88 32 : BUNreplace(lg->catalog_lid, (oid) p, &(lng){0}, false) != GDK_SUCCEED) {
89 0 : MT_rwlock_rdunlock(&cii.b->thashlock);
90 0 : return GDK_FAIL;
91 : }
92 32 : break;
93 : }
94 : }
95 32 : MT_rwlock_rdunlock(&cii.b->thashlock);
96 32 : if ((rc = BUNappend(lg->catalog_id, &colid, true)) == GDK_SUCCEED &&
97 32 : (rc = BUNappend(lg->catalog_bid, &newcol->batCacheid, true)) == GDK_SUCCEED &&
98 32 : (rc = BUNappend(lg->catalog_lid, &lng_nil, false)) == GDK_SUCCEED &&
99 32 : (rc = BUNappend(lg->catalog_cnt, &(lng){BATcount(newcol)}, false)) == GDK_SUCCEED) {
100 32 : BBPretain(newcol->batCacheid);
101 : }
102 : }
103 : return rc;
104 : }
105 : #endif
106 :
107 : static BAT *
108 1560 : log_temp_descriptor(log_bid b)
109 : {
110 1560 : if (b <= 0)
111 : return NULL;
112 1560 : return temp_descriptor(b);
113 : }
114 :
115 : #if defined CATALOG_JAN2022 || defined CATALOG_SEP2022 || defined CATALOG_AUG2024
116 : /* cannot use attribute((sentinel)) since sentinel is not a pointer */
117 : static gdk_return
118 112 : tabins(logger *lg, ...)
119 : {
120 112 : va_list va;
121 112 : int cid;
122 112 : const void *cval;
123 112 : gdk_return rc;
124 112 : BAT *b;
125 :
126 112 : va_start(va, lg);
127 112 : BATiter cni = bat_iterator(lg->catalog_id);
128 1504 : while ((cid = va_arg(va, int)) != 0) {
129 1392 : cval = va_arg(va, void *);
130 2784 : if ((b = log_temp_descriptor(log_find_bat(lg, cid))) == NULL) {
131 : rc = GDK_FAIL;
132 : break;
133 : }
134 1392 : rc = BUNappend(b, cval, true);
135 1392 : if (rc == GDK_SUCCEED) {
136 1392 : BUN p;
137 1392 : MT_rwlock_rdlock(&cni.b->thashlock);
138 1674 : HASHloop_int(cni, cni.b->thash, p, &cid) {
139 1392 : if (BUNfnd(lg->dcatalog, &(oid){p}) == BUN_NONE) {
140 1392 : rc = BUNreplace(lg->catalog_cnt, p, &(lng){BATcount(b)}, false);
141 1392 : break;
142 : }
143 : }
144 1392 : MT_rwlock_rdunlock(&cni.b->thashlock);
145 : }
146 1392 : bat_destroy(b);
147 1392 : if (rc != GDK_SUCCEED)
148 : break;
149 : }
150 112 : bat_iterator_end(&cni);
151 112 : va_end(va);
152 112 : return rc;
153 : }
154 : #endif
155 :
156 : static gdk_return
157 16 : bl_postversion(void *Store, logger *lg)
158 : {
159 16 : sqlstore *store = Store;
160 16 : gdk_return rc;
161 :
162 : #ifdef CATALOG_JUL2021
163 16 : if (store->catalog_version <= CATALOG_JUL2021) {
164 : /* change the language attribute in sys.functions for sys.env,
165 : * sys.var, and sys.db_users from SQL to MAL */
166 :
167 : /* sys.functions i.e. deleted rows */
168 0 : BAT *del_funcs = log_temp_descriptor(log_find_bat(lg, 2016));
169 0 : if (del_funcs == NULL)
170 0 : return GDK_FAIL;
171 0 : BAT *func_tid = BATmaskedcands(0, BATcount(del_funcs), del_funcs, false);
172 0 : bat_destroy(del_funcs);
173 : /* sys.functions.schema_id */
174 0 : BAT *func_schem = log_temp_descriptor(log_find_bat(lg, 2026));
175 0 : if (func_tid == NULL || func_schem == NULL) {
176 0 : bat_destroy(func_tid);
177 0 : bat_destroy(func_schem);
178 0 : return GDK_FAIL;
179 : }
180 : /* select * from sys.functions where schema_id = 2000 */
181 0 : BAT *cands = BATselect(func_schem, func_tid, &(int) {2000}, NULL, true, true, false, false);
182 0 : bat_destroy(func_schem);
183 0 : if (cands == NULL) {
184 0 : bat_destroy(func_tid);
185 0 : return GDK_FAIL;
186 : }
187 : /* the functions we need to change */
188 0 : BAT *funcs = COLnew(0, TYPE_str, 3, TRANSIENT);
189 0 : if (funcs == NULL ||
190 0 : BUNappend(funcs, "db_users", false) != GDK_SUCCEED ||
191 0 : BUNappend(funcs, "env", false) != GDK_SUCCEED ||
192 0 : BUNappend(funcs, "var", false) != GDK_SUCCEED) {
193 0 : bat_destroy(cands);
194 0 : bat_destroy(funcs);
195 0 : bat_destroy(func_tid);
196 0 : return GDK_FAIL;
197 : }
198 : /* sys.functions.name */
199 0 : BAT *func_name = log_temp_descriptor(log_find_bat(lg, 2018));
200 0 : if (func_name == NULL) {
201 0 : bat_destroy(cands);
202 0 : bat_destroy(funcs);
203 0 : bat_destroy(func_tid);
204 0 : return GDK_FAIL;
205 : }
206 : /* select * from sys.functions where schema_id = 2000 and name in (...) */
207 0 : BAT *b = BATintersect(func_name, funcs, cands, NULL, false, false, 3);
208 0 : bat_destroy(cands);
209 0 : bat_destroy(func_name);
210 0 : bat_destroy(funcs);
211 0 : cands = b;
212 0 : if (cands == NULL) {
213 0 : bat_destroy(func_tid);
214 0 : return GDK_FAIL;
215 : }
216 : /* sys.functions.language */
217 0 : BAT *func_lang = log_temp_descriptor(log_find_bat(lg, 2021));
218 0 : if (func_lang == NULL) {
219 0 : bat_destroy(cands);
220 0 : bat_destroy(func_tid);
221 0 : return GDK_FAIL;
222 : }
223 : /* select * from sys.functions where schema_id = 2000 and name in (...)
224 : * and language = FUNC_LANG_SQL */
225 0 : b = BATselect(func_lang, cands, &(int) {FUNC_LANG_SQL}, NULL, true, true, false, false);
226 0 : bat_destroy(cands);
227 0 : cands = b;
228 0 : if (cands == NULL) {
229 0 : bat_destroy(func_lang);
230 0 : bat_destroy(func_tid);
231 0 : return GDK_FAIL;
232 : }
233 0 : b = BATconstant(0, TYPE_int, &(int) {FUNC_LANG_MAL}, BATcount(cands), TRANSIENT);
234 0 : if (b == NULL) {
235 0 : bat_destroy(func_lang);
236 0 : bat_destroy(cands);
237 0 : bat_destroy(func_tid);
238 0 : return GDK_FAIL;
239 : }
240 0 : rc = GDK_FAIL;
241 0 : BAT *b2 = COLcopy(func_lang, func_lang->ttype, true, PERSISTENT);
242 0 : if (b2 == NULL ||
243 0 : BATreplace(b2, cands, b, false) != GDK_SUCCEED) {
244 0 : bat_destroy(b2);
245 0 : bat_destroy(cands);
246 0 : bat_destroy(b);
247 0 : bat_destroy(func_tid);
248 0 : bat_destroy(func_lang);
249 0 : return GDK_FAIL;
250 : }
251 0 : bat_destroy(b);
252 0 : bat_destroy(cands);
253 :
254 : /* additionally, update the language attribute for entries
255 : * that were declared using "EXTERNAL NAME" to be MAL functions
256 : * instead of SQL functions (a problem that seems to have
257 : * occurred in ancient databases) */
258 :
259 : /* sys.functions.func */
260 0 : BAT *func_func = log_temp_descriptor(log_find_bat(lg, 2019));
261 0 : if (func_func == NULL) {
262 0 : bat_destroy(func_tid);
263 0 : bat_destroy(b2);
264 0 : return GDK_FAIL;
265 : }
266 0 : cands = BATselect(func_lang, func_tid, &(int){FUNC_LANG_SQL}, NULL, true, true, false, false);
267 0 : bat_destroy(func_lang);
268 0 : bat_destroy(func_tid);
269 0 : if (cands == NULL) {
270 0 : bat_destroy(b2);
271 0 : bat_destroy(func_func);
272 0 : return GDK_FAIL;
273 : }
274 0 : struct canditer ci;
275 0 : canditer_init(&ci, func_func, cands);
276 0 : BATiter ffi = bat_iterator_nolock(func_func);
277 0 : for (BUN p = 0; p < ci.ncand; p++) {
278 0 : oid o = canditer_next(&ci);
279 0 : const char *f = BUNtvar(ffi, o - func_func->hseqbase);
280 0 : const char *e;
281 0 : if (!strNil(f) &&
282 0 : (e = strstr(f, "external")) != NULL &&
283 0 : e > f && isspace((unsigned char) e[-1]) && isspace((unsigned char) e[8]) && strncmp(e + 9, "name", 4) == 0 && isspace((unsigned char) e[13]) &&
284 0 : BUNreplace(b2, o, &(int){FUNC_LANG_MAL}, false) != GDK_SUCCEED) {
285 0 : bat_destroy(b2);
286 0 : bat_destroy(func_func);
287 0 : return GDK_FAIL;
288 : }
289 : }
290 0 : rc = replace_bat(lg, 2021, b2);
291 0 : bat_destroy(b2);
292 0 : if (rc != GDK_SUCCEED)
293 : return rc;
294 : }
295 16 : if (store->catalog_version <= CATALOG_JUL2021) {
296 : /* change the side_effects attribute in sys.functions for
297 : * selected functions */
298 :
299 : /* sys.functions i.e. deleted rows */
300 0 : BAT *del_funcs = log_temp_descriptor(log_find_bat(lg, 2016));
301 0 : if (del_funcs == NULL)
302 0 : return GDK_FAIL;
303 0 : BAT *func_tid = BATmaskedcands(0, BATcount(del_funcs), del_funcs, false);
304 0 : bat_destroy(del_funcs);
305 : /* sys.functions.schema_id */
306 0 : BAT *func_schem = log_temp_descriptor(log_find_bat(lg, 2026));
307 0 : if (func_tid == NULL || func_schem == NULL) {
308 0 : bat_destroy(func_tid);
309 0 : bat_destroy(func_schem);
310 0 : return GDK_FAIL;
311 : }
312 : /* select * from sys.functions where schema_id = 2000 */
313 0 : BAT *cands = BATselect(func_schem, func_tid, &(int) {2000}, NULL, true, true, false, false);
314 0 : bat_destroy(func_schem);
315 0 : bat_destroy(func_tid);
316 0 : if (cands == NULL) {
317 : return GDK_FAIL;
318 : }
319 : /* sys.functions.side_effect */
320 0 : BAT *func_se = log_temp_descriptor(log_find_bat(lg, 2023));
321 0 : if (func_se == NULL) {
322 0 : bat_destroy(cands);
323 0 : return GDK_FAIL;
324 : }
325 : /* make a copy that we can modify */
326 0 : BAT *b = COLcopy(func_se, func_se->ttype, true, PERSISTENT);
327 0 : bat_destroy(func_se);
328 0 : if (b == NULL) {
329 0 : bat_destroy(cands);
330 0 : return GDK_FAIL;
331 : }
332 0 : func_se = b;
333 : /* sys.functions.func */
334 0 : BAT *func_func = log_temp_descriptor(log_find_bat(lg, 2019));
335 0 : if (func_func == NULL) {
336 0 : bat_destroy(cands);
337 0 : bat_destroy(func_se);
338 0 : return GDK_FAIL;
339 : }
340 : /* the functions we need to change to FALSE */
341 0 : BAT *funcs = COLnew(0, TYPE_str, 1, TRANSIENT);
342 0 : if (funcs == NULL ||
343 0 : BUNappend(funcs, "sqlrand", false) != GDK_SUCCEED) {
344 0 : bat_destroy(cands);
345 0 : bat_destroy(func_se);
346 0 : bat_destroy(func_func);
347 0 : bat_destroy(funcs);
348 0 : return GDK_FAIL;
349 : }
350 : /* select * from sys.functions where schema_id = 2000 and func in (...) */
351 0 : b = BATintersect(func_func, funcs, cands, NULL, false, false, 1);
352 0 : bat_destroy(funcs);
353 0 : if (b == NULL) {
354 0 : bat_destroy(cands);
355 0 : bat_destroy(func_se);
356 0 : bat_destroy(func_func);
357 0 : return GDK_FAIL;
358 : }
359 : /* while we're at it, also change sys.env and sys.db_users to
360 : * being without side effect (legacy from ancient databases) */
361 : /* sys.functions.name */
362 0 : BAT *func_name = log_temp_descriptor(log_find_bat(lg, 2018));
363 0 : if (func_name == NULL) {
364 0 : bat_destroy(cands);
365 0 : bat_destroy(func_se);
366 0 : bat_destroy(func_func);
367 0 : bat_destroy(b);
368 0 : return GDK_FAIL;
369 : }
370 0 : BAT *b2 = BATselect(func_name, cands, "env", NULL, true, true, false, false);
371 0 : if (b2 == NULL || BATappend(b, b2, NULL, false) != GDK_SUCCEED) {
372 0 : bat_destroy(cands);
373 0 : bat_destroy(func_se);
374 0 : bat_destroy(func_func);
375 0 : bat_destroy(b);
376 0 : bat_destroy(func_name);
377 0 : bat_destroy(b2);
378 0 : return GDK_FAIL;
379 : }
380 0 : bat_destroy(b2);
381 0 : b2 = BATselect(func_name, cands, "db_users", NULL, true, true, false, false);
382 0 : bat_destroy(func_name);
383 0 : if (b2 == NULL || BATappend(b, b2, NULL, false) != GDK_SUCCEED) {
384 0 : bat_destroy(cands);
385 0 : bat_destroy(func_se);
386 0 : bat_destroy(func_func);
387 0 : bat_destroy(b);
388 0 : bat_destroy(b2);
389 0 : return GDK_FAIL;
390 : }
391 0 : bat_destroy(b2);
392 :
393 0 : BAT *vals = BATconstant(0, TYPE_bit, &(bit) {FALSE}, BATcount(b), TRANSIENT);
394 0 : if (vals == NULL) {
395 0 : bat_destroy(cands);
396 0 : bat_destroy(func_se);
397 0 : bat_destroy(func_func);
398 0 : bat_destroy(b);
399 0 : return GDK_FAIL;
400 : }
401 0 : rc = BATreplace(func_se, b, vals, false);
402 0 : bat_destroy(b);
403 0 : bat_destroy(vals);
404 0 : if (rc != GDK_SUCCEED) {
405 0 : bat_destroy(cands);
406 0 : bat_destroy(func_se);
407 0 : bat_destroy(func_func);
408 0 : return GDK_FAIL;
409 : }
410 : /* the functions we need to change to TRUE */
411 0 : funcs = COLnew(0, TYPE_str, 5, TRANSIENT);
412 0 : if (funcs == NULL ||
413 0 : BUNappend(funcs, "copy_from", false) != GDK_SUCCEED ||
414 0 : BUNappend(funcs, "next_value", false) != GDK_SUCCEED ||
415 0 : BUNappend(funcs, "update_schemas", false) != GDK_SUCCEED ||
416 0 : BUNappend(funcs, "update_tables", false) != GDK_SUCCEED) {
417 0 : bat_destroy(cands);
418 0 : bat_destroy(func_se);
419 0 : bat_destroy(func_func);
420 0 : bat_destroy(funcs);
421 0 : return GDK_FAIL;
422 : }
423 : /* select * from sys.functions where schema_id = 2000 and func in (...) */
424 0 : b = BATintersect(func_func, funcs, cands, NULL, false, false, 7);
425 0 : bat_destroy(funcs);
426 0 : bat_destroy(cands);
427 0 : bat_destroy(func_func);
428 0 : if (b == NULL) {
429 0 : bat_destroy(func_se);
430 0 : return GDK_FAIL;
431 : }
432 0 : vals = BATconstant(0, TYPE_bit, &(bit) {TRUE}, BATcount(b), TRANSIENT);
433 0 : if (vals == NULL) {
434 0 : bat_destroy(func_se);
435 0 : bat_destroy(b);
436 0 : return GDK_FAIL;
437 : }
438 0 : rc = BATreplace(func_se, b, vals, false);
439 0 : bat_destroy(b);
440 0 : bat_destroy(vals);
441 0 : if (rc != GDK_SUCCEED) {
442 0 : bat_destroy(func_se);
443 0 : return GDK_FAIL;
444 : }
445 : /* replace old column with modified copy */
446 0 : rc = replace_bat(lg, 2023, func_se);
447 0 : bat_destroy(func_se);
448 0 : if (rc != GDK_SUCCEED)
449 : return rc;
450 : }
451 16 : if (store->catalog_version <= CATALOG_JUL2021) {
452 : /* upgrade some columns in sys.sequences:
453 : * if increment is zero, set it to one (see ChangeLog);
454 : * if increment is greater than zero and maxvalue is zero,
455 : * set maxvalue to GDK_lng_max;
456 : * if increment is less than zero and minvalue is zero,
457 : * set minvalue to GDK_lng_min */
458 :
459 : /* sys.sequences i.e. deleted rows */
460 0 : BAT *del_seqs = log_temp_descriptor(log_find_bat(lg, 2037));
461 0 : if (del_seqs == NULL)
462 0 : return GDK_FAIL;
463 0 : BAT *seq_tid = BATmaskedcands(0, BATcount(del_seqs), del_seqs, false);
464 0 : bat_destroy(del_seqs);
465 0 : BAT *seq_min = log_temp_descriptor(log_find_bat(lg, 2042)); /* sys.sequences.minvalue */
466 0 : BAT *seq_max = log_temp_descriptor(log_find_bat(lg, 2043)); /* sys.sequences.maxvalue */
467 0 : BAT *seq_inc = log_temp_descriptor(log_find_bat(lg, 2044)); /* sys.sequences.increment */
468 0 : if (seq_tid == NULL || seq_min == NULL || seq_max == NULL || seq_inc == NULL) {
469 0 : bat_destroy(seq_tid);
470 0 : bat_destroy(seq_min);
471 0 : bat_destroy(seq_max);
472 0 : bat_destroy(seq_inc);
473 0 : return GDK_FAIL;
474 : }
475 : /* select * from sys.sequences where increment = 0 */
476 0 : BAT *inczero = BATselect(seq_inc, seq_tid, &(lng){0}, NULL, false, true, false, false);
477 0 : if (inczero == NULL) {
478 0 : bat_destroy(seq_tid);
479 0 : bat_destroy(seq_min);
480 0 : bat_destroy(seq_max);
481 0 : bat_destroy(seq_inc);
482 0 : return GDK_FAIL;
483 : }
484 0 : if (BATcount(inczero) > 0) {
485 0 : BAT *b = BATconstant(0, TYPE_lng, &(lng) {1}, BATcount(inczero), TRANSIENT);
486 0 : if (b == NULL) {
487 0 : bat_destroy(seq_tid);
488 0 : bat_destroy(seq_min);
489 0 : bat_destroy(seq_max);
490 0 : bat_destroy(seq_inc);
491 0 : bat_destroy(inczero);
492 0 : return GDK_FAIL;
493 : }
494 0 : BAT *b2 = COLcopy(seq_inc, seq_inc->ttype, true, PERSISTENT);
495 0 : rc = GDK_FAIL;
496 0 : if (b2 == NULL)
497 0 : rc = BATreplace(b2, inczero, b, false);
498 0 : bat_destroy(b);
499 0 : if (rc != GDK_SUCCEED) {
500 0 : bat_destroy(b2);
501 0 : bat_destroy(seq_tid);
502 0 : bat_destroy(seq_min);
503 0 : bat_destroy(seq_max);
504 0 : bat_destroy(seq_inc);
505 0 : bat_destroy(inczero);
506 0 : return GDK_FAIL;
507 : }
508 0 : rc = replace_bat(lg, 2044, b2);
509 0 : bat_destroy(seq_inc);
510 0 : seq_inc = b2;
511 0 : if (rc != GDK_SUCCEED) {
512 0 : bat_destroy(seq_tid);
513 0 : bat_destroy(seq_min);
514 0 : bat_destroy(seq_max);
515 0 : bat_destroy(seq_inc);
516 0 : bat_destroy(inczero);
517 0 : return rc;
518 : }
519 : }
520 0 : bat_destroy(inczero);
521 : /* select * from sys.sequences where increment > 0 */
522 0 : BAT *incpos = BATselect(seq_inc, seq_tid, &(lng){0}, &lng_nil, false, true, false, false);
523 0 : bat_destroy(seq_inc);
524 0 : if (incpos == NULL) {
525 0 : bat_destroy(seq_tid);
526 0 : bat_destroy(seq_min);
527 0 : bat_destroy(seq_max);
528 0 : return GDK_FAIL;
529 : }
530 : /* select * from sys.sequences where increment > 0 and maxvalue = 0 */
531 0 : BAT *cands = BATselect(seq_max, incpos, &(lng) {0}, NULL, true, true, false, false);
532 0 : bat_destroy(incpos);
533 0 : if (cands == NULL) {
534 0 : bat_destroy(seq_tid);
535 0 : bat_destroy(seq_min);
536 0 : bat_destroy(seq_max);
537 0 : return GDK_FAIL;
538 : }
539 0 : if (BATcount(cands) > 0) {
540 0 : BAT *b = BATconstant(0, TYPE_lng, &(lng){GDK_lng_max}, BATcount(cands), TRANSIENT);
541 0 : BAT *b2 = COLcopy(seq_max, seq_max->ttype, true, PERSISTENT);
542 0 : rc = GDK_FAIL;
543 0 : if (b != NULL && b2 != NULL)
544 0 : rc = BATreplace(b2, cands, b, false);
545 0 : bat_destroy(b);
546 0 : if (rc == GDK_SUCCEED)
547 0 : rc = replace_bat(lg, 2043, b2);
548 0 : bat_destroy(b2);
549 0 : if (rc != GDK_SUCCEED) {
550 0 : bat_destroy(cands);
551 0 : bat_destroy(seq_tid);
552 0 : bat_destroy(seq_min);
553 0 : bat_destroy(seq_max);
554 0 : return rc;
555 : }
556 : }
557 0 : bat_destroy(seq_max);
558 0 : bat_destroy(cands);
559 : /* select * from sys.sequences where increment < 0 */
560 0 : BAT *incneg = BATselect(seq_inc, seq_tid, &lng_nil, &(lng){0}, false, true, false, false);
561 0 : bat_destroy(seq_tid);
562 : /* select * from sys.sequences where increment < 0 and minvalue = 0 */
563 0 : cands = BATselect(seq_min, incneg, &(lng) {0}, NULL, true, true, false, false);
564 0 : bat_destroy(incneg);
565 0 : if (cands == NULL) {
566 0 : bat_destroy(seq_min);
567 0 : return GDK_FAIL;
568 : }
569 0 : if (BATcount(cands) > 0) {
570 0 : BAT *b = BATconstant(0, TYPE_lng, &(lng){GDK_lng_min}, BATcount(cands), TRANSIENT);
571 0 : BAT *b2 = COLcopy(seq_min, seq_min->ttype, true, PERSISTENT);
572 0 : rc = GDK_FAIL;
573 0 : if (b != NULL && b2 != NULL)
574 0 : rc = BATreplace(b2, cands, b, false);
575 0 : bat_destroy(b);
576 0 : if (rc == GDK_SUCCEED)
577 0 : rc = replace_bat(lg, 2042, b2);
578 0 : bat_destroy(b2);
579 0 : if (rc != GDK_SUCCEED) {
580 0 : bat_destroy(cands);
581 0 : bat_destroy(seq_min);
582 0 : return rc;
583 : }
584 : }
585 0 : bat_destroy(seq_min);
586 0 : bat_destroy(cands);
587 : }
588 : #endif
589 :
590 : #ifdef CATALOG_JAN2022
591 16 : if (store->catalog_version <= CATALOG_JAN2022) {
592 : /* GRANT SELECT ON sys.db_user_info TO monetdb;
593 : * except the grantor is 0 instead of user monetdb
594 : *
595 : * we need to find the IDs of the sys.db_user_info table and of
596 : * the sys.privileges table and its columns since none of these
597 : * have fixed IDs */
598 0 : BAT *b = log_temp_descriptor(log_find_bat(lg, 2067)); /* sys._tables */
599 0 : if (b == NULL)
600 0 : return GDK_FAIL;
601 0 : BAT *del_tabs = BATmaskedcands(0, BATcount(b), b, false);
602 0 : bat_destroy(b);
603 0 : if (del_tabs == NULL)
604 : return GDK_FAIL;
605 0 : b = log_temp_descriptor(log_find_bat(lg, 2076)); /* sys._columns */
606 0 : if (b == NULL) {
607 0 : bat_destroy(del_tabs);
608 0 : return GDK_FAIL;
609 : }
610 0 : BAT *del_cols = BATmaskedcands(0, BATcount(b), b, false);
611 0 : bat_destroy(b);
612 0 : b = log_temp_descriptor(log_find_bat(lg, 2070)); /* sys._tables.schema_id */
613 0 : if (del_cols == NULL || b == NULL) {
614 0 : bat_destroy(del_cols);
615 0 : bat_destroy(b);
616 0 : bat_destroy(del_tabs);
617 0 : return GDK_FAIL;
618 : }
619 0 : BAT *cands = BATselect(b, del_tabs, &(int) {2000}, NULL, true, true, false, false);
620 0 : bat_destroy(b);
621 0 : bat_destroy(del_tabs);
622 : /* cands contains undeleted rows from sys._tables for tables in
623 : * sys schema */
624 0 : BAT *tabnme = log_temp_descriptor(log_find_bat(lg, 2069)); /* sys._tables.name */
625 0 : if (cands == NULL || tabnme == NULL) {
626 0 : bat_destroy(cands);
627 0 : bat_destroy(tabnme);
628 0 : bat_destroy(del_cols);
629 0 : return GDK_FAIL;
630 : }
631 0 : b = BATselect(tabnme, cands, "db_user_info", NULL, true, true, false, false);
632 0 : if (b == NULL) {
633 0 : bat_destroy(cands);
634 0 : bat_destroy(tabnme);
635 0 : bat_destroy(del_cols);
636 0 : return GDK_FAIL;
637 : }
638 0 : oid dbpos = BUNtoid(b, 0);
639 0 : bat_destroy(b);
640 0 : b = BATselect(tabnme, cands, "privileges", NULL, true, true, false, false);
641 0 : bat_destroy(tabnme);
642 0 : bat_destroy(cands);
643 0 : BAT *tabid = log_temp_descriptor(log_find_bat(lg, 2068)); /* sys._tables.id */
644 0 : if (b == NULL || tabid == NULL) {
645 0 : bat_destroy(b);
646 0 : bat_destroy(tabid);
647 0 : bat_destroy(del_cols);
648 0 : return GDK_FAIL;
649 : }
650 0 : int dbid = ((int *) tabid->theap->base)[dbpos];
651 0 : int prid = ((int *) tabid->theap->base)[BUNtoid(b, 0)];
652 0 : BAT *coltid = log_temp_descriptor(log_find_bat(lg, 2082)); /* sys._columns.table_id */
653 0 : if (coltid == NULL) {
654 0 : bat_destroy(b);
655 0 : bat_destroy(del_cols);
656 0 : bat_destroy(tabid);
657 0 : return GDK_FAIL;
658 : }
659 0 : BAT *b1;
660 0 : rc = BATjoin(&b1, NULL, coltid, tabid, del_cols, b, false, 5);
661 0 : bat_destroy(coltid);
662 0 : bat_destroy(tabid);
663 0 : bat_destroy(del_cols);
664 0 : bat_destroy(b);
665 0 : BAT *colnr = log_temp_descriptor(log_find_bat(lg, 2085)); /* sys._columns.number */
666 0 : BAT *colid = log_temp_descriptor(log_find_bat(lg, 2077)); /* sys._columns.id */
667 0 : if (rc != GDK_SUCCEED || colnr == NULL || colid == NULL) {
668 0 : if (rc == GDK_SUCCEED)
669 0 : bat_destroy(b1);
670 0 : bat_destroy(colnr);
671 0 : bat_destroy(colid);
672 0 : return GDK_FAIL;
673 : }
674 : int privids[5];
675 0 : for (int i = 0; i < 5; i++) {
676 0 : oid p = BUNtoid(b1, i);
677 0 : privids[((int *) colnr->theap->base)[p]] = ((int *) colid->theap->base)[p];
678 : }
679 0 : bat_destroy(b1);
680 0 : bat_destroy(colnr);
681 0 : bat_destroy(colid);
682 0 : rc = tabins(lg,
683 0 : prid, &(msk) {false}, /* sys.privileges */
684 : privids[0], &dbid, /* sys.privileges.obj_id */
685 0 : privids[1], &(int) {USER_MONETDB}, /* sys.privileges.auth_id */
686 0 : privids[2], &(int) {PRIV_SELECT}, /* sys.privileges.privileges */
687 0 : privids[3], &(int) {0}, /* sys.privileges.grantor */
688 0 : privids[4], &(int) {0}, /* sys.privileges.grantee */
689 : 0);
690 0 : if (rc != GDK_SUCCEED)
691 : return rc;
692 : }
693 : #endif
694 :
695 : #ifdef CATALOG_SEP2022
696 16 : if (store->catalog_version <= CATALOG_SEP2022) {
697 : /* new STRING column sys.keys.check */
698 8 : BAT *b = log_temp_descriptor(log_find_bat(lg, 2088)); /* sys.keys.id */
699 8 : if (b == NULL)
700 0 : return GDK_FAIL;
701 8 : BAT *check = BATconstant(b->hseqbase, TYPE_str, ATOMnilptr(TYPE_str), BATcount(b), PERSISTENT);
702 8 : bat_destroy(b);
703 8 : if (check == NULL)
704 : return GDK_FAIL;
705 16 : if ((check = BATsetaccess(check, BAT_READ)) == NULL ||
706 : /* 2165 is sys.keys.check */
707 16 : BUNappend(lg->catalog_id, &(int) {2165}, true) != GDK_SUCCEED ||
708 16 : BUNappend(lg->catalog_bid, &check->batCacheid, true) != GDK_SUCCEED ||
709 16 : BUNappend(lg->catalog_lid, &lng_nil, false) != GDK_SUCCEED ||
710 8 : BUNappend(lg->catalog_cnt, &(lng){BATcount(check)}, false) != GDK_SUCCEED
711 : ) {
712 0 : bat_destroy(check);
713 0 : return GDK_FAIL;
714 : }
715 8 : BBPretain(check->batCacheid);
716 8 : bat_destroy(check);
717 :
718 8 : if (tabins(lg,
719 8 : 2076, &(msk) {false}, /* sys._columns */
720 : /* 2165 is sys.keys.check */
721 8 : 2077, &(int) {2165}, /* sys._columns.id */
722 : 2078, "check", /* sys._columns.name */
723 : 2079, "varchar", /* sys._columns.type */
724 8 : 2080, &(int) {2048}, /* sys._columns.type_digits */
725 8 : 2081, &(int) {0}, /* sys._columns.type_scale */
726 : /* 2087 is sys.keys */
727 8 : 2082, &(int) {2087}, /* sys._columns.table_id */
728 : 2083, str_nil, /* sys._columns.default */
729 8 : 2084, &(bit) {TRUE}, /* sys._columns.null */
730 8 : 2085, &(int) {6}, /* sys._columns.number */
731 : 2086, str_nil, /* sys._columns.storage */
732 : 0) != GDK_SUCCEED)
733 0 : return GDK_FAIL;
734 8 : if (tabins(lg,
735 8 : 2076, &(msk) {false}, /* sys._columns */
736 : /* 2166 is tmp.keys.check */
737 8 : 2077, &(int) {2166}, /* sys._columns.id */
738 : 2078, "check", /* sys._columns.name */
739 : 2079, "varchar", /* sys._columns.type */
740 8 : 2080, &(int) {2048}, /* sys._columns.type_digits */
741 8 : 2081, &(int) {0}, /* sys._columns.type_scale */
742 : /* 2135 is tmp.keys */
743 8 : 2082, &(int) {2135}, /* sys._columns.table_id */
744 : 2083, str_nil, /* sys._columns.default */
745 8 : 2084, &(bit) {TRUE}, /* sys._columns.null */
746 8 : 2085, &(int) {6}, /* sys._columns.number */
747 : 2086, str_nil, /* sys._columns.storage */
748 : 0) != GDK_SUCCEED)
749 0 : return GDK_FAIL;
750 : }
751 : #endif
752 :
753 : #ifdef CATALOG_AUG2024
754 16 : if (store->catalog_version <= CATALOG_AUG2024) {
755 : /* remove function sys.st_interiorrings and its arguments since
756 : * it references the now removed type GEOMETRYA */
757 16 : BAT *del_funcs = log_temp_descriptor(log_find_bat(lg, 2016)); /* sys.functions */
758 16 : if (del_funcs == NULL)
759 0 : return GDK_FAIL;
760 16 : BAT *dels = BATmaskedcands(0, BATcount(del_funcs), del_funcs, false);
761 16 : if (dels == NULL) {
762 0 : bat_destroy(del_funcs);
763 0 : return GDK_FAIL;
764 : }
765 16 : BAT *b = log_temp_descriptor(log_find_bat(lg, 2026)); /* sys.functions.schema_id */
766 16 : if (b == NULL) {
767 0 : bat_destroy(del_funcs);
768 0 : bat_destroy(dels);
769 0 : return GDK_FAIL;
770 : }
771 : /* select * from sys.functions where schema_id = 2000 */
772 16 : BAT *cands = BATselect(b, dels, &(int) {2000}, NULL, true, true, false, false);
773 16 : bat_destroy(b);
774 16 : bat_destroy(dels);
775 16 : b = log_temp_descriptor(log_find_bat(lg, 2018)); /* sys.functions.name */
776 16 : if (cands == NULL || b == NULL) {
777 0 : bat_destroy(del_funcs);
778 0 : bat_destroy(cands);
779 0 : bat_destroy(b);
780 0 : return GDK_FAIL;
781 : }
782 : /* select * from sys.functions where schema_id = 2000 and name = 'st_interiorrings' */
783 16 : BAT *funcs = BATselect(b, cands, "st_interiorrings", NULL, true, true, false, false);
784 16 : bat_destroy(cands);
785 16 : bat_destroy(b);
786 16 : if (funcs == NULL) {
787 0 : bat_destroy(del_funcs);
788 0 : return GDK_FAIL;
789 : }
790 : /* here, funcs contains the BUNs for the function
791 : * sys.st_interiorrings; if there are none, we're done */
792 16 : if (BATcount(funcs) > 0) {
793 16 : b = log_temp_descriptor(log_find_bat(lg, 2017)); /* sys.functions.id */
794 16 : if (b == NULL) {
795 0 : bat_destroy(del_funcs);
796 0 : bat_destroy(funcs);
797 0 : return GDK_FAIL;
798 : }
799 16 : BAT *del_args = log_temp_descriptor(log_find_bat(lg, 2028)); /* sys.args */
800 16 : if (del_args == NULL) {
801 0 : bat_destroy(del_funcs);
802 0 : bat_destroy(funcs);
803 0 : bat_destroy(b);
804 0 : return GDK_FAIL;
805 : }
806 16 : dels = BATmaskedcands(0, BATcount(del_args), del_args, false);
807 16 : if (dels == NULL) {
808 0 : bat_destroy(del_funcs);
809 0 : bat_destroy(del_args);
810 0 : bat_destroy(funcs);
811 0 : bat_destroy(b);
812 0 : return GDK_FAIL;
813 : }
814 16 : BAT *a = log_temp_descriptor(log_find_bat(lg, 2030)); /* sys.args.func_id */
815 16 : if (a == NULL) {
816 0 : bat_destroy(del_funcs);
817 0 : bat_destroy(del_args);
818 0 : bat_destroy(funcs);
819 0 : bat_destroy(b);
820 0 : return GDK_FAIL;
821 : }
822 16 : BAT *r1, *r2;
823 16 : gdk_return rc;
824 : /* find arguments to function sys.st_interiorrings */
825 16 : rc = BATjoin(&r1, &r2, b, a, funcs, dels, false, 10);
826 16 : bat_destroy(dels);
827 16 : bat_destroy(b);
828 16 : bat_destroy(a);
829 16 : if (rc != GDK_SUCCEED) {
830 0 : bat_destroy(del_funcs);
831 0 : bat_destroy(del_args);
832 0 : bat_destroy(funcs);
833 0 : return GDK_FAIL;
834 : }
835 16 : b = COLcopy(del_funcs, del_funcs->ttype, true, PERSISTENT);
836 16 : a = COLcopy(del_args, del_args->ttype, true, PERSISTENT);
837 16 : bat_destroy(del_funcs);
838 16 : bat_destroy(del_args);
839 16 : if (b == NULL || a == NULL) {
840 0 : bat_destroy(funcs);
841 0 : bat_destroy(r1);
842 0 : bat_destroy(r2);
843 0 : return GDK_FAIL;
844 : }
845 : /* now set the deleted bit for all functions and all
846 : * arguments that we've found (i.e. just the input and
847 : * output arg for sys.st_interiorrings and the function
848 : * itself) */
849 16 : BUN p, q;
850 48 : BATloop (r1, p, q) {
851 32 : oid o = BUNtoid(r1, p);
852 32 : if (BUNreplace(b, o, &(bool) {true}, false) != GDK_SUCCEED) {
853 0 : bat_destroy(funcs);
854 0 : bat_destroy(r1);
855 0 : bat_destroy(r2);
856 0 : bat_destroy(b);
857 0 : bat_destroy(a);
858 0 : return GDK_FAIL;
859 : }
860 32 : o = BUNtoid(r2, p);
861 32 : if (BUNreplace(a, o, &(bool) {true}, false) != GDK_SUCCEED) {
862 0 : bat_destroy(funcs);
863 0 : bat_destroy(r1);
864 0 : bat_destroy(r2);
865 0 : bat_destroy(b);
866 0 : bat_destroy(a);
867 0 : return GDK_FAIL;
868 : }
869 : }
870 16 : bat_destroy(r1);
871 16 : bat_destroy(r2);
872 16 : rc = replace_bat(lg, 2016, b);
873 16 : if (rc == GDK_SUCCEED)
874 16 : rc = replace_bat(lg, 2028, a);
875 16 : bat_destroy(b);
876 16 : bat_destroy(a);
877 16 : if (rc != GDK_SUCCEED) {
878 0 : bat_destroy(funcs);
879 0 : return rc;
880 : }
881 : }
882 16 : bat_destroy(funcs);
883 : }
884 16 : if (store->catalog_version <= CATALOG_AUG2024) {
885 : /* new TINYINT column sys.functions.order_specification */
886 16 : BAT *ftype = log_temp_descriptor(log_find_bat(lg, 2022)); /* sys.functions.type (int) */
887 16 : BAT *fname = log_temp_descriptor(log_find_bat(lg, 2018)); /* sys.functions.name (str) */
888 16 : if (ftype == NULL || fname == NULL)
889 0 : return GDK_FAIL;
890 16 : bte zero = 0;
891 16 : BAT *order_spec = BATconstant(ftype->hseqbase, TYPE_bte, &zero, BATcount(ftype), PERSISTENT);
892 : /* update functions set order_specification=1 where type == aggr and name in ('group_concat', 'listagg', 'xmlagg')
893 : * update functions set order_specification=2 where type == aggr and name = 'quantile' */
894 16 : if (order_spec == NULL) {
895 0 : bat_destroy(ftype);
896 0 : bat_destroy(fname);
897 0 : return GDK_FAIL;
898 : }
899 16 : bte *os = (bte*)Tloc(order_spec, 0);
900 16 : int *ft = (int*)Tloc(ftype, 0);
901 16 : BATiter fni = bat_iterator_nolock(fname);
902 52888 : for(BUN b = 0; b < BATcount(ftype); b++) {
903 52872 : if (ft[b] == F_AGGR) {
904 3746 : const char *f = BUNtvar(fni, b);
905 3746 : if (strcmp(f, "group_concat") == 0 || strcmp(f, "listagg") == 0 || strcmp(f, "xmlagg") == 0)
906 160 : os[b] = 1;
907 3586 : else if (strcmp(f, "quantile") == 0 || strcmp(f, "quantile_avg") == 0)
908 400 : os[b] = 2;
909 : }
910 : }
911 16 : bat_destroy(ftype);
912 16 : bat_destroy(fname);
913 32 : if ((order_spec = BATsetaccess(order_spec, BAT_READ)) == NULL ||
914 : /* 2167 is sys.functions.order_specification */
915 32 : BUNappend(lg->catalog_id, &(int) {2167}, true) != GDK_SUCCEED ||
916 32 : BUNappend(lg->catalog_bid, &order_spec->batCacheid, true) != GDK_SUCCEED ||
917 32 : BUNappend(lg->catalog_lid, &lng_nil, false) != GDK_SUCCEED ||
918 16 : BUNappend(lg->catalog_cnt, &(lng){BATcount(order_spec)}, false) != GDK_SUCCEED
919 : ) {
920 0 : bat_destroy(order_spec);
921 0 : return GDK_FAIL;
922 : }
923 16 : BBPretain(order_spec->batCacheid);
924 16 : bat_destroy(order_spec);
925 :
926 16 : if (tabins(lg,
927 16 : 2076, &(msk) {false}, /* sys._columns */
928 : /* 2167 is sys.functions.order_specification */
929 16 : 2077, &(int) {2167}, /* sys._columns.id */
930 : 2078, "order_specification", /* sys._columns.name */
931 : 2079, "tinyint", /* sys._columns.type */
932 16 : 2080, &(int) {7}, /* sys._columns.type_digits */
933 16 : 2081, &(int) {0}, /* sys._columns.type_scale */
934 : /* 2016 is sys.functions */
935 16 : 2082, &(int) {2016}, /* sys._columns.table_id */
936 : 2083, str_nil, /* sys._columns.default */
937 16 : 2084, &(bit) {TRUE}, /* sys._columns.null */
938 16 : 2085, &(int) {12}, /* sys._columns.number */
939 : 2086, str_nil, /* sys._columns.storage */
940 : 0) != GDK_SUCCEED)
941 0 : return GDK_FAIL;
942 : }
943 16 : if (store->catalog_version <= CATALOG_AUG2024) {
944 : /* new TINYINT column sys._columns.column_type */
945 16 : BAT *b = log_temp_descriptor(log_find_bat(lg, 2077)); /* sys._columns.id */
946 16 : if (b == NULL)
947 0 : return GDK_FAIL;
948 16 : BUN colcnt = BATcount(b); /* we'll need it a few times */
949 16 : bat_destroy(b);
950 16 : BAT *coltype = BATconstant(b->hseqbase, TYPE_bte, &(bte) {0}, colcnt, PERSISTENT);
951 16 : if (coltype == NULL)
952 : return GDK_FAIL;
953 32 : if ((coltype = BATsetaccess(coltype, BAT_READ)) == NULL ||
954 : /* 2168 is sys._columns.column_type */
955 32 : BUNappend(lg->catalog_id, &(int) {2168}, true) != GDK_SUCCEED ||
956 32 : BUNappend(lg->catalog_bid, &coltype->batCacheid, true) != GDK_SUCCEED ||
957 32 : BUNappend(lg->catalog_lid, &lng_nil, false) != GDK_SUCCEED ||
958 16 : BUNappend(lg->catalog_cnt, &(lng) {colcnt}, false) != GDK_SUCCEED
959 : ) {
960 0 : bat_destroy(coltype);
961 0 : return GDK_FAIL;
962 : }
963 16 : BBPretain(coltype->batCacheid);
964 16 : bat_destroy(coltype);
965 :
966 : /* new TINYINT column sys._columns.multiset */
967 16 : BAT *multiset = BATconstant(b->hseqbase, TYPE_bte, &(bte) {MS_VALUE}, colcnt, PERSISTENT);
968 16 : if (multiset == NULL)
969 : return GDK_FAIL;
970 32 : if ((multiset = BATsetaccess(multiset, BAT_READ)) == NULL ||
971 : /* 2170 is sys._columns.multiset */
972 32 : BUNappend(lg->catalog_id, &(int) {2170}, true) != GDK_SUCCEED ||
973 32 : BUNappend(lg->catalog_bid, &multiset->batCacheid, true) != GDK_SUCCEED ||
974 32 : BUNappend(lg->catalog_lid, &lng_nil, false) != GDK_SUCCEED ||
975 16 : BUNappend(lg->catalog_cnt, &(lng) {colcnt}, false) != GDK_SUCCEED
976 : ) {
977 0 : bat_destroy(multiset);
978 0 : return GDK_FAIL;
979 : }
980 16 : BBPretain(multiset->batCacheid);
981 16 : bat_destroy(multiset);
982 :
983 : /* new TINYINT column sys._columns.multiset */
984 16 : b = log_temp_descriptor(log_find_bat(lg, 2029)); /* sys.args.id */
985 16 : if (b == NULL)
986 0 : return GDK_FAIL;
987 16 : multiset = BATconstant(b->hseqbase, TYPE_bte, &(bte) {MS_VALUE}, BATcount(b), PERSISTENT);
988 16 : bat_destroy(b);
989 16 : if (multiset == NULL)
990 : return GDK_FAIL;
991 32 : if ((multiset = BATsetaccess(multiset, BAT_READ)) == NULL ||
992 : /* 2172 is sys.args.multiset */
993 32 : BUNappend(lg->catalog_id, &(int) {2172}, true) != GDK_SUCCEED ||
994 32 : BUNappend(lg->catalog_bid, &multiset->batCacheid, true) != GDK_SUCCEED ||
995 32 : BUNappend(lg->catalog_lid, &lng_nil, false) != GDK_SUCCEED ||
996 16 : BUNappend(lg->catalog_cnt, &(lng) {BATcount(multiset)}, false) != GDK_SUCCEED
997 : ) {
998 0 : bat_destroy(multiset);
999 0 : return GDK_FAIL;
1000 : }
1001 16 : BBPretain(multiset->batCacheid);
1002 16 : bat_destroy(multiset);
1003 :
1004 16 : if (tabins(lg,
1005 16 : 2076, &(msk) {false}, /* sys._columns */
1006 : /* 2168 is sys._columns.column_type */
1007 16 : 2077, &(int) {2168}, /* sys._columns.id */
1008 : 2078, "column_type", /* sys._columns.name */
1009 : 2079, "tinyint", /* sys._columns.type */
1010 16 : 2080, &(int) {7}, /* sys._columns.type_digits */
1011 16 : 2081, &(int) {0}, /* sys._columns.type_scale */
1012 : /* 2076 is sys._columns */
1013 16 : 2082, &(int) {2076}, /* sys._columns.table_id */
1014 : 2083, str_nil, /* sys._columns.default */
1015 16 : 2084, &(bit) {TRUE}, /* sys._columns.null */
1016 16 : 2085, &(int) {10}, /* sys._columns.number */
1017 : 2086, str_nil, /* sys._columns.storage */
1018 16 : 2168, &(bte) {0}, /* sys._columns.column_type */
1019 16 : 2170, &(bte) {MS_VALUE}, /* sys._columns.multiset */
1020 : 0) != GDK_SUCCEED)
1021 0 : return GDK_FAIL;
1022 16 : if (tabins(lg,
1023 16 : 2076, &(msk) {false}, /* sys._columns */
1024 : /* 2169 is tmp._columns.column_type */
1025 16 : 2077, &(int) {2169}, /* sys._columns.id */
1026 : 2078, "column_type", /* sys._columns.name */
1027 : 2079, "tinyint", /* sys._columns.type */
1028 16 : 2080, &(int) {7}, /* sys._columns.type_digits */
1029 16 : 2081, &(int) {0}, /* sys._columns.type_scale */
1030 : /* 2124 is tmp._columns */
1031 16 : 2082, &(int) {2124}, /* sys._columns.table_id */
1032 : 2083, str_nil, /* sys._columns.default */
1033 16 : 2084, &(bit) {TRUE}, /* sys._columns.null */
1034 16 : 2085, &(int) {10}, /* sys._columns.number */
1035 : 2086, str_nil, /* sys._columns.storage */
1036 16 : 2168, &(bte) {0}, /* sys._columns.column_type */
1037 16 : 2170, &(bte) {MS_VALUE}, /* sys._columns.multiset */
1038 : 0) != GDK_SUCCEED)
1039 0 : return GDK_FAIL;
1040 16 : if (tabins(lg,
1041 16 : 2076, &(msk) {false}, /* sys._columns */
1042 : /* 2170 is sys._columns.multiset */
1043 16 : 2077, &(int) {2170}, /* sys._columns.id */
1044 : 2078, "multiset", /* sys._columns.name */
1045 : 2079, "tinyint", /* sys._columns.type */
1046 16 : 2080, &(int) {7}, /* sys._columns.type_digits */
1047 16 : 2081, &(int) {0}, /* sys._columns.type_scale */
1048 : /* 2076 is sys._columns */
1049 16 : 2082, &(int) {2076}, /* sys._columns.table_id */
1050 : 2083, str_nil, /* sys._columns.default */
1051 16 : 2084, &(bit) {TRUE}, /* sys._columns.null */
1052 16 : 2085, &(int) {11}, /* sys._columns.number */
1053 : 2086, str_nil, /* sys._columns.storage */
1054 16 : 2168, &(bte) {0}, /* sys._columns.column_type */
1055 16 : 2170, &(bte) {MS_VALUE}, /* sys._columns.multiset */
1056 : 0) != GDK_SUCCEED)
1057 0 : return GDK_FAIL;
1058 16 : if (tabins(lg,
1059 16 : 2076, &(msk) {false}, /* sys._columns */
1060 : /* 2171 is tmp._columns.multiset */
1061 16 : 2077, &(int) {2171}, /* sys._columns.id */
1062 : 2078, "multiset", /* sys._columns.name */
1063 : 2079, "tinyint", /* sys._columns.type */
1064 16 : 2080, &(int) {7}, /* sys._columns.type_digits */
1065 16 : 2081, &(int) {0}, /* sys._columns.type_scale */
1066 : /* 2124 is tmp._columns */
1067 16 : 2082, &(int) {2124}, /* sys._columns.table_id */
1068 : 2083, str_nil, /* sys._columns.default */
1069 16 : 2084, &(bit) {TRUE}, /* sys._columns.null */
1070 16 : 2085, &(int) {11}, /* sys._columns.number */
1071 : 2086, str_nil, /* sys._columns.storage */
1072 16 : 2168, &(bte) {0}, /* sys._columns.column_type */
1073 16 : 2170, &(bte) {MS_VALUE}, /* sys._columns.multiset */
1074 : 0) != GDK_SUCCEED)
1075 0 : return GDK_FAIL;
1076 16 : if (tabins(lg,
1077 16 : 2076, &(msk) {false}, /* sys._columns */
1078 : /* 2172 is sy.args.multiset */
1079 16 : 2077, &(int) {2172}, /* sys._columns.id */
1080 : 2078, "multiset", /* sys._columns.name */
1081 : 2079, "tinyint", /* sys._columns.type */
1082 16 : 2080, &(int) {7}, /* sys._columns.type_digits */
1083 16 : 2081, &(int) {0}, /* sys._columns.type_scale */
1084 : /* 2028 is sy.args */
1085 16 : 2082, &(int) {2028}, /* sys._columns.table_id */
1086 : 2083, str_nil, /* sys._columns.default */
1087 16 : 2084, &(bit) {TRUE}, /* sys._columns.null */
1088 16 : 2085, &(int) {8}, /* sys._columns.number */
1089 : 2086, str_nil, /* sys._columns.storage */
1090 16 : 2168, &(bte) {0}, /* sys._columns.column_type */
1091 16 : 2170, &(bte) {MS_VALUE}, /* sys._columns.multiset */
1092 : 0) != GDK_SUCCEED)
1093 0 : return GDK_FAIL;
1094 : }
1095 : #endif
1096 :
1097 : return GDK_SUCCEED;
1098 : }
1099 :
1100 : static int
1101 358 : bl_create(sqlstore *store, int debug, const char *logdir, int cat_version)
1102 : {
1103 358 : if (store->logger)
1104 : return LOG_ERR;
1105 358 : store->logger = log_create(debug, "sql", logdir, cat_version, (preversionfix_fptr)&bl_preversion, (postversionfix_fptr)&bl_postversion, store);
1106 358 : if (store->logger)
1107 : return LOG_OK;
1108 : return LOG_ERR;
1109 : }
1110 :
1111 : static void
1112 357 : bl_destroy(sqlstore *store)
1113 : {
1114 357 : logger *l = store->logger;
1115 :
1116 357 : store->logger = NULL;
1117 357 : if (l)
1118 357 : log_destroy(l);
1119 357 : }
1120 :
1121 : static int
1122 1612 : bl_flush(sqlstore *store, lng save_id)
1123 : {
1124 1612 : if (store->logger)
1125 1612 : return log_flush(store->logger, save_id) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
1126 : return LOG_OK;
1127 : }
1128 :
1129 : static int
1130 7515 : bl_activate(sqlstore *store)
1131 : {
1132 7515 : if (store->logger)
1133 7515 : return log_activate(store->logger) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
1134 : return LOG_OK;
1135 : }
1136 :
1137 : static int
1138 8962 : bl_changes(sqlstore *store)
1139 : {
1140 8962 : return (int) MIN(log_changes(store->logger), GDK_int_max);
1141 : }
1142 :
1143 : static int
1144 505 : bl_get_sequence(sqlstore *store, int seq, lng *id)
1145 : {
1146 505 : return log_sequence(store->logger, seq, id);
1147 : }
1148 :
1149 : static int
1150 358 : bl_log_isnew(sqlstore *store)
1151 : {
1152 358 : logger *bat_logger = store->logger;
1153 358 : if (BATcount(bat_logger->catalog_bid) > 10) {
1154 128 : return 0;
1155 : }
1156 : return 1;
1157 : }
1158 :
1159 : static int
1160 62651 : bl_tstart(sqlstore *store, bool flush, ulng *log_file_id)
1161 : {
1162 62651 : return log_tstart(store->logger, flush, log_file_id) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
1163 : }
1164 :
1165 : static int
1166 62650 : bl_tend(sqlstore *store)
1167 : {
1168 62650 : return log_tend(store->logger) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
1169 : }
1170 :
1171 : static int
1172 62650 : bl_tflush(sqlstore *store, ulng log_file_id, ulng commit_ts)
1173 : {
1174 62650 : return log_tflush(store->logger, log_file_id, commit_ts) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
1175 : }
1176 :
1177 : static int
1178 7053 : bl_sequence(sqlstore *store, int seq, lng id)
1179 : {
1180 7053 : return log_tsequence(store->logger, seq, id) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
1181 : }
1182 :
1183 : /* Write a plan entry to copy part of the given file.
1184 : * That part of the file must remain unchanged until the plan is executed.
1185 : */
1186 : __attribute__((__warn_unused_result__))
1187 : static gdk_return
1188 1046 : snapshot_lazy_copy_file(stream *plan, const char *name, uint64_t extent)
1189 : {
1190 1046 : if (mnstr_printf(plan, "c %" PRIu64 " %s\n", extent, name) < 0) {
1191 0 : GDKerror("%s", mnstr_peek_error(plan));
1192 0 : return GDK_FAIL;
1193 : }
1194 : return GDK_SUCCEED;
1195 : }
1196 :
1197 : /* Write a plan entry to write the current contents of the given file.
1198 : * The contents are included in the plan so the source file is allowed to
1199 : * change in the mean time.
1200 : */
1201 : __attribute__((__warn_unused_result__))
1202 : static gdk_return
1203 10 : snapshot_immediate_copy_file(stream *plan, const char *path, const char *name)
1204 : {
1205 10 : gdk_return ret = GDK_FAIL;
1206 10 : const size_t bufsize = 64 * 1024;
1207 10 : struct stat statbuf;
1208 10 : char *buf = NULL;
1209 10 : stream *s = NULL;
1210 10 : size_t to_copy;
1211 :
1212 10 : if (MT_stat(path, &statbuf) < 0) {
1213 0 : GDKsyserror("stat failed on %s", path);
1214 0 : goto end;
1215 : }
1216 10 : to_copy = (size_t) statbuf.st_size;
1217 :
1218 10 : s = open_rstream(path);
1219 10 : if (!s) {
1220 0 : GDKerror("%s", mnstr_peek_error(NULL));
1221 0 : goto end;
1222 : }
1223 :
1224 10 : buf = GDKmalloc(bufsize);
1225 10 : if (!buf) {
1226 0 : GDKerror("GDKmalloc failed");
1227 0 : goto end;
1228 : }
1229 :
1230 10 : if (mnstr_printf(plan, "w %zu %s\n", to_copy, name) < 0) {
1231 0 : GDKerror("%s", mnstr_peek_error(plan));
1232 0 : goto end;
1233 : }
1234 :
1235 20 : while (to_copy > 0) {
1236 10 : size_t chunk = (to_copy <= bufsize) ? to_copy : bufsize;
1237 10 : ssize_t bytes_read = mnstr_read(s, buf, 1, chunk);
1238 10 : if (bytes_read < 0) {
1239 0 : GDKerror("Reading bytes of component %s failed: %s", path, mnstr_peek_error(s));
1240 0 : goto end;
1241 10 : } else if (bytes_read < (ssize_t) chunk) {
1242 0 : GDKerror("Read only %zu/%zu bytes of component %s: %s", (size_t) bytes_read, chunk, path, mnstr_peek_error(s));
1243 0 : goto end;
1244 : }
1245 :
1246 10 : ssize_t bytes_written = mnstr_write(plan, buf, 1, chunk);
1247 10 : if (bytes_written < 0) {
1248 0 : GDKerror("Writing to plan failed: %s", mnstr_peek_error(plan));
1249 0 : goto end;
1250 10 : } else if (bytes_written < (ssize_t) chunk) {
1251 0 : GDKerror("write to plan truncated");
1252 0 : goto end;
1253 : }
1254 10 : to_copy -= chunk;
1255 : }
1256 :
1257 : ret = GDK_SUCCEED;
1258 10 : end:
1259 10 : GDKfree(buf);
1260 10 : if (s)
1261 10 : close_stream(s);
1262 10 : return ret;
1263 : }
1264 :
1265 : /* Add plan entries for all relevant files in the Write Ahead Log */
1266 : __attribute__((__warn_unused_result__))
1267 : static gdk_return
1268 5 : snapshot_wal(logger *bat_logger, stream *plan, const char *db_dir)
1269 : {
1270 5 : char log_file[FILENAME_MAX];
1271 5 : int len;
1272 :
1273 5 : len = snprintf(log_file, sizeof(log_file), "%s/%s%s", db_dir, bat_logger->dir, LOGFILE);
1274 5 : if (len == -1 || (size_t)len >= sizeof(log_file)) {
1275 0 : GDKerror("Could not open %s, filename is too large", log_file);
1276 0 : return GDK_FAIL;
1277 : }
1278 5 : if (snapshot_immediate_copy_file(plan, log_file, log_file + strlen(db_dir) + 1) != GDK_SUCCEED)
1279 : return GDK_FAIL;
1280 :
1281 11 : for (ulng id = bat_logger->saved_id+1; id <= bat_logger->id; id++) {
1282 6 : struct stat statbuf;
1283 :
1284 6 : len = snprintf(log_file, sizeof(log_file), "%s/%s%s." LLFMT, db_dir, bat_logger->dir, LOGFILE, id);
1285 6 : if (len == -1 || (size_t)len >= sizeof(log_file)) {
1286 0 : GDKerror("Could not open %s, filename is too large", log_file);
1287 0 : return GDK_FAIL;
1288 : }
1289 6 : if (MT_stat(log_file, &statbuf) == 0) {
1290 6 : if (snapshot_lazy_copy_file(plan, log_file + strlen(db_dir) + 1, statbuf.st_size) != GDK_SUCCEED)
1291 : return GDK_FAIL;
1292 : } else {
1293 0 : GDKerror("Could not open %s", log_file);
1294 0 : return GDK_FAIL;
1295 : }
1296 : }
1297 : return GDK_SUCCEED;
1298 : }
1299 :
1300 : __attribute__((__warn_unused_result__))
1301 : static gdk_return
1302 1620 : snapshot_heap(stream *plan, const char *db_dir, bat batid, const char *filename, const char *suffix, uint64_t extent)
1303 : {
1304 1620 : char path1[FILENAME_MAX];
1305 1620 : char path2[FILENAME_MAX];
1306 1620 : const size_t offset = strlen(db_dir) + 1;
1307 1620 : struct stat statbuf;
1308 1620 : int len;
1309 :
1310 1620 : if (extent == 0) {
1311 : /* nothing to copy */
1312 : return GDK_SUCCEED;
1313 : }
1314 : // first check the backup dir
1315 1040 : len = snprintf(path1, FILENAME_MAX, "%s/%s/%o.%s", db_dir, BAKDIR, (int) batid, suffix);
1316 1040 : if (len == -1 || len >= FILENAME_MAX) {
1317 0 : path1[FILENAME_MAX - 1] = '\0';
1318 0 : GDKerror("Could not open %s, filename is too large", path1);
1319 0 : return GDK_FAIL;
1320 : }
1321 1040 : if (MT_stat(path1, &statbuf) == 0) {
1322 0 : return snapshot_lazy_copy_file(plan, path1 + offset, extent);
1323 : }
1324 1040 : if (errno != ENOENT) {
1325 0 : GDKsyserror("Error stat'ing %s", path1);
1326 0 : return GDK_FAIL;
1327 : }
1328 :
1329 : // then check the regular location
1330 1040 : len = snprintf(path2, FILENAME_MAX, "%s/%s/%s.%s", db_dir, BATDIR, filename, suffix);
1331 1040 : if (len == -1 || len >= FILENAME_MAX) {
1332 0 : path2[FILENAME_MAX - 1] = '\0';
1333 0 : GDKerror("Could not open %s, filename is too large", path2);
1334 0 : return GDK_FAIL;
1335 : }
1336 1040 : if (MT_stat(path2, &statbuf) == 0) {
1337 1040 : return snapshot_lazy_copy_file(plan, path2 + offset, extent);
1338 : }
1339 0 : if (errno != ENOENT) {
1340 0 : GDKsyserror("Error stat'ing %s", path2);
1341 0 : return GDK_FAIL;
1342 : }
1343 :
1344 0 : GDKerror("One of %s and %s must exist", path1, path2);
1345 0 : return GDK_FAIL;
1346 : }
1347 :
1348 : /* Add plan entries for all persistent BATs by looping over the BBP.dir.
1349 : * Also include the BBP.dir itself.
1350 : */
1351 : __attribute__((__warn_unused_result__))
1352 : static gdk_return
1353 5 : snapshot_bats(stream *plan, const char *db_dir)
1354 : {
1355 5 : char bbpdir[FILENAME_MAX];
1356 5 : FILE *fp = NULL;
1357 5 : int len;
1358 5 : gdk_return ret = GDK_FAIL;
1359 5 : int lineno = 0;
1360 5 : bat bbpsize = 0;
1361 5 : lng logno;
1362 5 : unsigned bbpversion;
1363 :
1364 5 : len = snprintf(bbpdir, FILENAME_MAX, "%s/%s/%s", db_dir, BAKDIR, "BBP.dir");
1365 5 : if (len == -1 || len >= FILENAME_MAX) {
1366 0 : GDKerror("Could not open %s, filename is too large", bbpdir);
1367 0 : return GDK_FAIL;
1368 : }
1369 5 : ret = snapshot_immediate_copy_file(plan, bbpdir, bbpdir + strlen(db_dir) + 1);
1370 5 : if (ret != GDK_SUCCEED)
1371 : return ret;
1372 :
1373 : // Open the catalog and parse the header
1374 5 : fp = fopen(bbpdir, "r");
1375 5 : if (fp == NULL) {
1376 0 : GDKerror("Could not open %s for reading: %s", bbpdir, mnstr_peek_error(NULL));
1377 0 : return GDK_FAIL;
1378 : }
1379 5 : bbpversion = BBPheader(fp, &lineno, &bbpsize, &logno, false);
1380 5 : if (bbpversion == 0)
1381 0 : goto end;
1382 5 : assert(bbpversion == GDKLIBRARY);
1383 :
1384 1280 : for (;;) {
1385 1285 : BAT b;
1386 1285 : Heap h;
1387 1285 : Heap vh;
1388 1285 : vh = h = (Heap) {
1389 : .free = 0,
1390 : };
1391 1285 : b = (BAT) {
1392 : .theap = &h,
1393 : .tvheap = &vh,
1394 : };
1395 1285 : char *options;
1396 1285 : char filename[sizeof(BBP_physical(0))];
1397 1285 : char batname[129];
1398 : #ifdef GDKLIBRARY_HASHASH
1399 1285 : int hashash;
1400 : #endif
1401 :
1402 1285 : switch (BBPreadBBPline(fp, bbpversion, &lineno, &b,
1403 : #ifdef GDKLIBRARY_HASHASH
1404 : &hashash,
1405 : #endif
1406 : batname, filename, &options)) {
1407 5 : case 0:
1408 : /* end of file */
1409 5 : fclose(fp);
1410 5 : return GDK_SUCCEED;
1411 : case 1:
1412 : /* successfully read an entry */
1413 1280 : break;
1414 0 : default:
1415 : /* error */
1416 0 : fclose(fp);
1417 0 : return GDK_FAIL;
1418 : }
1419 : #ifdef GDKLIBRARY_HASHASH
1420 1280 : assert(hashash == 0);
1421 : #endif
1422 1280 : if (ATOMvarsized(b.ttype)) {
1423 340 : ret = snapshot_heap(plan, db_dir, b.batCacheid, filename, "theap", b.tvheap->free);
1424 340 : if (ret != GDK_SUCCEED)
1425 0 : goto end;
1426 : }
1427 1280 : ret = snapshot_heap(plan, db_dir, b.batCacheid, filename, BATtailname(&b), b.theap->free);
1428 1280 : if (ret != GDK_SUCCEED)
1429 0 : goto end;
1430 : }
1431 :
1432 0 : end:
1433 0 : if (fp) {
1434 0 : fclose(fp);
1435 : }
1436 0 : return ret;
1437 : }
1438 :
1439 : __attribute__((__warn_unused_result__))
1440 : static gdk_return
1441 5 : snapshot_vaultkey(stream *plan, const char *db_dir)
1442 : {
1443 5 : char path[FILENAME_MAX];
1444 5 : struct stat statbuf;
1445 :
1446 5 : int len = snprintf(path, FILENAME_MAX, "%s/.vaultkey", db_dir);
1447 5 : if (len == -1 || len >= FILENAME_MAX) {
1448 0 : path[FILENAME_MAX - 1] = '\0';
1449 0 : GDKerror("Could not open %s, filename is too large", path);
1450 0 : return GDK_FAIL;
1451 : }
1452 5 : if (MT_stat(path, &statbuf) == 0) {
1453 0 : return snapshot_lazy_copy_file(plan, ".vaultkey", statbuf.st_size);
1454 : }
1455 5 : if (errno == ENOENT) {
1456 : // No .vaultkey? Fine.
1457 : return GDK_SUCCEED;
1458 : }
1459 :
1460 0 : GDKsyserror("Error stat'ing %s", path);
1461 0 : return GDK_FAIL;
1462 : }
1463 :
1464 : static gdk_return
1465 5 : bl_snapshot(sqlstore *store, stream *plan)
1466 : {
1467 5 : logger *bat_logger = store->logger;
1468 5 : gdk_return ret;
1469 5 : char db_dir[MAXPATH];
1470 5 : size_t db_dir_len;
1471 :
1472 : // Farm 0 is always the persistent farm.
1473 5 : if (GDKfilepath(db_dir, sizeof(db_dir), 0, NULL, "", NULL) != GDK_SUCCEED)
1474 : return GDK_FAIL;
1475 5 : db_dir_len = strlen(db_dir);
1476 5 : if (db_dir[db_dir_len - 1] == DIR_SEP)
1477 5 : db_dir[db_dir_len - 1] = '\0';
1478 :
1479 10 : if (mnstr_printf(plan, "%s\n", db_dir) < 0 ||
1480 : // Please monetdbd
1481 5 : mnstr_printf(plan, "w 0 .uplog\n") < 0) {
1482 0 : GDKerror("%s", mnstr_peek_error(plan));
1483 0 : ret = GDK_FAIL;
1484 0 : goto end;
1485 : }
1486 :
1487 5 : ret = snapshot_vaultkey(plan, db_dir);
1488 5 : if (ret != GDK_SUCCEED)
1489 0 : goto end;
1490 :
1491 5 : ret = snapshot_bats(plan, db_dir);
1492 5 : if (ret != GDK_SUCCEED)
1493 0 : goto end;
1494 :
1495 5 : ret = snapshot_wal(bat_logger, plan, db_dir);
1496 5 : if (ret != GDK_SUCCEED)
1497 : goto end;
1498 :
1499 : ret = GDK_SUCCEED;
1500 : end:
1501 : return ret;
1502 : }
1503 :
1504 : void
1505 358 : bat_logger_init( logger_functions *lf )
1506 : {
1507 358 : lf->create = bl_create;
1508 358 : lf->destroy = bl_destroy;
1509 358 : lf->flush = bl_flush;
1510 358 : lf->activate = bl_activate;
1511 358 : lf->changes = bl_changes;
1512 358 : lf->get_sequence = bl_get_sequence;
1513 358 : lf->log_isnew = bl_log_isnew;
1514 358 : lf->log_tstart = bl_tstart;
1515 358 : lf->log_tend = bl_tend;
1516 358 : lf->log_tflush = bl_tflush;
1517 358 : lf->log_tsequence = bl_sequence;
1518 358 : lf->get_snapshot_files = bl_snapshot;
1519 358 : }
|
