Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : #include "monetdb_config.h"
14 : #include "bat_logger.h"
15 : #include "bat_utils.h"
16 : #include "sql_types.h" /* EC_POS */
17 : #include "gdk_logger_internals.h"
18 : #include "mutils.h"
19 :
20 : #define CATALOG_JUL2021 52300 /* first in Jul2021 */
21 : #define CATALOG_JAN2022 52301 /* first in Jan2022 */
22 : #define CATALOG_SEP2022 52302 /* first in Sep2022 */
23 : #define CATALOG_AUG2024 52303 /* first in aug2024 */
24 :
25 : /* Note, CATALOG version 52300 is the first one where the basic system
26 : * tables (the ones created in store.c) have fixed and unchangeable
27 : * ids. */
28 :
29 : /* return GDK_SUCCEED if we can handle the upgrade from oldversion to
30 : * newversion */
31 : static gdk_return
32 16 : bl_preversion(sqlstore *store, int oldversion, int newversion)
33 : {
34 16 : (void)newversion;
35 :
36 : #ifdef CATALOG_JUL2021
37 16 : if (oldversion == CATALOG_JUL2021) {
38 : /* upgrade to default releases */
39 0 : store->catalog_version = oldversion;
40 0 : return GDK_SUCCEED;
41 : }
42 : #endif
43 :
44 : #ifdef CATALOG_JAN2022
45 16 : if (oldversion == CATALOG_JAN2022) {
46 : /* upgrade to default releases */
47 0 : store->catalog_version = oldversion;
48 0 : return GDK_SUCCEED;
49 : }
50 : #endif
51 :
52 : #ifdef CATALOG_SEP2022
53 16 : if (oldversion == CATALOG_SEP2022) {
54 : /* upgrade to default releases */
55 8 : store->catalog_version = oldversion;
56 8 : return GDK_SUCCEED;
57 : }
58 : #endif
59 :
60 : #ifdef CATALOG_AUG2024
61 8 : if (oldversion == CATALOG_AUG2024) {
62 : /* upgrade to default releases */
63 8 : store->catalog_version = oldversion;
64 8 : return GDK_SUCCEED;
65 : }
66 : #endif
67 : return GDK_FAIL;
68 : }
69 :
70 : #if defined CATALOG_JUL2021 || defined CATALOG_JAN2022
71 : /* replace a column in a system table with a new column
72 : * colid is the SQL id for the column, oldcolid is the BAT id of the
73 : * to-be-replaced BAT */
74 : static gdk_return
75 0 : replace_bat(logger *lg, int colid, BAT *newcol)
76 : {
77 0 : gdk_return rc;
78 0 : newcol = BATsetaccess(newcol, BAT_READ);
79 0 : if (newcol == NULL)
80 : return GDK_FAIL;
81 0 : if ((rc = BAThash(lg->catalog_id)) == GDK_SUCCEED) {
82 0 : BATiter cii = bat_iterator_nolock(lg->catalog_id);
83 0 : BUN p;
84 0 : MT_rwlock_rdlock(&cii.b->thashlock);
85 0 : HASHloop_int(cii, cii.b->thash, p, &colid) {
86 0 : if (BUNfnd(lg->dcatalog, &(oid){(oid)p}) == BUN_NONE) {
87 0 : if (BUNappend(lg->dcatalog, &(oid){(oid)p}, true) != GDK_SUCCEED ||
88 0 : BUNreplace(lg->catalog_lid, (oid) p, &(lng){0}, false) != GDK_SUCCEED) {
89 0 : MT_rwlock_rdunlock(&cii.b->thashlock);
90 0 : return GDK_FAIL;
91 : }
92 0 : break;
93 : }
94 : }
95 0 : MT_rwlock_rdunlock(&cii.b->thashlock);
96 0 : if ((rc = BUNappend(lg->catalog_id, &colid, true)) == GDK_SUCCEED &&
97 0 : (rc = BUNappend(lg->catalog_bid, &newcol->batCacheid, true)) == GDK_SUCCEED &&
98 0 : (rc = BUNappend(lg->catalog_lid, &lng_nil, false)) == GDK_SUCCEED &&
99 0 : (rc = BUNappend(lg->catalog_cnt, &(lng){BATcount(newcol)}, false)) == GDK_SUCCEED) {
100 0 : BBPretain(newcol->batCacheid);
101 : }
102 : }
103 : return rc;
104 : }
105 : #endif
106 :
107 : static BAT *
108 392 : log_temp_descriptor(log_bid b)
109 : {
110 392 : if (b <= 0)
111 : return NULL;
112 392 : return temp_descriptor(b);
113 : }
114 :
115 : #if defined CATALOG_JAN2022 || defined CATALOG_SEP2022 || defined CATALOG_AUG2024
116 : static gdk_return
117 32 : tabins(logger *lg, ...)
118 : {
119 32 : va_list va;
120 32 : int cid;
121 32 : const void *cval;
122 32 : gdk_return rc;
123 32 : BAT *b;
124 :
125 32 : va_start(va, lg);
126 32 : BATiter cni = bat_iterator(lg->catalog_id);
127 384 : while ((cid = va_arg(va, int)) != 0) {
128 352 : cval = va_arg(va, void *);
129 704 : if ((b = log_temp_descriptor(log_find_bat(lg, cid))) == NULL) {
130 : rc = GDK_FAIL;
131 : break;
132 : }
133 352 : rc = BUNappend(b, cval, true);
134 352 : if (rc == GDK_SUCCEED) {
135 352 : BUN p;
136 352 : MT_rwlock_rdlock(&cni.b->thashlock);
137 409 : HASHloop_int(cni, cni.b->thash, p, &cid) {
138 352 : if (BUNfnd(lg->dcatalog, &(oid){p}) == BUN_NONE) {
139 352 : rc = BUNreplace(lg->catalog_cnt, p, &(lng){BATcount(b)}, false);
140 352 : break;
141 : }
142 : }
143 352 : MT_rwlock_rdunlock(&cni.b->thashlock);
144 : }
145 352 : bat_destroy(b);
146 352 : if (rc != GDK_SUCCEED)
147 : break;
148 : }
149 32 : bat_iterator_end(&cni);
150 32 : va_end(va);
151 32 : return rc;
152 : }
153 : #endif
154 :
155 : static gdk_return
156 16 : bl_postversion(void *Store, logger *lg)
157 : {
158 16 : sqlstore *store = Store;
159 16 : gdk_return rc;
160 :
161 : #ifdef CATALOG_JUL2021
162 16 : if (store->catalog_version <= CATALOG_JUL2021) {
163 : /* change the language attribute in sys.functions for sys.env,
164 : * sys.var, and sys.db_users from SQL to MAL */
165 :
166 : /* sys.functions i.e. deleted rows */
167 0 : BAT *del_funcs = log_temp_descriptor(log_find_bat(lg, 2016));
168 0 : if (del_funcs == NULL)
169 0 : return GDK_FAIL;
170 0 : BAT *func_tid = BATmaskedcands(0, BATcount(del_funcs), del_funcs, false);
171 0 : bat_destroy(del_funcs);
172 : /* sys.functions.schema_id */
173 0 : BAT *func_schem = log_temp_descriptor(log_find_bat(lg, 2026));
174 0 : if (func_tid == NULL || func_schem == NULL) {
175 0 : bat_destroy(func_tid);
176 0 : bat_destroy(func_schem);
177 0 : return GDK_FAIL;
178 : }
179 : /* select * from sys.functions where schema_id = 2000 */
180 0 : BAT *cands = BATselect(func_schem, func_tid, &(int) {2000}, NULL, true, true, false, false);
181 0 : bat_destroy(func_schem);
182 0 : if (cands == NULL) {
183 0 : bat_destroy(func_tid);
184 0 : return GDK_FAIL;
185 : }
186 : /* the functions we need to change */
187 0 : BAT *funcs = COLnew(0, TYPE_str, 3, TRANSIENT);
188 0 : if (funcs == NULL ||
189 0 : BUNappend(funcs, "db_users", false) != GDK_SUCCEED ||
190 0 : BUNappend(funcs, "env", false) != GDK_SUCCEED ||
191 0 : BUNappend(funcs, "var", false) != GDK_SUCCEED) {
192 0 : bat_destroy(cands);
193 0 : bat_destroy(funcs);
194 0 : bat_destroy(func_tid);
195 0 : return GDK_FAIL;
196 : }
197 : /* sys.functions.name */
198 0 : BAT *func_name = log_temp_descriptor(log_find_bat(lg, 2018));
199 0 : if (func_name == NULL) {
200 0 : bat_destroy(cands);
201 0 : bat_destroy(funcs);
202 0 : bat_destroy(func_tid);
203 0 : return GDK_FAIL;
204 : }
205 : /* select * from sys.functions where schema_id = 2000 and name in (...) */
206 0 : BAT *b = BATintersect(func_name, funcs, cands, NULL, false, false, 3);
207 0 : bat_destroy(cands);
208 0 : bat_destroy(func_name);
209 0 : bat_destroy(funcs);
210 0 : cands = b;
211 0 : if (cands == NULL) {
212 0 : bat_destroy(func_tid);
213 0 : return GDK_FAIL;
214 : }
215 : /* sys.functions.language */
216 0 : BAT *func_lang = log_temp_descriptor(log_find_bat(lg, 2021));
217 0 : if (func_lang == NULL) {
218 0 : bat_destroy(cands);
219 0 : bat_destroy(func_tid);
220 0 : return GDK_FAIL;
221 : }
222 : /* select * from sys.functions where schema_id = 2000 and name in (...)
223 : * and language = FUNC_LANG_SQL */
224 0 : b = BATselect(func_lang, cands, &(int) {FUNC_LANG_SQL}, NULL, true, true, false, false);
225 0 : bat_destroy(cands);
226 0 : cands = b;
227 0 : if (cands == NULL) {
228 0 : bat_destroy(func_lang);
229 0 : bat_destroy(func_tid);
230 0 : return GDK_FAIL;
231 : }
232 0 : b = BATconstant(0, TYPE_int, &(int) {FUNC_LANG_MAL}, BATcount(cands), TRANSIENT);
233 0 : if (b == NULL) {
234 0 : bat_destroy(func_lang);
235 0 : bat_destroy(cands);
236 0 : bat_destroy(func_tid);
237 0 : return GDK_FAIL;
238 : }
239 0 : rc = GDK_FAIL;
240 0 : BAT *b2 = COLcopy(func_lang, func_lang->ttype, true, PERSISTENT);
241 0 : if (b2 == NULL ||
242 0 : BATreplace(b2, cands, b, false) != GDK_SUCCEED) {
243 0 : bat_destroy(b2);
244 0 : bat_destroy(cands);
245 0 : bat_destroy(b);
246 0 : bat_destroy(func_tid);
247 0 : bat_destroy(func_lang);
248 0 : return GDK_FAIL;
249 : }
250 0 : bat_destroy(b);
251 0 : bat_destroy(cands);
252 :
253 : /* additionally, update the language attribute for entries
254 : * that were declared using "EXTERNAL NAME" to be MAL functions
255 : * instead of SQL functions (a problem that seems to have
256 : * occurred in ancient databases) */
257 :
258 : /* sys.functions.func */
259 0 : BAT *func_func = log_temp_descriptor(log_find_bat(lg, 2019));
260 0 : if (func_func == NULL) {
261 0 : bat_destroy(func_tid);
262 0 : bat_destroy(b2);
263 0 : return GDK_FAIL;
264 : }
265 0 : cands = BATselect(func_lang, func_tid, &(int){FUNC_LANG_SQL}, NULL, true, true, false, false);
266 0 : bat_destroy(func_lang);
267 0 : bat_destroy(func_tid);
268 0 : if (cands == NULL) {
269 0 : bat_destroy(b2);
270 0 : bat_destroy(func_func);
271 0 : return GDK_FAIL;
272 : }
273 0 : struct canditer ci;
274 0 : canditer_init(&ci, func_func, cands);
275 0 : BATiter ffi = bat_iterator_nolock(func_func);
276 0 : for (BUN p = 0; p < ci.ncand; p++) {
277 0 : oid o = canditer_next(&ci);
278 0 : const char *f = BUNtvar(ffi, o - func_func->hseqbase);
279 0 : const char *e;
280 0 : if (!strNil(f) &&
281 0 : (e = strstr(f, "external")) != NULL &&
282 0 : e > f && isspace((unsigned char) e[-1]) && isspace((unsigned char) e[8]) && strncmp(e + 9, "name", 4) == 0 && isspace((unsigned char) e[13]) &&
283 0 : BUNreplace(b2, o, &(int){FUNC_LANG_MAL}, false) != GDK_SUCCEED) {
284 0 : bat_destroy(b2);
285 0 : bat_destroy(func_func);
286 0 : return GDK_FAIL;
287 : }
288 : }
289 0 : rc = replace_bat(lg, 2021, b2);
290 0 : bat_destroy(b2);
291 0 : if (rc != GDK_SUCCEED)
292 : return rc;
293 : }
294 16 : if (store->catalog_version <= CATALOG_JUL2021) {
295 : /* change the side_effects attribute in sys.functions for
296 : * selected functions */
297 :
298 : /* sys.functions i.e. deleted rows */
299 0 : BAT *del_funcs = log_temp_descriptor(log_find_bat(lg, 2016));
300 0 : if (del_funcs == NULL)
301 0 : return GDK_FAIL;
302 0 : BAT *func_tid = BATmaskedcands(0, BATcount(del_funcs), del_funcs, false);
303 0 : bat_destroy(del_funcs);
304 : /* sys.functions.schema_id */
305 0 : BAT *func_schem = log_temp_descriptor(log_find_bat(lg, 2026));
306 0 : if (func_tid == NULL || func_schem == NULL) {
307 0 : bat_destroy(func_tid);
308 0 : bat_destroy(func_schem);
309 0 : return GDK_FAIL;
310 : }
311 : /* select * from sys.functions where schema_id = 2000 */
312 0 : BAT *cands = BATselect(func_schem, func_tid, &(int) {2000}, NULL, true, true, false, false);
313 0 : bat_destroy(func_schem);
314 0 : bat_destroy(func_tid);
315 0 : if (cands == NULL) {
316 : return GDK_FAIL;
317 : }
318 : /* sys.functions.side_effect */
319 0 : BAT *func_se = log_temp_descriptor(log_find_bat(lg, 2023));
320 0 : if (func_se == NULL) {
321 0 : bat_destroy(cands);
322 0 : return GDK_FAIL;
323 : }
324 : /* make a copy that we can modify */
325 0 : BAT *b = COLcopy(func_se, func_se->ttype, true, PERSISTENT);
326 0 : bat_destroy(func_se);
327 0 : if (b == NULL) {
328 0 : bat_destroy(cands);
329 0 : return GDK_FAIL;
330 : }
331 0 : func_se = b;
332 : /* sys.functions.func */
333 0 : BAT *func_func = log_temp_descriptor(log_find_bat(lg, 2019));
334 0 : if (func_func == NULL) {
335 0 : bat_destroy(cands);
336 0 : bat_destroy(func_se);
337 0 : return GDK_FAIL;
338 : }
339 : /* the functions we need to change to FALSE */
340 0 : BAT *funcs = COLnew(0, TYPE_str, 1, TRANSIENT);
341 0 : if (funcs == NULL ||
342 0 : BUNappend(funcs, "sqlrand", false) != GDK_SUCCEED) {
343 0 : bat_destroy(cands);
344 0 : bat_destroy(func_se);
345 0 : bat_destroy(func_func);
346 0 : bat_destroy(funcs);
347 0 : return GDK_FAIL;
348 : }
349 : /* select * from sys.functions where schema_id = 2000 and func in (...) */
350 0 : b = BATintersect(func_func, funcs, cands, NULL, false, false, 1);
351 0 : bat_destroy(funcs);
352 0 : if (b == NULL) {
353 0 : bat_destroy(cands);
354 0 : bat_destroy(func_se);
355 0 : bat_destroy(func_func);
356 0 : return GDK_FAIL;
357 : }
358 : /* while we're at it, also change sys.env and sys.db_users to
359 : * being without side effect (legacy from ancient databases) */
360 : /* sys.functions.name */
361 0 : BAT *func_name = log_temp_descriptor(log_find_bat(lg, 2018));
362 0 : if (func_name == NULL) {
363 0 : bat_destroy(cands);
364 0 : bat_destroy(func_se);
365 0 : bat_destroy(func_func);
366 0 : bat_destroy(b);
367 0 : return GDK_FAIL;
368 : }
369 0 : BAT *b2 = BATselect(func_name, cands, "env", NULL, true, true, false, false);
370 0 : if (b2 == NULL || BATappend(b, b2, NULL, false) != GDK_SUCCEED) {
371 0 : bat_destroy(cands);
372 0 : bat_destroy(func_se);
373 0 : bat_destroy(func_func);
374 0 : bat_destroy(b);
375 0 : bat_destroy(func_name);
376 0 : bat_destroy(b2);
377 0 : return GDK_FAIL;
378 : }
379 0 : bat_destroy(b2);
380 0 : b2 = BATselect(func_name, cands, "db_users", NULL, true, true, false, false);
381 0 : bat_destroy(func_name);
382 0 : if (b2 == NULL || BATappend(b, b2, NULL, false) != GDK_SUCCEED) {
383 0 : bat_destroy(cands);
384 0 : bat_destroy(func_se);
385 0 : bat_destroy(func_func);
386 0 : bat_destroy(b);
387 0 : bat_destroy(b2);
388 0 : return GDK_FAIL;
389 : }
390 0 : bat_destroy(b2);
391 :
392 0 : BAT *vals = BATconstant(0, TYPE_bit, &(bit) {FALSE}, BATcount(b), TRANSIENT);
393 0 : if (vals == NULL) {
394 0 : bat_destroy(cands);
395 0 : bat_destroy(func_se);
396 0 : bat_destroy(func_func);
397 0 : bat_destroy(b);
398 0 : return GDK_FAIL;
399 : }
400 0 : rc = BATreplace(func_se, b, vals, false);
401 0 : bat_destroy(b);
402 0 : bat_destroy(vals);
403 0 : if (rc != GDK_SUCCEED) {
404 0 : bat_destroy(cands);
405 0 : bat_destroy(func_se);
406 0 : bat_destroy(func_func);
407 0 : return GDK_FAIL;
408 : }
409 : /* the functions we need to change to TRUE */
410 0 : funcs = COLnew(0, TYPE_str, 5, TRANSIENT);
411 0 : if (funcs == NULL ||
412 0 : BUNappend(funcs, "copy_from", false) != GDK_SUCCEED ||
413 0 : BUNappend(funcs, "next_value", false) != GDK_SUCCEED ||
414 0 : BUNappend(funcs, "update_schemas", false) != GDK_SUCCEED ||
415 0 : BUNappend(funcs, "update_tables", false) != GDK_SUCCEED) {
416 0 : bat_destroy(cands);
417 0 : bat_destroy(func_se);
418 0 : bat_destroy(func_func);
419 0 : bat_destroy(funcs);
420 0 : return GDK_FAIL;
421 : }
422 : /* select * from sys.functions where schema_id = 2000 and func in (...) */
423 0 : b = BATintersect(func_func, funcs, cands, NULL, false, false, 7);
424 0 : bat_destroy(funcs);
425 0 : bat_destroy(cands);
426 0 : bat_destroy(func_func);
427 0 : if (b == NULL) {
428 0 : bat_destroy(func_se);
429 0 : return GDK_FAIL;
430 : }
431 0 : vals = BATconstant(0, TYPE_bit, &(bit) {TRUE}, BATcount(b), TRANSIENT);
432 0 : if (vals == NULL) {
433 0 : bat_destroy(func_se);
434 0 : bat_destroy(b);
435 0 : return GDK_FAIL;
436 : }
437 0 : rc = BATreplace(func_se, b, vals, false);
438 0 : bat_destroy(b);
439 0 : bat_destroy(vals);
440 0 : if (rc != GDK_SUCCEED) {
441 0 : bat_destroy(func_se);
442 0 : return GDK_FAIL;
443 : }
444 : /* replace old column with modified copy */
445 0 : rc = replace_bat(lg, 2023, func_se);
446 0 : bat_destroy(func_se);
447 0 : if (rc != GDK_SUCCEED)
448 : return rc;
449 : }
450 16 : if (store->catalog_version <= CATALOG_JUL2021) {
451 : /* upgrade some columns in sys.sequences:
452 : * if increment is zero, set it to one (see ChangeLog);
453 : * if increment is greater than zero and maxvalue is zero,
454 : * set maxvalue to GDK_lng_max;
455 : * if increment is less than zero and minvalue is zero,
456 : * set minvalue to GDK_lng_min */
457 :
458 : /* sys.sequences i.e. deleted rows */
459 0 : BAT *del_seqs = log_temp_descriptor(log_find_bat(lg, 2037));
460 0 : if (del_seqs == NULL)
461 0 : return GDK_FAIL;
462 0 : BAT *seq_tid = BATmaskedcands(0, BATcount(del_seqs), del_seqs, false);
463 0 : bat_destroy(del_seqs);
464 0 : BAT *seq_min = log_temp_descriptor(log_find_bat(lg, 2042)); /* sys.sequences.minvalue */
465 0 : BAT *seq_max = log_temp_descriptor(log_find_bat(lg, 2043)); /* sys.sequences.maxvalue */
466 0 : BAT *seq_inc = log_temp_descriptor(log_find_bat(lg, 2044)); /* sys.sequences.increment */
467 0 : if (seq_tid == NULL || seq_min == NULL || seq_max == NULL || seq_inc == NULL) {
468 0 : bat_destroy(seq_tid);
469 0 : bat_destroy(seq_min);
470 0 : bat_destroy(seq_max);
471 0 : bat_destroy(seq_inc);
472 0 : return GDK_FAIL;
473 : }
474 : /* select * from sys.sequences where increment = 0 */
475 0 : BAT *inczero = BATselect(seq_inc, seq_tid, &(lng){0}, NULL, false, true, false, false);
476 0 : if (inczero == NULL) {
477 0 : bat_destroy(seq_tid);
478 0 : bat_destroy(seq_min);
479 0 : bat_destroy(seq_max);
480 0 : bat_destroy(seq_inc);
481 0 : return GDK_FAIL;
482 : }
483 0 : if (BATcount(inczero) > 0) {
484 0 : BAT *b = BATconstant(0, TYPE_lng, &(lng) {1}, BATcount(inczero), TRANSIENT);
485 0 : if (b == NULL) {
486 0 : bat_destroy(seq_tid);
487 0 : bat_destroy(seq_min);
488 0 : bat_destroy(seq_max);
489 0 : bat_destroy(seq_inc);
490 0 : bat_destroy(inczero);
491 0 : return GDK_FAIL;
492 : }
493 0 : BAT *b2 = COLcopy(seq_inc, seq_inc->ttype, true, PERSISTENT);
494 0 : rc = GDK_FAIL;
495 0 : if (b2 == NULL)
496 0 : rc = BATreplace(b2, inczero, b, false);
497 0 : bat_destroy(b);
498 0 : if (rc != GDK_SUCCEED) {
499 0 : bat_destroy(b2);
500 0 : bat_destroy(seq_tid);
501 0 : bat_destroy(seq_min);
502 0 : bat_destroy(seq_max);
503 0 : bat_destroy(seq_inc);
504 0 : bat_destroy(inczero);
505 0 : return GDK_FAIL;
506 : }
507 0 : rc = replace_bat(lg, 2044, b2);
508 0 : bat_destroy(seq_inc);
509 0 : seq_inc = b2;
510 0 : if (rc != GDK_SUCCEED) {
511 0 : bat_destroy(seq_tid);
512 0 : bat_destroy(seq_min);
513 0 : bat_destroy(seq_max);
514 0 : bat_destroy(seq_inc);
515 0 : bat_destroy(inczero);
516 0 : return rc;
517 : }
518 : }
519 0 : bat_destroy(inczero);
520 : /* select * from sys.sequences where increment > 0 */
521 0 : BAT *incpos = BATselect(seq_inc, seq_tid, &(lng){0}, &lng_nil, false, true, false, false);
522 0 : bat_destroy(seq_inc);
523 0 : if (incpos == NULL) {
524 0 : bat_destroy(seq_tid);
525 0 : bat_destroy(seq_min);
526 0 : bat_destroy(seq_max);
527 0 : return GDK_FAIL;
528 : }
529 : /* select * from sys.sequences where increment > 0 and maxvalue = 0 */
530 0 : BAT *cands = BATselect(seq_max, incpos, &(lng) {0}, NULL, true, true, false, false);
531 0 : bat_destroy(incpos);
532 0 : if (cands == NULL) {
533 0 : bat_destroy(seq_tid);
534 0 : bat_destroy(seq_min);
535 0 : bat_destroy(seq_max);
536 0 : return GDK_FAIL;
537 : }
538 0 : if (BATcount(cands) > 0) {
539 0 : BAT *b = BATconstant(0, TYPE_lng, &(lng){GDK_lng_max}, BATcount(cands), TRANSIENT);
540 0 : BAT *b2 = COLcopy(seq_max, seq_max->ttype, true, PERSISTENT);
541 0 : rc = GDK_FAIL;
542 0 : if (b != NULL && b2 != NULL)
543 0 : rc = BATreplace(b2, cands, b, false);
544 0 : bat_destroy(b);
545 0 : if (rc == GDK_SUCCEED)
546 0 : rc = replace_bat(lg, 2043, b2);
547 0 : bat_destroy(b2);
548 0 : if (rc != GDK_SUCCEED) {
549 0 : bat_destroy(cands);
550 0 : bat_destroy(seq_tid);
551 0 : bat_destroy(seq_min);
552 0 : bat_destroy(seq_max);
553 0 : return rc;
554 : }
555 : }
556 0 : bat_destroy(seq_max);
557 0 : bat_destroy(cands);
558 : /* select * from sys.sequences where increment < 0 */
559 0 : BAT *incneg = BATselect(seq_inc, seq_tid, &lng_nil, &(lng){0}, false, true, false, false);
560 0 : bat_destroy(seq_tid);
561 : /* select * from sys.sequences where increment < 0 and minvalue = 0 */
562 0 : cands = BATselect(seq_min, incneg, &(lng) {0}, NULL, true, true, false, false);
563 0 : bat_destroy(incneg);
564 0 : if (cands == NULL) {
565 0 : bat_destroy(seq_min);
566 0 : return GDK_FAIL;
567 : }
568 0 : if (BATcount(cands) > 0) {
569 0 : BAT *b = BATconstant(0, TYPE_lng, &(lng){GDK_lng_min}, BATcount(cands), TRANSIENT);
570 0 : BAT *b2 = COLcopy(seq_min, seq_min->ttype, true, PERSISTENT);
571 0 : rc = GDK_FAIL;
572 0 : if (b != NULL && b2 != NULL)
573 0 : rc = BATreplace(b2, cands, b, false);
574 0 : bat_destroy(b);
575 0 : if (rc == GDK_SUCCEED)
576 0 : rc = replace_bat(lg, 2042, b2);
577 0 : bat_destroy(b2);
578 0 : if (rc != GDK_SUCCEED) {
579 0 : bat_destroy(cands);
580 0 : bat_destroy(seq_min);
581 0 : return rc;
582 : }
583 : }
584 0 : bat_destroy(seq_min);
585 0 : bat_destroy(cands);
586 : }
587 : #endif
588 :
589 : #ifdef CATALOG_JAN2022
590 16 : if (store->catalog_version <= CATALOG_JAN2022) {
591 : /* GRANT SELECT ON sys.db_user_info TO monetdb;
592 : * except the grantor is 0 instead of user monetdb
593 : *
594 : * we need to find the IDs of the sys.db_user_info table and of
595 : * the sys.privileges table and its columns since none of these
596 : * have fixed IDs */
597 0 : BAT *b = log_temp_descriptor(log_find_bat(lg, 2067)); /* sys._tables */
598 0 : if (b == NULL)
599 0 : return GDK_FAIL;
600 0 : BAT *del_tabs = BATmaskedcands(0, BATcount(b), b, false);
601 0 : bat_destroy(b);
602 0 : if (del_tabs == NULL)
603 : return GDK_FAIL;
604 0 : b = log_temp_descriptor(log_find_bat(lg, 2076)); /* sys._columns */
605 0 : if (b == NULL) {
606 0 : bat_destroy(del_tabs);
607 0 : return GDK_FAIL;
608 : }
609 0 : BAT *del_cols = BATmaskedcands(0, BATcount(b), b, false);
610 0 : bat_destroy(b);
611 0 : b = log_temp_descriptor(log_find_bat(lg, 2070)); /* sys._tables.schema_id */
612 0 : if (del_cols == NULL || b == NULL) {
613 0 : bat_destroy(del_cols);
614 0 : bat_destroy(b);
615 0 : bat_destroy(del_tabs);
616 0 : return GDK_FAIL;
617 : }
618 0 : BAT *cands = BATselect(b, del_tabs, &(int) {2000}, NULL, true, true, false, false);
619 0 : bat_destroy(b);
620 0 : bat_destroy(del_tabs);
621 : /* cands contains undeleted rows from sys._tables for tables in
622 : * sys schema */
623 0 : BAT *tabnme = log_temp_descriptor(log_find_bat(lg, 2069)); /* sys._tables.name */
624 0 : if (cands == NULL || tabnme == NULL) {
625 0 : bat_destroy(cands);
626 0 : bat_destroy(tabnme);
627 0 : bat_destroy(del_cols);
628 0 : return GDK_FAIL;
629 : }
630 0 : b = BATselect(tabnme, cands, "db_user_info", NULL, true, true, false, false);
631 0 : if (b == NULL) {
632 0 : bat_destroy(cands);
633 0 : bat_destroy(tabnme);
634 0 : bat_destroy(del_cols);
635 0 : return GDK_FAIL;
636 : }
637 0 : oid dbpos = BUNtoid(b, 0);
638 0 : bat_destroy(b);
639 0 : b = BATselect(tabnme, cands, "privileges", NULL, true, true, false, false);
640 0 : bat_destroy(tabnme);
641 0 : bat_destroy(cands);
642 0 : BAT *tabid = log_temp_descriptor(log_find_bat(lg, 2068)); /* sys._tables.id */
643 0 : if (b == NULL || tabid == NULL) {
644 0 : bat_destroy(b);
645 0 : bat_destroy(tabid);
646 0 : bat_destroy(del_cols);
647 0 : return GDK_FAIL;
648 : }
649 0 : int dbid = ((int *) tabid->theap->base)[dbpos];
650 0 : int prid = ((int *) tabid->theap->base)[BUNtoid(b, 0)];
651 0 : BAT *coltid = log_temp_descriptor(log_find_bat(lg, 2082)); /* sys._columns.table_id */
652 0 : if (coltid == NULL) {
653 0 : bat_destroy(b);
654 0 : bat_destroy(del_cols);
655 0 : bat_destroy(tabid);
656 0 : return GDK_FAIL;
657 : }
658 0 : BAT *b1;
659 0 : rc = BATjoin(&b1, NULL, coltid, tabid, del_cols, b, false, 5);
660 0 : bat_destroy(coltid);
661 0 : bat_destroy(tabid);
662 0 : bat_destroy(del_cols);
663 0 : bat_destroy(b);
664 0 : BAT *colnr = log_temp_descriptor(log_find_bat(lg, 2085)); /* sys._columns.number */
665 0 : BAT *colid = log_temp_descriptor(log_find_bat(lg, 2077)); /* sys._columns.id */
666 0 : if (rc != GDK_SUCCEED || colnr == NULL || colid == NULL) {
667 0 : if (rc == GDK_SUCCEED)
668 0 : bat_destroy(b1);
669 0 : bat_destroy(colnr);
670 0 : bat_destroy(colid);
671 0 : return GDK_FAIL;
672 : }
673 : int privids[5];
674 0 : for (int i = 0; i < 5; i++) {
675 0 : oid p = BUNtoid(b1, i);
676 0 : privids[((int *) colnr->theap->base)[p]] = ((int *) colid->theap->base)[p];
677 : }
678 0 : bat_destroy(b1);
679 0 : bat_destroy(colnr);
680 0 : bat_destroy(colid);
681 0 : rc = tabins(lg,
682 0 : prid, &(msk) {false}, /* sys.privileges */
683 : privids[0], &dbid, /* sys.privileges.obj_id */
684 0 : privids[1], &(int) {USER_MONETDB}, /* sys.privileges.auth_id */
685 0 : privids[2], &(int) {PRIV_SELECT}, /* sys.privileges.privileges */
686 0 : privids[3], &(int) {0}, /* sys.privileges.grantor */
687 0 : privids[4], &(int) {0}, /* sys.privileges.grantee */
688 : 0);
689 0 : if (rc != GDK_SUCCEED)
690 : return rc;
691 : }
692 : #endif
693 :
694 : #ifdef CATALOG_SEP2022
695 16 : if (store->catalog_version <= CATALOG_SEP2022) {
696 : /* new STRING column sys.keys.check */
697 8 : BAT *b = log_temp_descriptor(log_find_bat(lg, 2088)); /* sys.keys.id */
698 8 : if (b == NULL)
699 0 : return GDK_FAIL;
700 8 : BAT *check = BATconstant(b->hseqbase, TYPE_str, ATOMnilptr(TYPE_str), BATcount(b), PERSISTENT);
701 8 : bat_destroy(b);
702 8 : if (check == NULL)
703 : return GDK_FAIL;
704 16 : if ((check = BATsetaccess(check, BAT_READ)) == NULL ||
705 : /* 2165 is sys.keys.check */
706 16 : BUNappend(lg->catalog_id, &(int) {2165}, true) != GDK_SUCCEED ||
707 16 : BUNappend(lg->catalog_bid, &check->batCacheid, true) != GDK_SUCCEED ||
708 16 : BUNappend(lg->catalog_lid, &lng_nil, false) != GDK_SUCCEED ||
709 8 : BUNappend(lg->catalog_cnt, &(lng){BATcount(check)}, false) != GDK_SUCCEED
710 : ) {
711 0 : bat_destroy(check);
712 0 : return GDK_FAIL;
713 : }
714 8 : BBPretain(check->batCacheid);
715 8 : bat_destroy(check);
716 :
717 8 : if (tabins(lg,
718 8 : 2076, &(msk) {false}, /* sys._columns */
719 : /* 2165 is sys.keys.check */
720 8 : 2077, &(int) {2165}, /* sys._columns.id */
721 : 2078, "check", /* sys._columns.name */
722 : 2079, "varchar", /* sys._columns.type */
723 8 : 2080, &(int) {2048}, /* sys._columns.type_digits */
724 8 : 2081, &(int) {0}, /* sys._columns.type_scale */
725 : /* 2087 is sys.keys */
726 8 : 2082, &(int) {2087}, /* sys._columns.table_id */
727 : 2083, str_nil, /* sys._columns.default */
728 8 : 2084, &(bit) {TRUE}, /* sys._columns.null */
729 8 : 2085, &(int) {6}, /* sys._columns.number */
730 : 2086, str_nil, /* sys._columns.storage */
731 : 0) != GDK_SUCCEED)
732 0 : return GDK_FAIL;
733 8 : if (tabins(lg,
734 8 : 2076, &(msk) {false}, /* sys._columns */
735 : /* 2166 is tmp.keys.check */
736 8 : 2077, &(int) {2166}, /* sys._columns.id */
737 : 2078, "check", /* sys._columns.name */
738 : 2079, "varchar", /* sys._columns.type */
739 8 : 2080, &(int) {2048}, /* sys._columns.type_digits */
740 8 : 2081, &(int) {0}, /* sys._columns.type_scale */
741 : /* 2135 is tmp.keys */
742 8 : 2082, &(int) {2135}, /* sys._columns.table_id */
743 : 2083, str_nil, /* sys._columns.default */
744 8 : 2084, &(bit) {TRUE}, /* sys._columns.null */
745 8 : 2085, &(int) {6}, /* sys._columns.number */
746 : 2086, str_nil, /* sys._columns.storage */
747 : 0) != GDK_SUCCEED)
748 0 : return GDK_FAIL;
749 : }
750 : #endif
751 :
752 : #ifdef CATALOG_AUG2024
753 16 : if (store->catalog_version <= CATALOG_AUG2024) {
754 : /* new TINYINT column sys.functions.order_specification */
755 16 : BAT *ftype = log_temp_descriptor(log_find_bat(lg, 2022)); /* sys.functions.type (int) */
756 16 : BAT *fname = log_temp_descriptor(log_find_bat(lg, 2018)); /* sys.functions.name (str) */
757 16 : if (ftype == NULL || fname == NULL)
758 0 : return GDK_FAIL;
759 16 : bte zero = 0;
760 16 : BAT *order_spec = BATconstant(ftype->hseqbase, TYPE_bte, &zero, BATcount(ftype), PERSISTENT);
761 : /* update functions set order_specification=1 where type == aggr and name in ('group_concat', 'listagg', 'xmlagg')
762 : * update functions set order_specification=2 where type == aggr and name = 'quantile' */
763 16 : if (order_spec == NULL) {
764 0 : bat_destroy(ftype);
765 0 : bat_destroy(fname);
766 0 : return GDK_FAIL;
767 : }
768 16 : bte *os = (bte*)Tloc(order_spec, 0);
769 16 : int *ft = (int*)Tloc(ftype, 0);
770 16 : BATiter fni = bat_iterator_nolock(fname);
771 52888 : for(BUN b = 0; b < BATcount(ftype); b++) {
772 52872 : if (ft[b] == F_AGGR) {
773 3746 : const char *f = BUNtvar(fni, b);
774 3746 : if (strcmp(f, "group_concat") == 0 || strcmp(f, "listagg") == 0 || strcmp(f, "xmlagg") == 0)
775 160 : os[b] = 1;
776 3586 : else if (strcmp(f, "quantile") == 0 || strcmp(f, "quantile_avg") == 0)
777 400 : os[b] = 2;
778 : }
779 : }
780 16 : bat_destroy(ftype);
781 16 : bat_destroy(fname);
782 32 : if ((order_spec = BATsetaccess(order_spec, BAT_READ)) == NULL ||
783 : /* 2167 is sys.functions.order_specification */
784 32 : BUNappend(lg->catalog_id, &(int) {2167}, true) != GDK_SUCCEED ||
785 32 : BUNappend(lg->catalog_bid, &order_spec->batCacheid, true) != GDK_SUCCEED ||
786 32 : BUNappend(lg->catalog_lid, &lng_nil, false) != GDK_SUCCEED ||
787 16 : BUNappend(lg->catalog_cnt, &(lng){BATcount(order_spec)}, false) != GDK_SUCCEED
788 : ) {
789 0 : bat_destroy(order_spec);
790 0 : return GDK_FAIL;
791 : }
792 16 : BBPretain(order_spec->batCacheid);
793 16 : bat_destroy(order_spec);
794 :
795 16 : if (tabins(lg,
796 16 : 2076, &(msk) {false}, /* sys._columns */
797 : /* 2167 is sys.functions.order_specification */
798 16 : 2077, &(int) {2167}, /* sys._columns.id */
799 : 2078, "order_specification", /* sys._columns.name */
800 : 2079, "tinyint", /* sys._columns.type */
801 16 : 2080, &(int) {7}, /* sys._columns.type_digits */
802 16 : 2081, &(int) {0}, /* sys._columns.type_scale */
803 : /* 2016 is sys.functions */
804 16 : 2082, &(int) {2016}, /* sys._columns.table_id */
805 : 2083, str_nil, /* sys._columns.default */
806 16 : 2084, &(bit) {TRUE}, /* sys._columns.null */
807 16 : 2085, &(int) {12}, /* sys._columns.number */
808 : 2086, str_nil, /* sys._columns.storage */
809 : 0) != GDK_SUCCEED)
810 0 : return GDK_FAIL;
811 : }
812 : #endif
813 : return GDK_SUCCEED;
814 : }
815 :
816 : static int
817 315 : bl_create(sqlstore *store, int debug, const char *logdir, int cat_version)
818 : {
819 315 : if (store->logger)
820 : return LOG_ERR;
821 315 : store->logger = log_create(debug, "sql", logdir, cat_version, (preversionfix_fptr)&bl_preversion, (postversionfix_fptr)&bl_postversion, store);
822 315 : if (store->logger)
823 : return LOG_OK;
824 : return LOG_ERR;
825 : }
826 :
827 : static void
828 298 : bl_destroy(sqlstore *store)
829 : {
830 298 : logger *l = store->logger;
831 :
832 298 : store->logger = NULL;
833 298 : if (l)
834 298 : log_destroy(l);
835 298 : }
836 :
837 : static int
838 1483 : bl_flush(sqlstore *store, lng save_id)
839 : {
840 1483 : if (store->logger)
841 1483 : return log_flush(store->logger, save_id) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
842 : return LOG_OK;
843 : }
844 :
845 : static int
846 5227 : bl_activate(sqlstore *store)
847 : {
848 5227 : if (store->logger)
849 5227 : return log_activate(store->logger) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
850 : return LOG_OK;
851 : }
852 :
853 : static int
854 7301 : bl_changes(sqlstore *store)
855 : {
856 7301 : return (int) MIN(log_changes(store->logger), GDK_int_max);
857 : }
858 :
859 : static int
860 430 : bl_get_sequence(sqlstore *store, int seq, lng *id)
861 : {
862 430 : return log_sequence(store->logger, seq, id);
863 : }
864 :
865 : static int
866 315 : bl_log_isnew(sqlstore *store)
867 : {
868 315 : logger *bat_logger = store->logger;
869 315 : if (BATcount(bat_logger->catalog_bid) > 10) {
870 89 : return 0;
871 : }
872 : return 1;
873 : }
874 :
875 : static int
876 62522 : bl_tstart(sqlstore *store, bool flush, ulng *log_file_id)
877 : {
878 62522 : return log_tstart(store->logger, flush, log_file_id) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
879 : }
880 :
881 : static int
882 62521 : bl_tend(sqlstore *store)
883 : {
884 62521 : return log_tend(store->logger) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
885 : }
886 :
887 : static int
888 62521 : bl_tflush(sqlstore *store, ulng log_file_id, ulng commit_ts)
889 : {
890 62521 : return log_tflush(store->logger, log_file_id, commit_ts) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
891 : }
892 :
893 : static int
894 6881 : bl_sequence(sqlstore *store, int seq, lng id)
895 : {
896 6881 : return log_tsequence(store->logger, seq, id) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
897 : }
898 :
899 : /* Write a plan entry to copy part of the given file.
900 : * That part of the file must remain unchanged until the plan is executed.
901 : */
902 : static gdk_return __attribute__((__warn_unused_result__))
903 1035 : snapshot_lazy_copy_file(stream *plan, const char *name, uint64_t extent)
904 : {
905 1035 : if (mnstr_printf(plan, "c %" PRIu64 " %s\n", extent, name) < 0) {
906 0 : GDKerror("%s", mnstr_peek_error(plan));
907 0 : return GDK_FAIL;
908 : }
909 : return GDK_SUCCEED;
910 : }
911 :
912 : /* Write a plan entry to write the current contents of the given file.
913 : * The contents are included in the plan so the source file is allowed to
914 : * change in the mean time.
915 : */
916 : static gdk_return __attribute__((__warn_unused_result__))
917 10 : snapshot_immediate_copy_file(stream *plan, const char *path, const char *name)
918 : {
919 10 : gdk_return ret = GDK_FAIL;
920 10 : const size_t bufsize = 64 * 1024;
921 10 : struct stat statbuf;
922 10 : char *buf = NULL;
923 10 : stream *s = NULL;
924 10 : size_t to_copy;
925 :
926 10 : if (MT_stat(path, &statbuf) < 0) {
927 0 : GDKsyserror("stat failed on %s", path);
928 0 : goto end;
929 : }
930 10 : to_copy = (size_t) statbuf.st_size;
931 :
932 10 : s = open_rstream(path);
933 10 : if (!s) {
934 0 : GDKerror("%s", mnstr_peek_error(NULL));
935 0 : goto end;
936 : }
937 :
938 10 : buf = GDKmalloc(bufsize);
939 10 : if (!buf) {
940 0 : GDKerror("GDKmalloc failed");
941 0 : goto end;
942 : }
943 :
944 10 : if (mnstr_printf(plan, "w %zu %s\n", to_copy, name) < 0) {
945 0 : GDKerror("%s", mnstr_peek_error(plan));
946 0 : goto end;
947 : }
948 :
949 20 : while (to_copy > 0) {
950 10 : size_t chunk = (to_copy <= bufsize) ? to_copy : bufsize;
951 10 : ssize_t bytes_read = mnstr_read(s, buf, 1, chunk);
952 10 : if (bytes_read < 0) {
953 0 : GDKerror("Reading bytes of component %s failed: %s", path, mnstr_peek_error(s));
954 0 : goto end;
955 10 : } else if (bytes_read < (ssize_t) chunk) {
956 0 : GDKerror("Read only %zu/%zu bytes of component %s: %s", (size_t) bytes_read, chunk, path, mnstr_peek_error(s));
957 0 : goto end;
958 : }
959 :
960 10 : ssize_t bytes_written = mnstr_write(plan, buf, 1, chunk);
961 10 : if (bytes_written < 0) {
962 0 : GDKerror("Writing to plan failed: %s", mnstr_peek_error(plan));
963 0 : goto end;
964 10 : } else if (bytes_written < (ssize_t) chunk) {
965 0 : GDKerror("write to plan truncated");
966 0 : goto end;
967 : }
968 10 : to_copy -= chunk;
969 : }
970 :
971 : ret = GDK_SUCCEED;
972 10 : end:
973 10 : GDKfree(buf);
974 10 : if (s)
975 10 : close_stream(s);
976 10 : return ret;
977 : }
978 :
979 : /* Add plan entries for all relevant files in the Write Ahead Log */
980 : static gdk_return __attribute__((__warn_unused_result__))
981 5 : snapshot_wal(logger *bat_logger, stream *plan, const char *db_dir)
982 : {
983 5 : char log_file[FILENAME_MAX];
984 5 : int len;
985 :
986 5 : len = snprintf(log_file, sizeof(log_file), "%s/%s%s", db_dir, bat_logger->dir, LOGFILE);
987 5 : if (len == -1 || (size_t)len >= sizeof(log_file)) {
988 0 : GDKerror("Could not open %s, filename is too large", log_file);
989 0 : return GDK_FAIL;
990 : }
991 5 : if (snapshot_immediate_copy_file(plan, log_file, log_file + strlen(db_dir) + 1) != GDK_SUCCEED)
992 : return GDK_FAIL;
993 :
994 15 : for (ulng id = bat_logger->saved_id+1; id <= bat_logger->id; id++) {
995 10 : struct stat statbuf;
996 :
997 10 : len = snprintf(log_file, sizeof(log_file), "%s/%s%s." LLFMT, db_dir, bat_logger->dir, LOGFILE, id);
998 10 : if (len == -1 || (size_t)len >= sizeof(log_file)) {
999 0 : GDKerror("Could not open %s, filename is too large", log_file);
1000 0 : return GDK_FAIL;
1001 : }
1002 10 : if (MT_stat(log_file, &statbuf) == 0) {
1003 10 : if (snapshot_lazy_copy_file(plan, log_file + strlen(db_dir) + 1, statbuf.st_size) != GDK_SUCCEED)
1004 : return GDK_FAIL;
1005 : } else {
1006 0 : GDKerror("Could not open %s", log_file);
1007 0 : return GDK_FAIL;
1008 : }
1009 : }
1010 : return GDK_SUCCEED;
1011 : }
1012 :
1013 : static gdk_return __attribute__((__warn_unused_result__))
1014 1605 : snapshot_heap(stream *plan, const char *db_dir, bat batid, const char *filename, const char *suffix, uint64_t extent)
1015 : {
1016 1605 : char path1[FILENAME_MAX];
1017 1605 : char path2[FILENAME_MAX];
1018 1605 : const size_t offset = strlen(db_dir) + 1;
1019 1605 : struct stat statbuf;
1020 1605 : int len;
1021 :
1022 1605 : if (extent == 0) {
1023 : /* nothing to copy */
1024 : return GDK_SUCCEED;
1025 : }
1026 : // first check the backup dir
1027 1025 : len = snprintf(path1, FILENAME_MAX, "%s/%s/%o.%s", db_dir, BAKDIR, (int) batid, suffix);
1028 1025 : if (len == -1 || len >= FILENAME_MAX) {
1029 0 : path1[FILENAME_MAX - 1] = '\0';
1030 0 : GDKerror("Could not open %s, filename is too large", path1);
1031 0 : return GDK_FAIL;
1032 : }
1033 1025 : if (MT_stat(path1, &statbuf) == 0) {
1034 0 : return snapshot_lazy_copy_file(plan, path1 + offset, extent);
1035 : }
1036 1025 : if (errno != ENOENT) {
1037 0 : GDKsyserror("Error stat'ing %s", path1);
1038 0 : return GDK_FAIL;
1039 : }
1040 :
1041 : // then check the regular location
1042 1025 : len = snprintf(path2, FILENAME_MAX, "%s/%s/%s.%s", db_dir, BATDIR, filename, suffix);
1043 1025 : if (len == -1 || len >= FILENAME_MAX) {
1044 0 : path2[FILENAME_MAX - 1] = '\0';
1045 0 : GDKerror("Could not open %s, filename is too large", path2);
1046 0 : return GDK_FAIL;
1047 : }
1048 1025 : if (MT_stat(path2, &statbuf) == 0) {
1049 1025 : return snapshot_lazy_copy_file(plan, path2 + offset, extent);
1050 : }
1051 0 : if (errno != ENOENT) {
1052 0 : GDKsyserror("Error stat'ing %s", path2);
1053 0 : return GDK_FAIL;
1054 : }
1055 :
1056 0 : GDKerror("One of %s and %s must exist", path1, path2);
1057 0 : return GDK_FAIL;
1058 : }
1059 :
1060 : /* Add plan entries for all persistent BATs by looping over the BBP.dir.
1061 : * Also include the BBP.dir itself.
1062 : */
1063 : static gdk_return __attribute__((__warn_unused_result__))
1064 5 : snapshot_bats(stream *plan, const char *db_dir)
1065 : {
1066 5 : char bbpdir[FILENAME_MAX];
1067 5 : FILE *fp = NULL;
1068 5 : int len;
1069 5 : gdk_return ret = GDK_FAIL;
1070 5 : int lineno = 0;
1071 5 : bat bbpsize = 0;
1072 5 : lng logno;
1073 5 : unsigned bbpversion;
1074 :
1075 5 : len = snprintf(bbpdir, FILENAME_MAX, "%s/%s/%s", db_dir, BAKDIR, "BBP.dir");
1076 5 : if (len == -1 || len >= FILENAME_MAX) {
1077 0 : GDKerror("Could not open %s, filename is too large", bbpdir);
1078 0 : return GDK_FAIL;
1079 : }
1080 5 : ret = snapshot_immediate_copy_file(plan, bbpdir, bbpdir + strlen(db_dir) + 1);
1081 5 : if (ret != GDK_SUCCEED)
1082 : return ret;
1083 :
1084 : // Open the catalog and parse the header
1085 5 : fp = fopen(bbpdir, "r");
1086 5 : if (fp == NULL) {
1087 0 : GDKerror("Could not open %s for reading: %s", bbpdir, mnstr_peek_error(NULL));
1088 0 : return GDK_FAIL;
1089 : }
1090 5 : bbpversion = BBPheader(fp, &lineno, &bbpsize, &logno, false);
1091 5 : if (bbpversion == 0)
1092 0 : goto end;
1093 5 : assert(bbpversion == GDKLIBRARY);
1094 :
1095 1265 : for (;;) {
1096 1270 : BAT b;
1097 1270 : Heap h;
1098 1270 : Heap vh;
1099 1270 : vh = h = (Heap) {
1100 : .free = 0,
1101 : };
1102 1270 : b = (BAT) {
1103 : .theap = &h,
1104 : .tvheap = &vh,
1105 : };
1106 1270 : char *options;
1107 1270 : char filename[sizeof(BBP_physical(0))];
1108 1270 : char batname[129];
1109 : #ifdef GDKLIBRARY_HASHASH
1110 1270 : int hashash;
1111 : #endif
1112 :
1113 1270 : switch (BBPreadBBPline(fp, bbpversion, &lineno, &b,
1114 : #ifdef GDKLIBRARY_HASHASH
1115 : &hashash,
1116 : #endif
1117 : batname, filename, &options)) {
1118 5 : case 0:
1119 : /* end of file */
1120 5 : fclose(fp);
1121 5 : return GDK_SUCCEED;
1122 : case 1:
1123 : /* successfully read an entry */
1124 1265 : break;
1125 0 : default:
1126 : /* error */
1127 0 : fclose(fp);
1128 0 : return GDK_FAIL;
1129 : }
1130 : #ifdef GDKLIBRARY_HASHASH
1131 1265 : assert(hashash == 0);
1132 : #endif
1133 1265 : if (ATOMvarsized(b.ttype)) {
1134 340 : ret = snapshot_heap(plan, db_dir, b.batCacheid, filename, "theap", b.tvheap->free);
1135 340 : if (ret != GDK_SUCCEED)
1136 0 : goto end;
1137 : }
1138 1265 : ret = snapshot_heap(plan, db_dir, b.batCacheid, filename, BATtailname(&b), b.theap->free);
1139 1265 : if (ret != GDK_SUCCEED)
1140 0 : goto end;
1141 : }
1142 :
1143 0 : end:
1144 0 : if (fp) {
1145 0 : fclose(fp);
1146 : }
1147 0 : return ret;
1148 : }
1149 :
1150 : static gdk_return __attribute__((__warn_unused_result__))
1151 5 : snapshot_vaultkey(stream *plan, const char *db_dir)
1152 : {
1153 5 : char path[FILENAME_MAX];
1154 5 : struct stat statbuf;
1155 :
1156 5 : int len = snprintf(path, FILENAME_MAX, "%s/.vaultkey", db_dir);
1157 5 : if (len == -1 || len >= FILENAME_MAX) {
1158 0 : path[FILENAME_MAX - 1] = '\0';
1159 0 : GDKerror("Could not open %s, filename is too large", path);
1160 0 : return GDK_FAIL;
1161 : }
1162 5 : if (MT_stat(path, &statbuf) == 0) {
1163 0 : return snapshot_lazy_copy_file(plan, ".vaultkey", statbuf.st_size);
1164 : }
1165 5 : if (errno == ENOENT) {
1166 : // No .vaultkey? Fine.
1167 : return GDK_SUCCEED;
1168 : }
1169 :
1170 0 : GDKsyserror("Error stat'ing %s", path);
1171 0 : return GDK_FAIL;
1172 : }
1173 :
1174 : static gdk_return
1175 5 : bl_snapshot(sqlstore *store, stream *plan)
1176 : {
1177 5 : logger *bat_logger = store->logger;
1178 5 : gdk_return ret;
1179 5 : char *db_dir = NULL;
1180 5 : size_t db_dir_len;
1181 :
1182 : // Farm 0 is always the persistent farm.
1183 5 : db_dir = GDKfilepath(0, NULL, "", NULL);
1184 5 : if (db_dir == NULL)
1185 : return GDK_FAIL;
1186 5 : db_dir_len = strlen(db_dir);
1187 5 : if (db_dir[db_dir_len - 1] == DIR_SEP)
1188 5 : db_dir[db_dir_len - 1] = '\0';
1189 :
1190 10 : if (mnstr_printf(plan, "%s\n", db_dir) < 0 ||
1191 : // Please monetdbd
1192 5 : mnstr_printf(plan, "w 0 .uplog\n") < 0) {
1193 0 : GDKerror("%s", mnstr_peek_error(plan));
1194 0 : ret = GDK_FAIL;
1195 0 : goto end;
1196 : }
1197 :
1198 5 : ret = snapshot_vaultkey(plan, db_dir);
1199 5 : if (ret != GDK_SUCCEED)
1200 0 : goto end;
1201 :
1202 5 : ret = snapshot_bats(plan, db_dir);
1203 5 : if (ret != GDK_SUCCEED)
1204 0 : goto end;
1205 :
1206 5 : ret = snapshot_wal(bat_logger, plan, db_dir);
1207 5 : if (ret != GDK_SUCCEED)
1208 : goto end;
1209 :
1210 : ret = GDK_SUCCEED;
1211 5 : end:
1212 5 : GDKfree(db_dir);
1213 5 : return ret;
1214 : }
1215 :
1216 : void
1217 315 : bat_logger_init( logger_functions *lf )
1218 : {
1219 315 : lf->create = bl_create;
1220 315 : lf->destroy = bl_destroy;
1221 315 : lf->flush = bl_flush;
1222 315 : lf->activate = bl_activate;
1223 315 : lf->changes = bl_changes;
1224 315 : lf->get_sequence = bl_get_sequence;
1225 315 : lf->log_isnew = bl_log_isnew;
1226 315 : lf->log_tstart = bl_tstart;
1227 315 : lf->log_tend = bl_tend;
1228 315 : lf->log_tflush = bl_tflush;
1229 315 : lf->log_tsequence = bl_sequence;
1230 315 : lf->get_snapshot_files = bl_snapshot;
1231 315 : }
|
