LCOV - code coverage report
Current view: top level - sql/storage/bat - bat_logger.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 258 750 34.4 %
Date: 2024-10-04 20:04:04 Functions: 23 23 100.0 %

          Line data    Source code
       1             : /*
       2             :  * SPDX-License-Identifier: MPL-2.0
       3             :  *
       4             :  * This Source Code Form is subject to the terms of the Mozilla Public
       5             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       6             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       7             :  *
       8             :  * Copyright 2024 MonetDB Foundation;
       9             :  * Copyright August 2008 - 2023 MonetDB B.V.;
      10             :  * Copyright 1997 - July 2008 CWI.
      11             :  */
      12             : 
      13             : #include "monetdb_config.h"
      14             : #include "bat_logger.h"
      15             : #include "bat_utils.h"
      16             : #include "sql_types.h" /* EC_POS */
      17             : #include "gdk_logger_internals.h"
      18             : #include "mutils.h"
      19             : 
      20             : #define CATALOG_JUL2021 52300   /* first in Jul2021 */
      21             : #define CATALOG_JAN2022 52301   /* first in Jan2022 */
      22             : #define CATALOG_SEP2022 52302   /* first in Sep2022 */
      23             : 
      24             : /* Note, CATALOG version 52300 is the first one where the basic system
      25             :  * tables (the ones created in store.c) have fixed and unchangeable
      26             :  * ids. */
      27             : 
      28             : /* return GDK_SUCCEED if we can handle the upgrade from oldversion to
      29             :  * newversion */
      30             : static gdk_return
      31           8 : bl_preversion(sqlstore *store, int oldversion, int newversion)
      32             : {
      33           8 :         (void)newversion;
      34             : 
      35             : #ifdef CATALOG_JUL2021
      36           8 :         if (oldversion == CATALOG_JUL2021) {
      37             :                 /* upgrade to default releases */
      38           0 :                 store->catalog_version = oldversion;
      39           0 :                 return GDK_SUCCEED;
      40             :         }
      41             : #endif
      42             : 
      43             : #ifdef CATALOG_JAN2022
      44           8 :         if (oldversion == CATALOG_JAN2022) {
      45             :                 /* upgrade to default releases */
      46           0 :                 store->catalog_version = oldversion;
      47           0 :                 return GDK_SUCCEED;
      48             :         }
      49             : #endif
      50             : 
      51             : #ifdef CATALOG_SEP2022
      52           8 :         if (oldversion == CATALOG_SEP2022) {
      53             :                 /* upgrade to default releases */
      54           8 :                 store->catalog_version = oldversion;
      55           8 :                 return GDK_SUCCEED;
      56             :         }
      57             : #endif
      58             : 
      59             :         return GDK_FAIL;
      60             : }
      61             : 
      62             : #if defined CATALOG_JUL2021 || defined CATALOG_JAN2022
      63             : /* replace a column in a system table with a new column
      64             :  * colid is the SQL id for the column, oldcolid is the BAT id of the
      65             :  * to-be-replaced BAT */
      66             : static gdk_return
      67          88 : replace_bat(logger *lg, int colid, BAT *newcol)
      68             : {
      69          88 :         gdk_return rc;
      70          88 :         newcol = BATsetaccess(newcol, BAT_READ);
      71          88 :         if (newcol == NULL)
      72             :                 return GDK_FAIL;
      73          88 :         if ((rc = BAThash(lg->catalog_id)) == GDK_SUCCEED) {
      74          88 :                 BATiter cii = bat_iterator_nolock(lg->catalog_id);
      75          88 :                 BUN p;
      76          88 :                 MT_rwlock_rdlock(&cii.b->thashlock);
      77          94 :                 HASHloop_int(cii, cii.b->thash, p, &colid) {
      78          88 :                         if (BUNfnd(lg->dcatalog, &(oid){(oid)p}) == BUN_NONE) {
      79         176 :                                 if (BUNappend(lg->dcatalog, &(oid){(oid)p}, true) != GDK_SUCCEED ||
      80          88 :                                         BUNreplace(lg->catalog_lid, (oid) p, &(lng){0}, false) != GDK_SUCCEED) {
      81           0 :                                         MT_rwlock_rdunlock(&cii.b->thashlock);
      82           0 :                                         return GDK_FAIL;
      83             :                                 }
      84          88 :                                 break;
      85             :                         }
      86             :                 }
      87          88 :                 MT_rwlock_rdunlock(&cii.b->thashlock);
      88          88 :                 if ((rc = BUNappend(lg->catalog_id, &colid, true)) == GDK_SUCCEED &&
      89          88 :                         (rc = BUNappend(lg->catalog_bid, &newcol->batCacheid, true)) == GDK_SUCCEED &&
      90          88 :                         (rc = BUNappend(lg->catalog_lid, &lng_nil, false)) == GDK_SUCCEED &&
      91          88 :                         (rc = BUNappend(lg->catalog_cnt, &(lng){BATcount(newcol)}, false)) == GDK_SUCCEED) {
      92          88 :                         BBPretain(newcol->batCacheid);
      93             :                 }
      94             :         }
      95             :         return rc;
      96             : }
      97             : #endif
      98             : 
      99             : static BAT *
     100         184 : log_temp_descriptor(log_bid b)
     101             : {
     102         184 :         if (b <= 0)
     103             :                 return NULL;
     104         184 :         return temp_descriptor(b);
     105             : }
     106             : 
     107             : #if defined CATALOG_JAN2022 || defined CATALOG_SEP2022
     108             : static gdk_return
     109          16 : tabins(logger *lg, bool first, int tt, int nid, ...)
     110             : {
     111          16 :         va_list va;
     112          16 :         int cid;
     113          16 :         const void *cval;
     114          16 :         gdk_return rc;
     115          16 :         BAT *b;
     116             : 
     117          16 :         va_start(va, nid);
     118         192 :         while ((cid = va_arg(va, int)) != 0) {
     119         176 :                 cval = va_arg(va, void *);
     120         352 :                 if ((b = log_temp_descriptor(log_find_bat(lg, cid))) == NULL) {
     121           0 :                         va_end(va);
     122           0 :                         return GDK_FAIL;
     123             :                 }
     124         176 :                 if (first) {
     125          88 :                         BAT *bn = COLcopy(b, b->ttype, true, PERSISTENT);
     126          88 :                         if (bn == NULL) {
     127           0 :                                 va_end(va);
     128           0 :                                 bat_destroy(b);
     129           0 :                                 return GDK_FAIL;
     130             :                         }
     131          88 :                         if (replace_bat(lg, cid, bn) != GDK_SUCCEED) {
     132           0 :                                 va_end(va);
     133           0 :                                 bat_destroy(b);
     134           0 :                                 bat_destroy(bn);
     135           0 :                                 return GDK_FAIL;
     136             :                         }
     137             :                         /* logical refs of b stay the same: it is moved from catalog_bid to del */
     138          88 :                         bat_destroy(b);
     139          88 :                         b = bn;
     140             :                 }
     141         176 :                 rc = BUNappend(b, cval, true);
     142         176 :                 bat_destroy(b);
     143         176 :                 if (rc != GDK_SUCCEED) {
     144           0 :                         va_end(va);
     145           0 :                         return rc;
     146             :                 }
     147             :         }
     148          16 :         va_end(va);
     149             : 
     150          16 :         if (tt >= 0) {
     151           0 :                 if ((b = COLnew(0, tt, 0, PERSISTENT)) == NULL)
     152             :                         return GDK_FAIL;
     153           0 :                 rc = log_bat_persists(lg, b, nid);
     154           0 :                 bat_destroy(b);
     155           0 :                 if (rc != GDK_SUCCEED)
     156             :                         return rc;
     157             :         }
     158             :         return GDK_SUCCEED;
     159             : }
     160             : #endif
     161             : 
     162             : static gdk_return
     163           8 : bl_postversion(void *Store, logger *lg)
     164             : {
     165           8 :         sqlstore *store = Store;
     166           8 :         gdk_return rc;
     167           8 :         bool tabins_first = true;
     168             : 
     169             : #ifdef CATALOG_JUL2021
     170           8 :         if (store->catalog_version <= CATALOG_JUL2021) {
     171             :                 /* change the language attribute in sys.functions for sys.env,
     172             :                  * sys.var, and sys.db_users from SQL to MAL */
     173             : 
     174             :                 /* sys.functions i.e. deleted rows */
     175           0 :                 BAT *del_funcs = log_temp_descriptor(log_find_bat(lg, 2016));
     176           0 :                 if (del_funcs == NULL)
     177           0 :                         return GDK_FAIL;
     178           0 :                 BAT *func_tid = BATmaskedcands(0, BATcount(del_funcs), del_funcs, false);
     179           0 :                 bat_destroy(del_funcs);
     180             :                 /* sys.functions.schema_id */
     181           0 :                 BAT *func_schem = log_temp_descriptor(log_find_bat(lg, 2026));
     182           0 :                 if (func_tid == NULL || func_schem == NULL) {
     183           0 :                         bat_destroy(func_tid);
     184           0 :                         bat_destroy(func_schem);
     185           0 :                         return GDK_FAIL;
     186             :                 }
     187             :                 /* select * from sys.functions where schema_id = 2000 */
     188           0 :                 BAT *cands = BATselect(func_schem, func_tid, &(int) {2000}, NULL, true, true, false, false);
     189           0 :                 bat_destroy(func_schem);
     190           0 :                 if (cands == NULL) {
     191           0 :                         bat_destroy(func_tid);
     192           0 :                         return GDK_FAIL;
     193             :                 }
     194             :                 /* the functions we need to change */
     195           0 :                 BAT *funcs = COLnew(0, TYPE_str, 3, TRANSIENT);
     196           0 :                 if (funcs == NULL ||
     197           0 :                         BUNappend(funcs, "db_users", false) != GDK_SUCCEED ||
     198           0 :                         BUNappend(funcs, "env", false) != GDK_SUCCEED ||
     199           0 :                         BUNappend(funcs, "var", false) != GDK_SUCCEED) {
     200           0 :                         bat_destroy(cands);
     201           0 :                         bat_destroy(funcs);
     202           0 :                         bat_destroy(func_tid);
     203           0 :                         return GDK_FAIL;
     204             :                 }
     205             :                 /* sys.functions.name */
     206           0 :                 BAT *func_name = log_temp_descriptor(log_find_bat(lg, 2018));
     207           0 :                 if (func_name == NULL) {
     208           0 :                         bat_destroy(cands);
     209           0 :                         bat_destroy(funcs);
     210           0 :                         bat_destroy(func_tid);
     211           0 :                         return GDK_FAIL;
     212             :                 }
     213             :                 /* select * from sys.functions where schema_id = 2000 and name in (...) */
     214           0 :                 BAT *b = BATintersect(func_name, funcs, cands, NULL, false, false, 3);
     215           0 :                 bat_destroy(cands);
     216           0 :                 bat_destroy(func_name);
     217           0 :                 bat_destroy(funcs);
     218           0 :                 cands = b;
     219           0 :                 if (cands == NULL) {
     220           0 :                         bat_destroy(func_tid);
     221           0 :                         return GDK_FAIL;
     222             :                 }
     223             :                 /* sys.functions.language */
     224           0 :                 BAT *func_lang = log_temp_descriptor(log_find_bat(lg, 2021));
     225           0 :                 if (func_lang == NULL) {
     226           0 :                         bat_destroy(cands);
     227           0 :                         bat_destroy(func_tid);
     228           0 :                         return GDK_FAIL;
     229             :                 }
     230             :                 /* select * from sys.functions where schema_id = 2000 and name in (...)
     231             :                  * and language = FUNC_LANG_SQL */
     232           0 :                 b = BATselect(func_lang, cands, &(int) {FUNC_LANG_SQL}, NULL, true, true, false, false);
     233           0 :                 bat_destroy(cands);
     234           0 :                 cands = b;
     235           0 :                 if (cands == NULL) {
     236           0 :                         bat_destroy(func_lang);
     237           0 :                         bat_destroy(func_tid);
     238           0 :                         return GDK_FAIL;
     239             :                 }
     240           0 :                 b = BATconstant(0, TYPE_int, &(int) {FUNC_LANG_MAL}, BATcount(cands), TRANSIENT);
     241           0 :                 if (b == NULL) {
     242           0 :                         bat_destroy(func_lang);
     243           0 :                         bat_destroy(cands);
     244           0 :                         bat_destroy(func_tid);
     245           0 :                         return GDK_FAIL;
     246             :                 }
     247           0 :                 rc = GDK_FAIL;
     248           0 :                 BAT *b2 = COLcopy(func_lang, func_lang->ttype, true, PERSISTENT);
     249           0 :                 if (b2 == NULL ||
     250           0 :                         BATreplace(b2, cands, b, false) != GDK_SUCCEED) {
     251           0 :                         bat_destroy(b2);
     252           0 :                         bat_destroy(cands);
     253           0 :                         bat_destroy(b);
     254           0 :                         bat_destroy(func_tid);
     255           0 :                         bat_destroy(func_lang);
     256           0 :                         return GDK_FAIL;
     257             :                 }
     258           0 :                 bat_destroy(b);
     259           0 :                 bat_destroy(cands);
     260             : 
     261             :                 /* additionally, update the language attribute for entries
     262             :                  * that were declared using "EXTERNAL NAME" to be MAL functions
     263             :                  * instead of SQL functions (a problem that seems to have
     264             :                  * occurred in ancient databases) */
     265             : 
     266             :                 /* sys.functions.func */
     267           0 :                 BAT *func_func = log_temp_descriptor(log_find_bat(lg, 2019));
     268           0 :                 if (func_func == NULL) {
     269           0 :                         bat_destroy(func_tid);
     270           0 :                         bat_destroy(b2);
     271           0 :                         return GDK_FAIL;
     272             :                 }
     273           0 :                 cands = BATselect(func_lang, func_tid, &(int){FUNC_LANG_SQL}, NULL, true, true, false, false);
     274           0 :                 bat_destroy(func_lang);
     275           0 :                 bat_destroy(func_tid);
     276           0 :                 if (cands == NULL) {
     277           0 :                         bat_destroy(b2);
     278           0 :                         bat_destroy(func_func);
     279           0 :                         return GDK_FAIL;
     280             :                 }
     281           0 :                 struct canditer ci;
     282           0 :                 canditer_init(&ci, func_func, cands);
     283           0 :                 BATiter ffi = bat_iterator_nolock(func_func);
     284           0 :                 for (BUN p = 0; p < ci.ncand; p++) {
     285           0 :                         oid o = canditer_next(&ci);
     286           0 :                         const char *f = BUNtvar(ffi, o - func_func->hseqbase);
     287           0 :                         const char *e;
     288           0 :                         if (!strNil(f) &&
     289           0 :                                 (e = strstr(f, "external")) != NULL &&
     290           0 :                                 e > f && isspace((unsigned char) e[-1]) && isspace((unsigned char) e[8]) && strncmp(e + 9, "name", 4) == 0 && isspace((unsigned char) e[13]) &&
     291           0 :                                 BUNreplace(b2, o, &(int){FUNC_LANG_MAL}, false) != GDK_SUCCEED) {
     292           0 :                                 bat_destroy(b2);
     293           0 :                                 bat_destroy(func_func);
     294           0 :                                 return GDK_FAIL;
     295             :                         }
     296             :                 }
     297           0 :                 rc = replace_bat(lg, 2021, b2);
     298           0 :                 bat_destroy(b2);
     299           0 :                 if (rc != GDK_SUCCEED)
     300             :                         return rc;
     301             :         }
     302           8 :         if (store->catalog_version <= CATALOG_JUL2021) {
     303             :                 /* change the side_effects attribute in sys.functions for
     304             :                  * selected functions */
     305             : 
     306             :                 /* sys.functions i.e. deleted rows */
     307           0 :                 BAT *del_funcs = log_temp_descriptor(log_find_bat(lg, 2016));
     308           0 :                 if (del_funcs == NULL)
     309           0 :                         return GDK_FAIL;
     310           0 :                 BAT *func_tid = BATmaskedcands(0, BATcount(del_funcs), del_funcs, false);
     311           0 :                 bat_destroy(del_funcs);
     312             :                 /* sys.functions.schema_id */
     313           0 :                 BAT *func_schem = log_temp_descriptor(log_find_bat(lg, 2026));
     314           0 :                 if (func_tid == NULL || func_schem == NULL) {
     315           0 :                         bat_destroy(func_tid);
     316           0 :                         bat_destroy(func_schem);
     317           0 :                         return GDK_FAIL;
     318             :                 }
     319             :                 /* select * from sys.functions where schema_id = 2000 */
     320           0 :                 BAT *cands = BATselect(func_schem, func_tid, &(int) {2000}, NULL, true, true, false, false);
     321           0 :                 bat_destroy(func_schem);
     322           0 :                 bat_destroy(func_tid);
     323           0 :                 if (cands == NULL) {
     324             :                         return GDK_FAIL;
     325             :                 }
     326             :                 /* sys.functions.side_effect */
     327           0 :                 BAT *func_se = log_temp_descriptor(log_find_bat(lg, 2023));
     328           0 :                 if (func_se == NULL) {
     329           0 :                         bat_destroy(cands);
     330           0 :                         return GDK_FAIL;
     331             :                 }
     332             :                 /* make a copy that we can modify */
     333           0 :                 BAT *b = COLcopy(func_se, func_se->ttype, true, PERSISTENT);
     334           0 :                 bat_destroy(func_se);
     335           0 :                 if (b == NULL) {
     336           0 :                         bat_destroy(cands);
     337           0 :                         return GDK_FAIL;
     338             :                 }
     339           0 :                 func_se = b;
     340             :                 /* sys.functions.func */
     341           0 :                 BAT *func_func = log_temp_descriptor(log_find_bat(lg, 2019));
     342           0 :                 if (func_func == NULL) {
     343           0 :                         bat_destroy(cands);
     344           0 :                         bat_destroy(func_se);
     345           0 :                         return GDK_FAIL;
     346             :                 }
     347             :                 /* the functions we need to change to FALSE */
     348           0 :                 BAT *funcs = COLnew(0, TYPE_str, 1, TRANSIENT);
     349           0 :                 if (funcs == NULL ||
     350           0 :                         BUNappend(funcs, "sqlrand", false) != GDK_SUCCEED) {
     351           0 :                         bat_destroy(cands);
     352           0 :                         bat_destroy(func_se);
     353           0 :                         bat_destroy(func_func);
     354           0 :                         bat_destroy(funcs);
     355           0 :                         return GDK_FAIL;
     356             :                 }
     357             :                 /* select * from sys.functions where schema_id = 2000 and func in (...) */
     358           0 :                 b = BATintersect(func_func, funcs, cands, NULL, false, false, 1);
     359           0 :                 bat_destroy(funcs);
     360           0 :                 if (b == NULL) {
     361           0 :                         bat_destroy(cands);
     362           0 :                         bat_destroy(func_se);
     363           0 :                         bat_destroy(func_func);
     364           0 :                         return GDK_FAIL;
     365             :                 }
     366             :                 /* while we're at it, also change sys.env and sys.db_users to
     367             :                  * being without side effect (legacy from ancient databases) */
     368             :                 /* sys.functions.name */
     369           0 :                 BAT *func_name = log_temp_descriptor(log_find_bat(lg, 2018));
     370           0 :                 if (func_name == NULL) {
     371           0 :                         bat_destroy(cands);
     372           0 :                         bat_destroy(func_se);
     373           0 :                         bat_destroy(func_func);
     374           0 :                         bat_destroy(b);
     375           0 :                         return GDK_FAIL;
     376             :                 }
     377           0 :                 BAT *b2 = BATselect(func_name, cands, "env", NULL, true, true, false, false);
     378           0 :                 if (b2 == NULL || BATappend(b, b2, NULL, false) != GDK_SUCCEED) {
     379           0 :                         bat_destroy(cands);
     380           0 :                         bat_destroy(func_se);
     381           0 :                         bat_destroy(func_func);
     382           0 :                         bat_destroy(b);
     383           0 :                         bat_destroy(func_name);
     384           0 :                         bat_destroy(b2);
     385           0 :                         return GDK_FAIL;
     386             :                 }
     387           0 :                 bat_destroy(b2);
     388           0 :                 b2 = BATselect(func_name, cands, "db_users", NULL, true, true, false, false);
     389           0 :                 bat_destroy(func_name);
     390           0 :                 if (b2 == NULL || BATappend(b, b2, NULL, false) != GDK_SUCCEED) {
     391           0 :                         bat_destroy(cands);
     392           0 :                         bat_destroy(func_se);
     393           0 :                         bat_destroy(func_func);
     394           0 :                         bat_destroy(b);
     395           0 :                         bat_destroy(b2);
     396           0 :                         return GDK_FAIL;
     397             :                 }
     398           0 :                 bat_destroy(b2);
     399             : 
     400           0 :                 BAT *vals = BATconstant(0, TYPE_bit, &(bit) {FALSE}, BATcount(b), TRANSIENT);
     401           0 :                 if (vals == NULL) {
     402           0 :                         bat_destroy(cands);
     403           0 :                         bat_destroy(func_se);
     404           0 :                         bat_destroy(func_func);
     405           0 :                         bat_destroy(b);
     406           0 :                         return GDK_FAIL;
     407             :                 }
     408           0 :                 rc = BATreplace(func_se, b, vals, false);
     409           0 :                 bat_destroy(b);
     410           0 :                 bat_destroy(vals);
     411           0 :                 if (rc != GDK_SUCCEED) {
     412           0 :                         bat_destroy(cands);
     413           0 :                         bat_destroy(func_se);
     414           0 :                         bat_destroy(func_func);
     415           0 :                         return GDK_FAIL;
     416             :                 }
     417             :                 /* the functions we need to change to TRUE */
     418           0 :                 funcs = COLnew(0, TYPE_str, 5, TRANSIENT);
     419           0 :                 if (funcs == NULL ||
     420           0 :                         BUNappend(funcs, "copy_from", false) != GDK_SUCCEED ||
     421           0 :                         BUNappend(funcs, "next_value", false) != GDK_SUCCEED ||
     422           0 :                         BUNappend(funcs, "update_schemas", false) != GDK_SUCCEED ||
     423           0 :                         BUNappend(funcs, "update_tables", false) != GDK_SUCCEED) {
     424           0 :                         bat_destroy(cands);
     425           0 :                         bat_destroy(func_se);
     426           0 :                         bat_destroy(func_func);
     427           0 :                         bat_destroy(funcs);
     428           0 :                         return GDK_FAIL;
     429             :                 }
     430             :                 /* select * from sys.functions where schema_id = 2000 and func in (...) */
     431           0 :                 b = BATintersect(func_func, funcs, cands, NULL, false, false, 7);
     432           0 :                 bat_destroy(funcs);
     433           0 :                 bat_destroy(cands);
     434           0 :                 bat_destroy(func_func);
     435           0 :                 if (b == NULL) {
     436           0 :                         bat_destroy(func_se);
     437           0 :                         return GDK_FAIL;
     438             :                 }
     439           0 :                 vals = BATconstant(0, TYPE_bit, &(bit) {TRUE}, BATcount(b), TRANSIENT);
     440           0 :                 if (vals == NULL) {
     441           0 :                         bat_destroy(func_se);
     442           0 :                         bat_destroy(b);
     443           0 :                         return GDK_FAIL;
     444             :                 }
     445           0 :                 rc = BATreplace(func_se, b, vals, false);
     446           0 :                 bat_destroy(b);
     447           0 :                 bat_destroy(vals);
     448           0 :                 if (rc != GDK_SUCCEED) {
     449           0 :                         bat_destroy(func_se);
     450           0 :                         return GDK_FAIL;
     451             :                 }
     452             :                 /* replace old column with modified copy */
     453           0 :                 rc = replace_bat(lg, 2023, func_se);
     454           0 :                 bat_destroy(func_se);
     455           0 :                 if (rc != GDK_SUCCEED)
     456             :                         return rc;
     457             :         }
     458           8 :         if (store->catalog_version <= CATALOG_JUL2021) {
     459             :                 /* upgrade some columns in sys.sequences:
     460             :                  * if increment is zero, set it to one (see ChangeLog);
     461             :                  * if increment is greater than zero and maxvalue is zero,
     462             :                  * set maxvalue to GDK_lng_max;
     463             :                  * if increment is less than zero and minvalue is zero,
     464             :                  * set minvalue to GDK_lng_min */
     465             : 
     466             :                 /* sys.sequences i.e. deleted rows */
     467           0 :                 BAT *del_seqs = log_temp_descriptor(log_find_bat(lg, 2037));
     468           0 :                 if (del_seqs == NULL)
     469           0 :                         return GDK_FAIL;
     470           0 :                 BAT *seq_tid = BATmaskedcands(0, BATcount(del_seqs), del_seqs, false);
     471           0 :                 bat_destroy(del_seqs);
     472           0 :                 BAT *seq_min = log_temp_descriptor(log_find_bat(lg, 2042)); /* sys.sequences.minvalue */
     473           0 :                 BAT *seq_max = log_temp_descriptor(log_find_bat(lg, 2043)); /* sys.sequences.maxvalue */
     474           0 :                 BAT *seq_inc = log_temp_descriptor(log_find_bat(lg, 2044)); /* sys.sequences.increment */
     475           0 :                 if (seq_tid == NULL || seq_min == NULL || seq_max == NULL || seq_inc == NULL) {
     476           0 :                         bat_destroy(seq_tid);
     477           0 :                         bat_destroy(seq_min);
     478           0 :                         bat_destroy(seq_max);
     479           0 :                         bat_destroy(seq_inc);
     480           0 :                         return GDK_FAIL;
     481             :                 }
     482             :                 /* select * from sys.sequences where increment = 0 */
     483           0 :                 BAT *inczero = BATselect(seq_inc, seq_tid, &(lng){0}, NULL, false, true, false, false);
     484           0 :                 if (inczero == NULL) {
     485           0 :                         bat_destroy(seq_tid);
     486           0 :                         bat_destroy(seq_min);
     487           0 :                         bat_destroy(seq_max);
     488           0 :                         bat_destroy(seq_inc);
     489           0 :                         return GDK_FAIL;
     490             :                 }
     491           0 :                 if (BATcount(inczero) > 0) {
     492           0 :                         BAT *b = BATconstant(0, TYPE_lng, &(lng) {1}, BATcount(inczero), TRANSIENT);
     493           0 :                         if (b == NULL) {
     494           0 :                                 bat_destroy(seq_tid);
     495           0 :                                 bat_destroy(seq_min);
     496           0 :                                 bat_destroy(seq_max);
     497           0 :                                 bat_destroy(seq_inc);
     498           0 :                                 bat_destroy(inczero);
     499           0 :                                 return GDK_FAIL;
     500             :                         }
     501           0 :                         BAT *b2 = COLcopy(seq_inc, seq_inc->ttype, true, PERSISTENT);
     502           0 :                         rc = GDK_FAIL;
     503           0 :                         if (b2 == NULL)
     504           0 :                                 rc = BATreplace(b2, inczero, b, false);
     505           0 :                         bat_destroy(b);
     506           0 :                         if (rc != GDK_SUCCEED) {
     507           0 :                                 bat_destroy(b2);
     508           0 :                                 bat_destroy(seq_tid);
     509           0 :                                 bat_destroy(seq_min);
     510           0 :                                 bat_destroy(seq_max);
     511           0 :                                 bat_destroy(seq_inc);
     512           0 :                                 bat_destroy(inczero);
     513           0 :                                 return GDK_FAIL;
     514             :                         }
     515           0 :                         rc = replace_bat(lg, 2044, b2);
     516           0 :                         bat_destroy(seq_inc);
     517           0 :                         seq_inc = b2;
     518           0 :                         if (rc != GDK_SUCCEED) {
     519           0 :                                 bat_destroy(seq_tid);
     520           0 :                                 bat_destroy(seq_min);
     521           0 :                                 bat_destroy(seq_max);
     522           0 :                                 bat_destroy(seq_inc);
     523           0 :                                 bat_destroy(inczero);
     524           0 :                                 return rc;
     525             :                         }
     526             :                 }
     527           0 :                 bat_destroy(inczero);
     528             :                 /* select * from sys.sequences where increment > 0 */
     529           0 :                 BAT *incpos = BATselect(seq_inc, seq_tid, &(lng){0}, &lng_nil, false, true, false, false);
     530           0 :                 bat_destroy(seq_inc);
     531           0 :                 if (incpos == NULL) {
     532           0 :                         bat_destroy(seq_tid);
     533           0 :                         bat_destroy(seq_min);
     534           0 :                         bat_destroy(seq_max);
     535           0 :                         return GDK_FAIL;
     536             :                 }
     537             :                 /* select * from sys.sequences where increment > 0 and maxvalue = 0 */
     538           0 :                 BAT *cands = BATselect(seq_max, incpos, &(lng) {0}, NULL, true, true, false, false);
     539           0 :                 bat_destroy(incpos);
     540           0 :                 if (cands == NULL) {
     541           0 :                         bat_destroy(seq_tid);
     542           0 :                         bat_destroy(seq_min);
     543           0 :                         bat_destroy(seq_max);
     544           0 :                         return GDK_FAIL;
     545             :                 }
     546           0 :                 if (BATcount(cands) > 0) {
     547           0 :                         BAT *b = BATconstant(0, TYPE_lng, &(lng){GDK_lng_max}, BATcount(cands), TRANSIENT);
     548           0 :                         BAT *b2 = COLcopy(seq_max, seq_max->ttype, true, PERSISTENT);
     549           0 :                         rc = GDK_FAIL;
     550           0 :                         if (b != NULL && b2 != NULL)
     551           0 :                                 rc = BATreplace(b2, cands, b, false);
     552           0 :                         bat_destroy(b);
     553           0 :                         if (rc == GDK_SUCCEED)
     554           0 :                                 rc = replace_bat(lg, 2043, b2);
     555           0 :                         bat_destroy(b2);
     556           0 :                         if (rc != GDK_SUCCEED) {
     557           0 :                                 bat_destroy(cands);
     558           0 :                                 bat_destroy(seq_tid);
     559           0 :                                 bat_destroy(seq_min);
     560           0 :                                 bat_destroy(seq_max);
     561           0 :                                 return rc;
     562             :                         }
     563             :                 }
     564           0 :                 bat_destroy(seq_max);
     565           0 :                 bat_destroy(cands);
     566             :                 /* select * from sys.sequences where increment < 0 */
     567           0 :                 BAT *incneg = BATselect(seq_inc, seq_tid, &lng_nil, &(lng){0}, false, true, false, false);
     568           0 :                 bat_destroy(seq_tid);
     569             :                 /* select * from sys.sequences where increment < 0 and minvalue = 0 */
     570           0 :                 cands = BATselect(seq_min, incneg, &(lng) {0}, NULL, true, true, false, false);
     571           0 :                 bat_destroy(incneg);
     572           0 :                 if (cands == NULL) {
     573           0 :                         bat_destroy(seq_min);
     574           0 :                         return GDK_FAIL;
     575             :                 }
     576           0 :                 if (BATcount(cands) > 0) {
     577           0 :                         BAT *b = BATconstant(0, TYPE_lng, &(lng){GDK_lng_min}, BATcount(cands), TRANSIENT);
     578           0 :                         BAT *b2 = COLcopy(seq_min, seq_min->ttype, true, PERSISTENT);
     579           0 :                         rc = GDK_FAIL;
     580           0 :                         if (b != NULL && b2 != NULL)
     581           0 :                                 rc = BATreplace(b2, cands, b, false);
     582           0 :                         bat_destroy(b);
     583           0 :                         if (rc == GDK_SUCCEED)
     584           0 :                                 rc = replace_bat(lg, 2042, b2);
     585           0 :                         bat_destroy(b2);
     586           0 :                         if (rc != GDK_SUCCEED) {
     587           0 :                                 bat_destroy(cands);
     588           0 :                                 bat_destroy(seq_min);
     589           0 :                                 return rc;
     590             :                         }
     591             :                 }
     592           0 :                 bat_destroy(seq_min);
     593           0 :                 bat_destroy(cands);
     594             :         }
     595             : #endif
     596             : 
     597             : #ifdef CATALOG_JAN2022
     598           8 :         if (store->catalog_version <= CATALOG_JAN2022) {
     599             :                 /* GRANT SELECT ON sys.db_user_info TO monetdb;
     600             :                  * except the grantor is 0 instead of user monetdb
     601             :                  *
     602             :                  * we need to find the IDs of the sys.db_user_info table and of
     603             :                  * the sys.privileges table and its columns since none of these
     604             :                  * have fixed IDs */
     605           0 :                 BAT *b = log_temp_descriptor(log_find_bat(lg, 2067)); /* sys._tables */
     606           0 :                 if (b == NULL)
     607           0 :                         return GDK_FAIL;
     608           0 :                 BAT *del_tabs = BATmaskedcands(0, BATcount(b), b, false);
     609           0 :                 bat_destroy(b);
     610           0 :                 if (del_tabs == NULL)
     611             :                         return GDK_FAIL;
     612           0 :                 b = log_temp_descriptor(log_find_bat(lg, 2076)); /* sys._columns */
     613           0 :                 if (b == NULL) {
     614           0 :                         bat_destroy(del_tabs);
     615           0 :                         return GDK_FAIL;
     616             :                 }
     617           0 :                 BAT *del_cols = BATmaskedcands(0, BATcount(b), b, false);
     618           0 :                 bat_destroy(b);
     619           0 :                 b = log_temp_descriptor(log_find_bat(lg, 2070)); /* sys._tables.schema_id */
     620           0 :                 if (del_cols == NULL || b == NULL) {
     621           0 :                         bat_destroy(del_cols);
     622           0 :                         bat_destroy(b);
     623           0 :                         bat_destroy(del_tabs);
     624           0 :                         return GDK_FAIL;
     625             :                 }
     626           0 :                 BAT *cands = BATselect(b, del_tabs, &(int) {2000}, NULL, true, true, false, false);
     627           0 :                 bat_destroy(b);
     628           0 :                 bat_destroy(del_tabs);
     629             :                 /* cands contains undeleted rows from sys._tables for tables in
     630             :                  * sys schema */
     631           0 :                 BAT *tabnme = log_temp_descriptor(log_find_bat(lg, 2069)); /* sys._tables.name */
     632           0 :                 if (cands == NULL || tabnme == NULL) {
     633           0 :                         bat_destroy(cands);
     634           0 :                         bat_destroy(tabnme);
     635           0 :                         bat_destroy(del_cols);
     636           0 :                         return GDK_FAIL;
     637             :                 }
     638           0 :                 b = BATselect(tabnme, cands, "db_user_info", NULL, true, true, false, false);
     639           0 :                 if (b == NULL) {
     640           0 :                         bat_destroy(cands);
     641           0 :                         bat_destroy(tabnme);
     642           0 :                         bat_destroy(del_cols);
     643           0 :                         return GDK_FAIL;
     644             :                 }
     645           0 :                 oid dbpos = BUNtoid(b, 0);
     646           0 :                 bat_destroy(b);
     647           0 :                 b = BATselect(tabnme, cands, "privileges", NULL, true, true, false, false);
     648           0 :                 bat_destroy(tabnme);
     649           0 :                 bat_destroy(cands);
     650           0 :                 BAT *tabid = log_temp_descriptor(log_find_bat(lg, 2068)); /* sys._tables.id */
     651           0 :                 if (b == NULL || tabid == NULL) {
     652           0 :                         bat_destroy(b);
     653           0 :                         bat_destroy(tabid);
     654           0 :                         bat_destroy(del_cols);
     655           0 :                         return GDK_FAIL;
     656             :                 }
     657           0 :                 int dbid = ((int *) tabid->theap->base)[dbpos];
     658           0 :                 int prid = ((int *) tabid->theap->base)[BUNtoid(b, 0)];
     659           0 :                 BAT *coltid = log_temp_descriptor(log_find_bat(lg, 2082)); /* sys._columns.table_id */
     660           0 :                 if (coltid == NULL) {
     661           0 :                         bat_destroy(b);
     662           0 :                         bat_destroy(del_cols);
     663           0 :                         bat_destroy(tabid);
     664           0 :                         return GDK_FAIL;
     665             :                 }
     666           0 :                 BAT *b1;
     667           0 :                 rc = BATjoin(&b1, NULL, coltid, tabid, del_cols, b, false, 5);
     668           0 :                 bat_destroy(coltid);
     669           0 :                 bat_destroy(tabid);
     670           0 :                 bat_destroy(del_cols);
     671           0 :                 bat_destroy(b);
     672           0 :                 BAT *colnr = log_temp_descriptor(log_find_bat(lg, 2085)); /* sys._columns.number */
     673           0 :                 BAT *colid = log_temp_descriptor(log_find_bat(lg, 2077)); /* sys._columns.id */
     674           0 :                 if (rc != GDK_SUCCEED || colnr == NULL || colid == NULL) {
     675           0 :                         if (rc == GDK_SUCCEED)
     676           0 :                                 bat_destroy(b1);
     677           0 :                         bat_destroy(colnr);
     678           0 :                         bat_destroy(colid);
     679           0 :                         return GDK_FAIL;
     680             :                 }
     681             :                 int privids[5];
     682           0 :                 for (int i = 0; i < 5; i++) {
     683           0 :                         oid p = BUNtoid(b1, i);
     684           0 :                         privids[((int *) colnr->theap->base)[p]] = ((int *) colid->theap->base)[p];
     685             :                 }
     686           0 :                 bat_destroy(b1);
     687           0 :                 bat_destroy(colnr);
     688           0 :                 bat_destroy(colid);
     689           0 :                 rc = tabins(lg, true, -1, 0,
     690           0 :                                         prid, &(msk) {false}, /* sys.privileges */
     691             :                                         privids[0], &dbid, /* sys.privileges.obj_id */
     692           0 :                                         privids[1], &(int) {USER_MONETDB}, /* sys.privileges.auth_id */
     693           0 :                                         privids[2], &(int) {PRIV_SELECT}, /* sys.privileges.privileges */
     694           0 :                                         privids[3], &(int) {0}, /* sys.privileges.grantor */
     695           0 :                                         privids[4], &(int) {0}, /* sys.privileges.grantee */
     696             :                                         0);
     697           0 :                 if (rc != GDK_SUCCEED)
     698             :                         return rc;
     699             :         }
     700             : #endif
     701             : 
     702             : #ifdef CATALOG_SEP2022
     703           8 :         if (store->catalog_version <= CATALOG_SEP2022) {
     704             :                         /* new STRING column sys.keys.check */
     705           8 :                         BAT *b = log_temp_descriptor(log_find_bat(lg, 2088)); /* sys.keys.id */
     706           8 :                         if (b == NULL)
     707           0 :                                 return GDK_FAIL;
     708           8 :                         BAT *check = BATconstant(b->hseqbase, TYPE_str, ATOMnilptr(TYPE_str), BATcount(b), PERSISTENT);
     709           8 :                         bat_destroy(b);
     710           8 :                         if (check == NULL)
     711             :                                 return GDK_FAIL;
     712          16 :                         if ((check = BATsetaccess(check, BAT_READ)) == NULL ||
     713             :                                 /* 2165 is sys.keys.check */
     714          16 :                                 BUNappend(lg->catalog_id, &(int) {2165}, true) != GDK_SUCCEED ||
     715          16 :                                 BUNappend(lg->catalog_bid, &check->batCacheid, true) != GDK_SUCCEED ||
     716          16 :                                 BUNappend(lg->catalog_lid, &lng_nil, false) != GDK_SUCCEED ||
     717           8 :                                 BUNappend(lg->catalog_cnt, &(lng){BATcount(check)}, false) != GDK_SUCCEED
     718             :                                 ) {
     719           0 :                                 bat_destroy(check);
     720           0 :                                 return GDK_FAIL;
     721             :                         }
     722           8 :                         BBPretain(check->batCacheid);
     723           8 :                         bat_destroy(check);
     724             : 
     725           8 :                         if (tabins(lg, tabins_first, -1, 0,
     726           8 :                                            2076, &(msk) {false},    /* sys._columns */
     727             :                                            /* 2165 is sys.keys.check */
     728           8 :                                            2077, &(int) {2165},             /* sys._columns.id */
     729             :                                            2078, "check",                     /* sys._columns.name */
     730             :                                            2079, "varchar",                   /* sys._columns.type */
     731           8 :                                            2080, &(int) {2048},             /* sys._columns.type_digits */
     732           8 :                                            2081, &(int) {0},                /* sys._columns.type_scale */
     733             :                                            /* 2087 is sys.keys */
     734           8 :                                            2082, &(int) {2087},             /* sys._columns.table_id */
     735             :                                            2083, str_nil,                       /* sys._columns.default */
     736           8 :                                            2084, &(bit) {TRUE},             /* sys._columns.null */
     737           8 :                                            2085, &(int) {6},                /* sys._columns.number */
     738             :                                            2086, str_nil,                       /* sys._columns.storage */
     739             :                                            0) != GDK_SUCCEED)
     740           0 :                                 return GDK_FAIL;
     741           8 :                         tabins_first = false;
     742           8 :                         if (tabins(lg, tabins_first, -1, 0,
     743           8 :                                            2076, &(msk) {false},    /* sys._columns */
     744             :                                            /* 2165 is tmp.keys.check */
     745           8 :                                            2077, &(int) {2166},             /* sys._columns.id */
     746             :                                            2078, "check",                     /* sys._columns.name */
     747             :                                            2079, "varchar",                   /* sys._columns.type */
     748           8 :                                            2080, &(int) {2048},             /* sys._columns.type_digits */
     749           8 :                                            2081, &(int) {0},                /* sys._columns.type_scale */
     750             :                                            /* 2135 is tmp.keys */
     751           8 :                                            2082, &(int) {2135},             /* sys._columns.table_id */
     752             :                                            2083, str_nil,                       /* sys._columns.default */
     753           8 :                                            2084, &(bit) {TRUE},             /* sys._columns.null */
     754           8 :                                            2085, &(int) {6},                /* sys._columns.number */
     755             :                                            2086, str_nil,                       /* sys._columns.storage */
     756             :                                            0) != GDK_SUCCEED)
     757           0 :                                 return GDK_FAIL;
     758             :         }
     759             : #endif
     760             : 
     761             :         return GDK_SUCCEED;
     762             : }
     763             : 
     764             : static int
     765         330 : bl_create(sqlstore *store, int debug, const char *logdir, int cat_version)
     766             : {
     767         330 :         if (store->logger)
     768             :                 return LOG_ERR;
     769         330 :         store->logger = log_create(debug, "sql", logdir, cat_version, (preversionfix_fptr)&bl_preversion, (postversionfix_fptr)&bl_postversion, store);
     770         330 :         if (store->logger)
     771             :                 return LOG_OK;
     772             :         return LOG_ERR;
     773             : }
     774             : 
     775             : static void
     776         329 : bl_destroy(sqlstore *store)
     777             : {
     778         329 :         logger *l = store->logger;
     779             : 
     780         329 :         store->logger = NULL;
     781         329 :         if (l)
     782         329 :                 log_destroy(l);
     783         329 : }
     784             : 
     785             : static int
     786        2776 : bl_flush(sqlstore *store, lng save_id)
     787             : {
     788        2776 :         if (store->logger)
     789        2776 :                 return log_flush(store->logger, save_id) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
     790             :         return LOG_OK;
     791             : }
     792             : 
     793             : static int
     794       10465 : bl_activate(sqlstore *store)
     795             : {
     796       10465 :         if (store->logger)
     797       10465 :                 return log_activate(store->logger) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
     798             :         return LOG_OK;
     799             : }
     800             : 
     801             : static int
     802       14598 : bl_changes(sqlstore *store)
     803             : {
     804       14598 :         return (int) MIN(log_changes(store->logger), GDK_int_max);
     805             : }
     806             : 
     807             : static int
     808         461 : bl_get_sequence(sqlstore *store, int seq, lng *id)
     809             : {
     810         461 :         return log_sequence(store->logger, seq, id);
     811             : }
     812             : 
     813             : static int
     814         330 : bl_log_isnew(sqlstore *store)
     815             : {
     816         330 :         logger *bat_logger = store->logger;
     817         330 :         if (BATcount(bat_logger->catalog_bid) > 10) {
     818         104 :                 return 0;
     819             :         }
     820             :         return 1;
     821             : }
     822             : 
     823             : static int
     824       62446 : bl_tstart(sqlstore *store, bool flush, ulng *log_file_id)
     825             : {
     826       62446 :         return log_tstart(store->logger, flush, log_file_id) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
     827             : }
     828             : 
     829             : static int
     830       62445 : bl_tend(sqlstore *store)
     831             : {
     832       62445 :         return log_tend(store->logger) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
     833             : }
     834             : 
     835             : static int
     836       62445 : bl_tflush(sqlstore *store, ulng log_file_id, ulng commit_ts)
     837             : {
     838       62445 :         return log_tflush(store->logger, log_file_id, commit_ts) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
     839             : }
     840             : 
     841             : static int
     842        6889 : bl_sequence(sqlstore *store, int seq, lng id)
     843             : {
     844        6889 :         return log_tsequence(store->logger, seq, id) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
     845             : }
     846             : 
     847             : /* Write a plan entry to copy part of the given file.
     848             :  * That part of the file must remain unchanged until the plan is executed.
     849             :  */
     850             : static gdk_return __attribute__((__warn_unused_result__))
     851        1026 : snapshot_lazy_copy_file(stream *plan, const char *name, uint64_t extent)
     852             : {
     853        1026 :         if (mnstr_printf(plan, "c %" PRIu64 " %s\n", extent, name) < 0) {
     854           0 :                 GDKerror("%s", mnstr_peek_error(plan));
     855           0 :                 return GDK_FAIL;
     856             :         }
     857             :         return GDK_SUCCEED;
     858             : }
     859             : 
     860             : /* Write a plan entry to write the current contents of the given file.
     861             :  * The contents are included in the plan so the source file is allowed to
     862             :  * change in the mean time.
     863             :  */
     864             : static gdk_return __attribute__((__warn_unused_result__))
     865          10 : snapshot_immediate_copy_file(stream *plan, const char *path, const char *name)
     866             : {
     867          10 :         gdk_return ret = GDK_FAIL;
     868          10 :         const size_t bufsize = 64 * 1024;
     869          10 :         struct stat statbuf;
     870          10 :         char *buf = NULL;
     871          10 :         stream *s = NULL;
     872          10 :         size_t to_copy;
     873             : 
     874          10 :         if (MT_stat(path, &statbuf) < 0) {
     875           0 :                 GDKsyserror("stat failed on %s", path);
     876           0 :                 goto end;
     877             :         }
     878          10 :         to_copy = (size_t) statbuf.st_size;
     879             : 
     880          10 :         s = open_rstream(path);
     881          10 :         if (!s) {
     882           0 :                 GDKerror("%s", mnstr_peek_error(NULL));
     883           0 :                 goto end;
     884             :         }
     885             : 
     886          10 :         buf = GDKmalloc(bufsize);
     887          10 :         if (!buf) {
     888           0 :                 GDKerror("GDKmalloc failed");
     889           0 :                 goto end;
     890             :         }
     891             : 
     892          10 :         if (mnstr_printf(plan, "w %zu %s\n", to_copy, name) < 0) {
     893           0 :                 GDKerror("%s", mnstr_peek_error(plan));
     894           0 :                 goto end;
     895             :         }
     896             : 
     897          20 :         while (to_copy > 0) {
     898          10 :                 size_t chunk = (to_copy <= bufsize) ? to_copy : bufsize;
     899          10 :                 ssize_t bytes_read = mnstr_read(s, buf, 1, chunk);
     900          10 :                 if (bytes_read < 0) {
     901           0 :                         GDKerror("Reading bytes of component %s failed: %s", path, mnstr_peek_error(s));
     902           0 :                         goto end;
     903          10 :                 } else if (bytes_read < (ssize_t) chunk) {
     904           0 :                         GDKerror("Read only %zu/%zu bytes of component %s: %s", (size_t) bytes_read, chunk, path, mnstr_peek_error(s));
     905           0 :                         goto end;
     906             :                 }
     907             : 
     908          10 :                 ssize_t bytes_written = mnstr_write(plan, buf, 1, chunk);
     909          10 :                 if (bytes_written < 0) {
     910           0 :                         GDKerror("Writing to plan failed: %s", mnstr_peek_error(plan));
     911           0 :                         goto end;
     912          10 :                 } else if (bytes_written < (ssize_t) chunk) {
     913           0 :                         GDKerror("write to plan truncated");
     914           0 :                         goto end;
     915             :                 }
     916          10 :                 to_copy -= chunk;
     917             :         }
     918             : 
     919             :         ret = GDK_SUCCEED;
     920          10 : end:
     921          10 :         GDKfree(buf);
     922          10 :         if (s)
     923          10 :                 close_stream(s);
     924          10 :         return ret;
     925             : }
     926             : 
     927             : /* Add plan entries for all relevant files in the Write Ahead Log */
     928             : static gdk_return __attribute__((__warn_unused_result__))
     929           5 : snapshot_wal(logger *bat_logger, stream *plan, const char *db_dir)
     930             : {
     931           5 :         char log_file[FILENAME_MAX];
     932           5 :         int len;
     933             : 
     934           5 :         len = snprintf(log_file, sizeof(log_file), "%s/%s%s", db_dir, bat_logger->dir, LOGFILE);
     935           5 :         if (len == -1 || (size_t)len >= sizeof(log_file)) {
     936           0 :                 GDKerror("Could not open %s, filename is too large", log_file);
     937           0 :                 return GDK_FAIL;
     938             :         }
     939           5 :         if (snapshot_immediate_copy_file(plan, log_file, log_file + strlen(db_dir) + 1) != GDK_SUCCEED)
     940             :                 return GDK_FAIL;
     941             : 
     942          11 :         for (ulng id = bat_logger->saved_id+1; id <= bat_logger->id; id++) {
     943           6 :                 struct stat statbuf;
     944             : 
     945           6 :                 len = snprintf(log_file, sizeof(log_file), "%s/%s%s." LLFMT, db_dir, bat_logger->dir, LOGFILE, id);
     946           6 :                 if (len == -1 || (size_t)len >= sizeof(log_file)) {
     947           0 :                         GDKerror("Could not open %s, filename is too large", log_file);
     948           0 :                         return GDK_FAIL;
     949             :                 }
     950           6 :                 if (MT_stat(log_file, &statbuf) == 0) {
     951           6 :                         if (snapshot_lazy_copy_file(plan, log_file + strlen(db_dir) + 1, statbuf.st_size) != GDK_SUCCEED)
     952             :                                 return GDK_FAIL;
     953             :                 } else {
     954           0 :                         GDKerror("Could not open %s", log_file);
     955           0 :                         return GDK_FAIL;
     956             :                 }
     957             :         }
     958             :         return GDK_SUCCEED;
     959             : }
     960             : 
     961             : static gdk_return __attribute__((__warn_unused_result__))
     962        1600 : snapshot_heap(stream *plan, const char *db_dir, bat batid, const char *filename, const char *suffix, uint64_t extent)
     963             : {
     964        1600 :         char path1[FILENAME_MAX];
     965        1600 :         char path2[FILENAME_MAX];
     966        1600 :         const size_t offset = strlen(db_dir) + 1;
     967        1600 :         struct stat statbuf;
     968        1600 :         int len;
     969             : 
     970        1600 :         if (extent == 0) {
     971             :                 /* nothing to copy */
     972             :                 return GDK_SUCCEED;
     973             :         }
     974             :         // first check the backup dir
     975        1020 :         len = snprintf(path1, FILENAME_MAX, "%s/%s/%o.%s", db_dir, BAKDIR, (int) batid, suffix);
     976        1020 :         if (len == -1 || len >= FILENAME_MAX) {
     977           0 :                 path1[FILENAME_MAX - 1] = '\0';
     978           0 :                 GDKerror("Could not open %s, filename is too large", path1);
     979           0 :                 return GDK_FAIL;
     980             :         }
     981        1020 :         if (MT_stat(path1, &statbuf) == 0) {
     982           0 :                 return snapshot_lazy_copy_file(plan, path1 + offset, extent);
     983             :         }
     984        1020 :         if (errno != ENOENT) {
     985           0 :                 GDKsyserror("Error stat'ing %s", path1);
     986           0 :                 return GDK_FAIL;
     987             :         }
     988             : 
     989             :         // then check the regular location
     990        1020 :         len = snprintf(path2, FILENAME_MAX, "%s/%s/%s.%s", db_dir, BATDIR, filename, suffix);
     991        1020 :         if (len == -1 || len >= FILENAME_MAX) {
     992           0 :                 path2[FILENAME_MAX - 1] = '\0';
     993           0 :                 GDKerror("Could not open %s, filename is too large", path2);
     994           0 :                 return GDK_FAIL;
     995             :         }
     996        1020 :         if (MT_stat(path2, &statbuf) == 0) {
     997        1020 :                 return snapshot_lazy_copy_file(plan, path2 + offset, extent);
     998             :         }
     999           0 :         if (errno != ENOENT) {
    1000           0 :                 GDKsyserror("Error stat'ing %s", path2);
    1001           0 :                 return GDK_FAIL;
    1002             :         }
    1003             : 
    1004           0 :         GDKerror("One of %s and %s must exist", path1, path2);
    1005           0 :         return GDK_FAIL;
    1006             : }
    1007             : 
    1008             : /* Add plan entries for all persistent BATs by looping over the BBP.dir.
    1009             :  * Also include the BBP.dir itself.
    1010             :  */
    1011             : static gdk_return __attribute__((__warn_unused_result__))
    1012           5 : snapshot_bats(stream *plan, const char *db_dir)
    1013             : {
    1014           5 :         char bbpdir[FILENAME_MAX];
    1015           5 :         FILE *fp = NULL;
    1016           5 :         int len;
    1017           5 :         gdk_return ret = GDK_FAIL;
    1018           5 :         int lineno = 0;
    1019           5 :         bat bbpsize = 0;
    1020           5 :         lng logno;
    1021           5 :         unsigned bbpversion;
    1022             : 
    1023           5 :         len = snprintf(bbpdir, FILENAME_MAX, "%s/%s/%s", db_dir, BAKDIR, "BBP.dir");
    1024           5 :         if (len == -1 || len >= FILENAME_MAX) {
    1025           0 :                 GDKerror("Could not open %s, filename is too large", bbpdir);
    1026           0 :                 return GDK_FAIL;
    1027             :         }
    1028           5 :         ret = snapshot_immediate_copy_file(plan, bbpdir, bbpdir + strlen(db_dir) + 1);
    1029           5 :         if (ret != GDK_SUCCEED)
    1030             :                 return ret;
    1031             : 
    1032             :         // Open the catalog and parse the header
    1033           5 :         fp = fopen(bbpdir, "r");
    1034           5 :         if (fp == NULL) {
    1035           0 :                 GDKerror("Could not open %s for reading: %s", bbpdir, mnstr_peek_error(NULL));
    1036           0 :                 return GDK_FAIL;
    1037             :         }
    1038           5 :         bbpversion = BBPheader(fp, &lineno, &bbpsize, &logno, false);
    1039           5 :         if (bbpversion == 0)
    1040           0 :                 goto end;
    1041           5 :         assert(bbpversion == GDKLIBRARY);
    1042             : 
    1043        1260 :         for (;;) {
    1044        1265 :                 BAT b;
    1045        1265 :                 Heap h;
    1046        1265 :                 Heap vh;
    1047        1265 :                 vh = h = (Heap) {
    1048             :                         .free = 0,
    1049             :                 };
    1050        1265 :                 b = (BAT) {
    1051             :                         .theap = &h,
    1052             :                         .tvheap = &vh,
    1053             :                 };
    1054        1265 :                 char *options;
    1055        1265 :                 char filename[sizeof(BBP_physical(0))];
    1056        1265 :                 char batname[129];
    1057             : #ifdef GDKLIBRARY_HASHASH
    1058        1265 :                 int hashash;
    1059             : #endif
    1060             : 
    1061        1265 :                 switch (BBPreadBBPline(fp, bbpversion, &lineno, &b,
    1062             : #ifdef GDKLIBRARY_HASHASH
    1063             :                                                            &hashash,
    1064             : #endif
    1065             :                                                            batname, filename, &options)) {
    1066           5 :                 case 0:
    1067             :                         /* end of file */
    1068           5 :                         fclose(fp);
    1069           5 :                         return GDK_SUCCEED;
    1070             :                 case 1:
    1071             :                         /* successfully read an entry */
    1072        1260 :                         break;
    1073           0 :                 default:
    1074             :                         /* error */
    1075           0 :                         fclose(fp);
    1076           0 :                         return GDK_FAIL;
    1077             :                 }
    1078             : #ifdef GDKLIBRARY_HASHASH
    1079        1260 :                 assert(hashash == 0);
    1080             : #endif
    1081        1260 :                 if (ATOMvarsized(b.ttype)) {
    1082         340 :                         ret = snapshot_heap(plan, db_dir, b.batCacheid, filename, "theap", b.tvheap->free);
    1083         340 :                         if (ret != GDK_SUCCEED)
    1084           0 :                                 goto end;
    1085             :                 }
    1086        1260 :                 ret = snapshot_heap(plan, db_dir, b.batCacheid, filename, BATtailname(&b), b.theap->free);
    1087        1260 :                 if (ret != GDK_SUCCEED)
    1088           0 :                         goto end;
    1089             :         }
    1090             : 
    1091           0 : end:
    1092           0 :         if (fp) {
    1093           0 :                 fclose(fp);
    1094             :         }
    1095           0 :         return ret;
    1096             : }
    1097             : 
    1098             : static gdk_return __attribute__((__warn_unused_result__))
    1099           5 : snapshot_vaultkey(stream *plan, const char *db_dir)
    1100             : {
    1101           5 :         char path[FILENAME_MAX];
    1102           5 :         struct stat statbuf;
    1103             : 
    1104           5 :         int len = snprintf(path, FILENAME_MAX, "%s/.vaultkey", db_dir);
    1105           5 :         if (len == -1 || len >= FILENAME_MAX) {
    1106           0 :                 path[FILENAME_MAX - 1] = '\0';
    1107           0 :                 GDKerror("Could not open %s, filename is too large", path);
    1108           0 :                 return GDK_FAIL;
    1109             :         }
    1110           5 :         if (MT_stat(path, &statbuf) == 0) {
    1111           0 :                 return snapshot_lazy_copy_file(plan, ".vaultkey", statbuf.st_size);
    1112             :         }
    1113           5 :         if (errno == ENOENT) {
    1114             :                 // No .vaultkey? Fine.
    1115             :                 return GDK_SUCCEED;
    1116             :         }
    1117             : 
    1118           0 :         GDKsyserror("Error stat'ing %s", path);
    1119           0 :         return GDK_FAIL;
    1120             : }
    1121             : 
    1122             : static gdk_return
    1123           5 : bl_snapshot(sqlstore *store, stream *plan)
    1124             : {
    1125           5 :         logger *bat_logger = store->logger;
    1126           5 :         gdk_return ret;
    1127           5 :         char *db_dir = NULL;
    1128           5 :         size_t db_dir_len;
    1129             : 
    1130             :         // Farm 0 is always the persistent farm.
    1131           5 :         db_dir = GDKfilepath(0, NULL, "", NULL);
    1132           5 :         if (db_dir == NULL)
    1133             :                 return GDK_FAIL;
    1134           5 :         db_dir_len = strlen(db_dir);
    1135           5 :         if (db_dir[db_dir_len - 1] == DIR_SEP)
    1136           5 :                 db_dir[db_dir_len - 1] = '\0';
    1137             : 
    1138          10 :         if (mnstr_printf(plan, "%s\n", db_dir) < 0 ||
    1139             :                 // Please monetdbd
    1140           5 :                 mnstr_printf(plan, "w 0 .uplog\n") < 0) {
    1141           0 :                 GDKerror("%s", mnstr_peek_error(plan));
    1142           0 :                 ret = GDK_FAIL;
    1143           0 :                 goto end;
    1144             :         }
    1145             : 
    1146           5 :         ret = snapshot_vaultkey(plan, db_dir);
    1147           5 :         if (ret != GDK_SUCCEED)
    1148           0 :                 goto end;
    1149             : 
    1150           5 :         ret = snapshot_bats(plan, db_dir);
    1151           5 :         if (ret != GDK_SUCCEED)
    1152           0 :                 goto end;
    1153             : 
    1154           5 :         ret = snapshot_wal(bat_logger, plan, db_dir);
    1155           5 :         if (ret != GDK_SUCCEED)
    1156             :                 goto end;
    1157             : 
    1158             :         ret = GDK_SUCCEED;
    1159           5 : end:
    1160           5 :         GDKfree(db_dir);
    1161           5 :         return ret;
    1162             : }
    1163             : 
    1164             : void
    1165         330 : bat_logger_init( logger_functions *lf )
    1166             : {
    1167         330 :         lf->create = bl_create;
    1168         330 :         lf->destroy = bl_destroy;
    1169         330 :         lf->flush = bl_flush;
    1170         330 :         lf->activate = bl_activate;
    1171         330 :         lf->changes = bl_changes;
    1172         330 :         lf->get_sequence = bl_get_sequence;
    1173         330 :         lf->log_isnew = bl_log_isnew;
    1174         330 :         lf->log_tstart = bl_tstart;
    1175         330 :         lf->log_tend = bl_tend;
    1176         330 :         lf->log_tflush = bl_tflush;
    1177         330 :         lf->log_tsequence = bl_sequence;
    1178         330 :         lf->get_snapshot_files = bl_snapshot;
    1179         330 : }

Generated by: LCOV version 1.14