Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : /*
14 : * @a M. L. Kersten, P. Boncz, N. Nes
15 : *
16 : * @* Utilities
17 : * The utility section contains functions to initialize the Monet
18 : * database system, memory allocation details, and a basic system
19 : * logging scheme.
20 : */
21 : #include "monetdb_config.h"
22 : #include "monet_options.h"
23 :
24 : #include "gdk.h"
25 : #include "gdk_private.h"
26 : #include "mutils.h"
27 :
28 : static BAT *GDKkey = NULL;
29 : static BAT *GDKval = NULL;
30 : ATOMIC_TYPE GDKdebug = ATOMIC_VAR_INIT(0);
31 :
32 : #include <signal.h>
33 :
34 : #ifdef HAVE_FCNTL_H
35 : #include <fcntl.h>
36 : #endif
37 :
38 : #ifdef HAVE_PWD_H
39 : # include <pwd.h>
40 : #endif
41 :
42 : #ifdef HAVE_SYS_PARAM_H
43 : # include <sys/param.h> /* prerequisite of sys/sysctl on OpenBSD */
44 : #endif
45 : #ifdef BSD /* BSD macro is defined in sys/param.h */
46 : # include <sys/sysctl.h>
47 : #endif
48 : #if defined(HAVE_SYS_RESOURCE_H) && defined(HAVE_GETRLIMIT)
49 : #include <sys/resource.h>
50 : #endif
51 :
52 : #ifdef __CYGWIN__
53 : #include <sysinfoapi.h>
54 : #endif
55 :
56 : static ATOMIC_TYPE GDKstopped = ATOMIC_VAR_INIT(0);
57 : static void GDKunlockHome(int farmid);
58 :
59 : #undef malloc
60 : #undef calloc
61 : #undef realloc
62 : #undef free
63 :
64 : /* when the number of updates to a BAT is less than 1 in this number, we
65 : * keep the unique_est property */
66 : BUN gdk_unique_estimate_keep_fraction = GDK_UNIQUE_ESTIMATE_KEEP_FRACTION; /* should become a define once */
67 : /* if the number of unique values is less than 1 in this number, we
68 : * destroy the hash rather than update it in HASH{append,insert,delete} */
69 : BUN hash_destroy_uniques_fraction = HASH_DESTROY_UNIQUES_FRACTION; /* likewise */
70 : /* if the estimated number of unique values is less than 1 in this
71 : * number, don't build a hash table to do a hashselect */
72 : dbl no_hash_select_fraction = NO_HASH_SELECT_FRACTION; /* same here */
73 : /* if the hash chain is longer than this number, we delete the hash
74 : * rather than maintaining it in HASHdelete */
75 : BUN hash_destroy_chain_length = HASH_DESTROY_CHAIN_LENGTH;
76 :
77 : /*
78 : * @+ Monet configuration file
79 : * Parse a possible MonetDB config file (if specified by command line
80 : * option -c/--config) to extract pre-settings of system variables.
81 : * Un-recognized parameters are simply skipped, because they may be
82 : * picked up by other components of the system. The consequence is
83 : * that making a typing error in the configuration file may be
84 : * unnoticed for a long time. Syntax errors are immediately flagged,
85 : * though.
86 : *
87 : * Since the GDK kernel moves into the database directory, we need to
88 : * keep the absolute path to the MonetDB config file for top-levels to
89 : * access its information.
90 : */
91 :
92 : static bool
93 337 : GDKenvironment(const char *dbpath)
94 : {
95 337 : if (dbpath == NULL) {
96 0 : TRC_CRITICAL(GDK, "Database name missing.\n");
97 0 : return false;
98 : }
99 337 : if (strlen(dbpath) >= FILENAME_MAX) {
100 0 : TRC_CRITICAL(GDK, "Database name too long.\n");
101 0 : return false;
102 : }
103 337 : if (!GDKembedded() && !MT_path_absolute(dbpath)) {
104 0 : TRC_CRITICAL(GDK, "Directory not an absolute path: %s.\n", dbpath);
105 0 : return false;
106 : }
107 : return true;
108 : }
109 :
110 : static struct orig_value {
111 : struct orig_value *next;
112 : char *value;
113 : char key[];
114 : } *orig_value;
115 : static MT_Lock GDKenvlock = MT_LOCK_INITIALIZER(GDKenvlock);
116 :
117 : const char *
118 460132 : GDKgetenv(const char *name)
119 : {
120 460132 : MT_lock_set(&GDKenvlock);
121 460133 : for (struct orig_value *ov = orig_value; ov; ov = ov->next) {
122 0 : if (strcmp(ov->key, name) == 0) {
123 0 : MT_lock_unset(&GDKenvlock);
124 0 : return ov->value;
125 : }
126 : }
127 460133 : MT_lock_unset(&GDKenvlock);
128 460133 : if (GDKkey && GDKval) {
129 459479 : BUN b = BUNfnd(GDKkey, name);
130 :
131 459478 : if (b != BUN_NONE) {
132 321857 : BATiter GDKenvi = bat_iterator(GDKval);
133 321858 : const char *v = BUNtvar(GDKenvi, b);
134 321858 : bat_iterator_end(&GDKenvi);
135 321858 : return v;
136 : }
137 : }
138 : return NULL;
139 : }
140 :
141 : bool
142 277657 : GDKgetenv_istext(const char *name, const char *text)
143 : {
144 277657 : const char *val = GDKgetenv(name);
145 :
146 277658 : return val && strcasecmp(val, text) == 0;
147 : }
148 :
149 : bool
150 38917 : GDKgetenv_isyes(const char *name)
151 : {
152 38917 : return GDKgetenv_istext(name, "yes");
153 : }
154 :
155 : bool
156 238741 : GDKgetenv_istrue(const char *name)
157 : {
158 238741 : return GDKgetenv_istext(name, "true");
159 : }
160 :
161 : int
162 93082 : GDKgetenv_int(const char *name, int def)
163 : {
164 93082 : const char *val = GDKgetenv(name);
165 :
166 93082 : if (val)
167 324 : return atoi(val);
168 : return def;
169 : }
170 :
171 : #define ESCAPE_CHAR '%'
172 :
173 : static bool
174 8204 : isutf8(const char *v, size_t *esclen)
175 : {
176 8204 : size_t n = 1;
177 8204 : int nutf8 = 0;
178 8204 : int m = 0;
179 152064 : for (size_t i = 0; v[i]; i++) {
180 143860 : if (nutf8 > 0) {
181 8 : if ((v[i] & 0xC0) != 0x80 ||
182 0 : (m != 0 && (v[i] & m) == 0))
183 0 : goto badutf8;
184 8 : m = 0;
185 8 : nutf8--;
186 143852 : } else if ((v[i] & 0xE0) == 0xC0) {
187 0 : nutf8 = 1;
188 0 : if ((v[i] & 0x1E) == 0)
189 0 : goto badutf8;
190 143852 : } else if ((v[i] & 0xF0) == 0xE0) {
191 4 : nutf8 = 2;
192 4 : if ((v[i] & 0x0F) == 0)
193 0 : m = 0x20;
194 143848 : } else if ((v[i] & 0xF8) == 0xF0) {
195 0 : nutf8 = 3;
196 0 : if ((v[i] & 0x07) == 0)
197 0 : m = 0x30;
198 143848 : } else if ((v[i] & 0x80) != 0) {
199 0 : goto badutf8;
200 : }
201 : }
202 8204 : *esclen = 0;
203 8204 : return true;
204 0 : badutf8:
205 0 : for (size_t i = 0; v[i]; i++) {
206 0 : if (v[i] & 0x80 || v[i] == ESCAPE_CHAR)
207 0 : n += 3;
208 : else
209 0 : n++;
210 : }
211 0 : *esclen = n;
212 0 : return false;
213 : }
214 :
215 : gdk_return
216 8204 : GDKsetenv(const char *name, const char *value)
217 : {
218 8204 : static const char hexdigits[] = "0123456789abcdef";
219 8204 : char *conval = NULL;
220 8204 : size_t esclen = 0;
221 8204 : if (!isutf8(value, &esclen)) {
222 0 : size_t j = strlen(name) + 1;
223 0 : struct orig_value *ov = GDKmalloc(offsetof(struct orig_value, key) + j + strlen(value) + 1);
224 0 : if (ov == NULL)
225 : return GDK_FAIL;
226 0 : strcpy(ov->key, name);
227 0 : ov->value = ov->key + j;
228 0 : strcpy(ov->value, value);
229 0 : conval = GDKmalloc(esclen);
230 0 : if (conval == NULL) {
231 0 : GDKfree(ov);
232 0 : return GDK_FAIL;
233 : }
234 : j = 0;
235 0 : for (size_t i = 0; value[i]; i++) {
236 0 : if (value[i] & 0x80 || value[i] == ESCAPE_CHAR) {
237 0 : conval[j++] = ESCAPE_CHAR;
238 0 : conval[j++] = hexdigits[(unsigned char) value[i] >> 4];
239 0 : conval[j++] = hexdigits[(unsigned char) value[i] & 0xF];
240 : } else {
241 0 : conval[j++] = value[i];
242 : }
243 : }
244 0 : conval[j] = 0;
245 0 : MT_lock_set(&GDKenvlock);
246 0 : ov->next = orig_value;
247 0 : orig_value = ov;
248 : /* remove previous value if present (later in list) */
249 0 : for (ov = orig_value; ov->next; ov = ov->next) {
250 0 : if (strcmp(ov->next->key, name) == 0) {
251 0 : struct orig_value *ovn = ov->next;
252 0 : ov->next = ovn->next;
253 0 : GDKfree(ovn);
254 : }
255 : }
256 0 : MT_lock_unset(&GDKenvlock);
257 : } else {
258 : /* remove previous value if present */
259 8204 : MT_lock_set(&GDKenvlock);
260 8204 : for (struct orig_value **ovp = &orig_value; *ovp; ovp = &(*ovp)->next) {
261 0 : if (strcmp((*ovp)->key, name) == 0) {
262 0 : struct orig_value *ov = *ovp;
263 0 : *ovp = ov->next;
264 0 : GDKfree(ov);
265 0 : break;
266 : }
267 : }
268 8204 : MT_lock_unset(&GDKenvlock);
269 : }
270 8204 : BUN p = BUNfnd(GDKkey, name);
271 8204 : gdk_return rc;
272 8204 : if (p != BUN_NONE) {
273 1938 : rc = BUNreplace(GDKval, p + GDKval->hseqbase,
274 : conval ? conval : value, false);
275 : } else {
276 7235 : rc = BUNappend(GDKkey, name, false);
277 7235 : if (rc == GDK_SUCCEED) {
278 14470 : rc = BUNappend(GDKval, conval ? conval : value, false);
279 7235 : if (rc != GDK_SUCCEED) {
280 : /* undo earlier successful append to
281 : * keep bats aligned (this can't really
282 : * fail, but we must check the result
283 : * anyway) */
284 0 : if (BUNdelete(GDKkey, GDKkey->hseqbase + GDKkey->batCount - 1) != GDK_SUCCEED)
285 0 : GDKerror("deleting key failed after failed value append");
286 : }
287 : }
288 : }
289 8204 : assert(BATcount(GDKval) == BATcount(GDKkey));
290 8204 : GDKfree(conval);
291 8204 : return rc;
292 : }
293 :
294 : gdk_return
295 33 : GDKcopyenv(BAT **key, BAT **val, bool writable)
296 : {
297 33 : BAT *k, *v;
298 :
299 33 : if (key == NULL || val == NULL) {
300 0 : GDKerror("called incorrectly.\n");
301 0 : return GDK_FAIL;
302 : }
303 33 : k = COLcopy(GDKkey, GDKkey->ttype, writable, TRANSIENT);
304 33 : v = COLcopy(GDKval, GDKval->ttype, writable, TRANSIENT);
305 33 : if (k == NULL || v == NULL) {
306 0 : BBPreclaim(k);
307 0 : BBPreclaim(v);
308 0 : return GDK_FAIL;
309 : }
310 33 : *key = k;
311 33 : *val = v;
312 33 : return GDK_SUCCEED;
313 : }
314 :
315 :
316 : /*
317 : * @+ System logging
318 : * Per database a log file can be maintained for collection of system
319 : * management information. Its contents is driven by the upper layers,
320 : * which encode information such as who logged on and how long the
321 : * session went on. The lower layers merely store error information
322 : * on the file. It should not be used for crash recovery, because
323 : * this should be dealt with on a per client basis.
324 : *
325 : * A system log can be maintained in the database to keep track of
326 : * session and crash information. It should regularly be refreshed to
327 : * avoid disk overflow.
328 : */
329 : #define GDKLOCK ".gdk_lock"
330 :
331 : #define GET_GDKLOCK(x) BBPfarms[BBPselectfarm((x), 0, offheap)].lock_file
332 :
333 : #define GDKLOGOFF "LOGOFF"
334 : #define GDKFOUNDDEAD "FOUND DEAD"
335 : #define GDKLOGON "LOGON"
336 : #define GDKCRASH "CRASH"
337 :
338 : /*
339 : * Single-lined comments can now be logged safely, together with
340 : * process, thread and user ID, and the current time.
341 : */
342 : static void __attribute__((__format__(__printf__, 2, 3)))
343 842 : GDKlog(FILE *lockFile, const char *format, ...)
344 : {
345 842 : va_list ap;
346 842 : char *p = 0, buf[1024];
347 842 : time_t tm = time(0);
348 : #if defined(HAVE_CTIME_R3) || defined(HAVE_CTIME_R)
349 842 : char tbuf[26];
350 : #endif
351 842 : char *ctm;
352 :
353 842 : if (MT_pagesize() == 0 || lockFile == NULL)
354 1 : return;
355 :
356 841 : va_start(ap, format);
357 841 : vsnprintf(buf, sizeof(buf), format, ap);
358 841 : va_end(ap);
359 :
360 : /* remove forbidden characters from message */
361 841 : for (p = buf; (p = strchr(p, '\n')) != NULL; *p = ' ')
362 : ;
363 841 : for (p = buf; (p = strchr(p, '@')) != NULL; *p = ' ')
364 : ;
365 :
366 841 : fseek(lockFile, 0, SEEK_END);
367 : #ifndef HAVE_GETUID
368 : #define getuid() 0
369 : #endif
370 : #ifdef HAVE_CTIME_R3
371 : ctm = ctime_r(&tm, tbuf, sizeof(tbuf));
372 : #else
373 841 : ctm = ctime_r(&tm, tbuf);
374 : #endif
375 841 : fprintf(lockFile, "USR=%d PID=%d TIME=%.24s @ %s\n", (int) getuid(), (int) getpid(), ctm, buf);
376 841 : fflush(lockFile);
377 : }
378 :
379 : /*
380 : * @+ Interrupt handling
381 : * The current version simply catches signals and prints a warning.
382 : * It should be extended to cope with the specifics of the interrupt
383 : * received.
384 : */
385 : #if 0 /* these are unused */
386 : static void
387 : BATSIGignore(int nr)
388 : {
389 : (void) nr;
390 : GDKsyserror("! ERROR signal %d caught by thread %zu\n", nr, (size_t) MT_getpid());
391 : }
392 : #endif
393 :
394 : #ifdef WIN32
395 : static void
396 : BATSIGabort(int nr)
397 : {
398 : (void) nr;
399 : _Exit(3); /* emulate Windows exit code without pop-up */
400 : }
401 : #endif
402 :
403 : #ifndef NATIVE_WIN32
404 : static void
405 338 : BATSIGinit(void)
406 : {
407 : #ifdef HAVE_SIGACTION
408 338 : struct sigaction sa;
409 338 : sigemptyset(&sa.sa_mask);
410 338 : sa.sa_flags = 0;
411 : #ifdef SIGPIPE
412 338 : sa.sa_handler = SIG_IGN;
413 338 : sigaction(SIGPIPE, &sa, NULL);
414 : #endif
415 : #ifdef SIGHUP
416 338 : sa.sa_handler = GDKtracer_reinit_basic;
417 338 : sigaction(SIGHUP, &sa, NULL);
418 : #endif
419 : #ifdef WIN32
420 : sa.sa_handler = BATSIGabort;
421 : sigaction(SIGABRT, &sa, NULL);
422 : #endif
423 : #else
424 : #ifdef SIGPIPE
425 : (void) signal(SIGPIPE, SIG_IGN);
426 : #endif
427 : #ifdef SIGHUP
428 : // Register signal to GDKtracer (logrotate)
429 : (void) signal(SIGHUP, GDKtracer_reinit_basic);
430 : #endif
431 : #ifdef WIN32
432 : (void) signal(SIGABRT, BATSIGabort);
433 : #endif
434 : #endif
435 : #if defined(WIN32) && !defined(__MINGW32__) && !defined(__CYGWIN__)
436 : _set_abort_behavior(0, _CALL_REPORTFAULT | _WRITE_ABORT_MSG);
437 : _set_error_mode(_OUT_TO_STDERR);
438 : #endif
439 338 : }
440 : #endif /* NATIVE_WIN32 */
441 :
442 : /* memory thresholds; these values some "sane" constants only, really
443 : * set in GDKinit() */
444 : #define MMAP_MINSIZE_PERSISTENT ((size_t) 1 << 18)
445 : #if SIZEOF_SIZE_T == 4
446 : #define MMAP_MINSIZE_TRANSIENT ((size_t) 1 << 20)
447 : #else
448 : #define MMAP_MINSIZE_TRANSIENT ((size_t) 1 << 32)
449 : #endif
450 : #define MMAP_PAGESIZE ((size_t) 1 << 16)
451 : size_t GDK_mmap_minsize_persistent = MMAP_MINSIZE_PERSISTENT;
452 : size_t GDK_mmap_minsize_transient = MMAP_MINSIZE_TRANSIENT;
453 : size_t GDK_mmap_pagesize = MMAP_PAGESIZE; /* mmap granularity */
454 : size_t GDK_mem_maxsize = GDK_VM_MAXSIZE;
455 : size_t GDK_vm_maxsize = GDK_VM_MAXSIZE;
456 :
457 : #define SEG_SIZE(x) ((ssize_t) (((x) + _MT_pagesize - 1) & ~(_MT_pagesize - 1)))
458 :
459 : /* This block is to provide atomic addition and subtraction to select
460 : * variables. We use intrinsic functions (recognized and inlined by
461 : * the compiler) for both the GNU C compiler and Microsoft Visual
462 : * Studio. By doing this, we avoid locking overhead. There is also a
463 : * fall-back for other compilers. */
464 : #include "matomic.h"
465 : static ATOMIC_TYPE GDK_mallocedbytes_estimate = ATOMIC_VAR_INIT(0);
466 : #ifndef NDEBUG
467 : static volatile lng GDK_malloc_success_count = -1;
468 : #endif
469 : static ATOMIC_TYPE GDK_vm_cursize = ATOMIC_VAR_INIT(0);
470 :
471 : size_t _MT_pagesize = 0; /* variable holding page size */
472 : size_t _MT_npages = 0; /* variable holding memory size in pages */
473 :
474 : static lng programepoch;
475 :
476 : void
477 339 : MT_init(void)
478 : {
479 339 : programepoch = GDKusec();
480 : #ifdef _MSC_VER
481 : {
482 : SYSTEM_INFO sysInfo;
483 :
484 : GetSystemInfo(&sysInfo);
485 : _MT_pagesize = sysInfo.dwPageSize;
486 : }
487 : #elif defined(BSD) && defined(HW_PAGESIZE)
488 : {
489 : int size;
490 : size_t len = sizeof(int);
491 : int mib[2];
492 :
493 : /* Everyone should have permission to make this call,
494 : * if we get a failure something is really wrong. */
495 : mib[0] = CTL_HW;
496 : mib[1] = HW_PAGESIZE;
497 : sysctl(mib, 2, &size, &len, NULL, 0);
498 : _MT_pagesize = size;
499 : }
500 : #elif defined(HAVE_SYSCONF) && defined(_SC_PAGESIZE)
501 339 : _MT_pagesize = (size_t)sysconf(_SC_PAGESIZE);
502 : #endif
503 339 : if (_MT_pagesize <= 0)
504 0 : _MT_pagesize = 4096; /* default */
505 :
506 : #ifdef WIN32
507 : {
508 : MEMORYSTATUSEX memStatEx;
509 :
510 : memStatEx.dwLength = sizeof(memStatEx);
511 : if (GlobalMemoryStatusEx(&memStatEx))
512 : _MT_npages = (size_t) (memStatEx.ullTotalPhys / _MT_pagesize);
513 : }
514 : #elif defined(BSD) && defined(HW_MEMSIZE) && SIZEOF_SIZE_T == SIZEOF_LNG
515 : /* Darwin, 64-bits */
516 : {
517 : uint64_t size = 0;
518 : size_t len = sizeof(size);
519 : int mib[2];
520 :
521 : /* Everyone should have permission to make this call,
522 : * if we get a failure something is really wrong. */
523 : mib[0] = CTL_HW;
524 : mib[1] = HW_MEMSIZE;
525 : sysctl(mib, 2, &size, &len, NULL, 0);
526 : _MT_npages = size / _MT_pagesize;
527 : }
528 : #elif defined(BSD) && defined (HW_PHYSMEM64) && SIZEOF_SIZE_T == SIZEOF_LNG
529 : /* OpenBSD, 64-bits */
530 : {
531 : int64_t size = 0;
532 : size_t len = sizeof(size);
533 : int mib[2];
534 :
535 : /* Everyone should have permission to make this call,
536 : * if we get a failure something is really wrong. */
537 : mib[0] = CTL_HW;
538 : mib[1] = HW_PHYSMEM64;
539 : sysctl(mib, 2, &size, &len, NULL, 0);
540 : _MT_npages = size / _MT_pagesize;
541 : }
542 : #elif defined(BSD) && defined(HW_PHYSMEM)
543 : /* NetBSD, OpenBSD, Darwin, 32-bits; FreeBSD 32 & 64-bits */
544 : {
545 : # ifdef __FreeBSD__
546 : unsigned long size = 0; /* type long required by sysctl() (?) */
547 : # else
548 : int size = 0;
549 : # endif
550 : size_t len = sizeof(size);
551 : int mib[2];
552 :
553 : /* Everyone should have permission to make this call,
554 : * if we get a failure something is really wrong. */
555 : mib[0] = CTL_HW;
556 : mib[1] = HW_PHYSMEM;
557 : sysctl(mib, 2, &size, &len, NULL, 0);
558 : _MT_npages = size / _MT_pagesize;
559 : }
560 : #elif defined(HAVE_SYSCONF) && defined(_SC_PHYS_PAGES)
561 339 : _MT_npages = (size_t)sysconf(_SC_PHYS_PAGES);
562 : # if SIZEOF_SIZE_T == SIZEOF_INT
563 : /* Bug #2935: the value returned here can be more than what can be
564 : * addressed on Solaris, so cap the value */
565 : if (UINT_MAX / _MT_pagesize < _MT_npages)
566 : _MT_npages = UINT_MAX / _MT_pagesize;
567 : # endif
568 : #else
569 : # error "don't know how to get the amount of physical memory for your OS"
570 : #endif
571 :
572 : #ifdef __linux__
573 : /* limit values to whatever cgroups gives us */
574 339 : FILE *fc;
575 339 : char buf[1024];
576 339 : char cgr1[1024] = "/sys/fs/cgroup/memory";
577 339 : char cgr2[1024] = "/sys/fs/cgroup";
578 339 : fc = fopen("/proc/self/mountinfo", "r");
579 339 : if (fc != NULL) {
580 11526 : while (fgets(buf, (int) sizeof(buf), fc) != NULL) {
581 11187 : char *p, *cgr;
582 11187 : if ((p = strstr(buf, " - cgroup ")) != NULL &&
583 0 : strstr(p, "memory") != NULL)
584 : cgr = cgr1;
585 11187 : else if (strstr(buf, " - cgroup2 ") != NULL)
586 : cgr = cgr2;
587 : else
588 10848 : continue;
589 : /* buf points at mount ID */
590 339 : p = strchr(buf, ' ');
591 339 : if (p == NULL)
592 : break;
593 339 : p++;
594 : /* p points at parent ID */
595 339 : p = strchr(p, ' ');
596 339 : if (p == NULL)
597 : break;
598 339 : p++;
599 : /* p points at major:minor */
600 339 : p = strchr(p, ' ');
601 339 : if (p == NULL)
602 : break;
603 339 : p++;
604 : /* p points at root */
605 339 : p = strchr(p, ' ');
606 339 : if (p == NULL)
607 : break;
608 339 : p++;
609 : /* p points at mount point */
610 339 : char *dir = p;
611 339 : p = strchr(p, ' ');
612 339 : if (p == NULL)
613 : break;
614 339 : *p = 0;
615 339 : strcpy_len(cgr, dir, 1024);
616 : }
617 339 : fclose(fc);
618 : }
619 339 : fc = fopen("/proc/self/cgroup", "r");
620 339 : if (fc != NULL) {
621 : /* each line is of the form:
622 : * hierarchy-ID:controller-list:cgroup-path
623 : *
624 : * For cgroup v1, the hierarchy-ID refers to the
625 : * second column in /proc/cgroups (which we ignore)
626 : * and the controller-list is a comma-separated list
627 : * of the controllers bound to the hierarchy. We look
628 : * for the "memory" controller and use its
629 : * cgroup-path. We ignore the other lines.
630 : *
631 : * For cgroup v2, the hierarchy-ID is 0 and the
632 : * controller-list is empty. We just use the
633 : * cgroup-path.
634 : *
635 : * We use the first line that we can match (either v1
636 : * or v2) and for which we can open any of the files
637 : * that we are looking for.
638 : */
639 339 : while (fgets(buf, (int) sizeof(buf), fc) != NULL) {
640 339 : char pth[1024];
641 339 : char *p, *q;
642 339 : bool success = false; /* true if we can open any file */
643 339 : FILE *f;
644 339 : uint64_t mem;
645 :
646 339 : p = strchr(buf, '\n');
647 339 : if (p == NULL)
648 : break;
649 339 : *p = 0;
650 339 : if (strncmp(buf, "0::", 3) == 0) {
651 : /* cgroup v2 entry */
652 339 : p = stpcpy(pth, cgr2);
653 339 : q = stpcpy(stpcpy(p, buf + 3), "/");
654 : /* hard limit */
655 339 : strcpy(q, "memory.max");
656 339 : f = fopen(pth, "r");
657 339 : while (f == NULL && q > p) {
658 : /* go up the hierarchy until we
659 : * find the file or the
660 : * hierarchy runs out */
661 0 : *--q = 0; /* zap the slash */
662 0 : q = strrchr(p, '/');
663 0 : if (q == NULL || q == p) {
664 : /* position after the slash */
665 0 : q = p + 1;
666 0 : break;
667 : }
668 0 : strcpy(++q, "memory.max");
669 0 : f = fopen(pth, "r");
670 : }
671 339 : if (f != NULL) {
672 339 : if (fscanf(f, "%" SCNu64, &mem) == 1
673 0 : && mem > 0
674 0 : && mem < (uint64_t) _MT_pagesize * _MT_npages) {
675 0 : _MT_npages = (size_t) (mem / _MT_pagesize);
676 : }
677 339 : success = true;
678 : /* assume "max" if not a number */
679 339 : fclose(f);
680 : }
681 : /* soft high limit */
682 339 : strcpy(q, "memory.high");
683 339 : f = fopen(pth, "r");
684 339 : if (f != NULL) {
685 339 : if (fscanf(f, "%" SCNu64, &mem) == 1
686 0 : && mem > 0
687 0 : && mem < (uint64_t) _MT_pagesize * _MT_npages) {
688 0 : _MT_npages = (size_t) (mem / _MT_pagesize);
689 : }
690 339 : success = true;
691 : /* assume "max" if not a number */
692 339 : fclose(f);
693 : }
694 : /* limit of swap usage, hard limit
695 : * we use this, together with
696 : * memory.high, as maximum virtual
697 : * memory size */
698 339 : strcpy(q, "memory.swap.max");
699 339 : f = fopen(pth, "r");
700 339 : if (f != NULL) {
701 339 : if (fscanf(f, "%" SCNu64, &mem) == 1
702 0 : && mem > 0
703 0 : && (mem += _MT_npages * _MT_pagesize) < (uint64_t) GDK_vm_maxsize) {
704 0 : GDK_vm_maxsize = (size_t) mem;
705 : }
706 339 : success = true;
707 339 : fclose(f);
708 : }
709 : #if 0 /* not sure about using this one */
710 : /* limit of swap usage, soft limit */
711 : strcpy(q, "memory.swap.high");
712 : f = fopen(pth, "r");
713 : if (f != NULL) {
714 : if (fscanf(f, "%" SCNu64, &mem) == 1
715 : && mem > 0
716 : && (mem += _MT_npages * _MT_pagesize) < (uint64_t) GDK_vm_maxsize) {
717 : GDK_vm_maxsize = (size_t) mem;
718 : }
719 : success = true;
720 : fclose(f);
721 : }
722 : #endif
723 : } else {
724 : /* cgroup v1 entry */
725 0 : p = strchr(buf, ':');
726 0 : if (p == NULL)
727 : break;
728 0 : q = p + 1;
729 0 : p = strchr(q, ':');
730 0 : if (p == NULL)
731 : break;
732 0 : *p++ = 0;
733 0 : if (strstr(q, "memory") == NULL)
734 0 : continue;
735 : /* limit of memory usage */
736 0 : strconcat_len(pth, sizeof(pth),
737 : cgr1, p,
738 : "/memory.limit_in_bytes",
739 : NULL);
740 0 : f = fopen(pth, "r");
741 0 : if (f == NULL) {
742 0 : strconcat_len(pth, sizeof(pth),
743 : cgr1,
744 : "/memory.limit_in_bytes",
745 : NULL);
746 0 : f = fopen(pth, "r");
747 : }
748 0 : if (f != NULL) {
749 0 : if (fscanf(f, "%" SCNu64, &mem) == 1
750 0 : && mem > 0
751 0 : && mem < (uint64_t) _MT_pagesize * _MT_npages) {
752 0 : _MT_npages = (size_t) (mem / _MT_pagesize);
753 : }
754 0 : success = true;
755 0 : fclose(f);
756 : }
757 : /* soft limit of memory usage */
758 0 : strconcat_len(pth, sizeof(pth),
759 : cgr1, p,
760 : "/memory.soft_limit_in_bytes",
761 : NULL);
762 0 : f = fopen(pth, "r");
763 0 : if (f == NULL) {
764 0 : strconcat_len(pth, sizeof(pth),
765 : cgr1,
766 : "/memory.soft_limit_in_bytes",
767 : NULL);
768 0 : f = fopen(pth, "r");
769 : }
770 0 : if (f != NULL) {
771 0 : if (fscanf(f, "%" SCNu64, &mem) == 1
772 0 : && mem > 0
773 0 : && mem < (uint64_t) _MT_pagesize * _MT_npages) {
774 0 : _MT_npages = (size_t) (mem / _MT_pagesize);
775 : }
776 0 : success = true;
777 0 : fclose(f);
778 : }
779 : /* limit of memory+swap usage
780 : * we use this as maximum virtual memory size */
781 0 : strconcat_len(pth, sizeof(pth),
782 : cgr1, p,
783 : "/memory.memsw.limit_in_bytes",
784 : NULL);
785 0 : f = fopen(pth, "r");
786 0 : if (f == NULL) {
787 0 : strconcat_len(pth, sizeof(pth),
788 : cgr1,
789 : "/memory.memsw.limit_in_bytes",
790 : NULL);
791 0 : f = fopen(pth, "r");
792 : }
793 0 : if (f != NULL) {
794 0 : if (fscanf(f, "%" SCNu64, &mem) == 1
795 0 : && mem > 0
796 0 : && mem < (uint64_t) GDK_vm_maxsize) {
797 0 : GDK_vm_maxsize = (size_t) mem;
798 : }
799 0 : success = true;
800 0 : fclose(f);
801 : }
802 : }
803 339 : if (success)
804 : break;
805 : }
806 339 : fclose(fc);
807 : }
808 : #endif
809 :
810 : #if defined(HAVE_SYS_RESOURCE_H) && defined(HAVE_GETRLIMIT) && defined(RLIMIT_AS)
811 339 : struct rlimit l;
812 : /* address space (virtual memory) limit */
813 339 : if (getrlimit(RLIMIT_AS, &l) == 0
814 339 : && l.rlim_cur != (rlim_t)RLIM_INFINITY
815 0 : && (size_t)l.rlim_cur < GDK_vm_maxsize) {
816 0 : GDK_vm_maxsize = l.rlim_cur;
817 : }
818 : #endif
819 339 : }
820 :
821 : /*
822 : * @+ Session Initialization
823 : * The interface code to the operating system is highly dependent on
824 : * the processing environment. It can be filtered away with
825 : * compile-time flags. Suicide is necessary due to some system
826 : * implementation errors.
827 : *
828 : * The kernel requires file descriptors for I/O with the user. They
829 : * are thread specific and should be obtained by a function.
830 : *
831 : * The arguments relevant for the kernel are extracted from the list.
832 : * Their value is turned into a blanc space.
833 : */
834 :
835 : #define CATNAP 50 /* time to sleep in ms for catnaps */
836 :
837 : static int THRinit(void);
838 : static gdk_return GDKlockHome(int farmid);
839 :
840 : #ifndef __COVERITY__
841 : #ifndef NDEBUG
842 : static MT_Lock mallocsuccesslock = MT_LOCK_INITIALIZER(mallocsuccesslock);
843 : #endif
844 : #endif
845 :
846 : void
847 346 : GDKsetdebug(unsigned debug)
848 : {
849 346 : ATOMIC_SET(&GDKdebug, debug);
850 346 : if (debug & ACCELMASK)
851 0 : GDKtracer_set_component_level("accelerator", "debug");
852 : else
853 346 : GDKtracer_reset_component_level("accelerator");
854 346 : if (debug & ALGOMASK)
855 0 : GDKtracer_set_component_level("algo", "debug");
856 : else
857 346 : GDKtracer_reset_component_level("algo");
858 346 : if (debug & ALLOCMASK)
859 0 : GDKtracer_set_component_level("alloc", "debug");
860 : else
861 346 : GDKtracer_reset_component_level("alloc");
862 346 : if (debug & BATMASK)
863 0 : GDKtracer_set_component_level("bat", "debug");
864 : else
865 346 : GDKtracer_reset_component_level("bat");
866 346 : if (debug & CHECKMASK)
867 344 : GDKtracer_set_component_level("check", "debug");
868 : else
869 2 : GDKtracer_reset_component_level("check");
870 346 : if (debug & DELTAMASK)
871 0 : GDKtracer_set_component_level("delta", "debug");
872 : else
873 346 : GDKtracer_reset_component_level("delta");
874 346 : if (debug & HEAPMASK)
875 0 : GDKtracer_set_component_level("heap", "debug");
876 : else
877 346 : GDKtracer_reset_component_level("heap");
878 346 : if (debug & IOMASK)
879 0 : GDKtracer_set_component_level("io", "debug");
880 : else
881 346 : GDKtracer_reset_component_level("io");
882 346 : if (debug & PARMASK)
883 0 : GDKtracer_set_component_level("par", "debug");
884 : else
885 346 : GDKtracer_reset_component_level("par");
886 346 : if (debug & PERFMASK)
887 0 : GDKtracer_set_component_level("perf", "debug");
888 : else
889 346 : GDKtracer_reset_component_level("perf");
890 346 : if (debug & TEMMASK)
891 0 : GDKtracer_set_component_level("tem", "debug");
892 : else
893 346 : GDKtracer_reset_component_level("tem");
894 346 : if (debug & THRDMASK)
895 0 : GDKtracer_set_component_level("thrd", "debug");
896 : else
897 346 : GDKtracer_reset_component_level("thrd");
898 346 : }
899 :
900 : unsigned
901 20 : GDKgetdebug(void)
902 : {
903 20 : ATOMIC_BASE_TYPE debug = ATOMIC_GET(&GDKdebug);
904 20 : const char *lvl;
905 20 : lvl = GDKtracer_get_component_level("accelerator");
906 20 : if (lvl && strcmp(lvl, "debug") == 0)
907 0 : debug |= ACCELMASK;
908 20 : lvl = GDKtracer_get_component_level("algo");
909 20 : if (lvl && strcmp(lvl, "debug") == 0)
910 0 : debug |= ALGOMASK;
911 20 : lvl = GDKtracer_get_component_level("alloc");
912 20 : if (lvl && strcmp(lvl, "debug") == 0)
913 0 : debug |= ALLOCMASK;
914 20 : lvl = GDKtracer_get_component_level("bat");
915 20 : if (lvl && strcmp(lvl, "debug") == 0)
916 0 : debug |= BATMASK;
917 20 : lvl = GDKtracer_get_component_level("check");
918 20 : if (lvl && strcmp(lvl, "debug") == 0)
919 0 : debug |= CHECKMASK;
920 20 : lvl = GDKtracer_get_component_level("delta");
921 20 : if (lvl && strcmp(lvl, "debug") == 0)
922 0 : debug |= DELTAMASK;
923 20 : lvl = GDKtracer_get_component_level("heap");
924 20 : if (lvl && strcmp(lvl, "debug") == 0)
925 0 : debug |= HEAPMASK;
926 20 : lvl = GDKtracer_get_component_level("io");
927 20 : if (lvl && strcmp(lvl, "debug") == 0)
928 0 : debug |= IOMASK;
929 20 : lvl = GDKtracer_get_component_level("par");
930 20 : if (lvl && strcmp(lvl, "debug") == 0)
931 0 : debug |= PARMASK;
932 20 : lvl = GDKtracer_get_component_level("perf");
933 20 : if (lvl && strcmp(lvl, "debug") == 0)
934 0 : debug |= PERFMASK;
935 20 : lvl = GDKtracer_get_component_level("tem");
936 20 : if (lvl && strcmp(lvl, "debug") == 0)
937 0 : debug |= TEMMASK;
938 20 : lvl = GDKtracer_get_component_level("thrd");
939 20 : if (lvl && strcmp(lvl, "debug") == 0)
940 0 : debug |= THRDMASK;
941 20 : return (unsigned) debug;
942 : }
943 :
944 : static bool Mbedded = true;
945 : bool
946 18153119 : GDKembedded(void)
947 : {
948 18153119 : return Mbedded;
949 : }
950 :
951 : static MT_Id mainpid;
952 :
953 : gdk_return
954 338 : GDKinit(opt *set, int setlen, bool embedded, const char *caller_revision)
955 : {
956 338 : static bool first = true;
957 338 : const char *dbpath;
958 338 : const char *dbtrace;
959 338 : const char *p;
960 338 : opt *n;
961 338 : int i, nlen = 0;
962 338 : char buf[16];
963 :
964 338 : if (caller_revision) {
965 327 : p = mercurial_revision();
966 327 : if (p && strcmp(p, caller_revision) != 0) {
967 0 : GDKerror("incompatible versions: caller is %s, GDK is %s\n", caller_revision, p);
968 0 : return GDK_FAIL;
969 : }
970 : }
971 :
972 338 : ATOMIC_SET(&GDKstopped, 0);
973 :
974 338 : if (BBPchkfarms() != GDK_SUCCEED)
975 : return GDK_FAIL;
976 :
977 338 : if (GDKinmemory(0)) {
978 : dbpath = dbtrace = NULL;
979 : } else {
980 337 : dbpath = mo_find_option(set, setlen, "gdk_dbpath");
981 337 : dbtrace = mo_find_option(set, setlen, "gdk_dbtrace");
982 : }
983 338 : Mbedded = embedded;
984 : /* some sanity checks (should also find if symbols are not defined) */
985 338 : static_assert(sizeof(int) == sizeof(int32_t),
986 : "int is not equal in size to int32_t");
987 338 : static_assert(sizeof(char) == SIZEOF_CHAR,
988 : "error in configure: bad value for SIZEOF_CHAR");
989 338 : static_assert(sizeof(short) == SIZEOF_SHORT,
990 : "error in configure: bad value for SIZEOF_SHORT");
991 338 : static_assert(sizeof(int) == SIZEOF_INT,
992 : "error in configure: bad value for SIZEOF_INT");
993 338 : static_assert(sizeof(long) == SIZEOF_LONG,
994 : "error in configure: bad value for SIZEOF_LONG");
995 338 : static_assert(sizeof(lng) == SIZEOF_LNG,
996 : "error in configure: bad value for SIZEOF_LNG");
997 : #ifdef HAVE_HGE
998 338 : static_assert(sizeof(hge) == SIZEOF_HGE,
999 : "error in configure: bad value for SIZEOF_HGE");
1000 : #endif
1001 338 : static_assert(sizeof(dbl) == SIZEOF_DOUBLE,
1002 : "error in configure: bad value for SIZEOF_DOUBLE");
1003 338 : static_assert(sizeof(oid) == SIZEOF_OID,
1004 : "error in configure: bad value for SIZEOF_OID");
1005 338 : static_assert(sizeof(void *) == SIZEOF_VOID_P,
1006 : "error in configure: bad value for SIZEOF_VOID_P");
1007 338 : static_assert(sizeof(size_t) == SIZEOF_SIZE_T,
1008 : "error in configure: bad value for SIZEOF_SIZE_T");
1009 338 : static_assert(SIZEOF_OID == SIZEOF_INT || SIZEOF_OID == SIZEOF_LNG,
1010 : "SIZEOF_OID should be equal to SIZEOF_INT or SIZEOF_LNG");
1011 338 : static_assert(sizeof(uuid) == 16,
1012 : "sizeof(uuid) should be equal to 16");
1013 :
1014 338 : if (first) {
1015 : /* some things are really only initialized once */
1016 328 : if (!MT_thread_init()) {
1017 0 : TRC_CRITICAL(GDK, "MT_thread_init failed\n");
1018 0 : return GDK_FAIL;
1019 : }
1020 :
1021 2687304 : for (i = 0; i <= BBP_BATMASK; i++) {
1022 2686976 : char name[MT_NAME_LEN];
1023 2686976 : snprintf(name, sizeof(name), "GDKswapLock%d", i);
1024 2686976 : MT_lock_init(&GDKbatLock[i].swap, name);
1025 : }
1026 328 : if (mnstr_init() < 0) {
1027 0 : TRC_CRITICAL(GDK, "mnstr_init failed\n");
1028 0 : return GDK_FAIL;
1029 : }
1030 : } else {
1031 : /* BBP was locked by BBPexit() */
1032 : //BBPunlock();
1033 338 : }
1034 338 : mainpid = MT_getpid();
1035 :
1036 338 : GDKtracer_init(dbpath, dbtrace);
1037 338 : errno = 0;
1038 338 : if (!GDKinmemory(0) && !GDKenvironment(dbpath))
1039 : return GDK_FAIL;
1040 :
1041 338 : MT_init_posix();
1042 338 : if (THRinit() < 0)
1043 : return GDK_FAIL;
1044 : #ifndef NATIVE_WIN32
1045 338 : BATSIGinit();
1046 : #endif
1047 338 : MT_init();
1048 :
1049 : /* now try to lock the database: go through all farms, and if
1050 : * we see a new directory, lock it */
1051 11090 : for (int farmid = 0; farmid < MAXFARMS; farmid++) {
1052 10754 : if (BBPfarms[farmid].dirname != NULL) {
1053 1342 : bool skip = false;
1054 1342 : for (int j = 0; j < farmid; j++) {
1055 832 : if (BBPfarms[j].dirname != NULL &&
1056 832 : strcmp(BBPfarms[farmid].dirname, BBPfarms[j].dirname) == 0) {
1057 : skip = true;
1058 : break;
1059 : }
1060 : }
1061 996 : if (!skip && GDKlockHome(farmid) != GDK_SUCCEED)
1062 : return GDK_FAIL;
1063 : }
1064 : }
1065 :
1066 : /* Mserver by default takes 80% of all memory as a default */
1067 : #if SIZEOF_SIZE_T == 4
1068 : if ((double) MT_npages() * (double) MT_pagesize() * 0.815 >= (double) GDK_VM_MAXSIZE)
1069 : GDK_mem_maxsize = GDK_VM_MAXSIZE;
1070 : else
1071 : #endif
1072 336 : GDK_mem_maxsize = (size_t) ((double) MT_npages() * (double) MT_pagesize() * 0.815);
1073 336 : const char *allow = mo_find_option(set, setlen, "allow_hge_upgrade");
1074 664 : if (BBPinit(allow && strcmp(allow, "yes") == 0) != GDK_SUCCEED)
1075 : return GDK_FAIL;
1076 336 : first = false;
1077 :
1078 336 : if (GDK_mem_maxsize / 16 < GDK_mmap_minsize_transient) {
1079 326 : GDK_mmap_minsize_transient = MAX(MMAP_PAGESIZE, GDK_mem_maxsize / 16);
1080 326 : if (GDK_mmap_minsize_persistent > GDK_mmap_minsize_transient)
1081 0 : GDK_mmap_minsize_persistent = GDK_mmap_minsize_transient;
1082 : }
1083 :
1084 336 : n = (opt *) malloc(setlen * sizeof(opt));
1085 336 : if (n == NULL) {
1086 0 : GDKsyserror("malloc failed\n");
1087 0 : return GDK_FAIL;
1088 : }
1089 :
1090 4047 : for (i = 0; i < setlen; i++) {
1091 16638 : bool done = false;
1092 :
1093 16638 : for (int j = 0; j < nlen; j++) {
1094 13589 : if (strcmp(n[j].name, set[i].name) == 0) {
1095 662 : if (n[j].kind < set[i].kind) {
1096 662 : n[j] = set[i];
1097 : }
1098 : done = true;
1099 : break;
1100 : }
1101 : }
1102 3049 : if (!done) {
1103 3049 : n[nlen] = set[i];
1104 3049 : nlen++;
1105 : }
1106 : }
1107 : /* check some options before creating our first BAT */
1108 3385 : for (i = 0; i < nlen; i++) {
1109 3049 : if (strcmp("gdk_mem_maxsize", n[i].name) == 0) {
1110 0 : GDK_mem_maxsize = (size_t) strtoll(n[i].value, NULL, 10);
1111 0 : GDK_mem_maxsize = MAX(1 << 26, GDK_mem_maxsize);
1112 0 : if (GDK_mem_maxsize / 16 < GDK_mmap_minsize_transient)
1113 0 : GDK_mmap_minsize_transient = MAX(MMAP_PAGESIZE, GDK_mem_maxsize / 16);
1114 0 : if (GDK_mmap_minsize_persistent > GDK_mmap_minsize_transient)
1115 0 : GDK_mmap_minsize_persistent = GDK_mmap_minsize_transient;
1116 3049 : } else if (strcmp("gdk_vm_maxsize", n[i].name) == 0) {
1117 179 : GDK_vm_maxsize = (size_t) strtoll(n[i].value, NULL, 10);
1118 179 : GDK_vm_maxsize = MAX(1 << 30, GDK_vm_maxsize);
1119 2870 : } else if (strcmp("gdk_mmap_minsize_persistent", n[i].name) == 0) {
1120 0 : GDK_mmap_minsize_persistent = (size_t) strtoll(n[i].value, NULL, 10);
1121 2870 : } else if (strcmp("gdk_mmap_minsize_transient", n[i].name) == 0) {
1122 0 : GDK_mmap_minsize_transient = (size_t) strtoll(n[i].value, NULL, 10);
1123 2870 : } else if (strcmp("gdk_mmap_pagesize", n[i].name) == 0) {
1124 0 : GDK_mmap_pagesize = (size_t) strtoll(n[i].value, NULL, 10);
1125 0 : if (GDK_mmap_pagesize < 1 << 12 ||
1126 0 : GDK_mmap_pagesize > 1 << 20 ||
1127 : /* x & (x - 1): turn off rightmost 1 bit;
1128 : * i.e. if result is zero, x is power of
1129 : * two */
1130 0 : (GDK_mmap_pagesize & (GDK_mmap_pagesize - 1)) != 0) {
1131 0 : free(n);
1132 0 : TRC_CRITICAL(GDK, "gdk_mmap_pagesize must be power of 2 between 2**12 and 2**20\n");
1133 0 : return GDK_FAIL;
1134 : }
1135 : }
1136 : }
1137 :
1138 336 : GDKkey = COLnew(0, TYPE_str, 100, TRANSIENT);
1139 336 : GDKval = COLnew(0, TYPE_str, 100, TRANSIENT);
1140 336 : if (GDKkey == NULL || GDKval == NULL) {
1141 0 : free(n);
1142 0 : TRC_CRITICAL(GDK, "Could not create environment BATs");
1143 0 : return GDK_FAIL;
1144 : }
1145 672 : if (BBPrename(GDKkey, "environment_key") != 0 ||
1146 336 : BBPrename(GDKval, "environment_val") != 0) {
1147 0 : free(n);
1148 0 : TRC_CRITICAL(GDK, "BBPrename of environment BATs failed");
1149 0 : return GDK_FAIL;
1150 : }
1151 336 : BBP_pid(GDKkey->batCacheid) = 0;
1152 336 : BBP_pid(GDKval->batCacheid) = 0;
1153 :
1154 : /* store options into environment BATs */
1155 3385 : for (i = 0; i < nlen; i++)
1156 3049 : if (GDKsetenv(n[i].name, n[i].value) != GDK_SUCCEED) {
1157 0 : TRC_CRITICAL(GDK, "GDKsetenv %s failed", n[i].name);
1158 0 : free(n);
1159 0 : return GDK_FAIL;
1160 : }
1161 336 : free(n);
1162 :
1163 336 : GDKnr_threads = GDKgetenv_int("gdk_nr_threads", 0);
1164 336 : if (GDKnr_threads == 0) {
1165 335 : GDKnr_threads = MT_check_nr_cores();
1166 335 : snprintf(buf, sizeof(buf), "%d", GDKnr_threads);
1167 335 : if (GDKsetenv("gdk_nr_threads", buf) != GDK_SUCCEED) {
1168 0 : TRC_CRITICAL(GDK, "GDKsetenv gdk_nr_threads failed");
1169 0 : return GDK_FAIL;
1170 : }
1171 : }
1172 336 : if (GDKnr_threads > THREADS)
1173 0 : GDKnr_threads = THREADS;
1174 :
1175 336 : if (!GDKinmemory(0)) {
1176 335 : if ((p = GDKgetenv("gdk_dbpath")) != NULL &&
1177 335 : (p = strrchr(p, DIR_SEP)) != NULL) {
1178 335 : if (GDKsetenv("gdk_dbname", p + 1) != GDK_SUCCEED) {
1179 0 : TRC_CRITICAL(GDK, "GDKsetenv gdk_dbname failed");
1180 0 : return GDK_FAIL;
1181 : }
1182 : #if DIR_SEP != '/' /* on Windows look for different separator */
1183 : } else if ((p = GDKgetenv("gdk_dbpath")) != NULL &&
1184 : (p = strrchr(p, '/')) != NULL) {
1185 : if (GDKsetenv("gdk_dbname", p + 1) != GDK_SUCCEED) {
1186 : TRC_CRITICAL(GDK, "GDKsetenv gdk_dbname failed");
1187 : return GDK_FAIL;
1188 : }
1189 : #endif
1190 : }
1191 1 : } else if (GDKgetenv("gdk_dbname") == NULL) {
1192 1 : if (GDKsetenv("gdk_dbname", "in-memory") != GDK_SUCCEED) {
1193 0 : TRC_CRITICAL(GDK, "GDKsetenv gdk_dbname failed");
1194 0 : return GDK_FAIL;
1195 : }
1196 : }
1197 336 : if (GDKgetenv("gdk_vm_maxsize") == NULL) {
1198 157 : snprintf(buf, sizeof(buf), "%zu", GDK_vm_maxsize);
1199 157 : if (GDKsetenv("gdk_vm_maxsize", buf) != GDK_SUCCEED) {
1200 0 : TRC_CRITICAL(GDK, "GDKsetenv gdk_vm_maxsize failed");
1201 0 : return GDK_FAIL;
1202 : }
1203 : }
1204 336 : if (GDKgetenv("gdk_mem_maxsize") == NULL) {
1205 336 : snprintf(buf, sizeof(buf), "%zu", GDK_mem_maxsize);
1206 336 : if (GDKsetenv("gdk_mem_maxsize", buf) != GDK_SUCCEED) {
1207 0 : TRC_CRITICAL(GDK, "GDKsetenv gdk_mem_maxsize failed");
1208 0 : return GDK_FAIL;
1209 : }
1210 : }
1211 336 : if (GDKgetenv("gdk_mmap_minsize_persistent") == NULL) {
1212 336 : snprintf(buf, sizeof(buf), "%zu", GDK_mmap_minsize_persistent);
1213 336 : if (GDKsetenv("gdk_mmap_minsize_persistent", buf) != GDK_SUCCEED) {
1214 0 : TRC_CRITICAL(GDK, "GDKsetenv gdk_mmap_minsize_persistent failed");
1215 0 : return GDK_FAIL;
1216 : }
1217 : }
1218 336 : if (GDKgetenv("gdk_mmap_minsize_transient") == NULL) {
1219 336 : snprintf(buf, sizeof(buf), "%zu", GDK_mmap_minsize_transient);
1220 336 : if (GDKsetenv("gdk_mmap_minsize_transient", buf) != GDK_SUCCEED) {
1221 0 : TRC_CRITICAL(GDK, "GDKsetenv gdk_mmap_minsize_transient failed");
1222 0 : return GDK_FAIL;
1223 : }
1224 : }
1225 336 : if (GDKgetenv("gdk_mmap_pagesize") == NULL) {
1226 336 : snprintf(buf, sizeof(buf), "%zu", GDK_mmap_pagesize);
1227 336 : if (GDKsetenv("gdk_mmap_pagesize", buf) != GDK_SUCCEED) {
1228 0 : TRC_CRITICAL(GDK, "GDKsetenv gdk_mmap_pagesize failed");
1229 0 : return GDK_FAIL;
1230 : }
1231 : }
1232 336 : if (GDKgetenv("monet_pid") == NULL) {
1233 336 : snprintf(buf, sizeof(buf), "%d", (int) getpid());
1234 336 : if (GDKsetenv("monet_pid", buf) != GDK_SUCCEED) {
1235 0 : TRC_CRITICAL(GDK, "GDKsetenv monet_pid failed");
1236 0 : return GDK_FAIL;
1237 : }
1238 : }
1239 336 : if (GDKsetenv("revision", mercurial_revision()) != GDK_SUCCEED) {
1240 0 : TRC_CRITICAL(GDK, "GDKsetenv revision failed");
1241 0 : return GDK_FAIL;
1242 : }
1243 336 : gdk_unique_estimate_keep_fraction = 0;
1244 336 : if ((p = GDKgetenv("gdk_unique_estimate_keep_fraction")) != NULL)
1245 0 : gdk_unique_estimate_keep_fraction = (BUN) strtoll(p, NULL, 10);
1246 336 : if (gdk_unique_estimate_keep_fraction == 0)
1247 336 : gdk_unique_estimate_keep_fraction = GDK_UNIQUE_ESTIMATE_KEEP_FRACTION;
1248 336 : hash_destroy_uniques_fraction = 0;
1249 336 : if ((p = GDKgetenv("hash_destroy_uniques_fraction")) != NULL)
1250 0 : hash_destroy_uniques_fraction = (BUN) strtoll(p, NULL, 10);
1251 336 : if (hash_destroy_uniques_fraction == 0)
1252 336 : hash_destroy_uniques_fraction = HASH_DESTROY_UNIQUES_FRACTION;
1253 336 : no_hash_select_fraction = 0;
1254 336 : if ((p = GDKgetenv("no_hash_select_fraction")) != NULL)
1255 0 : no_hash_select_fraction = (dbl) strtoll(p, NULL, 10);
1256 336 : if (no_hash_select_fraction == 0)
1257 336 : no_hash_select_fraction = NO_HASH_SELECT_FRACTION;
1258 336 : hash_destroy_chain_length = 0;
1259 336 : if ((p = GDKgetenv("hash_destroy_chain_length")) != NULL)
1260 0 : hash_destroy_chain_length = (BUN) strtoll(p, NULL, 10);
1261 336 : if (hash_destroy_chain_length == 0)
1262 336 : hash_destroy_chain_length = HASH_DESTROY_CHAIN_LENGTH;
1263 :
1264 : return GDK_SUCCEED;
1265 : }
1266 :
1267 : int GDKnr_threads = 0;
1268 : static ATOMIC_TYPE GDKnrofthreads = ATOMIC_VAR_INIT(0);
1269 : struct threadStruct {
1270 : ATOMIC_TYPE pid; /* thread id, 0 = unallocated */
1271 : };
1272 :
1273 : bool
1274 11496532 : GDKexiting(void)
1275 : {
1276 11496532 : return (bool) (ATOMIC_GET(&GDKstopped) > 0);
1277 : }
1278 :
1279 : void
1280 335 : GDKprepareExit(void)
1281 : {
1282 335 : ATOMIC_ADD(&GDKstopped, 1);
1283 :
1284 335 : if (MT_getpid() == mainpid) {
1285 334 : TRC_DEBUG_IF(THRD)
1286 0 : dump_threads();
1287 334 : join_detached_threads();
1288 : }
1289 335 : }
1290 :
1291 : void
1292 334 : GDKreset(int status)
1293 : {
1294 334 : assert(GDKexiting());
1295 :
1296 334 : if (GDKembedded())
1297 : // In the case of a restarted embedded database, GDKstopped has to be reset as well.
1298 11 : ATOMIC_SET(&GDKstopped, 0);
1299 :
1300 334 : if (GDKkey) {
1301 334 : BBPunfix(GDKkey->batCacheid);
1302 334 : GDKkey = NULL;
1303 : }
1304 334 : if (GDKval) {
1305 334 : BBPunfix(GDKval->batCacheid);
1306 334 : GDKval = NULL;
1307 : }
1308 :
1309 334 : join_detached_threads();
1310 :
1311 334 : MT_lock_set(&GDKenvlock);
1312 334 : while (orig_value) {
1313 0 : struct orig_value *ov = orig_value;
1314 0 : orig_value = orig_value->next;
1315 0 : GDKfree(ov);
1316 : }
1317 334 : MT_lock_unset(&GDKenvlock);
1318 :
1319 334 : if (status == 0) {
1320 : /* they had their chance, now kill them */
1321 334 : bool killed = MT_kill_threads();
1322 : /* all threads ceased running, now we can clean up */
1323 334 : if (!killed) {
1324 : /* we can't clean up after killing threads */
1325 334 : BBPexit();
1326 : }
1327 334 : GDKlog(GET_GDKLOCK(PERSISTENT), GDKLOGOFF);
1328 :
1329 11022 : for (int farmid = 0; farmid < MAXFARMS; farmid++) {
1330 10688 : if (BBPfarms[farmid].dirname != NULL) {
1331 1968 : bool skip = false;
1332 1968 : for (int j = 0; j < farmid; j++) {
1333 979 : if (BBPfarms[j].dirname != NULL &&
1334 0 : strcmp(BBPfarms[farmid].dirname, BBPfarms[j].dirname) == 0) {
1335 : skip = true;
1336 : break;
1337 : }
1338 : }
1339 989 : if (!skip)
1340 989 : GDKunlockHome(farmid);
1341 989 : if (BBPfarms[farmid].dirname) {
1342 989 : GDKfree((char*)BBPfarms[farmid].dirname);
1343 989 : BBPfarms[farmid].dirname = NULL;
1344 : }
1345 : }
1346 : }
1347 :
1348 : #ifdef LOCK_STATS
1349 : TRC_DEBUG_IF(TEM) GDKlockstatistics(1);
1350 : #endif
1351 334 : ATOMIC_SET(&GDKdebug, 0);
1352 334 : GDK_mmap_minsize_persistent = MMAP_MINSIZE_PERSISTENT;
1353 334 : GDK_mmap_minsize_transient = MMAP_MINSIZE_TRANSIENT;
1354 334 : GDK_mmap_pagesize = MMAP_PAGESIZE;
1355 334 : GDK_mem_maxsize = (size_t) ((double) MT_npages() * (double) MT_pagesize() * 0.815);
1356 334 : GDK_vm_maxsize = GDK_VM_MAXSIZE;
1357 334 : GDKatomcnt = TYPE_blob + 1;
1358 :
1359 334 : if (GDK_mem_maxsize / 16 < GDK_mmap_minsize_transient) {
1360 334 : GDK_mmap_minsize_transient = GDK_mem_maxsize / 16;
1361 334 : if (GDK_mmap_minsize_persistent > GDK_mmap_minsize_transient)
1362 0 : GDK_mmap_minsize_persistent = GDK_mmap_minsize_transient;
1363 : }
1364 :
1365 334 : GDKnr_threads = 0;
1366 334 : ATOMIC_SET(&GDKnrofthreads, 0);
1367 334 : close_stream(GDKstdout);
1368 334 : close_stream(GDKstdin);
1369 334 : GDKstdout = NULL;
1370 334 : GDKstdin = NULL;
1371 :
1372 334 : gdk_bbp_reset();
1373 : }
1374 334 : ATOMunknown_clean();
1375 :
1376 : /* stop GDKtracer */
1377 334 : GDKtracer_stop();
1378 334 : }
1379 :
1380 : /*
1381 : * All semaphores used by the application should be mentioned here.
1382 : * They are initialized during system initialization.
1383 : */
1384 :
1385 : batlock_t GDKbatLock[BBP_BATMASK + 1];
1386 :
1387 : /*
1388 : * @+ Concurrency control
1389 : * Concurrency control requires actions at several levels of the
1390 : * system. First, it should be ensured that each database is
1391 : * controlled by a single server process (group). Subsequent attempts
1392 : * should be stopped. This is regulated through file locking against
1393 : * ".gdk_lock".
1394 : *
1395 : * Before the locks and threads are initiated, we cannot use the
1396 : * normal routines yet. So we have a local fatal here instead of
1397 : * GDKfatal.
1398 : */
1399 : static gdk_return
1400 510 : GDKlockHome(int farmid)
1401 : {
1402 510 : int fd;
1403 510 : struct stat st;
1404 510 : char *gdklockpath;
1405 510 : FILE *GDKlockFile;
1406 :
1407 510 : assert(BBPfarms[farmid].dirname != NULL);
1408 510 : assert(BBPfarms[farmid].lock_file == NULL);
1409 :
1410 510 : if ((gdklockpath = GDKfilepath(farmid, NULL, GDKLOCK, NULL)) == NULL) {
1411 : return GDK_FAIL;
1412 : }
1413 :
1414 : /*
1415 : * Obtain the global database lock.
1416 : */
1417 510 : if (MT_stat(BBPfarms[farmid].dirname, &st) < 0 &&
1418 0 : GDKcreatedir(gdklockpath) != GDK_SUCCEED) {
1419 0 : TRC_CRITICAL(GDK, "could not create %s\n",
1420 : BBPfarms[farmid].dirname);
1421 0 : GDKfree(gdklockpath);
1422 0 : return GDK_FAIL;
1423 : }
1424 510 : if ((fd = MT_lockf(gdklockpath, F_TLOCK)) < 0) {
1425 2 : TRC_CRITICAL(GDK, "Database lock '%s' denied\n",
1426 : gdklockpath);
1427 2 : GDKfree(gdklockpath);
1428 2 : return GDK_FAIL;
1429 : }
1430 :
1431 : /* now we have the lock on the database and are the only
1432 : * process allowed in this section */
1433 :
1434 508 : if ((GDKlockFile = fdopen(fd, "r+")) == NULL) {
1435 0 : GDKsyserror("Could not fdopen %s\n", gdklockpath);
1436 0 : close(fd);
1437 0 : GDKfree(gdklockpath);
1438 0 : return GDK_FAIL;
1439 : }
1440 :
1441 : /*
1442 : * Print the new process list in the global lock file.
1443 : */
1444 508 : if (fseek(GDKlockFile, 0, SEEK_SET) == -1) {
1445 0 : fclose(GDKlockFile);
1446 0 : TRC_CRITICAL(GDK, "Error while setting the file pointer on %s\n", gdklockpath);
1447 0 : GDKfree(gdklockpath);
1448 0 : return GDK_FAIL;
1449 : }
1450 508 : if (ftruncate(fileno(GDKlockFile), 0) < 0) {
1451 0 : fclose(GDKlockFile);
1452 0 : TRC_CRITICAL(GDK, "Could not truncate %s\n", gdklockpath);
1453 0 : GDKfree(gdklockpath);
1454 0 : return GDK_FAIL;
1455 : }
1456 508 : if (fflush(GDKlockFile) == EOF) {
1457 0 : fclose(GDKlockFile);
1458 0 : TRC_CRITICAL(GDK, "Could not flush %s\n", gdklockpath);
1459 0 : GDKfree(gdklockpath);
1460 0 : return GDK_FAIL;
1461 : }
1462 508 : GDKlog(GDKlockFile, GDKLOGON);
1463 508 : GDKfree(gdklockpath);
1464 508 : BBPfarms[farmid].lock_file = GDKlockFile;
1465 508 : return GDK_SUCCEED;
1466 : }
1467 :
1468 :
1469 : static void
1470 989 : GDKunlockHome(int farmid)
1471 : {
1472 989 : if (BBPfarms[farmid].lock_file) {
1473 506 : char *gdklockpath = GDKfilepath(farmid, NULL, GDKLOCK, NULL);
1474 506 : if (gdklockpath)
1475 506 : MT_lockf(gdklockpath, F_ULOCK);
1476 506 : fclose(BBPfarms[farmid].lock_file);
1477 506 : BBPfarms[farmid].lock_file = NULL;
1478 506 : GDKfree(gdklockpath);
1479 : }
1480 989 : }
1481 :
1482 : /*
1483 : * @+ Error handling
1484 : * Errors come in three flavors: warnings, non-fatal and fatal errors.
1485 : * A fatal error leaves a core dump behind after trying to safe the
1486 : * content of the relation. A non-fatal error returns a message to
1487 : * the user and aborts the current transaction. Fatal errors are also
1488 : * recorded on the system log for post-mortem analysis.
1489 : * In non-silent mode the errors are immediately sent to output, which
1490 : * makes it hard for upper layers to detect if an error was produced
1491 : * in the process. To facilitate such testing, a global error count is
1492 : * maintained on a thread basis, which can be read out by the function
1493 : * GDKerrorCount(); Furthermore, threads may have set their private
1494 : * error buffer.
1495 : */
1496 :
1497 : #define GDKERRLEN (1024+512)
1498 :
1499 : void
1500 7112853 : GDKclrerr(void)
1501 : {
1502 7112853 : char *buf;
1503 :
1504 7112853 : buf = GDKerrbuf;
1505 7112505 : if (buf)
1506 7061002 : *buf = 0;
1507 7112505 : }
1508 :
1509 : jmp_buf GDKfataljump;
1510 : str GDKfatalmsg;
1511 : bit GDKfataljumpenable = 0;
1512 :
1513 : /* coverity[+kill] */
1514 : void
1515 0 : GDKfatal(const char *format, ...)
1516 : {
1517 0 : char message[GDKERRLEN];
1518 0 : size_t len = strlen(GDKFATAL);
1519 0 : va_list ap;
1520 :
1521 0 : GDKtracer_set_component_level("io", "debug");
1522 : #ifndef NATIVE_WIN32
1523 0 : BATSIGinit();
1524 : #endif
1525 0 : if (!strncmp(format, GDKFATAL, len)) {
1526 : len = 0;
1527 : } else {
1528 0 : strcpy(message, GDKFATAL);
1529 : }
1530 0 : va_start(ap, format);
1531 0 : vsnprintf(message + len, sizeof(message) - (len + 2), format, ap);
1532 0 : va_end(ap);
1533 :
1534 : #ifndef __COVERITY__
1535 0 : if (GDKfataljumpenable) {
1536 : // in embedded mode, we really don't want to kill our host
1537 0 : GDKfatalmsg = GDKstrdup(message);
1538 0 : longjmp(GDKfataljump, 42);
1539 : } else
1540 : #endif
1541 : {
1542 0 : fputs(message, stderr);
1543 0 : fputs("\n", stderr);
1544 0 : fflush(stderr);
1545 :
1546 : /*
1547 : * Real errors should be saved in the log file for post-crash
1548 : * inspection.
1549 : */
1550 0 : if (GDKexiting()) {
1551 0 : fflush(stdout);
1552 0 : exit(1);
1553 : } else {
1554 0 : GDKlog(GET_GDKLOCK(PERSISTENT), "%s", message);
1555 : #ifdef COREDUMP
1556 : abort();
1557 : #else
1558 0 : exit(1);
1559 : #endif
1560 : }
1561 : }
1562 : }
1563 :
1564 :
1565 : lng
1566 151788042 : GDKusec(void)
1567 : {
1568 : /* Return the time in microseconds since an epoch. The epoch
1569 : * is currently midnight at the start of January 1, 1970, UTC. */
1570 : #if defined(NATIVE_WIN32)
1571 : FILETIME ft;
1572 : ULARGE_INTEGER f;
1573 : GetSystemTimeAsFileTime(&ft); /* time since Jan 1, 1601 */
1574 : f.LowPart = ft.dwLowDateTime;
1575 : f.HighPart = ft.dwHighDateTime;
1576 : /* there are 369 years, of which 89 are leap years from
1577 : * January 1, 1601 to January 1, 1970 which makes 134774 days;
1578 : * multiply that with the number of seconds in a day and the
1579 : * number of 100ns units in a second; subtract that from the
1580 : * value for the current time since January 1, 1601 to get the
1581 : * time since the Unix epoch */
1582 : f.QuadPart -= LL_CONSTANT(134774) * 24 * 60 * 60 * 10000000;
1583 : /* and convert to microseconds */
1584 : return (lng) (f.QuadPart / 10);
1585 : #elif defined(HAVE_CLOCK_GETTIME)
1586 151788042 : struct timespec ts;
1587 151788042 : (void) clock_gettime(CLOCK_REALTIME, &ts);
1588 151786179 : return (lng) (ts.tv_sec * LL_CONSTANT(1000000) + ts.tv_nsec / 1000);
1589 : #elif defined(HAVE_GETTIMEOFDAY)
1590 : struct timeval tv;
1591 : gettimeofday(&tv, NULL);
1592 : return (lng) (tv.tv_sec * LL_CONSTANT(1000000) + tv.tv_usec);
1593 : #elif defined(HAVE_FTIME)
1594 : struct timeb tb;
1595 : ftime(&tb);
1596 : return (lng) (tb.time * LL_CONSTANT(1000000) + tb.millitm * LL_CONSTANT(1000));
1597 : #else
1598 : /* last resort */
1599 : return (lng) (time(NULL) * LL_CONSTANT(1000000));
1600 : #endif
1601 : }
1602 :
1603 :
1604 : int
1605 0 : GDKms(void)
1606 : {
1607 : /* wraps around after a bit over 24 days */
1608 0 : return (int) ((GDKusec() - programepoch) / 1000);
1609 : }
1610 :
1611 :
1612 : /*
1613 : * @+ Logical Thread management
1614 : *
1615 : * All semaphores used by the application should be mentioned here.
1616 : * They are initialized during system initialization.
1617 : *
1618 : * The first action upon thread creation is to add it to the pool of
1619 : * known threads. This should be done by the thread itself.
1620 : * Note that the users should have gained exclusive access already. A
1621 : * new entry is initialized automatically when not found. Its file
1622 : * descriptors are the same as for the server and should be
1623 : * subsequently reset.
1624 : */
1625 : stream *GDKstdout;
1626 : stream *GDKstdin;
1627 :
1628 : static int
1629 338 : THRinit(void)
1630 : {
1631 338 : if ((GDKstdout = stdout_wastream()) == NULL) {
1632 0 : TRC_CRITICAL(GDK, "malloc for stdout failed\n");
1633 0 : return -1;
1634 : }
1635 338 : if ((GDKstdin = stdin_rastream()) == NULL) {
1636 0 : TRC_CRITICAL(GDK, "malloc for stdin failed\n");
1637 0 : mnstr_destroy(GDKstdout);
1638 0 : GDKstdout = NULL;
1639 0 : return -1;
1640 : }
1641 338 : struct freebats *t = MT_thread_getfreebats();
1642 338 : t->freebats = 0;
1643 338 : t->nfreebats = 0;
1644 338 : return 0;
1645 : }
1646 :
1647 : const char *
1648 325 : GDKversion(void)
1649 : {
1650 325 : return MONETDB_VERSION;
1651 : }
1652 :
1653 : const char *
1654 661 : GDKlibversion(void)
1655 : {
1656 661 : return GDK_VERSION;
1657 : }
1658 :
1659 : inline size_t
1660 151885167 : GDKmem_cursize(void)
1661 : {
1662 : /* RAM/swapmem that Monet is really using now */
1663 151885167 : return (size_t) ATOMIC_GET(&GDK_mallocedbytes_estimate);
1664 : }
1665 :
1666 : inline size_t
1667 144437630 : GDKvm_cursize(void)
1668 : {
1669 : /* current Monet VM address space usage */
1670 144437630 : return (size_t) ATOMIC_GET(&GDK_vm_cursize) + GDKmem_cursize();
1671 : }
1672 :
1673 : #define heapinc(_memdelta) \
1674 : ATOMIC_ADD(&GDK_mallocedbytes_estimate, _memdelta)
1675 : #ifndef NDEBUG
1676 : #define heapdec(_memdelta) \
1677 : do { \
1678 : ATOMIC_BASE_TYPE old = ATOMIC_SUB(&GDK_mallocedbytes_estimate, _memdelta); \
1679 : assert(old >= (ATOMIC_BASE_TYPE) _memdelta); \
1680 : } while (0)
1681 : #else
1682 : #define heapdec(_memdelta) \
1683 : ATOMIC_SUB(&GDK_mallocedbytes_estimate, _memdelta)
1684 : #endif
1685 :
1686 : #define meminc(vmdelta) \
1687 : ATOMIC_ADD(&GDK_vm_cursize, SEG_SIZE(vmdelta))
1688 : #ifndef NDEBUG
1689 : #define memdec(vmdelta) \
1690 : do { \
1691 : ssize_t diff = SEG_SIZE(vmdelta); \
1692 : ATOMIC_BASE_TYPE old = ATOMIC_SUB(&GDK_vm_cursize, diff); \
1693 : assert(old >= (ATOMIC_BASE_TYPE) diff); \
1694 : } while (0)
1695 : #else
1696 : #define memdec(vmdelta) \
1697 : ATOMIC_SUB(&GDK_vm_cursize, SEG_SIZE(vmdelta))
1698 : #endif
1699 :
1700 : /* Memory allocation
1701 : *
1702 : * The functions GDKmalloc, GDKzalloc, GDKrealloc, GDKstrdup, and
1703 : * GDKfree are used throughout to allocate and free memory. These
1704 : * functions are almost directly mapped onto the system
1705 : * malloc/realloc/free functions, but they give us some extra
1706 : * debugging hooks.
1707 : *
1708 : * When allocating memory, we allocate a bit more than was asked for.
1709 : * The extra space is added onto the front of the memory area that is
1710 : * returned, and in debug builds also some at the end. The area in
1711 : * front is used to store the actual size of the allocated area. The
1712 : * most important use is to be able to keep statistics on how much
1713 : * memory is being used. In debug builds, the size is also used to
1714 : * make sure that we don't write outside of the allocated arena. This
1715 : * is also where the extra space at the end comes in.
1716 : */
1717 :
1718 : /* we allocate extra space and return a pointer offset by this amount */
1719 : #define MALLOC_EXTRA_SPACE (2 * SIZEOF_VOID_P)
1720 :
1721 : #if defined(NDEBUG) || defined(SANITIZER)
1722 : #define DEBUG_SPACE 0
1723 : #else
1724 : #define DEBUG_SPACE 16
1725 : #endif
1726 :
1727 : /* malloc smaller than this aren't subject to the GDK_vm_maxsize test */
1728 : #define SMALL_MALLOC 256
1729 :
1730 : static void *
1731 177938815 : GDKmalloc_internal(size_t size, bool clear)
1732 : {
1733 177938815 : void *s;
1734 177938815 : size_t nsize;
1735 :
1736 177938815 : assert(size != 0);
1737 : #ifndef NDEBUG
1738 : /* fail malloc for testing purposes depending on set limit */
1739 177938815 : if (GDK_malloc_success_count > 0) {
1740 0 : MT_lock_set(&mallocsuccesslock);
1741 0 : if (GDK_malloc_success_count > 0)
1742 0 : GDK_malloc_success_count--;
1743 0 : MT_lock_unset(&mallocsuccesslock);
1744 : }
1745 177938815 : if (GDK_malloc_success_count == 0) {
1746 0 : GDKerror("allocation failed because of testing limit\n");
1747 0 : return NULL;
1748 : }
1749 : #endif
1750 : #ifndef SIZE_CHECK_IN_HEAPS_ONLY
1751 : if (size > SMALL_MALLOC &&
1752 : GDKvm_cursize() + size >= GDK_vm_maxsize &&
1753 : !MT_thread_override_limits()) {
1754 : GDKerror("allocating too much memory\n");
1755 : return NULL;
1756 : }
1757 : #endif
1758 :
1759 : /* pad to multiple of eight bytes and add some extra space to
1760 : * write real size in front; when debugging, also allocate
1761 : * extra space for check bytes */
1762 177938815 : nsize = (size + 7) & ~7;
1763 177938815 : if (clear)
1764 36018653 : s = calloc(nsize + MALLOC_EXTRA_SPACE + DEBUG_SPACE, 1);
1765 : else
1766 141920162 : s = malloc(nsize + MALLOC_EXTRA_SPACE + DEBUG_SPACE);
1767 177938815 : if (s == NULL) {
1768 0 : GDKsyserror("malloc failed; memory requested: %zu, memory in use: %zu, virtual memory in use: %zu\n", size, GDKmem_cursize(), GDKvm_cursize());;
1769 0 : return NULL;
1770 : }
1771 177938815 : s = (void *) ((char *) s + MALLOC_EXTRA_SPACE);
1772 :
1773 177938815 : heapinc(nsize + MALLOC_EXTRA_SPACE + DEBUG_SPACE);
1774 :
1775 : /* just before the pointer that we return, write how much we
1776 : * asked of malloc */
1777 177938815 : ((size_t *) s)[-1] = nsize + MALLOC_EXTRA_SPACE + DEBUG_SPACE;
1778 : #if !defined(NDEBUG) && !defined(SANITIZER)
1779 : /* just before that, write how much was asked of us */
1780 177938815 : ((size_t *) s)[-2] = size;
1781 : /* write pattern to help find out-of-bounds writes */
1782 177938815 : memset((char *) s + size, '\xBD', nsize + DEBUG_SPACE - size);
1783 : #endif
1784 177938815 : return s;
1785 : }
1786 :
1787 : #undef GDKmalloc
1788 : void *
1789 137254917 : GDKmalloc(size_t size)
1790 : {
1791 137254917 : void *s;
1792 :
1793 137254917 : if ((s = GDKmalloc_internal(size, false)) == NULL)
1794 : return NULL;
1795 : #if !defined(NDEBUG) && !defined(SANITIZER)
1796 : /* write a pattern to help make sure all data is properly
1797 : * initialized by the caller */
1798 137233030 : DEADBEEFCHK memset(s, '\xBD', size);
1799 : #endif
1800 : return s;
1801 : }
1802 :
1803 : #undef GDKzalloc
1804 : void *
1805 36019023 : GDKzalloc(size_t size)
1806 : {
1807 36019023 : return GDKmalloc_internal(size, true);
1808 : }
1809 :
1810 : #undef GDKstrdup
1811 : char *
1812 5274395 : GDKstrdup(const char *s)
1813 : {
1814 5274395 : size_t size;
1815 5274395 : char *p;
1816 :
1817 5274395 : if (s == NULL)
1818 : return NULL;
1819 4679126 : size = strlen(s) + 1;
1820 :
1821 4679126 : if ((p = GDKmalloc_internal(size, false)) == NULL)
1822 : return NULL;
1823 4679177 : memcpy(p, s, size); /* including terminating NULL byte */
1824 4679177 : return p;
1825 : }
1826 :
1827 : #undef GDKstrndup
1828 : char *
1829 126 : GDKstrndup(const char *s, size_t size)
1830 : {
1831 126 : char *p;
1832 :
1833 126 : if (s == NULL)
1834 : return NULL;
1835 126 : if ((p = GDKmalloc_internal(size + 1, false)) == NULL)
1836 : return NULL;
1837 126 : if (size > 0)
1838 126 : memcpy(p, s, size);
1839 126 : p[size] = '\0'; /* make sure it's NULL terminated */
1840 126 : return p;
1841 : }
1842 :
1843 : #undef GDKfree
1844 : void
1845 214380803 : GDKfree(void *s)
1846 : {
1847 214380803 : size_t asize;
1848 :
1849 214380803 : if (s == NULL)
1850 : return;
1851 :
1852 177611783 : asize = ((size_t *) s)[-1]; /* how much allocated last */
1853 :
1854 : #if !defined(NDEBUG) && !defined(SANITIZER)
1855 177611783 : size_t *p = s;
1856 177611783 : assert((asize & 2) == 0); /* check against duplicate free */
1857 177611783 : size_t size = p[-2];
1858 177611783 : assert(((size + 7) & ~7) + MALLOC_EXTRA_SPACE + DEBUG_SPACE == asize);
1859 : /* check for out-of-bounds writes */
1860 3355596202 : for (size_t i = size; i < asize - MALLOC_EXTRA_SPACE; i++)
1861 3177984419 : assert(((char *) s)[i] == '\xBD');
1862 177611783 : p[-1] |= 2; /* indicate area is freed */
1863 :
1864 : /* overwrite memory that is to be freed with a pattern that
1865 : * will help us recognize access to already freed memory in
1866 : * the debugger */
1867 177611783 : DEADBEEFCHK memset(s, '\xDB', asize - MALLOC_EXTRA_SPACE);
1868 : #endif
1869 :
1870 177611783 : free((char *) s - MALLOC_EXTRA_SPACE);
1871 177611783 : heapdec((ssize_t) asize);
1872 : }
1873 :
1874 : #undef GDKrealloc
1875 : void *
1876 206098 : GDKrealloc(void *s, size_t size)
1877 : {
1878 206098 : size_t nsize, asize;
1879 : #if !defined(NDEBUG) && !defined(SANITIZER)
1880 206098 : size_t osize;
1881 : #endif
1882 206098 : size_t *os = s;
1883 :
1884 206098 : assert(size != 0);
1885 :
1886 206098 : if (s == NULL)
1887 858 : return GDKmalloc(size);
1888 :
1889 205240 : nsize = (size + 7) & ~7;
1890 205240 : asize = os[-1]; /* how much allocated last */
1891 :
1892 : #ifndef SIZE_CHECK_IN_HEAPS_ONLY
1893 : if (size > SMALL_MALLOC &&
1894 : nsize > asize &&
1895 : GDKvm_cursize() + nsize - asize >= GDK_vm_maxsize &&
1896 : !MT_thread_override_limits()) {
1897 : GDKerror("allocating too much memory\n");
1898 : return NULL;
1899 : }
1900 : #endif
1901 : #if !defined(NDEBUG) && !defined(SANITIZER)
1902 205240 : assert((asize & 2) == 0); /* check against duplicate free */
1903 : /* check for out-of-bounds writes */
1904 205240 : osize = os[-2]; /* how much asked for last */
1905 205240 : assert(((osize + 7) & ~7) + MALLOC_EXTRA_SPACE + DEBUG_SPACE == asize);
1906 3539709 : for (size_t i = osize; i < asize - MALLOC_EXTRA_SPACE; i++)
1907 3334469 : assert(((char *) s)[i] == '\xBD');
1908 : /* if shrinking, write debug pattern into to-be-freed memory */
1909 205240 : DEADBEEFCHK if (size < osize)
1910 30354 : memset((char *) s + size, '\xDB', osize - size);
1911 205240 : os[-1] |= 2; /* indicate area is freed */
1912 : #endif
1913 205240 : s = realloc((char *) s - MALLOC_EXTRA_SPACE,
1914 : nsize + MALLOC_EXTRA_SPACE + DEBUG_SPACE);
1915 205240 : if (s == NULL) {
1916 : #if !defined(NDEBUG) && !defined(SANITIZER)
1917 0 : os[-1] &= ~2; /* not freed after all */
1918 0 : assert(os[-1] == asize);
1919 0 : assert(os[-2] == osize);
1920 : #endif
1921 0 : GDKsyserror("realloc failed; memory requested: %zu, memory in use: %zu, virtual memory in use: %zu\n", size, GDKmem_cursize(), GDKvm_cursize());;
1922 0 : return NULL;
1923 : }
1924 205240 : s = (void *) ((char *) s + MALLOC_EXTRA_SPACE);
1925 : /* just before the pointer that we return, write how much we
1926 : * asked of malloc */
1927 205240 : ((size_t *) s)[-1] = nsize + MALLOC_EXTRA_SPACE + DEBUG_SPACE;
1928 : #if !defined(NDEBUG) && !defined(SANITIZER)
1929 : /* just before that, write how much was asked of us */
1930 205240 : ((size_t *) s)[-2] = size;
1931 : /* if growing, initialize new memory with debug pattern */
1932 205240 : DEADBEEFCHK if (size > osize)
1933 174811 : memset((char *) s + osize, '\xBD', size - osize);
1934 : /* write pattern to help find out-of-bounds writes */
1935 205240 : memset((char *) s + size, '\xBD', nsize + DEBUG_SPACE - size);
1936 : #endif
1937 :
1938 205240 : heapinc(nsize + MALLOC_EXTRA_SPACE + DEBUG_SPACE);
1939 205240 : heapdec((ssize_t) asize);
1940 :
1941 : return s;
1942 : }
1943 :
1944 : /* return how much memory was allocated; the argument must be a value
1945 : * returned by GDKmalloc, GDKzalloc, GDKrealloc, GDKstrdup, or
1946 : * GDKstrndup */
1947 : size_t
1948 179628 : GDKmallocated(const void *s)
1949 : {
1950 179628 : return ((const size_t *) s)[-1]; /* how much allocated last */
1951 : }
1952 :
1953 : void
1954 39525 : GDKsetmallocsuccesscount(lng count)
1955 : {
1956 39525 : (void) count;
1957 : #ifndef NDEBUG
1958 39525 : GDK_malloc_success_count = count;
1959 : #endif
1960 39525 : }
1961 :
1962 : /*
1963 : * @- virtual memory
1964 : * allocations affect only the logical VM resources.
1965 : */
1966 : #undef GDKmmap
1967 : void *
1968 2554 : GDKmmap(const char *path, int mode, size_t len)
1969 : {
1970 2554 : void *ret;
1971 :
1972 : #ifndef SIZE_CHECK_IN_HEAPS_ONLY
1973 : if (GDKvm_cursize() + len >= GDK_vm_maxsize &&
1974 : !MT_thread_override_limits()) {
1975 : GDKerror("requested too much virtual memory; memory requested: %zu, memory in use: %zu, virtual memory in use: %zu\n", len, GDKmem_cursize(), GDKvm_cursize());
1976 : return NULL;
1977 : }
1978 : #endif
1979 2554 : ret = MT_mmap(path, mode, len);
1980 2554 : if (ret != NULL) {
1981 2554 : if (mode & MMAP_COPY)
1982 0 : heapinc(len);
1983 : else
1984 2554 : meminc(len);
1985 : } else
1986 0 : GDKerror("requesting virtual memory failed; memory requested: %zu, memory in use: %zu, virtual memory in use: %zu\n", len, GDKmem_cursize(), GDKvm_cursize());
1987 2554 : return ret;
1988 : }
1989 :
1990 : #undef GDKmunmap
1991 : gdk_return
1992 2554 : GDKmunmap(void *addr, int mode, size_t size)
1993 : {
1994 2554 : int ret;
1995 :
1996 2554 : ret = MT_munmap(addr, size);
1997 2554 : if (ret == 0) {
1998 2554 : if (mode & MMAP_COPY)
1999 0 : heapdec(size);
2000 : else
2001 2554 : memdec(size);
2002 : }
2003 2554 : return ret == 0 ? GDK_SUCCEED : GDK_FAIL;
2004 : }
2005 :
2006 : #undef GDKmremap
2007 : void *
2008 568 : GDKmremap(const char *path, int mode, void *old_address, size_t old_size, size_t *new_size)
2009 : {
2010 568 : void *ret;
2011 :
2012 : #ifndef SIZE_CHECK_IN_HEAPS_ONLY
2013 : if (*new_size > old_size &&
2014 : GDKvm_cursize() + *new_size - old_size >= GDK_vm_maxsize &&
2015 : !MT_thread_override_limits()) {
2016 : GDKerror("requested too much virtual memory; memory requested: %zu, memory in use: %zu, virtual memory in use: %zu\n", *new_size, GDKmem_cursize(), GDKvm_cursize());
2017 : return NULL;
2018 : }
2019 : #endif
2020 568 : ret = MT_mremap(path, mode, old_address, old_size, new_size);
2021 568 : if (ret != NULL) {
2022 568 : if (mode & MMAP_COPY) {
2023 0 : heapdec(old_size);
2024 0 : heapinc(*new_size);
2025 : } else {
2026 568 : memdec(old_size);
2027 568 : meminc(*new_size);
2028 : }
2029 : } else {
2030 0 : GDKerror("requesting virtual memory failed; memory requested: %zu, memory in use: %zu, virtual memory in use: %zu\n", *new_size, GDKmem_cursize(), GDKvm_cursize());
2031 : }
2032 568 : return ret;
2033 : }
2034 :
2035 : /* print some potentially interesting information */
2036 : struct prinfocb {
2037 : struct prinfocb *next;
2038 : void (*func)(void);
2039 : } *prinfocb;
2040 :
2041 : void
2042 670 : GDKprintinforegister(void (*func)(void))
2043 : {
2044 670 : struct prinfocb *p = GDKmalloc(sizeof(struct prinfocb));
2045 670 : if (p == NULL) {
2046 0 : GDKerror("cannot register USR1 printing function.\n");
2047 0 : return;
2048 : }
2049 670 : p->func = func;
2050 670 : p->next = NULL;
2051 670 : struct prinfocb **pp = &prinfocb;
2052 1224 : while (*pp != NULL)
2053 554 : pp = &(*pp)->next;
2054 670 : *pp = p;
2055 : }
2056 :
2057 : void
2058 114 : GDKprintinfo(void)
2059 : {
2060 114 : size_t allocated = (size_t) ATOMIC_GET(&GDK_mallocedbytes_estimate);
2061 114 : size_t vmallocated = (size_t) ATOMIC_GET(&GDK_vm_cursize);
2062 :
2063 114 : printf("SIGUSR1 info start\n");
2064 114 : printf("Virtual memory allocated: %zu, of which %zu with malloc\n",
2065 : vmallocated + allocated, allocated);
2066 114 : printf("gdk_vm_maxsize: %zu, gdk_mem_maxsize: %zu\n",
2067 : GDK_vm_maxsize, GDK_mem_maxsize);
2068 114 : printf("gdk_mmap_minsize_persistent %zu, gdk_mmap_minsize_transient %zu\n",
2069 : GDK_mmap_minsize_persistent, GDK_mmap_minsize_transient);
2070 : #ifdef __linux__
2071 114 : int fd = open("/proc/self/statm", O_RDONLY | O_CLOEXEC);
2072 114 : if (fd >= 0) {
2073 114 : char buf[512];
2074 114 : ssize_t s = read(fd, buf, sizeof(buf) - 1);
2075 114 : close(fd);
2076 114 : if (s > 0) {
2077 114 : assert((size_t) s < sizeof(buf));
2078 114 : size_t size, resident, shared;
2079 114 : buf[s] = 0;
2080 114 : if (sscanf(buf, "%zu %zu %zu", &size, &resident, &shared) == 3) {
2081 114 : size *= MT_pagesize();
2082 114 : resident *= MT_pagesize();
2083 114 : shared *= MT_pagesize();
2084 114 : printf("Virtual size: %zu, anonymous RSS: %zu, shared RSS: %zu (together: %zu)\n",
2085 : size, resident - shared, shared, resident);
2086 : }
2087 : }
2088 : }
2089 : #endif
2090 114 : BBPprintinfo();
2091 : #ifdef LOCK_STATS
2092 : GDKlockstatistics(3);
2093 : #endif
2094 114 : dump_threads();
2095 342 : for (struct prinfocb *p = prinfocb; p; p = p->next)
2096 228 : (*p->func)();
2097 114 : printf("SIGUSR1 info end\n");
2098 114 : }
|
