Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : /* Author(s) M.L. Kersten
14 : * The MAL Runtime Profiler and system queue
15 : * This little helper module is used to perform instruction based profiling.
16 : * The QRYqueue is only update at the start/finish of a query.
17 : * It is also the place to keep track on the number of workers
18 : * The current code relies on a scan rather than a hash.
19 : */
20 :
21 : #include "monetdb_config.h"
22 : #include "mal_utils.h"
23 : #include "mal_runtime.h"
24 : #include "mal_interpreter.h"
25 : #include "mal_function.h"
26 : #include "mal_profiler.h"
27 : #include "mal_listing.h"
28 : #include "mal_authorize.h"
29 : #include "mal_resource.h"
30 : #include "mal_internal.h"
31 : #include "mal_private.h"
32 :
33 :
34 : QueryQueue QRYqueue = NULL;
35 : size_t qsize = 0;
36 : static oid qtag = 1; // A unique query identifier
37 :
38 : UserStats USRstats = NULL;
39 : size_t usrstatscnt = 0;
40 :
41 : static inline void
42 23304 : clearUSRstats(size_t idx)
43 : {
44 23304 : USRstats[idx] = (struct USERSTAT) { 0 };
45 : }
46 :
47 : /*
48 : * Find the index of the given 'user' in USRstats.
49 : * For a new 'user' return a new free slot.
50 : * If USRstats is full, extend it.
51 : */
52 : static size_t
53 265062 : getUSRstatsIdx(MalBlkPtr mb, oid user)
54 : {
55 265062 : size_t i = 0;
56 265062 : UserStats tmp = NULL;
57 :
58 267188 : for (i = 0; i < usrstatscnt; i++)
59 : /* The array is dense, so we either find the user or an empty slot. */
60 267187 : if (USRstats[i].user == user || USRstats[i].username == NULL)
61 265061 : return i;
62 :
63 : /* expand USRstats */
64 1 : tmp = (UserStats) GDKrealloc(USRstats,
65 : sizeof(struct USERSTAT) *
66 : (size_t) (usrstatscnt += MAL_MAXCLIENTS));
67 1 : if (tmp == NULL) {
68 : /* It's not a fatal error if we can't extend USRstats.
69 : * We don't want to affect existing USRstats. */
70 0 : addMalException(mb, "getUSRstatsIdx" MAL_MALLOC_FAIL);
71 0 : return (size_t) -1;
72 : }
73 1 : USRstats = tmp;
74 5 : for (; i < usrstatscnt; i++)
75 4 : clearUSRstats(i);
76 1 : return usrstatscnt - MAL_MAXCLIENTS;
77 : }
78 :
79 : static void
80 581227 : updateUserStats(Client cntxt, MalBlkPtr mb, lng ticks, time_t started,
81 : time_t finished, str query)
82 : {
83 : // don't keep stats for context without username
84 581227 : if (cntxt->username == NULL)
85 : return;
86 :
87 265062 : size_t idx = getUSRstatsIdx(mb, cntxt->user);
88 :
89 265062 : if (idx == (size_t) -1) {
90 0 : addMalException(mb,
91 : "updateUserStats"
92 : "Failed to get an entry in user statistics");
93 0 : return;
94 : }
95 :
96 265062 : if (USRstats[idx].username == NULL || USRstats[idx].user != cntxt->user
97 264712 : || strcmp(USRstats[idx].username, cntxt->username) != 0) {
98 380 : GDKfree(USRstats[idx].username);
99 380 : GDKfree(USRstats[idx].maxquery);
100 380 : clearUSRstats(idx);
101 380 : USRstats[idx].user = cntxt->user;
102 380 : USRstats[idx].username = GDKstrdup(cntxt->username);
103 : }
104 265062 : USRstats[idx].querycount++;
105 265062 : USRstats[idx].totalticks += ticks;
106 265062 : if (ticks >= USRstats[idx].maxticks && query) {
107 1395 : USRstats[idx].started = started;
108 1395 : USRstats[idx].finished = finished;
109 1395 : USRstats[idx].maxticks = ticks;
110 1395 : GDKfree(USRstats[idx].maxquery);
111 1395 : USRstats[idx].maxquery = GDKstrdup(query);
112 : }
113 : }
114 :
115 : /*
116 : * Free up the whole USRstats before mserver5 exits.
117 : */
118 : static void
119 328 : dropUSRstats(void)
120 : {
121 328 : size_t i;
122 328 : MT_lock_set(&mal_delayLock);
123 23576 : for (i = 0; i < usrstatscnt; i++) {
124 22920 : GDKfree(USRstats[i].username);
125 22920 : GDKfree(USRstats[i].maxquery);
126 22920 : clearUSRstats(i);
127 : }
128 328 : GDKfree(USRstats);
129 328 : USRstats = NULL;
130 328 : usrstatscnt = 0;
131 328 : MT_lock_unset(&mal_delayLock);
132 328 : }
133 :
134 : static str
135 581227 : isaSQLquery(MalBlkPtr mb)
136 : {
137 581227 : if (mb) {
138 11917893 : for (int i = 1; i < mb->stop; i++) {
139 11591711 : InstrPtr p = getInstrPtr(mb, i);
140 11591711 : if (getModuleId(p) && idcmp(getModuleId(p), "querylog") == 0
141 255045 : && idcmp(getFunctionId(p), "define") == 0)
142 255045 : return getVarConstant(mb, getArg(p, 1)).val.sval;
143 : }
144 : }
145 : return NULL;
146 : }
147 :
148 : /*
149 : * Manage the runtime profiling information
150 : * It is organized as a circular buffer, head/tail.
151 : * Elements are removed from the buffer when it becomes full.
152 : * This way we keep the information a little longer for inspection.
153 : */
154 :
155 : /* clear the next entry for a new call unless it is a running query */
156 : static inline void
157 25228 : clearQRYqueue(size_t idx)
158 : {
159 25228 : QRYqueue[idx] = (struct QRYQUEUE) { 0 };
160 : }
161 :
162 : static void
163 328 : dropQRYqueue(void)
164 : {
165 328 : MT_lock_set(&mal_delayLock);
166 24400 : for (size_t i = 0; i < qsize; i++) {
167 24072 : GDKfree(QRYqueue[i].query);
168 24072 : GDKfree(QRYqueue[i].username);
169 24072 : clearQRYqueue(i);
170 : }
171 328 : GDKfree(QRYqueue);
172 328 : QRYqueue = NULL;
173 328 : qsize = 0;
174 328 : qtag = 1;
175 328 : MT_lock_unset(&mal_delayLock);
176 328 : }
177 :
178 : oid
179 255453 : runtimeProfileSetTag(Client cntxt)
180 : {
181 255453 : MT_lock_set(&mal_delayLock);
182 255453 : cntxt->curprg->def->tag = qtag++;
183 255453 : MT_lock_unset(&mal_delayLock);
184 :
185 255453 : return cntxt->curprg->def->tag;
186 : }
187 :
188 : /* At the start of every MAL block or SQL query */
189 : void
190 655774 : runtimeProfileInit(Client cntxt, MalBlkPtr mb, MalStkPtr stk)
191 : {
192 655774 : static size_t qlast = 0;
193 655774 : size_t i, paused = 0;
194 655774 : str q;
195 :
196 : /* Recursive calls don't change the query queue, but later we have to check
197 : how to stop/pause/resume queries doing recursive calls from multiple workers */
198 655774 : if (stk->up)
199 : return;
200 581227 : MT_lock_set(&mal_delayLock);
201 :
202 581227 : if (USRstats == NULL) {
203 329 : usrstatscnt = MAL_MAXCLIENTS;
204 329 : USRstats = (UserStats) GDKzalloc(sizeof(struct USERSTAT) * usrstatscnt);
205 329 : if (USRstats == NULL) {
206 0 : addMalException(mb, "runtimeProfileInit" MAL_MALLOC_FAIL);
207 0 : MT_lock_unset(&mal_delayLock);
208 0 : return;
209 : }
210 : }
211 :
212 581227 : if (QRYqueue == NULL) {
213 329 : QRYqueue = (QueryQueue) GDKzalloc(sizeof(struct QRYQUEUE) *
214 : (qsize = MAL_MAXCLIENTS));
215 :
216 329 : if (QRYqueue == NULL) {
217 0 : addMalException(mb, "runtimeProfileInit" MAL_MALLOC_FAIL);
218 0 : MT_lock_unset(&mal_delayLock);
219 0 : return;
220 : }
221 : }
222 51782407 : for (i = 0; i < qsize; i++) {
223 102109877 : paused += QRYqueue[i].status && (QRYqueue[i].status[0] == 'p' || QRYqueue[i].status[0] == 'r'); /* running, prepared or paused */
224 : }
225 581227 : if (qsize - paused < (size_t) MAL_MAXCLIENTS) {
226 19 : qsize += MAL_MAXCLIENTS;
227 19 : QueryQueue tmp;
228 19 : tmp = (QueryQueue) GDKrealloc(QRYqueue,
229 : sizeof(struct QRYQUEUE) * qsize);
230 19 : if (tmp == NULL) {
231 0 : addMalException(mb, "runtimeProfileInit" MAL_MALLOC_FAIL);
232 0 : qsize -= MAL_MAXCLIENTS; /* undo increment */
233 0 : MT_lock_unset(&mal_delayLock);
234 0 : return;
235 : }
236 19 : QRYqueue = tmp;
237 1175 : for (i = qsize - MAL_MAXCLIENTS; i < qsize; i++)
238 1156 : clearQRYqueue(i);
239 : }
240 : // add new invocation
241 581245 : for (i = 0; i < qsize; i++) {
242 581245 : size_t j = qlast;
243 581245 : if (++qlast >= qsize)
244 8063 : qlast = 0;
245 581245 : if (QRYqueue[j].stk == NULL ||
246 18 : QRYqueue[j].status == NULL ||
247 18 : (QRYqueue[j].status[0] != 'r' && QRYqueue[j].status[0] != 'p')) {
248 581227 : QRYqueue[j].mb = mb;
249 581227 : QRYqueue[j].tag = stk->tag = mb->tag;
250 581227 : QRYqueue[j].stk = stk; // for status pause 'p'/running '0'/ quitting 'q'
251 581227 : QRYqueue[j].finished = 0;
252 581227 : QRYqueue[j].start = time(0);
253 581227 : q = isaSQLquery(mb);
254 581227 : GDKfree(QRYqueue[j].query);
255 581227 : QRYqueue[j].query = GDKstrdup(q); /* NULL in, NULL out */
256 581227 : GDKfree(QRYqueue[j].username);
257 581227 : if (!GDKembedded())
258 579400 : QRYqueue[j].username = GDKstrdup(cntxt->username);
259 581227 : QRYqueue[j].idx = cntxt->idx;
260 : /* give the MB upperbound by addition of 1 MB */
261 581227 : QRYqueue[j].memory = 1 + (int) (stk->memory / LL_CONSTANT(1048576)); /* Convert to MB */
262 581227 : QRYqueue[j].workers = (int) 1; /* this is the first one */
263 581227 : QRYqueue[j].status = "running";
264 581227 : QRYqueue[j].cntxt = cntxt;
265 581227 : QRYqueue[j].ticks = GDKusec();
266 581227 : break;
267 : }
268 : }
269 581227 : MT_lock_unset(&mal_delayLock);
270 581227 : MT_lock_set(&mal_contextLock);
271 581227 : cntxt->idle = 0;
272 581227 : MT_lock_unset(&mal_contextLock);
273 : }
274 :
275 : /*
276 : * At the end of every MAL block or SQL query.
277 : *
278 : * Returning from a recursive call does not change the number of workers.
279 : */
280 : void
281 655774 : runtimeProfileFinish(Client cntxt, MalBlkPtr mb, MalStkPtr stk)
282 : {
283 655774 : size_t i;
284 655774 : bool found = false;
285 :
286 : /* Recursive calls don't change the query queue, but later we have to check
287 : how to stop/pause/resume queries doing recursive calls from multiple workers */
288 655774 : if (stk->up)
289 : return;
290 581227 : MT_lock_set(&mal_delayLock);
291 26020463 : for (i = 0; i < qsize; i++) {
292 25439236 : if (QRYqueue[i].stk == stk) {
293 581227 : if (QRYqueue[i].status[0] == 's')
294 1 : QRYqueue[i].status = "aborted";
295 : else
296 581226 : QRYqueue[i].status = "finished";
297 581227 : QRYqueue[i].finished = time(0);
298 581227 : QRYqueue[i].workers = (int) ATOMIC_GET(&mb->workers);
299 : /* give the MB upperbound by addition of 1 MB */
300 581227 : QRYqueue[i].memory = 1 + (int) (mb->memory / LL_CONSTANT(1048576));
301 581227 : QRYqueue[i].cntxt = NULL;
302 581227 : QRYqueue[i].stk = NULL;
303 581227 : QRYqueue[i].mb = NULL;
304 581227 : QRYqueue[i].ticks = GDKusec() - QRYqueue[i].ticks;
305 581227 : updateUserStats(cntxt, mb, QRYqueue[i].ticks, QRYqueue[i].start,
306 : QRYqueue[i].finished, QRYqueue[i].query);
307 : // assume that the user is now idle
308 581227 : MT_lock_unset(&mal_delayLock);
309 581227 : MT_lock_set(&mal_contextLock);
310 581227 : cntxt->idle = time(0);
311 581227 : MT_lock_unset(&mal_contextLock);
312 581227 : found = true;
313 581227 : break;
314 : }
315 : }
316 :
317 : // every query that has been started has an entry in QRYqueue. If this
318 : // finished query is not found, we want to print some informational
319 : // messages for debugging.
320 581227 : if (!found) {
321 0 : assert(0);
322 : TRC_INFO_IF(MAL_SERVER) {
323 : TRC_INFO_ENDIF(MAL_SERVER,
324 : "runtimeProfilerFinish: stk (%p) not found in QRYqueue",
325 : stk);
326 : for (i = 0; i < qsize; i++) {
327 : // print some info. of queries not "finished"
328 : if (strcmp(QRYqueue[i].status, "finished") != 0) {
329 : TRC_INFO_ENDIF(MAL_SERVER,
330 : "QRYqueue[%zu]: stk(%p), tag(" OIDFMT
331 : "), username(%s), start(%ld), status(%s), query(%s)",
332 : i, QRYqueue[i].stk, QRYqueue[i].tag,
333 : QRYqueue[i].username, QRYqueue[i].start,
334 : QRYqueue[i].status, QRYqueue[i].query);
335 : }
336 : }
337 : }
338 655774 : MT_lock_unset(&mal_delayLock);
339 : }
340 :
341 : }
342 :
343 : /* Used by mal_reset to do the grand final clean up of this area before MonetDB exits */
344 : void
345 328 : mal_runtime_reset(void)
346 : {
347 328 : dropQRYqueue();
348 328 : dropUSRstats();
349 328 : }
350 :
351 : /* At the start of each MAL stmt */
352 : void
353 50261087 : runtimeProfileBegin(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
354 : RuntimeProfile prof)
355 : {
356 50261087 : (void) cntxt;
357 50261087 : (void) mb;
358 50261087 : (void) stk;
359 50261087 : (void) pci;
360 :
361 50261087 : assert(pci);
362 : /* always collect the MAL instruction execution time */
363 50261087 : prof->ticks = GDKusec();
364 50260615 : }
365 :
366 : /* At the end of each MAL stmt */
367 : void
368 50310820 : runtimeProfileExit(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
369 : RuntimeProfile prof)
370 : {
371 50310820 : lng ticks = GDKusec();
372 :
373 50310579 : if (profilerStatus > 0)
374 45 : profilerEvent(&(struct MalEvent) { cntxt, mb, stk, pci, ticks,
375 45 : ticks - prof->ticks },
376 : NULL);
377 50310579 : if (cntxt->sqlprofiler)
378 843 : sqlProfilerEvent(cntxt, mb, stk, pci, ticks, ticks - prof->ticks);
379 50310579 : if (profilerStatus < 0) {
380 : /* delay profiling until you encounter start of MAL function */
381 0 : if (getInstrPtr(mb, 0) == pci)
382 0 : profilerStatus = 1;
383 : }
384 50310579 : }
385 :
386 : /*
387 : * For performance evaluation it is handy to estimate the
388 : * amount of bytes produced by an instruction.
389 : * The actual amount is harder to guess, because an instruction
390 : * may trigger a side effect, such as creating a hash-index.
391 : * Side effects are ignored.
392 : */
393 :
394 : lng
395 0 : getBatSpace(BAT *b)
396 : {
397 0 : lng space = 0;
398 0 : if (b == NULL)
399 : return 0;
400 0 : space += BATcount(b) << b->tshift;
401 0 : if (space) {
402 0 : MT_lock_set(&b->theaplock);
403 0 : if (b->tvheap)
404 0 : space += heapinfo(b->tvheap, b->batCacheid);
405 0 : MT_lock_unset(&b->theaplock);
406 0 : MT_rwlock_rdlock(&b->thashlock);
407 0 : space += hashinfo(b->thash, b->batCacheid);
408 0 : MT_rwlock_rdunlock(&b->thashlock);
409 : }
410 : return space;
411 : }
412 :
413 : lng
414 0 : getVolume(MalStkPtr stk, InstrPtr pci, int rd)
415 : {
416 0 : int i, limit;
417 0 : lng vol = 0;
418 0 : BAT *b;
419 :
420 0 : if (stk == NULL)
421 : return 0;
422 0 : limit = rd ? pci->argc : pci->retc;
423 0 : i = rd ? pci->retc : 0;
424 :
425 0 : for (; i < limit; i++) {
426 0 : if (stk->stk[getArg(pci, i)].bat) {
427 0 : oid cnt = 0;
428 :
429 0 : b = BBPquickdesc(stk->stk[getArg(pci, i)].val.bval);
430 0 : if (b == NULL)
431 0 : continue;
432 0 : cnt = BATcount(b);
433 : /* Usually reading views cost as much as full bats.
434 : But when we output a slice that is not the case. */
435 0 : if (rd)
436 0 : vol += (!isVIEW(b) && !VIEWtparent(b)) ? tailsize(b, cnt) : 0;
437 0 : else if (!isVIEW(b))
438 0 : vol += tailsize(b, cnt);
439 : }
440 : }
441 : return vol;
442 : }
|
