Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : /* Author(s) M.L. Kersten
14 : * The MAL Runtime Profiler and system queue
15 : * This little helper module is used to perform instruction based profiling.
16 : * The QRYqueue is only update at the start/finish of a query.
17 : * It is also the place to keep track on the number of workers
18 : * The current code relies on a scan rather than a hash.
19 : */
20 :
21 : #include "monetdb_config.h"
22 : #include "mal_utils.h"
23 : #include "mal_runtime.h"
24 : #include "mal_interpreter.h"
25 : #include "mal_function.h"
26 : #include "mal_profiler.h"
27 : #include "mal_listing.h"
28 : #include "mal_authorize.h"
29 : #include "mal_resource.h"
30 : #include "mal_internal.h"
31 : #include "mal_private.h"
32 :
33 :
34 : QueryQueue QRYqueue = NULL;
35 : size_t qsize = 0;
36 : static oid qtag = 1; // A unique query identifier
37 :
38 : UserStats USRstats = NULL;
39 : size_t usrstatscnt = 0;
40 :
41 : static inline void
42 21289 : clearUSRstats(size_t idx)
43 : {
44 21289 : USRstats[idx] = (struct USERSTAT) { 0 };
45 : }
46 :
47 : /*
48 : * Find the index of the given 'user' in USRstats.
49 : * For a new 'user' return a new free slot.
50 : * If USRstats is full, extend it.
51 : */
52 : static size_t
53 259707 : getUSRstatsIdx(MalBlkPtr mb, oid user)
54 : {
55 259707 : size_t i = 0;
56 259707 : UserStats tmp = NULL;
57 :
58 261833 : for (i = 0; i < usrstatscnt; i++)
59 : /* The array is dense, so we either find the user or an empty slot. */
60 261832 : if (USRstats[i].user == user || USRstats[i].username == NULL)
61 259706 : return i;
62 :
63 : /* expand USRstats */
64 1 : tmp = (UserStats) GDKrealloc(USRstats,
65 : sizeof(struct USERSTAT) *
66 : (size_t) (usrstatscnt += MAL_MAXCLIENTS));
67 1 : if (tmp == NULL) {
68 : /* It's not a fatal error if we can't extend USRstats.
69 : * We don't want to affect existing USRstats. */
70 0 : addMalException(mb, "getUSRstatsIdx" MAL_MALLOC_FAIL);
71 0 : return (size_t) -1;
72 : }
73 1 : USRstats = tmp;
74 5 : for (; i < usrstatscnt; i++)
75 4 : clearUSRstats(i);
76 1 : return usrstatscnt - MAL_MAXCLIENTS;
77 : }
78 :
79 : static void
80 531950 : updateUserStats(Client cntxt, MalBlkPtr mb, lng ticks, time_t started,
81 : time_t finished, str query)
82 : {
83 : // don't keep stats for context without username
84 531950 : if (cntxt->username == NULL)
85 : return;
86 :
87 259707 : size_t idx = getUSRstatsIdx(mb, cntxt->user);
88 :
89 259707 : if (idx == (size_t) -1) {
90 0 : addMalException(mb,
91 : "updateUserStats"
92 : "Failed to get an entry in user statistics");
93 0 : return;
94 : }
95 :
96 259707 : if (USRstats[idx].username == NULL || USRstats[idx].user != cntxt->user
97 259388 : || strcmp(USRstats[idx].username, cntxt->username) != 0) {
98 349 : GDKfree(USRstats[idx].username);
99 349 : GDKfree(USRstats[idx].maxquery);
100 349 : clearUSRstats(idx);
101 349 : USRstats[idx].user = cntxt->user;
102 349 : USRstats[idx].username = GDKstrdup(cntxt->username);
103 : }
104 259707 : USRstats[idx].querycount++;
105 259707 : USRstats[idx].totalticks += ticks;
106 259707 : if (ticks >= USRstats[idx].maxticks && query) {
107 1216 : USRstats[idx].started = started;
108 1216 : USRstats[idx].finished = finished;
109 1216 : USRstats[idx].maxticks = ticks;
110 1216 : GDKfree(USRstats[idx].maxquery);
111 1216 : USRstats[idx].maxquery = GDKstrdup(query);
112 : }
113 : }
114 :
115 : /*
116 : * Free up the whole USRstats before mserver5 exits.
117 : */
118 : static void
119 297 : dropUSRstats(void)
120 : {
121 297 : size_t i;
122 297 : MT_lock_set(&mal_delayLock);
123 21530 : for (i = 0; i < usrstatscnt; i++) {
124 20936 : GDKfree(USRstats[i].username);
125 20936 : GDKfree(USRstats[i].maxquery);
126 20936 : clearUSRstats(i);
127 : }
128 297 : GDKfree(USRstats);
129 297 : USRstats = NULL;
130 297 : usrstatscnt = 0;
131 297 : MT_lock_unset(&mal_delayLock);
132 297 : }
133 :
134 : static str
135 531950 : isaSQLquery(MalBlkPtr mb)
136 : {
137 531950 : if (mb) {
138 9886662 : for (int i = 1; i < mb->stop; i++) {
139 9604408 : InstrPtr p = getInstrPtr(mb, i);
140 9604408 : if (getModuleId(p) && idcmp(getModuleId(p), "querylog") == 0
141 249696 : && idcmp(getFunctionId(p), "define") == 0)
142 249696 : return getVarConstant(mb, getArg(p, 1)).val.sval;
143 : }
144 : }
145 : return NULL;
146 : }
147 :
148 : /*
149 : * Manage the runtime profiling information
150 : * It is organized as a circular buffer, head/tail.
151 : * Elements are removed from the buffer when it becomes full.
152 : * This way we keep the information a little longer for inspection.
153 : */
154 :
155 : /* clear the next entry for a new call unless it is a running query */
156 : static inline void
157 23244 : clearQRYqueue(size_t idx)
158 : {
159 23244 : QRYqueue[idx] = (struct QRYQUEUE) { 0 };
160 : }
161 :
162 : static void
163 297 : dropQRYqueue(void)
164 : {
165 297 : MT_lock_set(&mal_delayLock);
166 22385 : for (size_t i = 0; i < qsize; i++) {
167 22088 : GDKfree(QRYqueue[i].query);
168 22088 : GDKfree(QRYqueue[i].username);
169 22088 : clearQRYqueue(i);
170 : }
171 297 : GDKfree(QRYqueue);
172 297 : QRYqueue = NULL;
173 297 : qsize = 0;
174 297 : qtag = 1;
175 297 : MT_lock_unset(&mal_delayLock);
176 297 : }
177 :
178 : oid
179 249436 : runtimeProfileSetTag(Client cntxt)
180 : {
181 249436 : MT_lock_set(&mal_delayLock);
182 250103 : cntxt->curprg->def->tag = qtag++;
183 250103 : MT_lock_unset(&mal_delayLock);
184 :
185 250102 : return cntxt->curprg->def->tag;
186 : }
187 :
188 : /* At the start of every MAL block or SQL query */
189 : void
190 702765 : runtimeProfileInit(Client cntxt, MalBlkPtr mb, MalStkPtr stk)
191 : {
192 702765 : static size_t qlast = 0;
193 702765 : size_t i, paused = 0;
194 702765 : str q;
195 :
196 : /* Recursive calls don't change the query queue, but later we have to check
197 : how to stop/pause/resume queries doing recursive calls from multiple workers */
198 702765 : if (stk->up)
199 : return;
200 531913 : MT_lock_set(&mal_delayLock);
201 :
202 531950 : if (USRstats == NULL) {
203 314 : usrstatscnt = MAL_MAXCLIENTS;
204 314 : USRstats = (UserStats) GDKzalloc(sizeof(struct USERSTAT) * usrstatscnt);
205 314 : if (USRstats == NULL) {
206 0 : addMalException(mb, "runtimeProfileInit" MAL_MALLOC_FAIL);
207 0 : MT_lock_unset(&mal_delayLock);
208 0 : return;
209 : }
210 : }
211 :
212 531950 : if (QRYqueue == NULL) {
213 314 : QRYqueue = (QueryQueue) GDKzalloc(sizeof(struct QRYQUEUE) *
214 : (qsize = MAL_MAXCLIENTS));
215 :
216 314 : if (QRYqueue == NULL) {
217 0 : addMalException(mb, "runtimeProfileInit" MAL_MALLOC_FAIL);
218 0 : MT_lock_unset(&mal_delayLock);
219 0 : return;
220 : }
221 : }
222 48655874 : for (i = 0; i < qsize; i++) {
223 95922907 : paused += QRYqueue[i].status && (QRYqueue[i].status[0] == 'p' || QRYqueue[i].status[0] == 'r'); /* running, prepared or paused */
224 : }
225 531950 : if (qsize - paused < (size_t) MAL_MAXCLIENTS) {
226 19 : qsize += MAL_MAXCLIENTS;
227 19 : QueryQueue tmp;
228 19 : tmp = (QueryQueue) GDKrealloc(QRYqueue,
229 : sizeof(struct QRYQUEUE) * qsize);
230 19 : if (tmp == NULL) {
231 0 : addMalException(mb, "runtimeProfileInit" MAL_MALLOC_FAIL);
232 0 : qsize -= MAL_MAXCLIENTS; /* undo increment */
233 0 : MT_lock_unset(&mal_delayLock);
234 0 : return;
235 : }
236 19 : QRYqueue = tmp;
237 1175 : for (i = qsize - MAL_MAXCLIENTS; i < qsize; i++)
238 1156 : clearQRYqueue(i);
239 : }
240 : // add new invocation
241 531968 : for (i = 0; i < qsize; i++) {
242 531968 : size_t j = qlast;
243 531968 : if (++qlast >= qsize)
244 7297 : qlast = 0;
245 531968 : if (QRYqueue[j].stk == NULL ||
246 18 : QRYqueue[j].status == NULL ||
247 18 : (QRYqueue[j].status[0] != 'r' && QRYqueue[j].status[0] != 'p')) {
248 531950 : QRYqueue[j].mb = mb;
249 531950 : QRYqueue[j].tag = stk->tag = mb->tag;
250 531950 : QRYqueue[j].stk = stk; // for status pause 'p'/running '0'/ quitting 'q'
251 531950 : QRYqueue[j].finished = 0;
252 531950 : QRYqueue[j].start = time(0);
253 531950 : q = isaSQLquery(mb);
254 531950 : GDKfree(QRYqueue[j].query);
255 531950 : QRYqueue[j].query = GDKstrdup(q); /* NULL in, NULL out */
256 531950 : GDKfree(QRYqueue[j].username);
257 531950 : if (!GDKembedded())
258 530123 : QRYqueue[j].username = GDKstrdup(cntxt->username);
259 531950 : QRYqueue[j].idx = cntxt->idx;
260 : /* give the MB upperbound by addition of 1 MB */
261 531950 : QRYqueue[j].memory = 1 + (int) (stk->memory / LL_CONSTANT(1048576)); /* Convert to MB */
262 531950 : QRYqueue[j].workers = (int) 1; /* this is the first one */
263 531950 : QRYqueue[j].status = "running";
264 531950 : QRYqueue[j].cntxt = cntxt;
265 531950 : QRYqueue[j].ticks = GDKusec();
266 531950 : break;
267 : }
268 : }
269 531950 : MT_lock_unset(&mal_delayLock);
270 531948 : MT_lock_set(&mal_contextLock);
271 531950 : cntxt->idle = 0;
272 531950 : MT_lock_unset(&mal_contextLock);
273 : }
274 :
275 : /*
276 : * At the end of every MAL block or SQL query.
277 : *
278 : * Returning from a recursive call does not change the number of workers.
279 : */
280 : void
281 702177 : runtimeProfileFinish(Client cntxt, MalBlkPtr mb, MalStkPtr stk)
282 : {
283 702177 : size_t i;
284 702177 : bool found = false;
285 :
286 : /* Recursive calls don't change the query queue, but later we have to check
287 : how to stop/pause/resume queries doing recursive calls from multiple workers */
288 702177 : if (stk->up)
289 : return;
290 531578 : MT_lock_set(&mal_delayLock);
291 24388327 : for (i = 0; i < qsize; i++) {
292 23856377 : if (QRYqueue[i].stk == stk) {
293 531950 : if (QRYqueue[i].status[0] == 's')
294 1 : QRYqueue[i].status = "aborted";
295 : else
296 531949 : QRYqueue[i].status = "finished";
297 531950 : QRYqueue[i].finished = time(0);
298 531950 : QRYqueue[i].workers = (int) ATOMIC_GET(&mb->workers);
299 : /* give the MB upperbound by addition of 1 MB */
300 531950 : QRYqueue[i].memory = 1 + (int) (mb->memory / LL_CONSTANT(1048576));
301 531950 : QRYqueue[i].cntxt = NULL;
302 531950 : QRYqueue[i].stk = NULL;
303 531950 : QRYqueue[i].mb = NULL;
304 531950 : QRYqueue[i].ticks = GDKusec() - QRYqueue[i].ticks;
305 531950 : updateUserStats(cntxt, mb, QRYqueue[i].ticks, QRYqueue[i].start,
306 : QRYqueue[i].finished, QRYqueue[i].query);
307 : // assume that the user is now idle
308 531950 : MT_lock_unset(&mal_delayLock);
309 531949 : MT_lock_set(&mal_contextLock);
310 531950 : cntxt->idle = time(0);
311 531950 : MT_lock_unset(&mal_contextLock);
312 531950 : found = true;
313 531950 : break;
314 : }
315 : }
316 :
317 : // every query that has been started has an entry in QRYqueue. If this
318 : // finished query is not found, we want to print some informational
319 : // messages for debugging.
320 531950 : if (!found) {
321 0 : assert(0);
322 : TRC_INFO_IF(MAL_SERVER) {
323 : TRC_INFO_ENDIF(MAL_SERVER,
324 : "runtimeProfilerFinish: stk (%p) not found in QRYqueue",
325 : stk);
326 : for (i = 0; i < qsize; i++) {
327 : // print some info. of queries not "finished"
328 : if (strcmp(QRYqueue[i].status, "finished") != 0) {
329 : TRC_INFO_ENDIF(MAL_SERVER,
330 : "QRYqueue[%zu]: stk(%p), tag(" OIDFMT
331 : "), username(%s), start(%ld), status(%s), query(%s)",
332 : i, QRYqueue[i].stk, QRYqueue[i].tag,
333 : QRYqueue[i].username, QRYqueue[i].start,
334 : QRYqueue[i].status, QRYqueue[i].query);
335 : }
336 : }
337 : }
338 702549 : MT_lock_unset(&mal_delayLock);
339 : }
340 :
341 : }
342 :
343 : /* Used by mal_reset to do the grand final clean up of this area before MonetDB exits */
344 : void
345 297 : mal_runtime_reset(void)
346 : {
347 297 : dropQRYqueue();
348 297 : dropUSRstats();
349 297 : }
350 :
351 : /* At the start of each MAL stmt */
352 : void
353 65686811 : runtimeProfileBegin(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
354 : RuntimeProfile prof)
355 : {
356 65686811 : (void) cntxt;
357 65686811 : (void) mb;
358 65686811 : (void) stk;
359 65686811 : (void) pci;
360 :
361 65686811 : assert(pci);
362 : /* always collect the MAL instruction execution time */
363 65686811 : prof->ticks = GDKusec();
364 65707862 : }
365 :
366 : /* At the end of each MAL stmt */
367 : void
368 65738326 : runtimeProfileExit(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
369 : RuntimeProfile prof)
370 : {
371 65738326 : lng ticks = GDKusec();
372 :
373 65788788 : if (profilerStatus > 0)
374 45 : profilerEvent(&(struct MalEvent) { cntxt, mb, stk, pci, ticks,
375 45 : ticks - prof->ticks },
376 : NULL);
377 65788788 : if (cntxt->sqlprofiler)
378 939 : sqlProfilerEvent(cntxt, mb, stk, pci, ticks, ticks - prof->ticks);
379 65788792 : if (profilerStatus < 0) {
380 : /* delay profiling until you encounter start of MAL function */
381 0 : if (getInstrPtr(mb, 0) == pci)
382 0 : profilerStatus = 1;
383 : }
384 65788792 : }
385 :
386 : /*
387 : * For performance evaluation it is handy to estimate the
388 : * amount of bytes produced by an instruction.
389 : * The actual amount is harder to guess, because an instruction
390 : * may trigger a side effect, such as creating a hash-index.
391 : * Side effects are ignored.
392 : */
393 :
394 : lng
395 0 : getBatSpace(BAT *b)
396 : {
397 0 : lng space = 0;
398 0 : if (b == NULL)
399 : return 0;
400 0 : space += BATcount(b) << b->tshift;
401 0 : if (space) {
402 0 : MT_lock_set(&b->theaplock);
403 0 : if (b->tvheap)
404 0 : space += heapinfo(b->tvheap, b->batCacheid);
405 0 : MT_lock_unset(&b->theaplock);
406 0 : MT_rwlock_rdlock(&b->thashlock);
407 0 : space += hashinfo(b->thash, b->batCacheid);
408 0 : MT_rwlock_rdunlock(&b->thashlock);
409 : }
410 : return space;
411 : }
412 :
413 : lng
414 0 : getVolume(MalStkPtr stk, InstrPtr pci, int rd)
415 : {
416 0 : int i, limit;
417 0 : lng vol = 0;
418 0 : BAT *b;
419 :
420 0 : if (stk == NULL)
421 : return 0;
422 0 : limit = rd ? pci->argc : pci->retc;
423 0 : i = rd ? pci->retc : 0;
424 :
425 0 : for (; i < limit; i++) {
426 0 : if (stk->stk[getArg(pci, i)].bat) {
427 0 : oid cnt = 0;
428 :
429 0 : b = BBPquickdesc(stk->stk[getArg(pci, i)].val.bval);
430 0 : if (b == NULL)
431 0 : continue;
432 0 : cnt = BATcount(b);
433 : /* Usually reading views cost as much as full bats.
434 : But when we output a slice that is not the case. */
435 0 : if (rd)
436 0 : vol += (!isVIEW(b) && !VIEWtparent(b)) ? tailsize(b, cnt) : 0;
437 0 : else if (!isVIEW(b))
438 0 : vol += tailsize(b, cnt);
439 : }
440 : }
441 : return vol;
442 : }
|
