Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : /* Author(s) M.L. Kersten
14 : * The MAL Runtime Profiler and system queue
15 : * This little helper module is used to perform instruction based profiling.
16 : * The QRYqueue is only update at the start/finish of a query.
17 : * It is also the place to keep track on the number of workers
18 : * The current code relies on a scan rather than a hash.
19 : */
20 :
21 : #include "monetdb_config.h"
22 : #include "mal_utils.h"
23 : #include "mal_runtime.h"
24 : #include "mal_interpreter.h"
25 : #include "mal_function.h"
26 : #include "mal_profiler.h"
27 : #include "mal_listing.h"
28 : #include "mal_authorize.h"
29 : #include "mal_resource.h"
30 : #include "mal_internal.h"
31 : #include "mal_private.h"
32 :
33 :
34 : QueryQueue QRYqueue = NULL;
35 : size_t qsize = 0;
36 : static oid qtag = 1; // A unique query identifier
37 :
38 : UserStats USRstats = NULL;
39 : size_t usrstatscnt = 0;
40 :
41 : static inline void
42 23109 : clearUSRstats(size_t idx)
43 : {
44 23109 : USRstats[idx] = (struct USERSTAT) { 0 };
45 : }
46 :
47 : /*
48 : * Find the index of the given 'user' in USRstats.
49 : * For a new 'user' return a new free slot.
50 : * If USRstats is full, extend it.
51 : */
52 : static size_t
53 264951 : getUSRstatsIdx(MalBlkPtr mb, oid user)
54 : {
55 264951 : size_t i = 0;
56 264951 : UserStats tmp = NULL;
57 :
58 267077 : for (i = 0; i < usrstatscnt; i++)
59 : /* The array is dense, so we either find the user or an empty slot. */
60 267076 : if (USRstats[i].user == user || USRstats[i].username == NULL)
61 264950 : return i;
62 :
63 : /* expand USRstats */
64 1 : tmp = (UserStats) GDKrealloc(USRstats,
65 : sizeof(struct USERSTAT) *
66 : (size_t) (usrstatscnt += MAL_MAXCLIENTS));
67 1 : if (tmp == NULL) {
68 : /* It's not a fatal error if we can't extend USRstats.
69 : * We don't want to affect existing USRstats. */
70 0 : addMalException(mb, "getUSRstatsIdx" MAL_MALLOC_FAIL);
71 0 : return (size_t) -1;
72 : }
73 1 : USRstats = tmp;
74 5 : for (; i < usrstatscnt; i++)
75 4 : clearUSRstats(i);
76 1 : return usrstatscnt - MAL_MAXCLIENTS;
77 : }
78 :
79 : static void
80 569157 : updateUserStats(Client cntxt, MalBlkPtr mb, lng ticks, time_t started,
81 : time_t finished, str query)
82 : {
83 : // don't keep stats for context without username
84 569157 : if (cntxt->username == NULL)
85 : return;
86 :
87 264951 : size_t idx = getUSRstatsIdx(mb, cntxt->user);
88 :
89 264951 : if (idx == (size_t) -1) {
90 0 : addMalException(mb,
91 : "updateUserStats"
92 : "Failed to get an entry in user statistics");
93 0 : return;
94 : }
95 :
96 264951 : if (USRstats[idx].username == NULL || USRstats[idx].user != cntxt->user
97 264604 : || strcmp(USRstats[idx].username, cntxt->username) != 0) {
98 377 : GDKfree(USRstats[idx].username);
99 377 : GDKfree(USRstats[idx].maxquery);
100 377 : clearUSRstats(idx);
101 377 : USRstats[idx].user = cntxt->user;
102 377 : USRstats[idx].username = GDKstrdup(cntxt->username);
103 : }
104 264951 : USRstats[idx].querycount++;
105 264951 : USRstats[idx].totalticks += ticks;
106 264951 : if (ticks >= USRstats[idx].maxticks && query) {
107 1406 : USRstats[idx].started = started;
108 1406 : USRstats[idx].finished = finished;
109 1406 : USRstats[idx].maxticks = ticks;
110 1406 : GDKfree(USRstats[idx].maxquery);
111 1406 : USRstats[idx].maxquery = GDKstrdup(query);
112 : }
113 : }
114 :
115 : /*
116 : * Free up the whole USRstats before mserver5 exits.
117 : */
118 : static void
119 325 : dropUSRstats(void)
120 : {
121 325 : size_t i;
122 325 : MT_lock_set(&mal_delayLock);
123 23378 : for (i = 0; i < usrstatscnt; i++) {
124 22728 : GDKfree(USRstats[i].username);
125 22728 : GDKfree(USRstats[i].maxquery);
126 22728 : clearUSRstats(i);
127 : }
128 325 : GDKfree(USRstats);
129 325 : USRstats = NULL;
130 325 : usrstatscnt = 0;
131 325 : MT_lock_unset(&mal_delayLock);
132 325 : }
133 :
134 : static str
135 569157 : isaSQLquery(MalBlkPtr mb)
136 : {
137 569157 : if (mb) {
138 11358144 : for (int i = 1; i < mb->stop; i++) {
139 11043902 : InstrPtr p = getInstrPtr(mb, i);
140 11043902 : if (getModuleId(p) && idcmp(getModuleId(p), "querylog") == 0
141 254915 : && idcmp(getFunctionId(p), "define") == 0)
142 254915 : return getVarConstant(mb, getArg(p, 1)).val.sval;
143 : }
144 : }
145 : return NULL;
146 : }
147 :
148 : /*
149 : * Manage the runtime profiling information
150 : * It is organized as a circular buffer, head/tail.
151 : * Elements are removed from the buffer when it becomes full.
152 : * This way we keep the information a little longer for inspection.
153 : */
154 :
155 : /* clear the next entry for a new call unless it is a running query */
156 : static inline void
157 25164 : clearQRYqueue(size_t idx)
158 : {
159 25164 : QRYqueue[idx] = (struct QRYQUEUE) { 0 };
160 : }
161 :
162 : static void
163 325 : dropQRYqueue(void)
164 : {
165 325 : MT_lock_set(&mal_delayLock);
166 24269 : for (size_t i = 0; i < qsize; i++) {
167 23944 : GDKfree(QRYqueue[i].query);
168 23944 : GDKfree(QRYqueue[i].username);
169 23944 : clearQRYqueue(i);
170 : }
171 325 : GDKfree(QRYqueue);
172 325 : QRYqueue = NULL;
173 325 : qsize = 0;
174 325 : qtag = 1;
175 325 : MT_lock_unset(&mal_delayLock);
176 325 : }
177 :
178 : oid
179 255323 : runtimeProfileSetTag(Client cntxt)
180 : {
181 255323 : MT_lock_set(&mal_delayLock);
182 255323 : cntxt->curprg->def->tag = qtag++;
183 255323 : MT_lock_unset(&mal_delayLock);
184 :
185 255323 : return cntxt->curprg->def->tag;
186 : }
187 :
188 : /* At the start of every MAL block or SQL query */
189 : void
190 643440 : runtimeProfileInit(Client cntxt, MalBlkPtr mb, MalStkPtr stk)
191 : {
192 643440 : static size_t qlast = 0;
193 643440 : size_t i, paused = 0;
194 643440 : str q;
195 :
196 : /* Recursive calls don't change the query queue, but later we have to check
197 : how to stop/pause/resume queries doing recursive calls from multiple workers */
198 643440 : if (stk->up)
199 : return;
200 569157 : MT_lock_set(&mal_delayLock);
201 :
202 569157 : if (USRstats == NULL) {
203 326 : usrstatscnt = MAL_MAXCLIENTS;
204 326 : USRstats = (UserStats) GDKzalloc(sizeof(struct USERSTAT) * usrstatscnt);
205 326 : if (USRstats == NULL) {
206 0 : addMalException(mb, "runtimeProfileInit" MAL_MALLOC_FAIL);
207 0 : MT_lock_unset(&mal_delayLock);
208 0 : return;
209 : }
210 : }
211 :
212 569157 : if (QRYqueue == NULL) {
213 326 : QRYqueue = (QueryQueue) GDKzalloc(sizeof(struct QRYQUEUE) *
214 : (qsize = MAL_MAXCLIENTS));
215 :
216 326 : if (QRYqueue == NULL) {
217 0 : addMalException(mb, "runtimeProfileInit" MAL_MALLOC_FAIL);
218 0 : MT_lock_unset(&mal_delayLock);
219 0 : return;
220 : }
221 : }
222 51076045 : for (i = 0; i < qsize; i++) {
223 100711071 : paused += QRYqueue[i].status && (QRYqueue[i].status[0] == 'p' || QRYqueue[i].status[0] == 'r'); /* running, prepared or paused */
224 : }
225 569157 : if (qsize - paused < (size_t) MAL_MAXCLIENTS) {
226 20 : qsize += MAL_MAXCLIENTS;
227 20 : QueryQueue tmp;
228 20 : tmp = (QueryQueue) GDKrealloc(QRYqueue,
229 : sizeof(struct QRYQUEUE) * qsize);
230 20 : if (tmp == NULL) {
231 0 : addMalException(mb, "runtimeProfileInit" MAL_MALLOC_FAIL);
232 0 : qsize -= MAL_MAXCLIENTS; /* undo increment */
233 0 : MT_lock_unset(&mal_delayLock);
234 0 : return;
235 : }
236 20 : QRYqueue = tmp;
237 1240 : for (i = qsize - MAL_MAXCLIENTS; i < qsize; i++)
238 1220 : clearQRYqueue(i);
239 : }
240 : // add new invocation
241 569175 : for (i = 0; i < qsize; i++) {
242 569175 : size_t j = qlast;
243 569175 : if (++qlast >= qsize)
244 7856 : qlast = 0;
245 569175 : if (QRYqueue[j].stk == NULL ||
246 18 : QRYqueue[j].status == NULL ||
247 18 : (QRYqueue[j].status[0] != 'r' && QRYqueue[j].status[0] != 'p')) {
248 569157 : QRYqueue[j].mb = mb;
249 569157 : QRYqueue[j].tag = stk->tag = mb->tag;
250 569157 : QRYqueue[j].stk = stk; // for status pause 'p'/running '0'/ quitting 'q'
251 569157 : QRYqueue[j].finished = 0;
252 569157 : QRYqueue[j].start = time(0);
253 569157 : q = isaSQLquery(mb);
254 569157 : GDKfree(QRYqueue[j].query);
255 569157 : QRYqueue[j].query = GDKstrdup(q); /* NULL in, NULL out */
256 569157 : GDKfree(QRYqueue[j].username);
257 569157 : if (!GDKembedded())
258 567340 : QRYqueue[j].username = GDKstrdup(cntxt->username);
259 569157 : QRYqueue[j].idx = cntxt->idx;
260 : /* give the MB upperbound by addition of 1 MB */
261 569157 : QRYqueue[j].memory = 1 + (int) (stk->memory / LL_CONSTANT(1048576)); /* Convert to MB */
262 569157 : QRYqueue[j].workers = (int) 1; /* this is the first one */
263 569157 : QRYqueue[j].status = "running";
264 569157 : QRYqueue[j].cntxt = cntxt;
265 569157 : QRYqueue[j].ticks = GDKusec();
266 569157 : break;
267 : }
268 : }
269 569157 : MT_lock_unset(&mal_delayLock);
270 569157 : MT_lock_set(&mal_contextLock);
271 569157 : cntxt->idle = 0;
272 569157 : MT_lock_unset(&mal_contextLock);
273 : }
274 :
275 : /*
276 : * At the end of every MAL block or SQL query.
277 : *
278 : * Returning from a recursive call does not change the number of workers.
279 : */
280 : void
281 643440 : runtimeProfileFinish(Client cntxt, MalBlkPtr mb, MalStkPtr stk)
282 : {
283 643440 : size_t i;
284 643440 : bool found = false;
285 :
286 : /* Recursive calls don't change the query queue, but later we have to check
287 : how to stop/pause/resume queries doing recursive calls from multiple workers */
288 643440 : if (stk->up)
289 : return;
290 569157 : MT_lock_set(&mal_delayLock);
291 25657338 : for (i = 0; i < qsize; i++) {
292 25088181 : if (QRYqueue[i].stk == stk) {
293 569157 : if (QRYqueue[i].status[0] == 's')
294 1 : QRYqueue[i].status = "aborted";
295 : else
296 569156 : QRYqueue[i].status = "finished";
297 569157 : QRYqueue[i].finished = time(0);
298 569157 : QRYqueue[i].workers = (int) ATOMIC_GET(&mb->workers);
299 : /* give the MB upperbound by addition of 1 MB */
300 569157 : QRYqueue[i].memory = 1 + (int) (mb->memory / LL_CONSTANT(1048576));
301 569157 : QRYqueue[i].cntxt = NULL;
302 569157 : QRYqueue[i].stk = NULL;
303 569157 : QRYqueue[i].mb = NULL;
304 569157 : QRYqueue[i].ticks = GDKusec() - QRYqueue[i].ticks;
305 569157 : updateUserStats(cntxt, mb, QRYqueue[i].ticks, QRYqueue[i].start,
306 : QRYqueue[i].finished, QRYqueue[i].query);
307 : // assume that the user is now idle
308 569157 : MT_lock_unset(&mal_delayLock);
309 569157 : MT_lock_set(&mal_contextLock);
310 569157 : cntxt->idle = time(0);
311 569157 : MT_lock_unset(&mal_contextLock);
312 569157 : found = true;
313 569157 : break;
314 : }
315 : }
316 :
317 : // every query that has been started has an entry in QRYqueue. If this
318 : // finished query is not found, we want to print some informational
319 : // messages for debugging.
320 569157 : if (!found) {
321 0 : assert(0);
322 : TRC_INFO_IF(MAL_SERVER) {
323 : TRC_INFO_ENDIF(MAL_SERVER,
324 : "runtimeProfilerFinish: stk (%p) not found in QRYqueue",
325 : stk);
326 : for (i = 0; i < qsize; i++) {
327 : // print some info. of queries not "finished"
328 : if (strcmp(QRYqueue[i].status, "finished") != 0) {
329 : TRC_INFO_ENDIF(MAL_SERVER,
330 : "QRYqueue[%zu]: stk(%p), tag(" OIDFMT
331 : "), username(%s), start(%ld), status(%s), query(%s)",
332 : i, QRYqueue[i].stk, QRYqueue[i].tag,
333 : QRYqueue[i].username, QRYqueue[i].start,
334 : QRYqueue[i].status, QRYqueue[i].query);
335 : }
336 : }
337 : }
338 643440 : MT_lock_unset(&mal_delayLock);
339 : }
340 :
341 : }
342 :
343 : /* Used by mal_reset to do the grand final clean up of this area before MonetDB exits */
344 : void
345 325 : mal_runtime_reset(void)
346 : {
347 325 : dropQRYqueue();
348 325 : dropUSRstats();
349 325 : }
350 :
351 : /* At the start of each MAL stmt */
352 : void
353 54025298 : runtimeProfileBegin(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
354 : RuntimeProfile prof)
355 : {
356 54025298 : (void) cntxt;
357 54025298 : (void) mb;
358 54025298 : (void) stk;
359 54025298 : (void) pci;
360 :
361 54025298 : assert(pci);
362 : /* always collect the MAL instruction execution time */
363 54025298 : prof->ticks = GDKusec();
364 54025506 : }
365 :
366 : /* At the end of each MAL stmt */
367 : void
368 54075138 : runtimeProfileExit(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
369 : RuntimeProfile prof)
370 : {
371 54075138 : lng ticks = GDKusec();
372 :
373 54074552 : if (profilerStatus > 0)
374 45 : profilerEvent(&(struct MalEvent) { cntxt, mb, stk, pci, ticks,
375 45 : ticks - prof->ticks },
376 : NULL);
377 54074552 : if (cntxt->sqlprofiler)
378 843 : sqlProfilerEvent(cntxt, mb, stk, pci, ticks, ticks - prof->ticks);
379 54074552 : if (profilerStatus < 0) {
380 : /* delay profiling until you encounter start of MAL function */
381 0 : if (getInstrPtr(mb, 0) == pci)
382 0 : profilerStatus = 1;
383 : }
384 54074552 : }
385 :
386 : /*
387 : * For performance evaluation it is handy to estimate the
388 : * amount of bytes produced by an instruction.
389 : * The actual amount is harder to guess, because an instruction
390 : * may trigger a side effect, such as creating a hash-index.
391 : * Side effects are ignored.
392 : */
393 :
394 : lng
395 0 : getBatSpace(BAT *b)
396 : {
397 0 : lng space = 0;
398 0 : if (b == NULL)
399 : return 0;
400 0 : space += BATcount(b) << b->tshift;
401 0 : if (space) {
402 0 : MT_lock_set(&b->theaplock);
403 0 : if (b->tvheap)
404 0 : space += heapinfo(b->tvheap, b->batCacheid);
405 0 : MT_lock_unset(&b->theaplock);
406 0 : MT_rwlock_rdlock(&b->thashlock);
407 0 : space += hashinfo(b->thash, b->batCacheid);
408 0 : MT_rwlock_rdunlock(&b->thashlock);
409 0 : space += IMPSimprintsize(b);
410 : }
411 : return space;
412 : }
413 :
414 : lng
415 0 : getVolume(MalStkPtr stk, InstrPtr pci, int rd)
416 : {
417 0 : int i, limit;
418 0 : lng vol = 0;
419 0 : BAT *b;
420 :
421 0 : if (stk == NULL)
422 : return 0;
423 0 : limit = rd ? pci->argc : pci->retc;
424 0 : i = rd ? pci->retc : 0;
425 :
426 0 : for (; i < limit; i++) {
427 0 : if (stk->stk[getArg(pci, i)].bat) {
428 0 : oid cnt = 0;
429 :
430 0 : b = BBPquickdesc(stk->stk[getArg(pci, i)].val.bval);
431 0 : if (b == NULL)
432 0 : continue;
433 0 : cnt = BATcount(b);
434 : /* Usually reading views cost as much as full bats.
435 : But when we output a slice that is not the case. */
436 0 : if (rd)
437 0 : vol += (!isVIEW(b) && !VIEWtparent(b)) ? tailsize(b, cnt) : 0;
438 0 : else if (!isVIEW(b))
439 0 : vol += tailsize(b, cnt);
440 : }
441 : }
442 : return vol;
443 : }
|
