Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024, 2025 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : /* Author(s) M.L. Kersten
14 : * The MAL Runtime Profiler and system queue
15 : * This little helper module is used to perform instruction based profiling.
16 : * The QRYqueue is only update at the start/finish of a query.
17 : * It is also the place to keep track on the number of workers
18 : * The current code relies on a scan rather than a hash.
19 : */
20 :
21 : #include "monetdb_config.h"
22 : #include "mal_utils.h"
23 : #include "mal_runtime.h"
24 : #include "mal_interpreter.h"
25 : #include "mal_function.h"
26 : #include "mal_profiler.h"
27 : #include "mal_listing.h"
28 : #include "mal_authorize.h"
29 : #include "mal_resource.h"
30 : #include "mal_internal.h"
31 : #include "mal_private.h"
32 :
33 :
34 : QueryQueue QRYqueue = NULL;
35 : size_t qsize = 0;
36 : static oid qtag = 1; // A unique query identifier
37 :
38 : UserStats USRstats = NULL;
39 : size_t usrstatscnt = 0;
40 :
41 : static inline void
42 25127 : clearUSRstats(size_t idx)
43 : {
44 25127 : USRstats[idx] = (struct USERSTAT) { 0 };
45 : }
46 :
47 : /*
48 : * Find the index of the given 'user' in USRstats.
49 : * For a new 'user' return a new free slot.
50 : * If USRstats is full, extend it.
51 : */
52 : static size_t
53 302067 : getUSRstatsIdx(MalBlkPtr mb, oid user)
54 : {
55 302067 : size_t i = 0;
56 302067 : UserStats tmp = NULL;
57 :
58 304330 : for (i = 0; i < usrstatscnt; i++)
59 : /* The array is dense, so we either find the user or an empty slot. */
60 304329 : if (USRstats[i].user == user || USRstats[i].username == NULL)
61 302066 : return i;
62 :
63 : /* expand USRstats */
64 1 : tmp = (UserStats) GDKrealloc(USRstats,
65 : sizeof(struct USERSTAT) *
66 : (size_t) (usrstatscnt += MAL_MAXCLIENTS));
67 1 : if (tmp == NULL) {
68 : /* It's not a fatal error if we can't extend USRstats.
69 : * We don't want to affect existing USRstats. */
70 0 : addMalException(mb, "getUSRstatsIdx" MAL_MALLOC_FAIL);
71 0 : return (size_t) -1;
72 : }
73 1 : USRstats = tmp;
74 5 : for (; i < usrstatscnt; i++)
75 4 : clearUSRstats(i);
76 1 : return usrstatscnt - MAL_MAXCLIENTS;
77 : }
78 :
79 : static void
80 626898 : updateUserStats(Client cntxt, MalBlkPtr mb, lng ticks, time_t started,
81 : time_t finished, str query)
82 : {
83 : // don't keep stats for context without username
84 626898 : if (cntxt->username == NULL)
85 : return;
86 :
87 302067 : size_t idx = getUSRstatsIdx(mb, cntxt->user);
88 :
89 302067 : if (idx == (size_t) -1) {
90 0 : addMalException(mb,
91 : "updateUserStats"
92 : "Failed to get an entry in user statistics");
93 0 : return;
94 : }
95 :
96 302067 : if (USRstats[idx].username == NULL || USRstats[idx].user != cntxt->user
97 301688 : || strcmp(USRstats[idx].username, cntxt->username) != 0) {
98 411 : GDKfree(USRstats[idx].username);
99 411 : GDKfree(USRstats[idx].maxquery);
100 411 : clearUSRstats(idx);
101 411 : USRstats[idx].user = cntxt->user;
102 411 : USRstats[idx].username = GDKstrdup(cntxt->username);
103 : }
104 302067 : USRstats[idx].querycount++;
105 302067 : USRstats[idx].totalticks += ticks;
106 302067 : if (ticks >= USRstats[idx].maxticks && query) {
107 1544 : USRstats[idx].started = started;
108 1544 : USRstats[idx].finished = finished;
109 1544 : USRstats[idx].maxticks = ticks;
110 1544 : GDKfree(USRstats[idx].maxquery);
111 1544 : USRstats[idx].maxquery = GDKstrdup(query);
112 : }
113 : }
114 :
115 : /*
116 : * Free up the whole USRstats before mserver5 exits.
117 : */
118 : static void
119 356 : dropUSRstats(void)
120 : {
121 356 : size_t i;
122 356 : MT_lock_set(&mal_delayLock);
123 25424 : for (i = 0; i < usrstatscnt; i++) {
124 24712 : GDKfree(USRstats[i].username);
125 24712 : GDKfree(USRstats[i].maxquery);
126 24712 : clearUSRstats(i);
127 : }
128 356 : GDKfree(USRstats);
129 356 : USRstats = NULL;
130 356 : usrstatscnt = 0;
131 356 : MT_lock_unset(&mal_delayLock);
132 356 : }
133 :
134 : static str
135 626898 : isaSQLquery(MalBlkPtr mb)
136 : {
137 626898 : if (mb) {
138 12514970 : for (int i = 1; i < mb->stop; i++) {
139 12180068 : InstrPtr p = getInstrPtr(mb, i);
140 12180068 : if (getModuleId(p) && idcmp(getModuleId(p), "querylog") == 0
141 291996 : && idcmp(getFunctionId(p), "define") == 0)
142 291996 : return getVarConstant(mb, getArg(p, 1)).val.sval;
143 : }
144 : }
145 : return NULL;
146 : }
147 :
148 : /*
149 : * Manage the runtime profiling information
150 : * It is organized as a circular buffer, head/tail.
151 : * Elements are removed from the buffer when it becomes full.
152 : * This way we keep the information a little longer for inspection.
153 : */
154 :
155 : /* clear the next entry for a new call unless it is a running query */
156 : static inline void
157 27276 : clearQRYqueue(size_t idx)
158 : {
159 27276 : QRYqueue[idx] = (struct QRYQUEUE) { 0 };
160 : }
161 :
162 : static void
163 356 : dropQRYqueue(void)
164 : {
165 356 : MT_lock_set(&mal_delayLock);
166 26348 : for (size_t i = 0; i < qsize; i++) {
167 25992 : GDKfree(QRYqueue[i].query);
168 25992 : GDKfree(QRYqueue[i].username);
169 25992 : clearQRYqueue(i);
170 : }
171 356 : GDKfree(QRYqueue);
172 356 : QRYqueue = NULL;
173 356 : qsize = 0;
174 356 : qtag = 1;
175 356 : MT_lock_unset(&mal_delayLock);
176 356 : }
177 :
178 : oid
179 291527 : runtimeProfileSetTag(Client cntxt)
180 : {
181 291527 : MT_lock_set(&mal_delayLock);
182 292392 : cntxt->curprg->def->tag = qtag++;
183 292392 : MT_lock_unset(&mal_delayLock);
184 :
185 292392 : return cntxt->curprg->def->tag;
186 : }
187 :
188 : /* At the start of every MAL block or SQL query */
189 : void
190 798384 : runtimeProfileInit(Client cntxt, MalBlkPtr mb, MalStkPtr stk)
191 : {
192 798384 : static size_t qlast = 0;
193 798384 : size_t i, paused = 0;
194 798384 : str q;
195 :
196 : /* Recursive calls don't change the query queue, but later we have to check
197 : how to stop/pause/resume queries doing recursive calls from multiple workers */
198 798384 : if (stk->up)
199 : return;
200 626850 : MT_lock_set(&mal_delayLock);
201 :
202 626898 : if (USRstats == NULL) {
203 357 : usrstatscnt = MAL_MAXCLIENTS;
204 357 : USRstats = (UserStats) GDKzalloc(sizeof(struct USERSTAT) * usrstatscnt);
205 357 : if (USRstats == NULL) {
206 0 : addMalException(mb, "runtimeProfileInit" MAL_MALLOC_FAIL);
207 0 : MT_lock_unset(&mal_delayLock);
208 0 : return;
209 : }
210 : }
211 :
212 626898 : if (QRYqueue == NULL) {
213 357 : QRYqueue = (QueryQueue) GDKzalloc(sizeof(struct QRYQUEUE) *
214 : (qsize = MAL_MAXCLIENTS));
215 :
216 357 : if (QRYqueue == NULL) {
217 0 : addMalException(mb, "runtimeProfileInit" MAL_MALLOC_FAIL);
218 0 : MT_lock_unset(&mal_delayLock);
219 0 : return;
220 : }
221 : }
222 57554062 : for (i = 0; i < qsize; i++) {
223 113579973 : paused += QRYqueue[i].status && (QRYqueue[i].status[0] == 'p' || QRYqueue[i].status[0] == 'r'); /* running, prepared or paused */
224 : }
225 626898 : if (qsize - paused < (size_t) MAL_MAXCLIENTS) {
226 21 : qsize += MAL_MAXCLIENTS;
227 21 : QueryQueue tmp;
228 21 : tmp = (QueryQueue) GDKrealloc(QRYqueue,
229 : sizeof(struct QRYQUEUE) * qsize);
230 21 : if (tmp == NULL) {
231 0 : addMalException(mb, "runtimeProfileInit" MAL_MALLOC_FAIL);
232 0 : qsize -= MAL_MAXCLIENTS; /* undo increment */
233 0 : MT_lock_unset(&mal_delayLock);
234 0 : return;
235 : }
236 21 : QRYqueue = tmp;
237 1305 : for (i = qsize - MAL_MAXCLIENTS; i < qsize; i++)
238 1284 : clearQRYqueue(i);
239 : }
240 : // add new invocation
241 626917 : for (i = 0; i < qsize; i++) {
242 626917 : size_t j = qlast;
243 626917 : if (++qlast >= qsize)
244 8524 : qlast = 0;
245 626917 : if (QRYqueue[j].stk == NULL ||
246 19 : QRYqueue[j].status == NULL ||
247 19 : (QRYqueue[j].status[0] != 'r' && QRYqueue[j].status[0] != 'p')) {
248 626898 : QRYqueue[j].mb = mb;
249 626898 : QRYqueue[j].tag = stk->tag = mb->tag;
250 626898 : QRYqueue[j].stk = stk; // for status pause 'p'/running '0'/ quitting 'q'
251 626898 : QRYqueue[j].finished = 0;
252 626898 : QRYqueue[j].start = time(0);
253 626898 : q = isaSQLquery(mb);
254 626898 : GDKfree(QRYqueue[j].query);
255 626898 : QRYqueue[j].query = GDKstrdup(q); /* NULL in, NULL out */
256 626898 : GDKfree(QRYqueue[j].username);
257 626898 : if (!GDKembedded())
258 625059 : QRYqueue[j].username = GDKstrdup(cntxt->username);
259 626898 : QRYqueue[j].idx = cntxt->idx;
260 : /* give the MB upperbound by addition of 1 MB */
261 626898 : QRYqueue[j].memory = 1 + (int) (stk->memory / LL_CONSTANT(1048576)); /* Convert to MB */
262 626898 : QRYqueue[j].workers = (int) 1; /* this is the first one */
263 626898 : QRYqueue[j].status = "running";
264 626898 : QRYqueue[j].cntxt = cntxt;
265 626898 : QRYqueue[j].ticks = GDKusec();
266 626898 : break;
267 : }
268 : }
269 626898 : MT_lock_unset(&mal_delayLock);
270 626897 : MT_lock_set(&mal_contextLock);
271 626898 : cntxt->idle = 0;
272 626898 : MT_lock_unset(&mal_contextLock);
273 : }
274 :
275 : /*
276 : * At the end of every MAL block or SQL query.
277 : *
278 : * Returning from a recursive call does not change the number of workers.
279 : */
280 : void
281 797637 : runtimeProfileFinish(Client cntxt, MalBlkPtr mb, MalStkPtr stk)
282 : {
283 797637 : size_t i;
284 797637 : bool found = false;
285 :
286 : /* Recursive calls don't change the query queue, but later we have to check
287 : how to stop/pause/resume queries doing recursive calls from multiple workers */
288 797637 : if (stk->up)
289 : return;
290 626403 : MT_lock_set(&mal_delayLock);
291 28782038 : for (i = 0; i < qsize; i++) {
292 28155140 : if (QRYqueue[i].stk == stk) {
293 626898 : if (QRYqueue[i].status[0] == 's')
294 1 : QRYqueue[i].status = "aborted";
295 : else
296 626897 : QRYqueue[i].status = "finished";
297 626898 : QRYqueue[i].finished = time(0);
298 626898 : QRYqueue[i].workers = (int) ATOMIC_GET(&mb->workers);
299 : /* give the MB upperbound by addition of 1 MB */
300 626898 : QRYqueue[i].memory = 1 + (int) (mb->memory / LL_CONSTANT(1048576));
301 626898 : QRYqueue[i].cntxt = NULL;
302 626898 : QRYqueue[i].stk = NULL;
303 626898 : QRYqueue[i].mb = NULL;
304 626898 : QRYqueue[i].ticks = GDKusec() - QRYqueue[i].ticks;
305 626898 : updateUserStats(cntxt, mb, QRYqueue[i].ticks, QRYqueue[i].start,
306 : QRYqueue[i].finished, QRYqueue[i].query);
307 : // assume that the user is now idle
308 626898 : MT_lock_unset(&mal_delayLock);
309 626898 : MT_lock_set(&mal_contextLock);
310 626898 : cntxt->idle = time(0);
311 626898 : MT_lock_unset(&mal_contextLock);
312 626898 : found = true;
313 626898 : break;
314 : }
315 : }
316 :
317 : // every query that has been started has an entry in QRYqueue. If this
318 : // finished query is not found, we want to print some informational
319 : // messages for debugging.
320 626898 : if (!found) {
321 0 : assert(0);
322 : TRC_INFO_IF(MAL_SERVER) {
323 : TRC_INFO_ENDIF(MAL_SERVER,
324 : "runtimeProfilerFinish: stk (%p) not found in QRYqueue",
325 : stk);
326 : for (i = 0; i < qsize; i++) {
327 : // print some info. of queries not "finished"
328 : if (strcmp(QRYqueue[i].status, "finished") != 0) {
329 : struct tm tmp;
330 : char tbuf[64];
331 : (void) localtime_r(&QRYqueue[i].start, &tmp);
332 : strftime(tbuf, sizeof(tbuf), "%F %T", &tmp);
333 : TRC_INFO_ENDIF(MAL_SERVER,
334 : "QRYqueue[%zu]: stk(%p), tag(" OIDFMT
335 : "), username(%s), start(%s), status(%s), query(%s)",
336 : i, QRYqueue[i].stk, QRYqueue[i].tag,
337 : QRYqueue[i].username, tbuf,
338 : QRYqueue[i].status, QRYqueue[i].query);
339 : }
340 : }
341 : }
342 798132 : MT_lock_unset(&mal_delayLock);
343 : }
344 :
345 : }
346 :
347 : /* Used by mal_reset to do the grand final clean up of this area before MonetDB exits */
348 : void
349 356 : mal_runtime_reset(void)
350 : {
351 356 : dropQRYqueue();
352 356 : dropUSRstats();
353 356 : }
354 :
355 : /* At the start of each MAL stmt */
356 : void
357 68366358 : runtimeProfileBegin(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
358 : RuntimeProfile prof)
359 : {
360 68366358 : (void) cntxt;
361 68366358 : (void) mb;
362 68366358 : (void) stk;
363 68366358 : (void) pci;
364 :
365 68366358 : assert(pci);
366 : /* always collect the MAL instruction execution time */
367 68366358 : prof->ticks = GDKusec();
368 68387798 : }
369 :
370 : /* At the end of each MAL stmt */
371 : void
372 68381568 : runtimeProfileExit(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
373 : RuntimeProfile prof)
374 : {
375 68381568 : lng ticks = GDKusec();
376 :
377 68438461 : if (profilerStatus > 0)
378 45 : profilerEvent(&(struct MalEvent) { cntxt, mb, stk, pci, ticks,
379 45 : ticks - prof->ticks },
380 : NULL);
381 68438461 : if (cntxt->sqlprofiler)
382 938 : sqlProfilerEvent(cntxt, mb, stk, pci, ticks, ticks - prof->ticks);
383 68438467 : if (profilerStatus < 0) {
384 : /* delay profiling until you encounter start of MAL function */
385 0 : if (getInstrPtr(mb, 0) == pci)
386 0 : profilerStatus = 1;
387 : }
388 68438467 : }
|
