Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : #include "monetdb_config.h"
14 : #include <netcdf.h>
15 : #include "sql_mvc.h"
16 : #include "sql.h"
17 : #include "sql_execute.h"
18 : #include "sql_scenario.h"
19 : #include "mal_exception.h"
20 : #include "netcdf_vault.h"
21 :
22 : /* SQL statements for population of NetCDF catalog */
23 : #define INSFILE \
24 : "INSERT INTO netcdf_files(file_id,location) VALUES(%d, '%s');"
25 :
26 : #define INSDIM \
27 : "INSERT INTO netcdf_dims(dim_id,file_id,name,length) VALUES(%d, %d, '%s', %d);"
28 :
29 : #define INSVAR \
30 : "INSERT INTO netcdf_vars(var_id,file_id,name,vartype,ndim,coord_dim_id) VALUES(%d, %d, '%s', '%s', %d, %d);"
31 :
32 : #define INSVARDIM \
33 : "INSERT INTO netcdf_vardim (var_id,dim_id,file_id,dimpos) VALUES(%d, %d, %d, %d);"
34 :
35 : #define INSATTR \
36 : "INSERT INTO netcdf_attrs (obj_name,att_name,att_type,value,file_id,gr_name) VALUES('%s', '%s', '%s', '%s', %d, '%s');"
37 :
38 : #define LOAD_NCDF_VAR(tpe,ncdftpe) \
39 : { \
40 : tpe *databuf; \
41 : res = COLnew(0, TYPE_##tpe, sz, TRANSIENT); \
42 : if ( res == NULL ) \
43 : return createException(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL); \
44 : databuf = (tpe *)Tloc(res, 0); \
45 : if ( (retval = nc_get_var_##ncdftpe(ncid, varid, databuf)) ) \
46 : return createException(MAL, "netcdf.importvar", \
47 : SQLSTATE(NC000) "Cannot read variable %d values: %s", \
48 : varid, nc_strerror(retval)); \
49 : }
50 :
51 : static void
52 0 : fix_quote( char *n, int l)
53 : {
54 0 : int i;
55 :
56 0 : for(i=0;i<l;i++)
57 0 : if (n[i]=='\'')
58 0 : n[i] = ' ';
59 : }
60 :
61 :
62 : /* simple test for netcdf library */
63 : str
64 0 : NCDFtest(int *vars, str *fname)
65 : {
66 0 : int ncid; /* dataset id */
67 0 : int dims, ngatts, unlimdim;
68 0 : int retval;
69 :
70 0 : str msg = MAL_SUCCEED;
71 :
72 : /* Open NetCDF file */
73 0 : if ((retval = nc_open(*fname, NC_NOWRITE, &ncid)))
74 0 : return createException(MAL, "netcdf.test", SQLSTATE(NC000) "Cannot open NetCDF file %s: %s", *fname, nc_strerror(retval));
75 :
76 0 : if ((retval = nc_inq(ncid, &dims, vars, &ngatts, &unlimdim)))
77 0 : return createException(MAL, "netcdf.test", SQLSTATE(NC000) "Cannot read NetCDF header: %s", nc_strerror(retval));
78 :
79 0 : if ((retval = nc_close(ncid)))
80 0 : return createException(MAL, "netcdf.test", SQLSTATE(NC000) "Cannot close file %s: \
81 : %s", *fname, nc_strerror(retval));
82 :
83 : return msg;
84 : }
85 :
86 : /* the following function is from ncdump utility: NetCDF type number to name */
87 : static const char *
88 0 : prim_type_name(nc_type type)
89 : {
90 0 : switch (type) {
91 : case NC_BYTE:
92 : return "byte";
93 0 : case NC_CHAR:
94 0 : return "char";
95 0 : case NC_SHORT:
96 0 : return "short";
97 0 : case NC_INT:
98 0 : return "int";
99 0 : case NC_FLOAT:
100 0 : return "float";
101 0 : case NC_DOUBLE:
102 0 : return "double";
103 : #ifdef USE_NETCDF4
104 : case NC_UBYTE:
105 : return "ubyte";
106 : case NC_USHORT:
107 : return "ushort";
108 : case NC_UINT:
109 : return "uint";
110 : case NC_INT64:
111 : return "int64";
112 : case NC_UINT64:
113 : return "uint64";
114 : case NC_STRING:
115 : return "string";
116 : #endif /* USE_NETCDF4 */
117 0 : default:
118 0 : return "bad type";
119 : }
120 : }
121 :
122 : /* Mapping NetCDF to SQL data type */
123 :
124 : static const char *
125 0 : NCDF2SQL(nc_type type)
126 : {
127 0 : switch (type) {
128 : case NC_BYTE:
129 : return "tinyint";
130 0 : case NC_CHAR:
131 0 : return "char(1)";
132 0 : case NC_SHORT:
133 0 : return "smallint";
134 0 : case NC_INT:
135 0 : return "int";
136 0 : case NC_FLOAT:
137 0 : return "float";
138 0 : case NC_DOUBLE:
139 0 : return "double";
140 : #ifdef USE_NETCDF4
141 : /* ?? mapping of unsigned types */
142 : case NC_UBYTE:
143 : return "ubyte";
144 : case NC_USHORT:
145 : return "ushort";
146 : case NC_UINT:
147 : return "uint";
148 : case NC_INT64:
149 : return "bigint";
150 : case NC_UINT64:
151 : return "uint64";
152 : case NC_STRING:
153 : return "string";
154 : #endif /* USE_NETCDF4 */
155 0 : default:
156 0 : return "type not supported";
157 : }
158 : }
159 :
160 : #define array_series(sta, ste, sto, TYPE) { \
161 : int s,g; \
162 : TYPE i, *o = (TYPE*)Tloc(bn, 0); \
163 : TYPE start = sta, step = ste, stop = sto; \
164 : if ( start < stop && step > 0) { \
165 : for ( s = 0; s < series; s++) \
166 : for ( i = start; i < stop; i += step) \
167 : for( g = 0; g < group; g++){ \
168 : *o = i; \
169 : o++; \
170 : } \
171 : } else { \
172 : for ( s = 0; s < series; s++) \
173 : for ( i = start; i > stop; i += step) \
174 : for( g = 0; g < group; g++){ \
175 : *o = i; \
176 : o++; \
177 : } \
178 : } \
179 : }
180 :
181 : /* create and populate a dimension bat */
182 : static str
183 0 : NCDFARRAYseries(bat *bid, bte start, bte step, int stop, int group, int series)
184 : {
185 0 : BAT *bn = NULL;
186 0 : BUN cnt = 0;
187 :
188 0 : cnt = (BUN) ceil(((stop * 1.0 - start) / step)) * group * series ;
189 0 : if (stop <= (int) GDK_bte_max ) {
190 0 : bte sta = (bte) start, ste = (bte) step, sto = (bte) stop;
191 :
192 0 : bn = COLnew(0, TYPE_bte, cnt, TRANSIENT);
193 0 : if ( bn == NULL)
194 0 : throw(MAL, "ntcdf.loadvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
195 0 : array_series(sta, ste, sto, bte);
196 0 : } else if (stop <= (int) GDK_sht_max) {
197 0 : sht sta = (sht) start, ste = (sht) step, sto = (sht) stop;
198 :
199 0 : bn = COLnew(0, TYPE_sht, cnt, TRANSIENT);
200 0 : if ( bn == NULL)
201 0 : throw(MAL, "netcdf.loadvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
202 0 : array_series(sta, ste, sto, sht);
203 : } else {
204 0 : int sta = (int) start, ste = (int) step, sto = (int) stop;
205 :
206 0 : bn = COLnew(0, TYPE_int, cnt, TRANSIENT);
207 0 : if ( bn == NULL)
208 0 : throw(MAL, "netcdf.loadvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
209 0 : array_series(sta, ste, sto, int);
210 : }
211 :
212 0 : BATsetcount(bn, cnt);
213 0 : bn->tsorted = (cnt <= 1 || (series == 1 && step > 0));
214 0 : bn->trevsorted = (cnt <= 1 || (series == 1 && step < 0));
215 0 : bn->tnonil = true;
216 0 : *bid = bn->batCacheid;
217 0 : BBPkeepref(bn);
218 0 : return MAL_SUCCEED;
219 : }
220 :
221 : str
222 0 : NCDFattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
223 : {
224 0 : mvc *m = NULL;
225 0 : sql_schema *sch = NULL;
226 0 : sql_table *tfiles = NULL, *tdims = NULL, *tvars = NULL, *tvardim = NULL, *tattrs = NULL;
227 0 : sql_column *col;
228 0 : str msg = MAL_SUCCEED;
229 0 : str fname = *getArgReference_str(stk, pci, 1);
230 0 : char buf[BUFSIZ], *s= buf;
231 0 : oid fid, rid = oid_nil;
232 0 : sql_trans *tr;
233 :
234 0 : int ncid; /* dataset id */
235 0 : int ndims, nvars, ngatts, unlimdim;
236 0 : int didx, vidx, vndims, vnatts, i, aidx, coord_dim_id = -1;
237 0 : int vdims[NC_MAX_VAR_DIMS];
238 :
239 0 : size_t dlen, alen;
240 0 : char dname[NC_MAX_NAME+1], vname[NC_MAX_NAME+1], aname[NC_MAX_NAME +1],
241 : abuf[80], *aval;
242 0 : char **dims = NULL;
243 0 : nc_type vtype, atype; /* == int */
244 :
245 0 : int retval, avalint;
246 0 : float avalfl;
247 0 : double avaldbl;
248 0 : str esc_str0, esc_str1;
249 :
250 0 : msg = getSQLContext(cntxt, mb, &m, NULL);
251 0 : if (msg)
252 : return msg;
253 :
254 0 : tr = m->session->tr;
255 0 : sqlstore *store = tr->store;
256 0 : sch = mvc_bind_schema(m, "sys");
257 0 : if ( !sch )
258 0 : return createException(MAL, "netcdf.attach", SQLSTATE(NC000) "Cannot get schema sys\n");
259 :
260 0 : tfiles = mvc_bind_table(m, sch, "netcdf_files");
261 0 : tdims = mvc_bind_table(m, sch, "netcdf_dims");
262 0 : tvars = mvc_bind_table(m, sch, "netcdf_vars");
263 0 : tvardim = mvc_bind_table(m, sch, "netcdf_vardim");
264 0 : tattrs = mvc_bind_table(m, sch, "netcdf_attrs");
265 :
266 0 : if (tfiles == NULL || tdims == NULL || tvars == NULL ||
267 0 : tvardim == NULL || tattrs == NULL)
268 0 : return createException(MAL, "netcdf.attach", SQLSTATE(NC000) "Catalog table missing\n");
269 :
270 : /* check if the file is already attached */
271 0 : col = mvc_bind_column(m, tfiles, "location");
272 0 : rid = store->table_api.column_find_row(m->session->tr, col, fname, NULL);
273 0 : if (!is_oid_nil(rid))
274 0 : return createException(SQL, "netcdf.attach", SQLSTATE(NC000) "File %s is already attached\n", fname);
275 :
276 : /* Open NetCDF file */
277 0 : if ((retval = nc_open(fname, NC_NOWRITE, &ncid)))
278 0 : return createException(MAL, "netcdf.test", SQLSTATE(NC000) "Cannot open NetCDF \
279 : file %s: %s", fname, nc_strerror(retval));
280 :
281 0 : if ((retval = nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdim)))
282 0 : return createException(MAL, "netcdf.test", SQLSTATE(NC000) "Cannot read NetCDF \
283 : header: %s", nc_strerror(retval));
284 :
285 : /* Insert row into netcdf_files table */
286 0 : col = mvc_bind_column(m, tfiles, "file_id");
287 0 : fid = store->storage_api.count_col(tr, col, 1) + 1;
288 :
289 0 : esc_str0 = SQLescapeString(fname);
290 0 : if (!esc_str0) {
291 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
292 0 : goto finish;
293 : }
294 0 : snprintf(buf, BUFSIZ, INSFILE, (int)fid, esc_str0);
295 0 : GDKfree(esc_str0);
296 0 : if ( ( msg = SQLstatementIntern(cntxt, s, "netcdf.attach", TRUE, FALSE, NULL))
297 : != MAL_SUCCEED )
298 0 : goto finish;
299 :
300 : /* Read dimensions from NetCDF header and insert a row for each one into netcdf_dims table */
301 :
302 0 : dims = (char **)GDKzalloc(sizeof(char *) * ndims);
303 0 : if (!dims) {
304 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
305 0 : goto finish;
306 : }
307 0 : for (didx = 0; didx < ndims; didx++){
308 0 : if ((retval = nc_inq_dim(ncid, didx, dname, &dlen)) != 0)
309 0 : return createException(MAL, "netcdf.attach", SQLSTATE(NC000) "Cannot read dimension %d : %s", didx, nc_strerror(retval));
310 :
311 0 : esc_str0 = SQLescapeString(dname);
312 0 : if (!esc_str0) {
313 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
314 0 : goto finish;
315 : }
316 :
317 0 : snprintf(buf, BUFSIZ, INSDIM, didx, (int)fid, esc_str0, (int)dlen);
318 0 : GDKfree(esc_str0);
319 0 : if ( ( msg = SQLstatementIntern(cntxt, s, "netcdf.attach", TRUE, FALSE, NULL))
320 : != MAL_SUCCEED )
321 0 : goto finish;
322 :
323 0 : dims[didx] = GDKstrdup(dname);
324 0 : if (!dims[didx]) {
325 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
326 0 : goto finish;
327 : }
328 : }
329 :
330 : /* Read variables and attributes from the header and insert rows in netcdf_vars, netcdf_vardims, and netcdf_attrs tables */
331 0 : for (vidx = 0; vidx < nvars; vidx++){
332 0 : if ( (retval = nc_inq_var(ncid, vidx, vname, &vtype, &vndims, vdims, &vnatts)))
333 0 : return createException(MAL, "netcdf.attach",
334 : SQLSTATE(NC000) "Cannot read variable %d : %s",
335 : vidx, nc_strerror(retval));
336 :
337 : /* Check if this is coordinate variable */
338 0 : if ( (vndims == 1) && ( strcmp(vname, dims[vdims[0]]) == 0 ))
339 0 : coord_dim_id = vdims[0];
340 : else coord_dim_id = -1;
341 :
342 0 : esc_str0 = SQLescapeString(vname);
343 0 : if (!esc_str0) {
344 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
345 0 : goto finish;
346 : }
347 :
348 0 : snprintf(buf, BUFSIZ, INSVAR, vidx, (int)fid, esc_str0, prim_type_name(vtype), vndims, coord_dim_id);
349 0 : GDKfree(esc_str0);
350 0 : if ( ( msg = SQLstatementIntern(cntxt, s, "netcdf.attach", TRUE, FALSE, NULL))
351 : != MAL_SUCCEED )
352 0 : goto finish;
353 :
354 0 : if ( coord_dim_id < 0 ){
355 0 : for (i = 0; i < vndims; i++){
356 0 : snprintf(buf, BUFSIZ, INSVARDIM, vidx, vdims[i], (int)fid, i);
357 0 : if ( ( msg = SQLstatementIntern(cntxt, s, "netcdf.attach", TRUE, FALSE, NULL))
358 : != MAL_SUCCEED )
359 0 : goto finish;
360 : }
361 : }
362 :
363 0 : if ( vnatts > 0 ) { /* fill in netcdf_attrs table */
364 :
365 0 : for (aidx = 0; aidx < vnatts; aidx++){
366 0 : if ((retval = nc_inq_attname(ncid,vidx,aidx,aname)))
367 0 : return createException(MAL, "netcdf.attach",
368 : SQLSTATE(NC000) "Cannot read attribute %d of variable %d: %s",
369 : aidx, vidx, nc_strerror(retval));
370 :
371 0 : if ((retval = nc_inq_att(ncid,vidx,aname,&atype,&alen)))
372 0 : return createException(MAL, "netcdf.attach",
373 : SQLSTATE(NC000) "Cannot read attribute %s type and length: %s",
374 : aname, nc_strerror(retval));
375 :
376 0 : esc_str0 = SQLescapeString(vname);
377 0 : if (!esc_str0) {
378 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
379 0 : goto finish;
380 : }
381 0 : esc_str1 = SQLescapeString(aname);
382 0 : if (!esc_str1) {
383 0 : GDKfree(esc_str0);
384 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
385 0 : goto finish;
386 : }
387 0 : switch ( atype ) {
388 0 : case NC_CHAR:
389 0 : aval = (char *) GDKzalloc(alen + 1);
390 0 : if (!aval) {
391 0 : GDKfree(esc_str0);
392 0 : GDKfree(esc_str1);
393 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
394 0 : goto finish;
395 : }
396 0 : if ((retval = nc_get_att_text(ncid,vidx,aname,aval)))
397 0 : return createException(MAL, "netcdf.attach",
398 : SQLSTATE(NC000) "Cannot read attribute %s value: %s",
399 : aname, nc_strerror(retval));
400 0 : fix_quote(aval, alen);
401 0 : aval[alen] = '\0';
402 0 : snprintf(buf, BUFSIZ, INSATTR, esc_str0, esc_str1, "string", aval, (int)fid, "root");
403 0 : GDKfree(aval);
404 0 : break;
405 :
406 0 : case NC_INT:
407 0 : if ((retval = nc_get_att_int(ncid,vidx,aname,&avalint)))
408 0 : return createException(MAL, "netcdf.attach",
409 : SQLSTATE(NC000) "Cannot read attribute %s value: %s",
410 : aname, nc_strerror(retval));
411 0 : snprintf(abuf,80,"%d",avalint);
412 0 : snprintf(buf, BUFSIZ, INSATTR, esc_str0, esc_str1, prim_type_name(atype), abuf, (int)fid, "root");
413 0 : break;
414 :
415 0 : case NC_FLOAT:
416 0 : if ((retval = nc_get_att_float(ncid,vidx,aname,&avalfl)))
417 0 : return createException(MAL, "netcdf.attach",
418 : SQLSTATE(NC000) "Cannot read attribute %s value: %s",
419 : aname, nc_strerror(retval));
420 0 : snprintf(abuf,80,"%7.2f",avalfl);
421 0 : snprintf(buf, BUFSIZ, INSATTR, esc_str0, esc_str1, prim_type_name(atype), abuf, (int)fid, "root");
422 0 : break;
423 :
424 0 : case NC_DOUBLE:
425 0 : if ((retval = nc_get_att_double(ncid,vidx,aname,&avaldbl)))
426 0 : return createException(MAL, "netcdf.attach",
427 : SQLSTATE(NC000) "Cannot read attribute %s value: %s",
428 : aname, nc_strerror(retval));
429 0 : snprintf(abuf,80,"%7.2e",avaldbl);
430 0 : snprintf(buf, BUFSIZ, INSATTR, esc_str0, esc_str1, prim_type_name(atype), abuf, (int)fid, "root");
431 0 : break;
432 :
433 0 : default: continue; /* next attribute */
434 : }
435 0 : GDKfree(esc_str1);
436 0 : GDKfree(esc_str0);
437 :
438 0 : printf("statement: '%s'\n", s);
439 0 : if ( ( msg = SQLstatementIntern(cntxt, s, "netcdf.attach", TRUE, FALSE, NULL))
440 : != MAL_SUCCEED )
441 0 : goto finish;
442 :
443 : } /* attr loop */
444 :
445 : }
446 : } /* var loop */
447 :
448 : /* Extract global attributes */
449 :
450 0 : for (aidx = 0; aidx < ngatts; aidx++){
451 0 : if ((retval = nc_inq_attname(ncid,NC_GLOBAL,aidx,aname)) != 0)
452 0 : return createException(MAL, "netcdf.attach",
453 : SQLSTATE(NC000) "Cannot read global attribute %d: %s",
454 : aidx, nc_strerror(retval));
455 :
456 0 : if ((retval = nc_inq_att(ncid,NC_GLOBAL,aname,&atype,&alen)) != 0){
457 : if (dims != NULL ){
458 0 : for (didx = 0; didx < ndims; didx++)
459 0 : GDKfree(dims[didx]);
460 0 : GDKfree(dims);
461 : }
462 0 : return createException(MAL, "netcdf.attach",
463 : SQLSTATE(NC000) "Cannot read global attribute %s type and length: %s",
464 : aname, nc_strerror(retval));
465 : }
466 :
467 0 : esc_str0 = SQLescapeString(aname);
468 0 : if (!esc_str0) {
469 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
470 0 : goto finish;
471 : }
472 :
473 0 : switch ( atype ) {
474 0 : case NC_CHAR:
475 0 : aval = (char *) GDKzalloc(alen + 1);
476 0 : if (!aval) {
477 0 : GDKfree(esc_str0);
478 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
479 0 : goto finish;
480 : }
481 0 : if ((retval = nc_get_att_text(ncid,NC_GLOBAL,aname,aval))) {
482 0 : GDKfree(esc_str0);
483 0 : if (dims != NULL ){
484 0 : for (didx = 0; didx < ndims; didx++)
485 0 : GDKfree(dims[didx]);
486 0 : GDKfree(dims);
487 : }
488 0 : return createException(MAL, "netcdf.attach",
489 : SQLSTATE(NC000) "Cannot read global attribute %s value: %s",
490 : aname, nc_strerror(retval));
491 : }
492 0 : fix_quote(aval, alen);
493 0 : aval[alen] = '\0';
494 0 : snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", esc_str0, "string", aval, (int)fid, "root");
495 0 : GDKfree(aval);
496 0 : break;
497 :
498 0 : case NC_INT:
499 0 : if ((retval = nc_get_att_int(ncid,NC_GLOBAL,aname,&avalint))){
500 0 : GDKfree(esc_str0);
501 0 : if (dims != NULL ){
502 0 : for (didx = 0; didx < ndims; didx++)
503 0 : GDKfree(dims[didx]);
504 0 : GDKfree(dims);
505 : }
506 0 : return createException(MAL, "netcdf.attach",
507 : SQLSTATE(NC000) "Cannot read global attribute %s of type %s : %s",
508 : aname, prim_type_name(atype), nc_strerror(retval));
509 : }
510 0 : snprintf(abuf,80,"%d",avalint);
511 0 : snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", esc_str0, prim_type_name(atype), abuf, (int)fid, "root");
512 0 : break;
513 :
514 0 : case NC_FLOAT:
515 0 : if ((retval = nc_get_att_float(ncid,NC_GLOBAL,aname,&avalfl))){
516 0 : GDKfree(esc_str0);
517 0 : if (dims != NULL ){
518 0 : for (didx = 0; didx < ndims; didx++)
519 0 : GDKfree(dims[didx]);
520 0 : GDKfree(dims);
521 : }
522 0 : return createException(MAL, "netcdf.attach",
523 : SQLSTATE(NC000) "Cannot read global attribute %s of type %s: %s",
524 : aname, prim_type_name(atype), nc_strerror(retval));
525 : }
526 0 : snprintf(abuf,80,"%7.2f",avalfl);
527 0 : snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", esc_str0, prim_type_name(atype), abuf, (int)fid, "root");
528 0 : break;
529 :
530 0 : case NC_DOUBLE:
531 0 : if ((retval = nc_get_att_double(ncid,NC_GLOBAL,aname,&avaldbl))){
532 0 : GDKfree(esc_str0);
533 0 : if (dims != NULL ){
534 0 : for (didx = 0; didx < ndims; didx++)
535 0 : GDKfree(dims[didx]);
536 0 : GDKfree(dims);
537 : }
538 0 : return createException(MAL, "netcdf.attach",
539 : SQLSTATE(NC000) "Cannot read global attribute %s value: %s",
540 : aname, nc_strerror(retval));
541 : }
542 0 : snprintf(abuf,80,"%7.2e",avaldbl);
543 0 : snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", esc_str0, prim_type_name(atype), abuf, (int)fid, "root");
544 0 : break;
545 :
546 0 : default: continue; /* next attribute */
547 : }
548 0 : GDKfree(esc_str0);
549 :
550 0 : printf("global: '%s'\n", s);
551 0 : if ( ( msg = SQLstatementIntern(cntxt, s, "netcdf.attach", TRUE, FALSE, NULL))
552 : != MAL_SUCCEED )
553 0 : goto finish;
554 :
555 : } /* global attr loop */
556 :
557 :
558 0 : finish:
559 0 : nc_close(ncid);
560 :
561 0 : if (dims != NULL ){
562 0 : for (didx = 0; didx < ndims; didx++)
563 0 : GDKfree(dims[didx]);
564 0 : GDKfree(dims);
565 : }
566 :
567 : return msg;
568 : }
569 :
570 :
571 : /* Compose create table statement to create table representing NetCDF variable in the
572 : * database. Used for testing, can be removed from release. */
573 : str
574 0 : NCDFimportVarStmt(str *sciqlstmt, str *fname, int *varid)
575 : {
576 0 : int ncid; /* dataset id */
577 0 : int vndims, vnatts, i, j, retval;
578 0 : int vdims[NC_MAX_VAR_DIMS];
579 :
580 0 : size_t dlen;
581 0 : char dname[NC_MAX_NAME+1], vname[NC_MAX_NAME+1];
582 0 : nc_type vtype; /* == int */
583 :
584 0 : char buf[BUFSIZ];
585 0 : str msg = MAL_SUCCEED;
586 :
587 : /* Open NetCDF file */
588 0 : if ((retval = nc_open(*fname, NC_NOWRITE, &ncid)))
589 0 : return createException(MAL, "netcdf.importvar",
590 : SQLSTATE(NC000) "Cannot open NetCDF file %s: %s", *fname, nc_strerror(retval));
591 :
592 0 : if ( (retval = nc_inq_var(ncid, *varid, vname, &vtype, &vndims, vdims, &vnatts)))
593 0 : return createException(MAL, "netcdf.attach",
594 : SQLSTATE(NC000) "Cannot read variable %d : %s", *varid, nc_strerror(retval));
595 :
596 :
597 0 : j = snprintf(buf, BUFSIZ,"create table %s( ", vname);
598 :
599 0 : for (i = 0; i < vndims; i++){
600 0 : if ((retval = nc_inq_dim(ncid, vdims[i], dname, &dlen)))
601 0 : return createException(MAL, "netcdf.attach",
602 : SQLSTATE(NC000) "Cannot read dimension %d : %s", vdims[i], nc_strerror(retval));
603 :
604 0 : (void)dlen;
605 0 : j += snprintf(buf + j, BUFSIZ - j, "%s INTEGER, ", dname);
606 :
607 : }
608 :
609 0 : j += snprintf(buf + j, BUFSIZ - j, "value %s);", NCDF2SQL(vtype));
610 :
611 0 : nc_close(ncid);
612 :
613 0 : *sciqlstmt = GDKstrdup(buf);
614 0 : if(*sciqlstmt == NULL)
615 0 : return createException(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
616 : return msg;
617 : }
618 :
619 : /* Load variable varid from data set ncid into the bat v. Generate dimension
620 : * bats dim using NCDFARRAYseries */
621 : static str
622 0 : NCDFloadVar(bat **dim, bat *v, int ncid, int varid, nc_type vtype, int vndims, int *vdims)
623 : {
624 :
625 0 : BAT *res;
626 0 : bat vbid, *dim_bids;
627 0 : int retval, i, j;
628 0 : char *sermsg = NULL;
629 0 : size_t sz = 1;
630 0 : size_t *dlen = NULL, *val_rep = NULL, *grp_rep = NULL;
631 :
632 0 : if ( dim == NULL )
633 0 : return createException(MAL, "netcdf.importvar", SQLSTATE(NC000) "array of dimension bat is NULL");
634 0 : dim_bids = *dim;
635 :
636 0 : dlen = (size_t *)GDKzalloc(sizeof(size_t) * vndims);
637 0 : if (!dlen)
638 0 : return createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
639 :
640 0 : for (i = 0; i < vndims; i++){
641 0 : if ((retval = nc_inq_dimlen(ncid, vdims[i], &dlen[i])))
642 0 : return createException(MAL, "netcdf.importvar",
643 : SQLSTATE(NC000) "Cannot read dimension %d : %s",
644 : vdims[i], nc_strerror(retval));
645 0 : sz *= dlen[i];
646 : }
647 :
648 0 : switch (vtype) {
649 0 : case NC_INT:
650 : {
651 0 : LOAD_NCDF_VAR(int,int);
652 : break;
653 : }
654 0 : case NC_FLOAT:
655 : case NC_DOUBLE:
656 : {
657 0 : LOAD_NCDF_VAR(dbl,double);
658 : break;
659 : }
660 :
661 : default:
662 0 : GDKfree(dlen);
663 0 : return createException(MAL, "netcdf.importvar",
664 : SQLSTATE(NC000) "Type %s not supported yet",
665 : prim_type_name(vtype));
666 :
667 : }
668 :
669 0 : BATsetcount(res, sz);
670 0 : res->tnonil = true;
671 0 : res->tnil = false;
672 0 : res->tsorted = false;
673 0 : res->trevsorted = false;
674 0 : BATkey(res, false);
675 0 : vbid = res->batCacheid;
676 0 : BBPkeepref(res);
677 :
678 0 : res = NULL;
679 :
680 : /* Manually create dimensions with range [0:1:dlen[i]] */
681 0 : val_rep = (size_t *)GDKmalloc(sizeof(size_t) * vndims);
682 0 : grp_rep = (size_t *)GDKmalloc(sizeof(size_t) * vndims);
683 0 : if (val_rep == NULL || grp_rep == NULL) {
684 0 : GDKfree(dlen);
685 0 : GDKfree(val_rep);
686 0 : GDKfree(grp_rep);
687 0 : throw(MAL, "netcdf.loadvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
688 : }
689 :
690 : /* compute the repetition factor inside of the series (val_rep) and of series (grp_rep) */
691 0 : for (i = 0; i < vndims; i++) {
692 0 : val_rep[i] = grp_rep[i] = 1;
693 0 : for (j = 0; j < i; j++)
694 0 : grp_rep[i] *= dlen[j];
695 0 : for (j = i + 1; j < vndims; j++)
696 0 : val_rep[i] *= dlen[j];
697 : }
698 :
699 0 : for (i = 0; i < vndims; i++) {
700 0 : sermsg = NCDFARRAYseries(&dim_bids[i], 0, 1, dlen[i], val_rep[i], grp_rep[i]);
701 :
702 0 : if (sermsg != MAL_SUCCEED) {
703 0 : BBPrelease(vbid); /* undo the BBPkeepref(res) above */
704 0 : for ( j = 0; j < i; j++) /* undo log. ref of previous dimensions */
705 0 : BBPrelease(dim_bids[j]);
706 0 : GDKfree(dlen);
707 0 : GDKfree(val_rep);
708 0 : GDKfree(grp_rep);
709 0 : return sermsg;
710 : }
711 : }
712 : /* to do : is descriptor check of dim_bids is needed? */
713 :
714 0 : GDKfree(dlen);
715 0 : GDKfree(val_rep);
716 0 : GDKfree(grp_rep);
717 :
718 0 : *v = vbid;
719 :
720 0 : return MAL_SUCCEED;
721 : }
722 :
723 : /* import variable given file id and variable name */
724 : str
725 0 : NCDFimportVariable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
726 : {
727 0 : mvc *m = NULL;
728 0 : sql_schema *sch = NULL;
729 0 : sql_table *tfiles = NULL, *arr_table = NULL;
730 0 : sql_column *col;
731 :
732 0 : str msg = MAL_SUCCEED, vname = *getArgReference_str(stk, pci, 2);
733 0 : str fname = NULL, dimtype = NULL, aname_sys = NULL;
734 0 : int fid = *getArgReference_int(stk, pci, 1);
735 0 : int varid, vndims, vnatts, i, j, retval;
736 0 : char buf[BUFSIZ], *s= buf, aname[256], **dname;
737 0 : oid rid = oid_nil;
738 0 : int vdims[NC_MAX_VAR_DIMS];
739 0 : nc_type vtype;
740 0 : int ncid; /* dataset id */
741 0 : size_t dlen;
742 0 : bat vbatid = 0, *dim_bids;
743 0 : BAT *vbat = NULL, *dimbat;
744 :
745 0 : msg = getSQLContext(cntxt, mb, &m, NULL);
746 0 : if (msg)
747 : return msg;
748 :
749 0 : sqlstore *store = m->session->tr->store;
750 0 : sch = mvc_bind_schema(m, "sys");
751 0 : if ( !sch )
752 0 : return createException(MAL, "netcdf.importvar", SQLSTATE(NC000) "Cannot get schema sys\n");
753 :
754 0 : tfiles = mvc_bind_table(m, sch, "netcdf_files");
755 0 : if (tfiles == NULL)
756 0 : return createException(MAL, "netcdf.importvar", SQLSTATE(NC000) "Catalog table missing\n");
757 :
758 : /* get the name of the attached NetCDF file */
759 0 : col = mvc_bind_column(m, tfiles, "file_id");
760 0 : if (col == NULL)
761 0 : return createException(MAL, "netcdf.importvar", SQLSTATE(NC000) "Could not find \"netcdf_files\".\"file_id\"\n");
762 0 : rid = store->table_api.column_find_row(m->session->tr, col, (void *)&fid, NULL);
763 0 : if (is_oid_nil(rid))
764 0 : return createException(MAL, "netcdf.importvar", SQLSTATE(NC000) "File %d not in the NetCDF vault\n", fid);
765 :
766 :
767 0 : col = mvc_bind_column(m, tfiles, "location");
768 0 : fname = (str)store->table_api.column_find_value(m->session->tr, col, rid);
769 :
770 : /* Open NetCDF file */
771 0 : if ((retval = nc_open(fname, NC_NOWRITE, &ncid))) {
772 0 : char *msg = createException(MAL, "netcdf.importvar", SQLSTATE(NC000) "Cannot open NetCDF file %s: %s",
773 : fname, nc_strerror(retval));
774 0 : GDKfree(fname);
775 0 : return msg;
776 : }
777 0 : GDKfree(fname);
778 :
779 : /* Get info for variable vname from NetCDF file */
780 0 : if ( (retval = nc_inq_varid(ncid, vname, &varid)) ) {
781 0 : nc_close(ncid);
782 0 : return createException(MAL, "netcdf.importvar",
783 : SQLSTATE(NC000) "Cannot read variable %s: %s",
784 : vname, nc_strerror(retval));
785 : }
786 0 : if ( (retval = nc_inq_var(ncid, varid, vname, &vtype, &vndims, vdims, &vnatts))) {
787 0 : nc_close(ncid);
788 0 : return createException(MAL, "netcdf.importvar",
789 : SQLSTATE(NC000) "Cannot read variable %d : %s",
790 : varid, nc_strerror(retval));
791 : }
792 :
793 : /* compose 'create table' statement in the buffer */
794 0 : dname = (char **) GDKzalloc( sizeof(char *) * vndims);
795 0 : if (dname == NULL) {
796 0 : nc_close(ncid);
797 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
798 : }
799 0 : for (i = 0; i < vndims; i++) {
800 0 : dname[i] = (char *) GDKzalloc(NC_MAX_NAME + 1);
801 0 : if(!dname[i]) {
802 0 : for (j = 0; j < i; j++)
803 0 : GDKfree(dname[j]);
804 0 : GDKfree(dname);
805 0 : nc_close(ncid);
806 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
807 : }
808 : }
809 :
810 0 : snprintf(aname, 256, "%s%d", vname, fid);
811 :
812 0 : j = snprintf(buf, BUFSIZ,"create table %s.%s( ", sch->base.name, aname);
813 :
814 0 : for (i = 0; i < vndims; i++){
815 0 : if ((retval = nc_inq_dim(ncid, vdims[i], dname[i], &dlen))) {
816 0 : for (j = 0; j < vndims; j++)
817 0 : GDKfree(dname[j]);
818 0 : GDKfree(dname);
819 0 : nc_close(ncid);
820 0 : return createException(MAL, "netcdf.importvar",
821 : SQLSTATE(NC000) "Cannot read dimension %d : %s",
822 : vdims[i], nc_strerror(retval));
823 : }
824 :
825 0 : if ( dlen <= (int) GDK_bte_max )
826 : dimtype = "TINYINT";
827 0 : else if ( dlen <= (int) GDK_sht_max )
828 : dimtype = "SMALLINT";
829 : else
830 0 : dimtype = "INT";
831 :
832 0 : (void)dlen;
833 0 : j += snprintf(buf + j, BUFSIZ - j, "%s %s, ", dname[i], dimtype);
834 : }
835 :
836 0 : j += snprintf(buf + j, BUFSIZ - j, "value %s);", NCDF2SQL(vtype));
837 :
838 : /* execute 'create table ' */
839 0 : msg = SQLstatementIntern(cntxt, s, "netcdf.importvar", TRUE, FALSE, NULL);
840 0 : if (msg != MAL_SUCCEED){
841 0 : for (i = 0; i < vndims; i++)
842 0 : GDKfree(dname[i]);
843 0 : GDKfree(dname);
844 0 : nc_close(ncid);
845 0 : return msg;
846 : }
847 :
848 : /* load variable data */
849 0 : dim_bids = (bat *)GDKmalloc(sizeof(bat) * vndims);
850 0 : if (dim_bids == NULL){
851 0 : for (i = 0; i < vndims; i++)
852 0 : GDKfree(dname[i]);
853 0 : GDKfree(dname);
854 0 : nc_close(ncid);
855 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
856 : }
857 :
858 0 : msg = NCDFloadVar(&dim_bids, &vbatid, ncid, varid, vtype, vndims, vdims);
859 0 : if ( msg != MAL_SUCCEED ) {
860 0 : for (i = 0; i < vndims; i++)
861 0 : GDKfree(dname[i]);
862 0 : GDKfree(dname);
863 0 : GDKfree(dim_bids);
864 0 : nc_close(ncid);
865 0 : return msg;
866 : }
867 :
868 : /* associate columns in the table with loaded variable data */
869 0 : aname_sys = toLower(aname);
870 0 : arr_table = mvc_bind_table(m, sch, aname_sys);
871 0 : if (arr_table == NULL){
872 0 : for (i = 0; i < vndims; i++)
873 0 : GDKfree(dname[i]);
874 0 : GDKfree(dname);
875 0 : GDKfree(dim_bids);
876 0 : nc_close(ncid);
877 0 : throw(MAL, "netcdf.importvar", SQLSTATE(NC000) "netcdf table %s missing\n", aname_sys);
878 : }
879 :
880 0 : col = mvc_bind_column(m, arr_table, "value");
881 0 : if (col == NULL){
882 0 : for (i = 0; i < vndims; i++)
883 0 : GDKfree(dname[i]);
884 0 : GDKfree(dname);
885 0 : GDKfree(dim_bids);
886 0 : nc_close(ncid);
887 0 : throw(MAL, "netcdf.importvar", SQLSTATE(NC000) "Cannot find column %s.value\n", aname_sys);
888 : }
889 :
890 0 : vbat = BATdescriptor(vbatid);
891 0 : if(vbat == NULL) {
892 0 : for (i = 0; i < vndims; i++)
893 0 : GDKfree(dname[i]);
894 0 : GDKfree(dname);
895 0 : GDKfree(dim_bids);
896 0 : nc_close(ncid);
897 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
898 : }
899 0 : BUN offset;
900 0 : BAT *pos = NULL;
901 0 : if (store->storage_api.claim_tab(m->session->tr, arr_table, BATcount(vbat), &offset, &pos) != LOG_OK) {
902 0 : for (i = 0; i < vndims; i++)
903 0 : GDKfree(dname[i]);
904 0 : GDKfree(dname);
905 0 : GDKfree(dim_bids);
906 0 : BBPunfix(vbatid);
907 0 : BBPrelease(vbatid);
908 0 : nc_close(ncid);
909 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
910 : }
911 0 : if (!isNew(arr_table) && sql_trans_add_dependency_change(m->session->tr, arr_table->base.id, dml) != LOG_OK) {
912 0 : for (i = 0; i < vndims; i++)
913 0 : GDKfree(dname[i]);
914 0 : GDKfree(dname);
915 0 : GDKfree(dim_bids);
916 0 : BBPunfix(vbatid);
917 0 : BBPrelease(vbatid);
918 0 : bat_destroy(pos);
919 0 : nc_close(ncid);
920 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
921 : }
922 0 : store->storage_api.append_col(m->session->tr, col, offset, pos, vbat, BATcount(vbat), TYPE_bat);
923 0 : BBPunfix(vbatid);
924 0 : BBPrelease(vbatid);
925 0 : vbat = NULL;
926 :
927 : /* associate dimension bats */
928 0 : for (i = 0; i < vndims; i++){
929 0 : col = mvc_bind_column(m, arr_table, dname[i]);
930 0 : if (col == NULL){
931 0 : for (i = 0; i < vndims; i++)
932 0 : GDKfree(dname[i]);
933 0 : GDKfree(dname);
934 0 : GDKfree(dim_bids);
935 0 : bat_destroy(pos);
936 0 : nc_close(ncid);
937 0 : throw(MAL, "netcdf.importvar", SQLSTATE(NC000) "Cannot find column %s.%s\n", aname_sys, dname[i]);
938 : }
939 :
940 0 : dimbat = BATdescriptor(dim_bids[i]);
941 0 : if(dimbat == NULL) {
942 0 : for (i = 0; i < vndims; i++)
943 0 : GDKfree(dname[i]);
944 0 : GDKfree(dname);
945 0 : GDKfree(dim_bids);
946 0 : bat_destroy(pos);
947 0 : nc_close(ncid);
948 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
949 : }
950 0 : store->storage_api.append_col(m->session->tr, col, offset, pos, dimbat, BATcount(dimbat), TYPE_bat);
951 0 : BBPunfix(dim_bids[i]); /* phys. ref from BATdescriptor */
952 0 : BBPrelease(dim_bids[i]); /* log. ref. from loadVar */
953 0 : dimbat = NULL;
954 : }
955 :
956 0 : for (i = 0; i < vndims; i++)
957 0 : GDKfree(dname[i]);
958 0 : GDKfree(dname);
959 0 : GDKfree(dim_bids);
960 0 : bat_destroy(pos);
961 0 : nc_close(ncid);
962 0 : return msg;
963 : }
964 :
965 : #include "mel.h"
966 : static mel_func netcdf_init_funcs[] = {
967 : command("netcdf", "test", NCDFtest, false, "Returns number of variables in a given NetCDF dataset (file)", args(1,2, arg("",int),arg("filename",str))),
968 : pattern("netcdf", "attach", NCDFattach, true, "Register a NetCDF file in the vault", args(1,2, arg("",void),arg("filename",str))),
969 : command("netcdf", "importvar", NCDFimportVarStmt, true, "Import variable: compose create array string", args(1,3, arg("",str),arg("filename",str),arg("varid",int))),
970 : pattern("netcdf", "importvariable", NCDFimportVariable, true, "Import variable: create array and load data from variable varname of file fileid", args(1,3, arg("",void),arg("fileid",int),arg("varname",str))),
971 : { .imp=NULL }
972 : };
973 : #include "mal_import.h"
974 : #ifdef _MSC_VER
975 : #undef read
976 : #pragma section(".CRT$XCU",read)
977 : #endif
978 324 : LIB_STARTUP_FUNC(init_netcdf_mal)
979 324 : { mal_module("netcdf", NULL, netcdf_init_funcs); }
|
