Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : #include "monetdb_config.h"
14 : #include <netcdf.h>
15 : #include "sql_mvc.h"
16 : #include "sql.h"
17 : #include "sql_execute.h"
18 : #include "sql_scenario.h"
19 : #include "mal_exception.h"
20 : #include "netcdf_vault.h"
21 :
22 : /* SQL statements for population of NetCDF catalog */
23 : #define INSFILE \
24 : "INSERT INTO netcdf_files(file_id,location) VALUES(%d, '%s');"
25 :
26 : #define INSDIM \
27 : "INSERT INTO netcdf_dims(dim_id,file_id,name,length) VALUES(%d, %d, '%s', %d);"
28 :
29 : #define INSVAR \
30 : "INSERT INTO netcdf_vars(var_id,file_id,name,vartype,ndim,coord_dim_id) VALUES(%d, %d, '%s', '%s', %d, %d);"
31 :
32 : #define INSVARDIM \
33 : "INSERT INTO netcdf_vardim (var_id,dim_id,file_id,dimpos) VALUES(%d, %d, %d, %d);"
34 :
35 : #define INSATTR \
36 : "INSERT INTO netcdf_attrs (obj_name,att_name,att_type,value,file_id,gr_name) VALUES('%s', '%s', '%s', '%s', %d, '%s');"
37 :
38 : #define LOAD_NCDF_VAR(tpe,ncdftpe) \
39 : { \
40 : tpe *databuf; \
41 : res = COLnew(0, TYPE_##tpe, sz, TRANSIENT); \
42 : if ( res == NULL ) \
43 : return createException(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL); \
44 : databuf = (tpe *)Tloc(res, 0); \
45 : if ( (retval = nc_get_var_##ncdftpe(ncid, varid, databuf)) ) \
46 : return createException(MAL, "netcdf.importvar", \
47 : SQLSTATE(NC000) "Cannot read variable %d values: %s", \
48 : varid, nc_strerror(retval)); \
49 : }
50 :
51 : static void
52 0 : fix_quote( char *n, int l)
53 : {
54 0 : int i;
55 :
56 0 : for(i=0;i<l;i++)
57 0 : if (n[i]=='\'')
58 0 : n[i] = ' ';
59 : }
60 :
61 :
62 : /* simple test for netcdf library */
63 : str
64 0 : NCDFtest(int *vars, str *fname)
65 : {
66 0 : int ncid; /* dataset id */
67 0 : int dims, ngatts, unlimdim;
68 0 : int retval;
69 :
70 0 : str msg = MAL_SUCCEED;
71 :
72 : /* Open NetCDF file */
73 0 : if ((retval = nc_open(*fname, NC_NOWRITE, &ncid)))
74 0 : return createException(MAL, "netcdf.test", SQLSTATE(NC000) "Cannot open NetCDF file %s: %s", *fname, nc_strerror(retval));
75 :
76 0 : if ((retval = nc_inq(ncid, &dims, vars, &ngatts, &unlimdim)))
77 0 : return createException(MAL, "netcdf.test", SQLSTATE(NC000) "Cannot read NetCDF header: %s", nc_strerror(retval));
78 :
79 0 : if ((retval = nc_close(ncid)))
80 0 : return createException(MAL, "netcdf.test", SQLSTATE(NC000) "Cannot close file %s: \
81 : %s", *fname, nc_strerror(retval));
82 :
83 : return msg;
84 : }
85 :
86 : /* the following function is from ncdump utility: NetCDF type number to name */
87 : static const char *
88 0 : prim_type_name(nc_type type)
89 : {
90 0 : switch (type) {
91 : case NC_BYTE:
92 : return "byte";
93 0 : case NC_CHAR:
94 0 : return "char";
95 0 : case NC_SHORT:
96 0 : return "short";
97 0 : case NC_INT:
98 0 : return "int";
99 0 : case NC_FLOAT:
100 0 : return "float";
101 0 : case NC_DOUBLE:
102 0 : return "double";
103 : #ifdef USE_NETCDF4
104 : case NC_UBYTE:
105 : return "ubyte";
106 : case NC_USHORT:
107 : return "ushort";
108 : case NC_UINT:
109 : return "uint";
110 : case NC_INT64:
111 : return "int64";
112 : case NC_UINT64:
113 : return "uint64";
114 : case NC_STRING:
115 : return "string";
116 : #endif /* USE_NETCDF4 */
117 0 : default:
118 0 : return "bad type";
119 : }
120 : }
121 :
122 : /* Mapping NetCDF to SQL data type */
123 :
124 : static const char *
125 0 : NCDF2SQL(nc_type type)
126 : {
127 0 : switch (type) {
128 : case NC_BYTE:
129 : return "tinyint";
130 0 : case NC_CHAR:
131 0 : return "char(1)";
132 0 : case NC_SHORT:
133 0 : return "smallint";
134 0 : case NC_INT:
135 0 : return "int";
136 0 : case NC_FLOAT:
137 0 : return "float";
138 0 : case NC_DOUBLE:
139 0 : return "double";
140 : #ifdef USE_NETCDF4
141 : /* ?? mapping of unsigned types */
142 : case NC_UBYTE:
143 : return "ubyte";
144 : case NC_USHORT:
145 : return "ushort";
146 : case NC_UINT:
147 : return "uint";
148 : case NC_INT64:
149 : return "bigint";
150 : case NC_UINT64:
151 : return "uint64";
152 : case NC_STRING:
153 : return "string";
154 : #endif /* USE_NETCDF4 */
155 0 : default:
156 0 : return "type not supported";
157 : }
158 : }
159 :
160 : #define array_series(sta, ste, sto, TYPE) { \
161 : int s,g; \
162 : TYPE i, *o = (TYPE*)Tloc(bn, 0); \
163 : TYPE start = sta, step = ste, stop = sto; \
164 : if ( start < stop && step > 0) { \
165 : for ( s = 0; s < series; s++) \
166 : for ( i = start; i < stop; i += step) \
167 : for( g = 0; g < group; g++){ \
168 : *o = i; \
169 : o++; \
170 : } \
171 : } else { \
172 : for ( s = 0; s < series; s++) \
173 : for ( i = start; i > stop; i += step) \
174 : for( g = 0; g < group; g++){ \
175 : *o = i; \
176 : o++; \
177 : } \
178 : } \
179 : }
180 :
181 : /* create and populate a dimension bat */
182 : static str
183 0 : NCDFARRAYseries(bat *bid, bte start, bte step, int stop, int group, int series)
184 : {
185 0 : BAT *bn = NULL;
186 0 : BUN cnt = 0;
187 :
188 0 : cnt = (BUN) ceil(((stop * 1.0 - start) / step)) * group * series ;
189 0 : if (stop <= (int) GDK_bte_max ) {
190 0 : bte sta = (bte) start, ste = (bte) step, sto = (bte) stop;
191 :
192 0 : bn = COLnew(0, TYPE_bte, cnt, TRANSIENT);
193 0 : if ( bn == NULL)
194 0 : throw(MAL, "ntcdf.loadvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
195 0 : array_series(sta, ste, sto, bte);
196 0 : } else if (stop <= (int) GDK_sht_max) {
197 0 : sht sta = (sht) start, ste = (sht) step, sto = (sht) stop;
198 :
199 0 : bn = COLnew(0, TYPE_sht, cnt, TRANSIENT);
200 0 : if ( bn == NULL)
201 0 : throw(MAL, "netcdf.loadvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
202 0 : array_series(sta, ste, sto, sht);
203 : } else {
204 0 : int sta = (int) start, ste = (int) step, sto = (int) stop;
205 :
206 0 : bn = COLnew(0, TYPE_int, cnt, TRANSIENT);
207 0 : if ( bn == NULL)
208 0 : throw(MAL, "netcdf.loadvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
209 0 : array_series(sta, ste, sto, int);
210 : }
211 :
212 0 : BATsetcount(bn, cnt);
213 0 : bn->tsorted = (cnt <= 1 || (series == 1 && step > 0));
214 0 : bn->trevsorted = (cnt <= 1 || (series == 1 && step < 0));
215 0 : bn->tkey = (cnt <= 1);
216 0 : bn->tnonil = true;
217 0 : *bid = bn->batCacheid;
218 0 : BBPkeepref(bn);
219 0 : return MAL_SUCCEED;
220 : }
221 :
222 : str
223 0 : NCDFattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
224 : {
225 0 : mvc *m = NULL;
226 0 : sql_schema *sch = NULL;
227 0 : sql_table *tfiles = NULL, *tdims = NULL, *tvars = NULL, *tvardim = NULL, *tattrs = NULL;
228 0 : sql_column *col;
229 0 : str msg = MAL_SUCCEED;
230 0 : str fname = *getArgReference_str(stk, pci, 1);
231 0 : char buf[BUFSIZ], *s= buf;
232 0 : oid fid, rid = oid_nil;
233 0 : sql_trans *tr;
234 :
235 0 : int ncid; /* dataset id */
236 0 : int ndims, nvars, ngatts, unlimdim;
237 0 : int didx, vidx, vndims, vnatts, i, aidx, coord_dim_id = -1;
238 0 : int vdims[NC_MAX_VAR_DIMS];
239 :
240 0 : size_t dlen, alen;
241 0 : char dname[NC_MAX_NAME+1], vname[NC_MAX_NAME+1], aname[NC_MAX_NAME +1],
242 : abuf[80], *aval;
243 0 : char **dims = NULL;
244 0 : nc_type vtype, atype; /* == int */
245 :
246 0 : int retval, avalint;
247 0 : float avalfl;
248 0 : double avaldbl;
249 0 : str esc_str0, esc_str1;
250 :
251 0 : msg = getSQLContext(cntxt, mb, &m, NULL);
252 0 : if (msg)
253 : return msg;
254 :
255 0 : tr = m->session->tr;
256 0 : sqlstore *store = tr->store;
257 0 : sch = mvc_bind_schema(m, "sys");
258 0 : if ( !sch )
259 0 : return createException(MAL, "netcdf.attach", SQLSTATE(NC000) "Cannot get schema sys\n");
260 :
261 0 : tfiles = mvc_bind_table(m, sch, "netcdf_files");
262 0 : tdims = mvc_bind_table(m, sch, "netcdf_dims");
263 0 : tvars = mvc_bind_table(m, sch, "netcdf_vars");
264 0 : tvardim = mvc_bind_table(m, sch, "netcdf_vardim");
265 0 : tattrs = mvc_bind_table(m, sch, "netcdf_attrs");
266 :
267 0 : if (tfiles == NULL || tdims == NULL || tvars == NULL ||
268 0 : tvardim == NULL || tattrs == NULL)
269 0 : return createException(MAL, "netcdf.attach", SQLSTATE(NC000) "Catalog table missing\n");
270 :
271 : /* check if the file is already attached */
272 0 : col = mvc_bind_column(m, tfiles, "location");
273 0 : rid = store->table_api.column_find_row(m->session->tr, col, fname, NULL);
274 0 : if (!is_oid_nil(rid))
275 0 : return createException(SQL, "netcdf.attach", SQLSTATE(NC000) "File %s is already attached\n", fname);
276 :
277 : /* Open NetCDF file */
278 0 : if ((retval = nc_open(fname, NC_NOWRITE, &ncid)))
279 0 : return createException(MAL, "netcdf.test", SQLSTATE(NC000) "Cannot open NetCDF \
280 : file %s: %s", fname, nc_strerror(retval));
281 :
282 0 : if ((retval = nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdim)))
283 0 : return createException(MAL, "netcdf.test", SQLSTATE(NC000) "Cannot read NetCDF \
284 : header: %s", nc_strerror(retval));
285 :
286 : /* Insert row into netcdf_files table */
287 0 : col = mvc_bind_column(m, tfiles, "file_id");
288 0 : fid = store->storage_api.count_col(tr, col, 1) + 1;
289 :
290 0 : esc_str0 = SQLescapeString(fname);
291 0 : if (!esc_str0) {
292 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
293 0 : goto finish;
294 : }
295 0 : snprintf(buf, BUFSIZ, INSFILE, (int)fid, esc_str0);
296 0 : GDKfree(esc_str0);
297 0 : if ( ( msg = SQLstatementIntern(cntxt, s, "netcdf.attach", TRUE, FALSE, NULL))
298 : != MAL_SUCCEED )
299 0 : goto finish;
300 :
301 : /* Read dimensions from NetCDF header and insert a row for each one into netcdf_dims table */
302 :
303 0 : dims = (char **)GDKzalloc(sizeof(char *) * ndims);
304 0 : if (!dims) {
305 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
306 0 : goto finish;
307 : }
308 0 : for (didx = 0; didx < ndims; didx++){
309 0 : if ((retval = nc_inq_dim(ncid, didx, dname, &dlen)) != 0)
310 0 : return createException(MAL, "netcdf.attach", SQLSTATE(NC000) "Cannot read dimension %d : %s", didx, nc_strerror(retval));
311 :
312 0 : esc_str0 = SQLescapeString(dname);
313 0 : if (!esc_str0) {
314 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
315 0 : goto finish;
316 : }
317 :
318 0 : snprintf(buf, BUFSIZ, INSDIM, didx, (int)fid, esc_str0, (int)dlen);
319 0 : GDKfree(esc_str0);
320 0 : if ( ( msg = SQLstatementIntern(cntxt, s, "netcdf.attach", TRUE, FALSE, NULL))
321 : != MAL_SUCCEED )
322 0 : goto finish;
323 :
324 0 : dims[didx] = GDKstrdup(dname);
325 0 : if (!dims[didx]) {
326 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
327 0 : goto finish;
328 : }
329 : }
330 :
331 : /* Read variables and attributes from the header and insert rows in netcdf_vars, netcdf_vardims, and netcdf_attrs tables */
332 0 : for (vidx = 0; vidx < nvars; vidx++){
333 0 : if ( (retval = nc_inq_var(ncid, vidx, vname, &vtype, &vndims, vdims, &vnatts)))
334 0 : return createException(MAL, "netcdf.attach",
335 : SQLSTATE(NC000) "Cannot read variable %d : %s",
336 : vidx, nc_strerror(retval));
337 :
338 : /* Check if this is coordinate variable */
339 0 : if ( (vndims == 1) && ( strcmp(vname, dims[vdims[0]]) == 0 ))
340 0 : coord_dim_id = vdims[0];
341 : else coord_dim_id = -1;
342 :
343 0 : esc_str0 = SQLescapeString(vname);
344 0 : if (!esc_str0) {
345 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
346 0 : goto finish;
347 : }
348 :
349 0 : snprintf(buf, BUFSIZ, INSVAR, vidx, (int)fid, esc_str0, prim_type_name(vtype), vndims, coord_dim_id);
350 0 : GDKfree(esc_str0);
351 0 : if ( ( msg = SQLstatementIntern(cntxt, s, "netcdf.attach", TRUE, FALSE, NULL))
352 : != MAL_SUCCEED )
353 0 : goto finish;
354 :
355 0 : if ( coord_dim_id < 0 ){
356 0 : for (i = 0; i < vndims; i++){
357 0 : snprintf(buf, BUFSIZ, INSVARDIM, vidx, vdims[i], (int)fid, i);
358 0 : if ( ( msg = SQLstatementIntern(cntxt, s, "netcdf.attach", TRUE, FALSE, NULL))
359 : != MAL_SUCCEED )
360 0 : goto finish;
361 : }
362 : }
363 :
364 0 : if ( vnatts > 0 ) { /* fill in netcdf_attrs table */
365 :
366 0 : for (aidx = 0; aidx < vnatts; aidx++){
367 0 : if ((retval = nc_inq_attname(ncid,vidx,aidx,aname)))
368 0 : return createException(MAL, "netcdf.attach",
369 : SQLSTATE(NC000) "Cannot read attribute %d of variable %d: %s",
370 : aidx, vidx, nc_strerror(retval));
371 :
372 0 : if ((retval = nc_inq_att(ncid,vidx,aname,&atype,&alen)))
373 0 : return createException(MAL, "netcdf.attach",
374 : SQLSTATE(NC000) "Cannot read attribute %s type and length: %s",
375 : aname, nc_strerror(retval));
376 :
377 0 : esc_str0 = SQLescapeString(vname);
378 0 : if (!esc_str0) {
379 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
380 0 : goto finish;
381 : }
382 0 : esc_str1 = SQLescapeString(aname);
383 0 : if (!esc_str1) {
384 0 : GDKfree(esc_str0);
385 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
386 0 : goto finish;
387 : }
388 0 : switch ( atype ) {
389 0 : case NC_CHAR:
390 0 : aval = (char *) GDKzalloc(alen + 1);
391 0 : if (!aval) {
392 0 : GDKfree(esc_str0);
393 0 : GDKfree(esc_str1);
394 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
395 0 : goto finish;
396 : }
397 0 : if ((retval = nc_get_att_text(ncid,vidx,aname,aval)))
398 0 : return createException(MAL, "netcdf.attach",
399 : SQLSTATE(NC000) "Cannot read attribute %s value: %s",
400 : aname, nc_strerror(retval));
401 0 : fix_quote(aval, alen);
402 0 : aval[alen] = '\0';
403 0 : snprintf(buf, BUFSIZ, INSATTR, esc_str0, esc_str1, "string", aval, (int)fid, "root");
404 0 : GDKfree(aval);
405 0 : break;
406 :
407 0 : case NC_INT:
408 0 : if ((retval = nc_get_att_int(ncid,vidx,aname,&avalint)))
409 0 : return createException(MAL, "netcdf.attach",
410 : SQLSTATE(NC000) "Cannot read attribute %s value: %s",
411 : aname, nc_strerror(retval));
412 0 : snprintf(abuf,80,"%d",avalint);
413 0 : snprintf(buf, BUFSIZ, INSATTR, esc_str0, esc_str1, prim_type_name(atype), abuf, (int)fid, "root");
414 0 : break;
415 :
416 0 : case NC_FLOAT:
417 0 : if ((retval = nc_get_att_float(ncid,vidx,aname,&avalfl)))
418 0 : return createException(MAL, "netcdf.attach",
419 : SQLSTATE(NC000) "Cannot read attribute %s value: %s",
420 : aname, nc_strerror(retval));
421 0 : snprintf(abuf,80,"%7.2f",avalfl);
422 0 : snprintf(buf, BUFSIZ, INSATTR, esc_str0, esc_str1, prim_type_name(atype), abuf, (int)fid, "root");
423 0 : break;
424 :
425 0 : case NC_DOUBLE:
426 0 : if ((retval = nc_get_att_double(ncid,vidx,aname,&avaldbl)))
427 0 : return createException(MAL, "netcdf.attach",
428 : SQLSTATE(NC000) "Cannot read attribute %s value: %s",
429 : aname, nc_strerror(retval));
430 0 : snprintf(abuf,80,"%7.2e",avaldbl);
431 0 : snprintf(buf, BUFSIZ, INSATTR, esc_str0, esc_str1, prim_type_name(atype), abuf, (int)fid, "root");
432 0 : break;
433 :
434 0 : default: continue; /* next attribute */
435 : }
436 0 : GDKfree(esc_str1);
437 0 : GDKfree(esc_str0);
438 :
439 0 : printf("statement: '%s'\n", s);
440 0 : if ( ( msg = SQLstatementIntern(cntxt, s, "netcdf.attach", TRUE, FALSE, NULL))
441 : != MAL_SUCCEED )
442 0 : goto finish;
443 :
444 : } /* attr loop */
445 :
446 : }
447 : } /* var loop */
448 :
449 : /* Extract global attributes */
450 :
451 0 : for (aidx = 0; aidx < ngatts; aidx++){
452 0 : if ((retval = nc_inq_attname(ncid,NC_GLOBAL,aidx,aname)) != 0)
453 0 : return createException(MAL, "netcdf.attach",
454 : SQLSTATE(NC000) "Cannot read global attribute %d: %s",
455 : aidx, nc_strerror(retval));
456 :
457 0 : if ((retval = nc_inq_att(ncid,NC_GLOBAL,aname,&atype,&alen)) != 0){
458 : if (dims != NULL ){
459 0 : for (didx = 0; didx < ndims; didx++)
460 0 : GDKfree(dims[didx]);
461 0 : GDKfree(dims);
462 : }
463 0 : return createException(MAL, "netcdf.attach",
464 : SQLSTATE(NC000) "Cannot read global attribute %s type and length: %s",
465 : aname, nc_strerror(retval));
466 : }
467 :
468 0 : esc_str0 = SQLescapeString(aname);
469 0 : if (!esc_str0) {
470 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
471 0 : goto finish;
472 : }
473 :
474 0 : switch ( atype ) {
475 0 : case NC_CHAR:
476 0 : aval = (char *) GDKzalloc(alen + 1);
477 0 : if (!aval) {
478 0 : GDKfree(esc_str0);
479 0 : msg = createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
480 0 : goto finish;
481 : }
482 0 : if ((retval = nc_get_att_text(ncid,NC_GLOBAL,aname,aval))) {
483 0 : GDKfree(esc_str0);
484 0 : if (dims != NULL ){
485 0 : for (didx = 0; didx < ndims; didx++)
486 0 : GDKfree(dims[didx]);
487 0 : GDKfree(dims);
488 : }
489 0 : return createException(MAL, "netcdf.attach",
490 : SQLSTATE(NC000) "Cannot read global attribute %s value: %s",
491 : aname, nc_strerror(retval));
492 : }
493 0 : fix_quote(aval, alen);
494 0 : aval[alen] = '\0';
495 0 : snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", esc_str0, "string", aval, (int)fid, "root");
496 0 : GDKfree(aval);
497 0 : break;
498 :
499 0 : case NC_INT:
500 0 : if ((retval = nc_get_att_int(ncid,NC_GLOBAL,aname,&avalint))){
501 0 : GDKfree(esc_str0);
502 0 : if (dims != NULL ){
503 0 : for (didx = 0; didx < ndims; didx++)
504 0 : GDKfree(dims[didx]);
505 0 : GDKfree(dims);
506 : }
507 0 : return createException(MAL, "netcdf.attach",
508 : SQLSTATE(NC000) "Cannot read global attribute %s of type %s : %s",
509 : aname, prim_type_name(atype), nc_strerror(retval));
510 : }
511 0 : snprintf(abuf,80,"%d",avalint);
512 0 : snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", esc_str0, prim_type_name(atype), abuf, (int)fid, "root");
513 0 : break;
514 :
515 0 : case NC_FLOAT:
516 0 : if ((retval = nc_get_att_float(ncid,NC_GLOBAL,aname,&avalfl))){
517 0 : GDKfree(esc_str0);
518 0 : if (dims != NULL ){
519 0 : for (didx = 0; didx < ndims; didx++)
520 0 : GDKfree(dims[didx]);
521 0 : GDKfree(dims);
522 : }
523 0 : return createException(MAL, "netcdf.attach",
524 : SQLSTATE(NC000) "Cannot read global attribute %s of type %s: %s",
525 : aname, prim_type_name(atype), nc_strerror(retval));
526 : }
527 0 : snprintf(abuf,80,"%7.2f",avalfl);
528 0 : snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", esc_str0, prim_type_name(atype), abuf, (int)fid, "root");
529 0 : break;
530 :
531 0 : case NC_DOUBLE:
532 0 : if ((retval = nc_get_att_double(ncid,NC_GLOBAL,aname,&avaldbl))){
533 0 : GDKfree(esc_str0);
534 0 : if (dims != NULL ){
535 0 : for (didx = 0; didx < ndims; didx++)
536 0 : GDKfree(dims[didx]);
537 0 : GDKfree(dims);
538 : }
539 0 : return createException(MAL, "netcdf.attach",
540 : SQLSTATE(NC000) "Cannot read global attribute %s value: %s",
541 : aname, nc_strerror(retval));
542 : }
543 0 : snprintf(abuf,80,"%7.2e",avaldbl);
544 0 : snprintf(buf, BUFSIZ, INSATTR, "GLOBAL", esc_str0, prim_type_name(atype), abuf, (int)fid, "root");
545 0 : break;
546 :
547 0 : default: continue; /* next attribute */
548 : }
549 0 : GDKfree(esc_str0);
550 :
551 0 : printf("global: '%s'\n", s);
552 0 : if ( ( msg = SQLstatementIntern(cntxt, s, "netcdf.attach", TRUE, FALSE, NULL))
553 : != MAL_SUCCEED )
554 0 : goto finish;
555 :
556 : } /* global attr loop */
557 :
558 :
559 0 : finish:
560 0 : nc_close(ncid);
561 :
562 0 : if (dims != NULL ){
563 0 : for (didx = 0; didx < ndims; didx++)
564 0 : GDKfree(dims[didx]);
565 0 : GDKfree(dims);
566 : }
567 :
568 : return msg;
569 : }
570 :
571 :
572 : /* Compose create table statement to create table representing NetCDF variable in the
573 : * database. Used for testing, can be removed from release. */
574 : str
575 0 : NCDFimportVarStmt(str *sciqlstmt, str *fname, int *varid)
576 : {
577 0 : int ncid; /* dataset id */
578 0 : int vndims, vnatts, i, j, retval;
579 0 : int vdims[NC_MAX_VAR_DIMS];
580 :
581 0 : size_t dlen;
582 0 : char dname[NC_MAX_NAME+1], vname[NC_MAX_NAME+1];
583 0 : nc_type vtype; /* == int */
584 :
585 0 : char buf[BUFSIZ];
586 0 : str msg = MAL_SUCCEED;
587 :
588 : /* Open NetCDF file */
589 0 : if ((retval = nc_open(*fname, NC_NOWRITE, &ncid)))
590 0 : return createException(MAL, "netcdf.importvar",
591 : SQLSTATE(NC000) "Cannot open NetCDF file %s: %s", *fname, nc_strerror(retval));
592 :
593 0 : if ( (retval = nc_inq_var(ncid, *varid, vname, &vtype, &vndims, vdims, &vnatts)))
594 0 : return createException(MAL, "netcdf.attach",
595 : SQLSTATE(NC000) "Cannot read variable %d : %s", *varid, nc_strerror(retval));
596 :
597 :
598 0 : j = snprintf(buf, BUFSIZ,"create table %s( ", vname);
599 :
600 0 : for (i = 0; i < vndims; i++){
601 0 : if ((retval = nc_inq_dim(ncid, vdims[i], dname, &dlen)))
602 0 : return createException(MAL, "netcdf.attach",
603 : SQLSTATE(NC000) "Cannot read dimension %d : %s", vdims[i], nc_strerror(retval));
604 :
605 0 : (void)dlen;
606 0 : j += snprintf(buf + j, BUFSIZ - j, "%s INTEGER, ", dname);
607 :
608 : }
609 :
610 0 : j += snprintf(buf + j, BUFSIZ - j, "value %s);", NCDF2SQL(vtype));
611 :
612 0 : nc_close(ncid);
613 :
614 0 : *sciqlstmt = GDKstrdup(buf);
615 0 : if(*sciqlstmt == NULL)
616 0 : return createException(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
617 : return msg;
618 : }
619 :
620 : /* Load variable varid from data set ncid into the bat v. Generate dimension
621 : * bats dim using NCDFARRAYseries */
622 : static str
623 0 : NCDFloadVar(bat **dim, bat *v, int ncid, int varid, nc_type vtype, int vndims, int *vdims)
624 : {
625 :
626 0 : BAT *res;
627 0 : bat vbid, *dim_bids;
628 0 : int retval, i, j;
629 0 : char *sermsg = NULL;
630 0 : size_t sz = 1;
631 0 : size_t *dlen = NULL, *val_rep = NULL, *grp_rep = NULL;
632 :
633 0 : if ( dim == NULL )
634 0 : return createException(MAL, "netcdf.importvar", SQLSTATE(NC000) "array of dimension bat is NULL");
635 0 : dim_bids = *dim;
636 :
637 0 : dlen = (size_t *)GDKzalloc(sizeof(size_t) * vndims);
638 0 : if (!dlen)
639 0 : return createException(MAL, "netcdf.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
640 :
641 0 : for (i = 0; i < vndims; i++){
642 0 : if ((retval = nc_inq_dimlen(ncid, vdims[i], &dlen[i])))
643 0 : return createException(MAL, "netcdf.importvar",
644 : SQLSTATE(NC000) "Cannot read dimension %d : %s",
645 : vdims[i], nc_strerror(retval));
646 0 : sz *= dlen[i];
647 : }
648 :
649 0 : switch (vtype) {
650 0 : case NC_INT:
651 : {
652 0 : LOAD_NCDF_VAR(int,int);
653 : break;
654 : }
655 0 : case NC_FLOAT:
656 : case NC_DOUBLE:
657 : {
658 0 : LOAD_NCDF_VAR(dbl,double);
659 : break;
660 : }
661 :
662 : default:
663 0 : GDKfree(dlen);
664 0 : return createException(MAL, "netcdf.importvar",
665 : SQLSTATE(NC000) "Type %s not supported yet",
666 : prim_type_name(vtype));
667 :
668 : }
669 :
670 0 : BATsetcount(res, sz);
671 0 : res->tnonil = true;
672 0 : res->tnil = false;
673 0 : res->tsorted = false;
674 0 : res->trevsorted = false;
675 0 : BATkey(res, false);
676 0 : vbid = res->batCacheid;
677 0 : BBPkeepref(res);
678 :
679 0 : res = NULL;
680 :
681 : /* Manually create dimensions with range [0:1:dlen[i]] */
682 0 : val_rep = (size_t *)GDKmalloc(sizeof(size_t) * vndims);
683 0 : grp_rep = (size_t *)GDKmalloc(sizeof(size_t) * vndims);
684 0 : if (val_rep == NULL || grp_rep == NULL) {
685 0 : GDKfree(dlen);
686 0 : GDKfree(val_rep);
687 0 : GDKfree(grp_rep);
688 0 : throw(MAL, "netcdf.loadvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
689 : }
690 :
691 : /* compute the repetition factor inside of the series (val_rep) and of series (grp_rep) */
692 0 : for (i = 0; i < vndims; i++) {
693 0 : val_rep[i] = grp_rep[i] = 1;
694 0 : for (j = 0; j < i; j++)
695 0 : grp_rep[i] *= dlen[j];
696 0 : for (j = i + 1; j < vndims; j++)
697 0 : val_rep[i] *= dlen[j];
698 : }
699 :
700 0 : for (i = 0; i < vndims; i++) {
701 0 : sermsg = NCDFARRAYseries(&dim_bids[i], 0, 1, dlen[i], val_rep[i], grp_rep[i]);
702 :
703 0 : if (sermsg != MAL_SUCCEED) {
704 0 : BBPrelease(vbid); /* undo the BBPkeepref(res) above */
705 0 : for ( j = 0; j < i; j++) /* undo log. ref of previous dimensions */
706 0 : BBPrelease(dim_bids[j]);
707 0 : GDKfree(dlen);
708 0 : GDKfree(val_rep);
709 0 : GDKfree(grp_rep);
710 0 : return sermsg;
711 : }
712 : }
713 : /* to do : is descriptor check of dim_bids is needed? */
714 :
715 0 : GDKfree(dlen);
716 0 : GDKfree(val_rep);
717 0 : GDKfree(grp_rep);
718 :
719 0 : *v = vbid;
720 :
721 0 : return MAL_SUCCEED;
722 : }
723 :
724 : /* import variable given file id and variable name */
725 : str
726 0 : NCDFimportVariable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
727 : {
728 0 : mvc *m = NULL;
729 0 : sql_schema *sch = NULL;
730 0 : sql_table *tfiles = NULL, *arr_table = NULL;
731 0 : sql_column *col;
732 :
733 0 : str msg = MAL_SUCCEED, vname = *getArgReference_str(stk, pci, 2);
734 0 : str fname = NULL, dimtype = NULL, aname_sys = NULL;
735 0 : int fid = *getArgReference_int(stk, pci, 1);
736 0 : int varid, vndims, vnatts, i, j, retval;
737 0 : char buf[BUFSIZ], *s= buf, aname[256], **dname;
738 0 : oid rid = oid_nil;
739 0 : int vdims[NC_MAX_VAR_DIMS];
740 0 : nc_type vtype;
741 0 : int ncid; /* dataset id */
742 0 : size_t dlen;
743 0 : bat vbatid = 0, *dim_bids;
744 0 : BAT *vbat = NULL, *dimbat;
745 :
746 0 : msg = getSQLContext(cntxt, mb, &m, NULL);
747 0 : if (msg)
748 : return msg;
749 :
750 0 : sqlstore *store = m->session->tr->store;
751 0 : sch = mvc_bind_schema(m, "sys");
752 0 : if ( !sch )
753 0 : return createException(MAL, "netcdf.importvar", SQLSTATE(NC000) "Cannot get schema sys\n");
754 :
755 0 : tfiles = mvc_bind_table(m, sch, "netcdf_files");
756 0 : if (tfiles == NULL)
757 0 : return createException(MAL, "netcdf.importvar", SQLSTATE(NC000) "Catalog table missing\n");
758 :
759 : /* get the name of the attached NetCDF file */
760 0 : col = mvc_bind_column(m, tfiles, "file_id");
761 0 : if (col == NULL)
762 0 : return createException(MAL, "netcdf.importvar", SQLSTATE(NC000) "Could not find \"netcdf_files\".\"file_id\"\n");
763 0 : rid = store->table_api.column_find_row(m->session->tr, col, (void *)&fid, NULL);
764 0 : if (is_oid_nil(rid))
765 0 : return createException(MAL, "netcdf.importvar", SQLSTATE(NC000) "File %d not in the NetCDF vault\n", fid);
766 :
767 :
768 0 : col = mvc_bind_column(m, tfiles, "location");
769 0 : fname = (str)store->table_api.column_find_value(m->session->tr, col, rid);
770 :
771 : /* Open NetCDF file */
772 0 : if ((retval = nc_open(fname, NC_NOWRITE, &ncid))) {
773 0 : char *msg = createException(MAL, "netcdf.importvar", SQLSTATE(NC000) "Cannot open NetCDF file %s: %s",
774 : fname, nc_strerror(retval));
775 0 : GDKfree(fname);
776 0 : return msg;
777 : }
778 0 : GDKfree(fname);
779 :
780 : /* Get info for variable vname from NetCDF file */
781 0 : if ( (retval = nc_inq_varid(ncid, vname, &varid)) ) {
782 0 : nc_close(ncid);
783 0 : return createException(MAL, "netcdf.importvar",
784 : SQLSTATE(NC000) "Cannot read variable %s: %s",
785 : vname, nc_strerror(retval));
786 : }
787 0 : if ( (retval = nc_inq_var(ncid, varid, vname, &vtype, &vndims, vdims, &vnatts))) {
788 0 : nc_close(ncid);
789 0 : return createException(MAL, "netcdf.importvar",
790 : SQLSTATE(NC000) "Cannot read variable %d : %s",
791 : varid, nc_strerror(retval));
792 : }
793 :
794 : /* compose 'create table' statement in the buffer */
795 0 : dname = (char **) GDKzalloc( sizeof(char *) * vndims);
796 0 : if (dname == NULL) {
797 0 : nc_close(ncid);
798 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
799 : }
800 0 : for (i = 0; i < vndims; i++) {
801 0 : dname[i] = (char *) GDKzalloc(NC_MAX_NAME + 1);
802 0 : if(!dname[i]) {
803 0 : for (j = 0; j < i; j++)
804 0 : GDKfree(dname[j]);
805 0 : GDKfree(dname);
806 0 : nc_close(ncid);
807 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
808 : }
809 : }
810 :
811 0 : snprintf(aname, 256, "%s%d", vname, fid);
812 :
813 0 : j = snprintf(buf, BUFSIZ,"create table %s.%s( ", sch->base.name, aname);
814 :
815 0 : for (i = 0; i < vndims; i++){
816 0 : if ((retval = nc_inq_dim(ncid, vdims[i], dname[i], &dlen))) {
817 0 : for (j = 0; j < vndims; j++)
818 0 : GDKfree(dname[j]);
819 0 : GDKfree(dname);
820 0 : nc_close(ncid);
821 0 : return createException(MAL, "netcdf.importvar",
822 : SQLSTATE(NC000) "Cannot read dimension %d : %s",
823 : vdims[i], nc_strerror(retval));
824 : }
825 :
826 0 : if ( dlen <= (int) GDK_bte_max )
827 : dimtype = "TINYINT";
828 0 : else if ( dlen <= (int) GDK_sht_max )
829 : dimtype = "SMALLINT";
830 : else
831 0 : dimtype = "INT";
832 :
833 0 : (void)dlen;
834 0 : j += snprintf(buf + j, BUFSIZ - j, "%s %s, ", dname[i], dimtype);
835 : }
836 :
837 0 : j += snprintf(buf + j, BUFSIZ - j, "value %s);", NCDF2SQL(vtype));
838 :
839 : /* execute 'create table ' */
840 0 : msg = SQLstatementIntern(cntxt, s, "netcdf.importvar", TRUE, FALSE, NULL);
841 0 : if (msg != MAL_SUCCEED){
842 0 : for (i = 0; i < vndims; i++)
843 0 : GDKfree(dname[i]);
844 0 : GDKfree(dname);
845 0 : nc_close(ncid);
846 0 : return msg;
847 : }
848 :
849 : /* load variable data */
850 0 : dim_bids = (bat *)GDKmalloc(sizeof(bat) * vndims);
851 0 : if (dim_bids == NULL){
852 0 : for (i = 0; i < vndims; i++)
853 0 : GDKfree(dname[i]);
854 0 : GDKfree(dname);
855 0 : nc_close(ncid);
856 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
857 : }
858 :
859 0 : msg = NCDFloadVar(&dim_bids, &vbatid, ncid, varid, vtype, vndims, vdims);
860 0 : if ( msg != MAL_SUCCEED ) {
861 0 : for (i = 0; i < vndims; i++)
862 0 : GDKfree(dname[i]);
863 0 : GDKfree(dname);
864 0 : GDKfree(dim_bids);
865 0 : nc_close(ncid);
866 0 : return msg;
867 : }
868 :
869 : /* associate columns in the table with loaded variable data */
870 0 : aname_sys = toLower(aname);
871 0 : arr_table = mvc_bind_table(m, sch, aname_sys);
872 0 : if (arr_table == NULL){
873 0 : for (i = 0; i < vndims; i++)
874 0 : GDKfree(dname[i]);
875 0 : GDKfree(dname);
876 0 : GDKfree(dim_bids);
877 0 : nc_close(ncid);
878 0 : throw(MAL, "netcdf.importvar", SQLSTATE(NC000) "netcdf table %s missing\n", aname_sys);
879 : }
880 :
881 0 : col = mvc_bind_column(m, arr_table, "value");
882 0 : if (col == NULL){
883 0 : for (i = 0; i < vndims; i++)
884 0 : GDKfree(dname[i]);
885 0 : GDKfree(dname);
886 0 : GDKfree(dim_bids);
887 0 : nc_close(ncid);
888 0 : throw(MAL, "netcdf.importvar", SQLSTATE(NC000) "Cannot find column %s.value\n", aname_sys);
889 : }
890 :
891 0 : vbat = BATdescriptor(vbatid);
892 0 : if(vbat == NULL) {
893 0 : for (i = 0; i < vndims; i++)
894 0 : GDKfree(dname[i]);
895 0 : GDKfree(dname);
896 0 : GDKfree(dim_bids);
897 0 : nc_close(ncid);
898 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
899 : }
900 0 : BUN offset;
901 0 : BAT *pos = NULL;
902 0 : if (store->storage_api.claim_tab(m->session->tr, arr_table, BATcount(vbat), &offset, &pos) != LOG_OK) {
903 0 : for (i = 0; i < vndims; i++)
904 0 : GDKfree(dname[i]);
905 0 : GDKfree(dname);
906 0 : GDKfree(dim_bids);
907 0 : BBPunfix(vbatid);
908 0 : BBPrelease(vbatid);
909 0 : nc_close(ncid);
910 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
911 : }
912 0 : if (!isNew(arr_table) && sql_trans_add_dependency_change(m->session->tr, arr_table->base.id, dml) != LOG_OK) {
913 0 : for (i = 0; i < vndims; i++)
914 0 : GDKfree(dname[i]);
915 0 : GDKfree(dname);
916 0 : GDKfree(dim_bids);
917 0 : BBPunfix(vbatid);
918 0 : BBPrelease(vbatid);
919 0 : bat_destroy(pos);
920 0 : nc_close(ncid);
921 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY013) MAL_MALLOC_FAIL);
922 : }
923 0 : store->storage_api.append_col(m->session->tr, col, offset, pos, vbat, BATcount(vbat), true, vbat->ttype);
924 0 : BBPunfix(vbatid);
925 0 : BBPrelease(vbatid);
926 0 : vbat = NULL;
927 :
928 : /* associate dimension bats */
929 0 : for (i = 0; i < vndims; i++){
930 0 : col = mvc_bind_column(m, arr_table, dname[i]);
931 0 : if (col == NULL){
932 0 : for (i = 0; i < vndims; i++)
933 0 : GDKfree(dname[i]);
934 0 : GDKfree(dname);
935 0 : GDKfree(dim_bids);
936 0 : bat_destroy(pos);
937 0 : nc_close(ncid);
938 0 : throw(MAL, "netcdf.importvar", SQLSTATE(NC000) "Cannot find column %s.%s\n", aname_sys, dname[i]);
939 : }
940 :
941 0 : dimbat = BATdescriptor(dim_bids[i]);
942 0 : if(dimbat == NULL) {
943 0 : for (i = 0; i < vndims; i++)
944 0 : GDKfree(dname[i]);
945 0 : GDKfree(dname);
946 0 : GDKfree(dim_bids);
947 0 : bat_destroy(pos);
948 0 : nc_close(ncid);
949 0 : throw(MAL, "netcdf.importvar", SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
950 : }
951 0 : store->storage_api.append_col(m->session->tr, col, offset, pos, dimbat, BATcount(dimbat), true, dimbat->ttype);
952 0 : BBPunfix(dim_bids[i]); /* phys. ref from BATdescriptor */
953 0 : BBPrelease(dim_bids[i]); /* log. ref. from loadVar */
954 0 : dimbat = NULL;
955 : }
956 :
957 0 : for (i = 0; i < vndims; i++)
958 0 : GDKfree(dname[i]);
959 0 : GDKfree(dname);
960 0 : GDKfree(dim_bids);
961 0 : bat_destroy(pos);
962 0 : nc_close(ncid);
963 0 : return msg;
964 : }
965 :
966 : #include "mel.h"
967 : static mel_func netcdf_init_funcs[] = {
968 : command("netcdf", "test", NCDFtest, false, "Returns number of variables in a given NetCDF dataset (file)", args(1,2, arg("",int),arg("filename",str))),
969 : pattern("netcdf", "attach", NCDFattach, true, "Register a NetCDF file in the vault", args(1,2, arg("",void),arg("filename",str))),
970 : command("netcdf", "importvar", NCDFimportVarStmt, true, "Import variable: compose create array string", args(1,3, arg("",str),arg("filename",str),arg("varid",int))),
971 : pattern("netcdf", "importvariable", NCDFimportVariable, true, "Import variable: create array and load data from variable varname of file fileid", args(1,3, arg("",void),arg("fileid",int),arg("varname",str))),
972 : { .imp=NULL }
973 : };
974 : #include "mal_import.h"
975 : #ifdef _MSC_VER
976 : #undef read
977 : #pragma section(".CRT$XCU",read)
978 : #endif
979 316 : LIB_STARTUP_FUNC(init_netcdf_mal)
980 316 : { mal_module("netcdf", NULL, netcdf_init_funcs); }
|
