Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : /*
14 : * Authors: M. Ivanova, M. Kersten, N. Nes
15 : *
16 : * This module contains primitives for accessing data in FITS file format.
17 : */
18 :
19 : #include "monetdb_config.h"
20 : #include <glob.h>
21 :
22 : /* clash with GDK? */
23 : #undef ttype
24 : #include <fitsio.h>
25 : #include <fitsio2.h>
26 : #include <longnam.h>
27 :
28 : #include "fits.h"
29 : #include "mutils.h"
30 : #include "sql_mvc.h"
31 : #include "sql_scenario.h"
32 : #include "sql_execute.h"
33 : #include "sql.h"
34 : #include "mal_exception.h"
35 :
36 : #define FITS_INS_COL "INSERT INTO sys.fits_columns(id, name, type, units, number, table_id) \
37 : VALUES(%d,'%s','%s','%s',%d,%d);"
38 : #define ATTACHDIR "call sys.fitsattach('%s');"
39 :
40 : static void
41 0 : FITSinitCatalog(mvc *m)
42 : {
43 0 : sql_schema *sch;
44 0 : sql_table *fits_tp, *fits_fl, *fits_tbl, *fits_col;
45 0 : sql_column *col = NULL;
46 :
47 0 : sch = mvc_bind_schema(m, "sys");
48 :
49 0 : fits_fl = mvc_bind_table(m, sch, "fits_files");
50 0 : if (fits_fl == NULL) {
51 0 : mvc_create_table(&fits_fl, m, sch, "fits_files", tt_table, 0, SQL_PERSIST, 0, 2, 0);
52 0 : mvc_create_column_(&col, m, fits_fl, "id", "int", 32);
53 0 : mvc_create_column_(&col, m, fits_fl, "name", "varchar", 80);
54 : }
55 :
56 0 : fits_tbl = mvc_bind_table(m, sch, "fits_tables");
57 0 : if (fits_tbl == NULL) {
58 0 : mvc_create_table(&fits_tbl, m, sch, "fits_tables", tt_table, 0, SQL_PERSIST, 0, 8, 0);
59 0 : mvc_create_column_(&col, m, fits_tbl, "id", "int", 32);
60 0 : mvc_create_column_(&col, m, fits_tbl, "name", "varchar", 80);
61 0 : mvc_create_column_(&col, m, fits_tbl, "columns", "int", 32);
62 0 : mvc_create_column_(&col, m, fits_tbl, "file_id", "int", 32);
63 0 : mvc_create_column_(&col, m, fits_tbl, "hdu", "int", 32);
64 0 : mvc_create_column_(&col, m, fits_tbl, "date", "varchar", 80);
65 0 : mvc_create_column_(&col, m, fits_tbl, "origin", "varchar", 80);
66 0 : mvc_create_column_(&col, m, fits_tbl, "comment", "varchar", 80);
67 : }
68 :
69 0 : fits_col = mvc_bind_table(m, sch, "fits_columns");
70 0 : if (fits_col == NULL) {
71 0 : mvc_create_table(&fits_col, m, sch, "fits_columns", tt_table, 0, SQL_PERSIST, 0, 6, 0);
72 0 : mvc_create_column_(&col, m, fits_col, "id", "int", 32);
73 0 : mvc_create_column_(&col, m, fits_col, "name", "varchar", 80);
74 0 : mvc_create_column_(&col, m, fits_col, "type", "varchar", 80);
75 0 : mvc_create_column_(&col, m, fits_col, "units", "varchar", 80);
76 0 : mvc_create_column_(&col, m, fits_col, "number", "int", 32);
77 0 : mvc_create_column_(&col, m, fits_col, "table_id", "int", 32);
78 : }
79 :
80 0 : fits_tp = mvc_bind_table(m, sch, "fits_table_properties");
81 0 : if (fits_tp == NULL) {
82 0 : mvc_create_table(&fits_tp, m, sch, "fits_table_properties", tt_table, 0, SQL_PERSIST, 0, 5, 0);
83 0 : mvc_create_column_(&col, m, fits_tp, "table_id", "int", 32);
84 0 : mvc_create_column_(&col, m, fits_tp, "xtension", "varchar", 80);
85 0 : mvc_create_column_(&col, m, fits_tp, "bitpix", "int", 32);
86 0 : mvc_create_column_(&col, m, fits_tp, "stilvers", "varchar", 80);
87 0 : mvc_create_column_(&col, m, fits_tp, "stilclas", "varchar", 80);
88 : }
89 0 : }
90 :
91 : static int
92 0 : fits2mtype(int t, int rep)
93 : {
94 0 : if (rep > 1) {
95 : return TYPE_blob;
96 : }
97 0 : switch (t) {
98 : case TBIT:
99 : case TLOGICAL:
100 : return TYPE_bit;
101 : case TBYTE:
102 : case TSBYTE:
103 : return TYPE_bte;
104 : case TSTRING:
105 : return TYPE_str;
106 : case TUSHORT:
107 : case TSHORT:
108 : return TYPE_sht;
109 : case TUINT:
110 : case TINT:
111 : return TYPE_int;
112 : case TLONG:
113 : case TULONG:
114 : case TLONGLONG:
115 : return TYPE_lng;
116 : case TFLOAT:
117 : return TYPE_flt;
118 : case TDOUBLE:
119 : return TYPE_dbl;
120 : /* missing */
121 : case TCOMPLEX:
122 : case TDBLCOMPLEX:
123 : return -1;
124 : }
125 : return -1;
126 : }
127 :
128 : static int
129 0 : fits2subtype(sql_subtype *tpe, int t, long rep, long wid) /* type long used by fits library */
130 : {
131 0 : if (rep > 1) {
132 0 : sql_find_subtype(tpe, "blob", (unsigned int)rep*wid, 0);
133 0 : return 1;
134 : }
135 0 : switch (t) {
136 0 : case TBIT:
137 : case TLOGICAL:
138 0 : sql_find_subtype(tpe, "boolean", 0, 0);
139 0 : break;
140 0 : case TBYTE:
141 : case TSBYTE:
142 0 : sql_find_subtype(tpe, "char", 1, 0);
143 0 : break;
144 0 : case TSTRING:
145 0 : sql_find_subtype(tpe, "varchar", (unsigned int)wid, 0);
146 0 : break;
147 0 : case TUSHORT:
148 : case TSHORT:
149 0 : sql_find_subtype(tpe, "smallint", 16, 0);
150 0 : break;
151 0 : case TUINT:
152 : case TINT:
153 0 : sql_find_subtype(tpe, "int", 32, 0);
154 0 : break;
155 0 : case TULONG:
156 : case TLONG:
157 : case TLONGLONG:
158 0 : sql_find_subtype(tpe, "bigint", 64, 0);
159 0 : break;
160 0 : case TFLOAT:
161 0 : sql_find_subtype(tpe, "real", 32, 0);
162 0 : break;
163 0 : case TDOUBLE:
164 0 : sql_find_subtype(tpe, "double", 51, 0);
165 0 : break;
166 : /* missing */
167 : case TCOMPLEX:
168 : case TDBLCOMPLEX:
169 : return -1;
170 : }
171 : return 1;
172 : }
173 :
174 0 : str FITSexportTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
175 : {
176 0 : str msg = MAL_SUCCEED;
177 0 : str tname = *getArgReference_str(stk, pci, 1);
178 0 : mvc *m = NULL;
179 0 : sql_trans *tr;
180 0 : sql_schema *sch;
181 0 : sql_table *tbl, *column, *tables = NULL;
182 0 : sql_column *col;
183 0 : oid rid = oid_nil;
184 0 : str type, name, *colname, *tform;
185 0 : fitsfile *fptr;
186 0 : char filename[BUFSIZ];
187 0 : size_t nrows = 0;
188 0 : long optimal; /* type long used by fits library */
189 0 : rids * rs;
190 :
191 0 : lng tm0, texportboolean=0, texportchar=0, texportstring=0, texportshort=0, texportint=0, texportlng=0, texportfloat=0, texportdouble=0;
192 0 : size_t numberrow = 0, dimension = 0;
193 0 : int cc = 0, status = 0, j = 0, columns, *fid, block = 0;
194 0 : int boolcols = 0, charcols = 0, strcols = 0, shortcols = 0, intcols = 0, lngcols = 0, floatcols = 0, doublecols = 0;
195 0 : int hdutype;
196 :
197 0 : char *charvalue, *readcharrows;
198 0 : str strvalue; char **readstrrows;
199 0 : short *shortvalue, *readshortrows;
200 0 : int *intvalue, *readintrows;
201 0 : lng *lngvalue, *readlngrows;
202 0 : float *realvalue, *readfloatrows;
203 0 : double *doublevalue, *readdoublerows;
204 0 : _Bool *boolvalue, *readboolrows;
205 :
206 0 : if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
207 : return msg;
208 0 : if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
209 : return msg;
210 :
211 0 : tr = m->session->tr;
212 0 : sqlstore *store = tr->store;
213 0 : sch = mvc_bind_schema(m, "sys");
214 :
215 : /* First step: look if the table exists in the database. If the table is not in the database, the export function cannot continue */
216 :
217 0 : tbl = mvc_bind_table(m, sch, tname);
218 0 : if (tbl == NULL) {
219 0 : msg = createException (MAL, "fits.exporttable", SQLSTATE(FI000) "Table %s is missing.\n", tname);
220 0 : return msg;
221 : }
222 :
223 :
224 0 : columns = ol_length((*tbl).columns);
225 0 : colname = (str *) GDKmalloc(columns * sizeof(str));
226 0 : tform = (str *) GDKmalloc(columns * sizeof(str));
227 0 : if (colname == NULL || tform == NULL) {
228 0 : GDKfree(colname);
229 0 : GDKfree(tform);
230 0 : throw(MAL, "fits.exporttable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
231 : }
232 :
233 0 : TRC_DEBUG(FITS, "Number of columns: %d\n", columns);
234 :
235 0 : tables = mvc_bind_table(m, sch, "_tables");
236 0 : col = mvc_bind_column(m, tables, "name");
237 0 : rid = store->table_api.column_find_row(m->session->tr, col, tname, NULL);
238 :
239 0 : col = mvc_bind_column(m, tables, "id");
240 0 : fid = (int*) store->table_api.column_find_value(m->session->tr, col, rid);
241 :
242 0 : column = mvc_bind_table(m, sch, "_columns");
243 0 : col = mvc_bind_column(m, column, "table_id");
244 :
245 0 : rs = store->table_api.rids_select(m->session->tr, col, (void *) fid, (void *) fid, NULL);
246 0 : if (!rs)
247 0 : throw(MAL, "fits.exporttable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
248 0 : GDKfree(fid);
249 :
250 0 : while ((rid = store->table_api.rids_next(rs)), !is_oid_nil(rid))
251 : {
252 0 : col = mvc_bind_column(m, column, "name");
253 0 : name = (char *) store->table_api.column_find_value(m->session->tr, col, rid);
254 0 : colname[j] = toLower(name);
255 0 : GDKfree(name);
256 :
257 0 : col = mvc_bind_column(m, column, "type");
258 0 : type = (char *) store->table_api.column_find_value(m->session->tr, col, rid);
259 :
260 0 : if (strcmp(type,"boolean")==0) tform[j] = "1L";
261 :
262 0 : if (strcmp(type,"char")==0) tform[j] = "1S";
263 :
264 0 : if (strcmp(type,"varchar")==0) tform[j] = "8A";
265 :
266 0 : if (strcmp(type,"smallint")==0) tform[j] = "1I";
267 :
268 0 : if (strcmp(type,"int")==0) tform[j] = "1J";
269 :
270 0 : if (strcmp(type,"bigint")==0) tform[j] = "1K";
271 :
272 0 : if (strcmp(type,"real")==0) tform[j] = "1E";
273 :
274 0 : if (strcmp(type,"double")==0) tform[j] = "1D";
275 0 : GDKfree(type);
276 :
277 0 : j++;
278 : }
279 :
280 0 : col = mvc_bind_column(m, tbl, colname[0]);
281 :
282 0 : nrows = store->storage_api.count_col(tr, col, 0);
283 0 : assert(nrows <= (size_t) GDK_oid_max);
284 :
285 0 : snprintf(filename,BUFSIZ,"\n%s.fit",tname);
286 0 : TRC_INFO(FITS, "Filename: %s\n", filename);
287 :
288 0 : MT_remove(filename);
289 :
290 0 : status=0;
291 :
292 0 : fits_create_file(&fptr, filename, &status);
293 0 : fits_create_img(fptr, USHORT_IMG, 0, NULL, &status);
294 0 : fits_close_file(fptr, &status);
295 0 : fits_open_file(&fptr, filename, READWRITE, &status);
296 :
297 0 : fits_movabs_hdu(fptr, 1, &hdutype, &status);
298 0 : fits_create_tbl( fptr, BINARY_TBL, 0, columns, colname, tform, NULL, tname, &status);
299 :
300 0 : for (cc = 0; cc < columns; cc++)
301 : {
302 0 : char * columntype;
303 0 : col = mvc_bind_column(m, tbl, colname[cc]);
304 0 : columntype = col -> type.type->base.name;
305 :
306 0 : if (strcmp(columntype,"boolean")==0)
307 : {
308 0 : boolcols++; dimension = 0; block = 0;
309 0 : fits_get_rowsize(fptr,&optimal,&status);
310 0 : readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal);
311 :
312 0 : for (numberrow = 0; numberrow < nrows ; numberrow++)
313 : {
314 0 : boolvalue = (_Bool*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
315 0 : readboolrows[dimension] = *boolvalue;
316 0 : GDKfree(boolvalue);
317 0 : dimension++;
318 :
319 0 : if (dimension == (size_t) optimal)
320 : {
321 0 : dimension = 0;
322 0 : tm0 = GDKusec();
323 0 : fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, optimal, readboolrows, &status);
324 0 : texportboolean += GDKusec() - tm0;
325 0 : GDKfree(readboolrows);
326 0 : readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal);
327 0 : block++;
328 : }
329 : }
330 0 : tm0 = GDKusec();
331 0 : fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, dimension, readboolrows, &status);
332 0 : texportboolean += GDKusec() - tm0;
333 0 : GDKfree(readboolrows);
334 : }
335 :
336 0 : if (strcmp(columntype,"char")==0)
337 : {
338 0 : charcols++; dimension = 0; block = 0;
339 0 : fits_get_rowsize(fptr,&optimal,&status);
340 0 : readcharrows = (char *) GDKmalloc (sizeof(char) * optimal);
341 :
342 0 : for (numberrow = 0; numberrow < nrows ; numberrow++)
343 : {
344 0 : charvalue = (char*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
345 0 : readcharrows[dimension] = *charvalue;
346 0 : GDKfree(charvalue);
347 0 : dimension++;
348 :
349 0 : if (dimension == (size_t) optimal)
350 : {
351 0 : dimension = 0;
352 0 : tm0 = GDKusec();
353 0 : fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, optimal, readcharrows, &status);
354 0 : texportchar += GDKusec() - tm0;
355 0 : GDKfree(readcharrows);
356 0 : readcharrows = (char *) GDKmalloc (sizeof(char) * optimal);
357 0 : block++;
358 : }
359 : }
360 0 : tm0 = GDKusec();
361 0 : fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, dimension, readcharrows, &status);
362 0 : texportchar += GDKusec() - tm0;
363 0 : GDKfree(readcharrows);
364 : }
365 :
366 0 : if (strcmp(columntype,"varchar")==0)
367 : {
368 0 : strcols++; dimension=0; block=0;
369 0 : fits_get_rowsize(fptr,&optimal,&status);
370 0 : readstrrows = (char **) GDKmalloc (sizeof(char *) * optimal);
371 :
372 0 : for (numberrow = 0; numberrow < nrows ; numberrow++)
373 : {
374 0 : strvalue = (char *) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
375 0 : readstrrows[dimension] = strvalue;
376 0 : dimension++;
377 :
378 0 : if (dimension == (size_t) optimal)
379 : {
380 0 : dimension = 0;
381 0 : tm0 = GDKusec();
382 0 : fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, optimal, readstrrows, &status);
383 0 : texportstring += GDKusec() - tm0;
384 0 : for (dimension = 0; dimension < (size_t) optimal; dimension++)
385 0 : GDKfree(readstrrows[dimension]);
386 0 : dimension = 0;
387 0 : GDKfree(readstrrows);
388 0 : readstrrows = (char **) GDKmalloc(sizeof(char *) * optimal);
389 0 : block++;
390 : }
391 : }
392 0 : tm0 = GDKusec();
393 0 : fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, dimension, readstrrows, &status);
394 0 : texportstring += GDKusec() - tm0;
395 0 : for (numberrow = 0; numberrow < dimension; numberrow++)
396 0 : GDKfree(readstrrows[numberrow]);
397 0 : GDKfree(readstrrows);
398 : }
399 :
400 0 : if (strcmp(columntype,"smallint")==0)
401 : {
402 0 : shortcols++; dimension = 0; block = 0;
403 0 : fits_get_rowsize(fptr,&optimal,&status);
404 0 : readshortrows = (short *) GDKmalloc (sizeof(short) * optimal);
405 :
406 0 : for (numberrow = 0; numberrow < nrows ; numberrow++)
407 : {
408 0 : shortvalue = (short*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
409 0 : readshortrows[dimension] = *shortvalue;
410 0 : GDKfree(shortvalue);
411 0 : dimension++;
412 :
413 0 : if (dimension == (size_t) optimal)
414 : {
415 0 : dimension = 0;
416 0 : tm0 = GDKusec();
417 0 : fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, optimal, readshortrows, &status);
418 0 : texportshort += GDKusec() - tm0;
419 0 : GDKfree(readshortrows);
420 0 : readshortrows = (short *) GDKmalloc (sizeof(short) * optimal);
421 0 : block++;
422 : }
423 : }
424 0 : tm0 = GDKusec();
425 0 : fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, dimension, readshortrows, &status);
426 0 : texportshort += GDKusec() - tm0;
427 0 : GDKfree(readshortrows);
428 : }
429 :
430 0 : if (strcmp(columntype,"int")==0)
431 : {
432 0 : intcols++; dimension = 0; block = 0;
433 0 : fits_get_rowsize(fptr,&optimal,&status);
434 0 : readintrows = (int *) GDKmalloc (sizeof(int) * optimal);
435 :
436 0 : for (numberrow = 0; numberrow < nrows ; numberrow++)
437 : {
438 0 : intvalue = (int*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
439 0 : readintrows[dimension] = *intvalue;
440 0 : GDKfree(intvalue);
441 0 : dimension++;
442 :
443 0 : if (dimension == (size_t) optimal)
444 : {
445 0 : dimension = 0;
446 0 : tm0 = GDKusec();
447 0 : fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, optimal, readintrows, &status);
448 0 : texportint += GDKusec() - tm0;
449 0 : GDKfree(readintrows);
450 0 : readintrows = (int *) GDKmalloc (sizeof(int) * optimal);
451 0 : block++;
452 : }
453 : }
454 0 : tm0 = GDKusec();
455 0 : fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, dimension, readintrows, &status);
456 0 : texportint += GDKusec() - tm0;
457 0 : GDKfree(readintrows);
458 : }
459 :
460 0 : if (strcmp(columntype,"bigint")==0)
461 : {
462 0 : lngcols++; dimension = 0; block = 0;
463 0 : fits_get_rowsize(fptr,&optimal,&status);
464 0 : readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal);
465 :
466 0 : for (numberrow = 0; numberrow < nrows ; numberrow++)
467 : {
468 0 : lngvalue = (lng*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
469 0 : readlngrows[dimension] = *lngvalue;
470 0 : GDKfree(lngvalue);
471 0 : dimension++;
472 :
473 0 : if (dimension == (size_t) optimal)
474 : {
475 0 : dimension = 0;
476 0 : tm0 = GDKusec();
477 0 : fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, optimal, readlngrows, &status);
478 0 : texportlng += GDKusec() - tm0;
479 0 : GDKfree(readlngrows);
480 0 : readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal);
481 0 : block++;
482 : }
483 : }
484 0 : tm0 = GDKusec();
485 0 : fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, dimension, readlngrows, &status);
486 0 : texportlng += GDKusec() - tm0;
487 0 : GDKfree(readlngrows);
488 : }
489 :
490 0 : if (strcmp(columntype,"real")==0)
491 : {
492 0 : floatcols++; dimension = 0; block = 0;
493 0 : fits_get_rowsize(fptr,&optimal,&status);
494 0 : readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal);
495 :
496 0 : for (numberrow = 0; numberrow < nrows ; numberrow++)
497 : {
498 0 : realvalue = (float*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
499 0 : readfloatrows[dimension] = *realvalue;
500 0 : GDKfree(realvalue);
501 0 : dimension++;
502 :
503 0 : if (dimension == (size_t) optimal)
504 : {
505 0 : dimension = 0;
506 0 : tm0 = GDKusec();
507 0 : fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, optimal, readfloatrows, &status);
508 0 : texportfloat += GDKusec() - tm0;
509 0 : GDKfree(readfloatrows);
510 0 : readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal);
511 0 : block++;
512 : }
513 : }
514 0 : tm0 = GDKusec();
515 0 : fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, dimension, readfloatrows, &status);
516 0 : texportfloat += GDKusec() - tm0;
517 0 : GDKfree(readfloatrows);
518 : }
519 :
520 0 : if (strcmp(columntype,"double")==0)
521 : {
522 0 : doublecols++; dimension = 0; block = 0;
523 0 : fits_get_rowsize(fptr,&optimal,&status);
524 0 : readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal);
525 :
526 0 : for (numberrow = 0; numberrow < nrows ; numberrow++)
527 : {
528 0 : doublevalue = (double*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
529 0 : readdoublerows[dimension] = *doublevalue;
530 0 : GDKfree(doublevalue);
531 0 : dimension++;
532 :
533 0 : if (dimension == (size_t) optimal)
534 : {
535 0 : dimension = 0;
536 0 : tm0 = GDKusec();
537 0 : fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status);
538 0 : texportdouble += GDKusec() - tm0;
539 0 : GDKfree(readdoublerows);
540 0 : readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal);
541 0 : block++;
542 : }
543 : }
544 0 : tm0 = GDKusec();
545 0 : fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status);
546 0 : texportdouble += GDKusec() - tm0;
547 0 : GDKfree(readdoublerows);
548 : }
549 : }
550 :
551 :
552 : // print all the times that were needed to export each one of the columns
553 0 : if (texportboolean > 0) TRC_DEBUG(FITS, "%d Boolean\tcolumn(s) exported in "LLFMT" usec\n", boolcols, texportboolean);
554 0 : if (texportchar > 0) TRC_DEBUG(FITS, "%d Char\t\tcolumn(s) exported in "LLFMT" usec\n", charcols, texportchar);
555 0 : if (texportstring > 0) TRC_DEBUG(FITS, "%d String\tcolumn(s) exported in "LLFMT" usec\n", strcols, texportstring);
556 0 : if (texportshort > 0) TRC_DEBUG(FITS, "%d Short\t\tcolumn(s) exported in "LLFMT" usec\n", shortcols, texportshort);
557 0 : if (texportint > 0) TRC_DEBUG(FITS, "%d Integer\tcolumn(s) exported in "LLFMT" usec\n", intcols, texportint);
558 0 : if (texportlng > 0) TRC_DEBUG(FITS, "%d Long\t\tcolumn(s) exported in "LLFMT" usec\n", lngcols, texportlng);
559 0 : if (texportfloat > 0) TRC_DEBUG(FITS, "%d Float\t\tcolumn(s) exported in "LLFMT" usec\n", floatcols, texportfloat);
560 0 : if (texportdouble > 0) TRC_DEBUG(FITS, "%d Double\tcolumn(s) exported in "LLFMT" usec\n", doublecols, texportdouble);
561 :
562 :
563 0 : fits_close_file(fptr, &status);
564 0 : return msg;
565 : }
566 :
567 :
568 0 : str FITSdir(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
569 : {
570 0 : str msg = MAL_SUCCEED;
571 0 : str dir = *getArgReference_str(stk, pci, 1);
572 0 : DIR *dp;
573 0 : struct dirent *ep;
574 0 : fitsfile *fptr;
575 0 : char *s;
576 0 : int status = 0;
577 0 : (void)mb;
578 :
579 0 : dp = opendir(dir);
580 0 : if (dp != NULL) {
581 : char stmt[BUFSIZ];
582 : char fname[BUFSIZ];
583 :
584 : s = stmt;
585 :
586 0 : while ((ep = readdir(dp)) != NULL && !msg) {
587 0 : snprintf(fname, sizeof(fname), "%s/%s", dir, ep->d_name);
588 0 : status = 0;
589 0 : fits_open_file(&fptr, fname, READONLY, &status);
590 0 : if (status == 0) {
591 0 : char *filename = SQLescapeString(fname);
592 0 : if (!filename) {
593 0 : msg = createException(MAL, "fits.listdir", SQLSTATE(HY013) MAL_MALLOC_FAIL);
594 0 : break;
595 : }
596 0 : snprintf(stmt, sizeof(stmt), ATTACHDIR, filename);
597 0 : GDKfree(filename);
598 0 : TRC_DEBUG(FITS, "Executing: %s\n", s);
599 0 : msg = SQLstatementIntern(cntxt, s, "fits.listofdir", TRUE, FALSE, NULL);
600 0 : fits_close_file(fptr, &status);
601 : }
602 :
603 : }
604 0 : (void)closedir(dp);
605 : } else
606 0 : msg = createException(MAL, "listdir", SQLSTATE(FI000) "Couldn't open the directory");
607 :
608 0 : return msg;
609 : }
610 :
611 0 : str FITSdirpat(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
612 : {
613 0 : str msg = MAL_SUCCEED;
614 0 : str dir = *getArgReference_str(stk, pci, 1);
615 0 : str pat = *getArgReference_str(stk, pci, 2);
616 0 : fitsfile *fptr;
617 0 : char *s;
618 0 : int status = 0;
619 0 : glob_t globbuf;
620 0 : char fulldirectory[BUFSIZ];
621 0 : size_t j = 0;
622 :
623 0 : (void)mb;
624 :
625 0 : globbuf.gl_offs = 0;
626 0 : snprintf(fulldirectory, sizeof(fulldirectory), "%s/%s", dir, pat);
627 0 : glob(fulldirectory, GLOB_DOOFFS, NULL, &globbuf);
628 :
629 0 : TRC_DEBUG(FITS, "Fulldir: %s - Size: %zu\n", fulldirectory, globbuf.gl_pathc);
630 :
631 0 : if (globbuf.gl_pathc == 0)
632 0 : throw(MAL, "fits.listdirpat", SQLSTATE(FI000) "Couldn't open the directory or there are no files that match the pattern");
633 :
634 0 : for (j = 0; j < globbuf.gl_pathc; j++) {
635 0 : char stmt[BUFSIZ];
636 0 : char fname[BUFSIZ];
637 :
638 0 : s = stmt;
639 0 : strcpy_len(fname, globbuf.gl_pathv[j], sizeof(fname));
640 0 : status = 0;
641 0 : fits_open_file(&fptr, fname, READONLY, &status);
642 0 : if (status == 0) {
643 0 : char *filename = SQLescapeString(fname);
644 0 : if (!filename) {
645 0 : throw(MAL, "fits.listdirpat", SQLSTATE(HY013) MAL_MALLOC_FAIL);
646 : }
647 0 : snprintf(stmt, sizeof(stmt), ATTACHDIR, filename);
648 0 : GDKfree(filename);
649 0 : TRC_DEBUG(FITS, "Executing: %s\n", s);
650 0 : msg = SQLstatementIntern(cntxt, s, "fits.listofdirpat", TRUE, FALSE, NULL);
651 0 : fits_close_file(fptr, &status);
652 :
653 0 : break;
654 : }
655 : }
656 :
657 : return msg;
658 : }
659 :
660 :
661 : str
662 0 : FITStest(int *res, str *fname)
663 : {
664 0 : fitsfile *fptr; /* pointer to the FITS file, defined in fitsio.h */
665 0 : str msg = MAL_SUCCEED;
666 0 : int status = 0, hdutype;
667 :
668 0 : *res = 0;
669 0 : if (fits_open_file(&fptr, *fname, READONLY, &status))
670 0 : msg = createException(MAL, "fits.test", SQLSTATE(FI000) "Missing FITS file %s", *fname);
671 : else {
672 0 : fits_movabs_hdu(fptr, 2, &hdutype, &status);
673 0 : *res = hdutype;
674 0 : fits_close_file(fptr, &status);
675 : }
676 :
677 0 : return msg;
678 : }
679 :
680 0 : str FITSattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
681 : {
682 0 : mvc *m = NULL;
683 0 : sql_trans *tr;
684 0 : sql_schema *sch;
685 0 : sql_table *fits_tp, *fits_fl, *fits_tbl, *fits_col, *tbl = NULL;
686 0 : sql_column *col;
687 0 : str msg = MAL_SUCCEED;
688 0 : str fname = *getArgReference_str(stk, pci, 1);
689 0 : fitsfile *fptr; /* pointer to the FITS file */
690 0 : int status = 0, i, j, hdutype, hdunum = 1, cnum = 0, bitpixnumber = 0;
691 0 : oid fid, tid, cid, rid = oid_nil;
692 0 : char tname[BUFSIZ], *tname_low = NULL, *s, bname[BUFSIZ-100], stmt[BUFSIZ];
693 0 : long tbcol; /* type long used by fits library */
694 0 : char cname[BUFSIZ], tform[BUFSIZ], tunit[BUFSIZ], tnull[BUFSIZ], tdisp[BUFSIZ];
695 0 : char *esc_cname, *esc_tform, *esc_tunit;
696 0 : double tscal, tzero;
697 0 : char xtensionname[BUFSIZ] = "", stilversion[BUFSIZ] = "";
698 0 : char stilclass[BUFSIZ] = "", tdate[BUFSIZ] = "", orig[BUFSIZ] = "", comm[BUFSIZ] = "";
699 :
700 0 : if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
701 : return msg;
702 0 : if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
703 : return msg;
704 :
705 0 : if (fits_open_file(&fptr, fname, READONLY, &status)) {
706 0 : msg = createException(MAL, "fits.attach", SQLSTATE(FI000) "Missing FITS file %s.\n", fname);
707 0 : return msg;
708 : }
709 :
710 0 : tr = m->session->tr;
711 0 : sch = mvc_bind_schema(m, "sys");
712 0 : sqlstore *store = tr->store;
713 :
714 0 : fits_fl = mvc_bind_table(m, sch, "fits_files");
715 0 : if (fits_fl == NULL)
716 0 : FITSinitCatalog(m);
717 :
718 0 : fits_fl = mvc_bind_table(m, sch, "fits_files");
719 0 : fits_tbl = mvc_bind_table(m, sch, "fits_tables");
720 0 : fits_col = mvc_bind_table(m, sch, "fits_columns");
721 0 : fits_tp = mvc_bind_table(m, sch, "fits_table_properties");
722 :
723 : /* check if the file is already attached */
724 0 : col = mvc_bind_column(m, fits_fl, "name");
725 0 : rid = store->table_api.column_find_row(m->session->tr, col, fname, NULL);
726 0 : if (!is_oid_nil(rid)) {
727 0 : fits_close_file(fptr, &status);
728 0 : throw(MAL, "fits.attach", SQLSTATE(FI000) "File %s already attached\n", fname);
729 : }
730 :
731 : /* add row in the fits_files catalog table */
732 0 : BUN offset;
733 0 : if (store->storage_api.claim_tab(m->session->tr, fits_fl, 1, &offset, NULL) != LOG_OK) {
734 0 : fits_close_file(fptr, &status);
735 0 : throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
736 : }
737 0 : if (!isNew(fits_fl) && sql_trans_add_dependency_change(m->session->tr, fits_fl->base.id, dml) != LOG_OK) {
738 0 : fits_close_file(fptr, &status);
739 0 : throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
740 : }
741 0 : col = mvc_bind_column(m, fits_fl, "id");
742 0 : fid = store->storage_api.count_col(tr, col, 0) + 1;
743 0 : store->storage_api.append_col(m->session->tr,
744 : mvc_bind_column(m, fits_fl, "id"), offset, NULL, &fid, 1, TYPE_int);
745 0 : store->storage_api.append_col(m->session->tr,
746 : mvc_bind_column(m, fits_fl, "name"), offset, NULL, &fname, 1, TYPE_str);
747 0 : col = mvc_bind_column(m, fits_tbl, "id");
748 0 : tid = store->storage_api.count_col(tr, col, 0) + 1;
749 :
750 0 : if ((s = strrchr(fname, DIR_SEP)) == NULL)
751 : s = fname;
752 : else
753 0 : s++;
754 0 : if (strcpy_len(bname, s, sizeof(bname)) >= sizeof(bname)) {
755 0 : fits_close_file(fptr, &status);
756 0 : throw(MAL, "fits.attach", SQLSTATE(FI000) "File name too long\n");
757 : }
758 0 : strcpy(bname, s);
759 0 : s = strrchr(bname, '.');
760 0 : if (s) *s = 0;
761 :
762 0 : fits_get_num_hdus(fptr, &hdunum, &status);
763 0 : for (i = 1; i <= hdunum; i++) {
764 0 : fits_movabs_hdu(fptr, i, &hdutype, &status);
765 0 : if (hdutype != ASCII_TBL && hdutype != BINARY_TBL)
766 0 : continue;
767 :
768 : /* SQL table name - the name of FITS extention */
769 0 : fits_read_key(fptr, TSTRING, "EXTNAME", tname, NULL, &status);
770 0 : if (status) {
771 0 : snprintf(tname, sizeof(tname), "%s_%d", bname, i);
772 0 : tname_low = toLower(tname);
773 0 : status = 0;
774 : }else { /* check table name for existence in the fits catalog */
775 0 : tname_low = toLower(tname);
776 0 : col = mvc_bind_column(m, fits_tbl, "name");
777 0 : rid = store->table_api.column_find_row(m->session->tr, col, tname_low, NULL);
778 : /* or as regular SQL table */
779 0 : tbl = mvc_bind_table(m, sch, tname_low);
780 0 : if (!is_oid_nil(rid) || tbl) {
781 0 : snprintf(tname, sizeof(tname), "%s_%d", bname, i);
782 0 : tname_low = toLower(tname);
783 : }
784 : }
785 :
786 0 : fits_read_key(fptr, TSTRING, "BITPIX", &bitpixnumber, NULL, &status);
787 0 : if (status) {
788 0 : status = 0;
789 : }
790 0 : fits_read_key(fptr, TSTRING, "DATE-HDU", tdate, NULL, &status);
791 0 : if (status) {
792 0 : status = 0;
793 : }
794 0 : fits_read_key(fptr, TSTRING, "XTENSION", xtensionname, NULL, &status);
795 0 : if (status) {
796 0 : status = 0;
797 : }
798 0 : fits_read_key(fptr, TSTRING, "STILVERS", stilversion, NULL, &status);
799 0 : if (status) {
800 0 : status = 0;
801 : }
802 0 : fits_read_key(fptr, TSTRING, "STILCLAS", stilclass, NULL, &status);
803 0 : if (status) {
804 0 : status = 0;
805 : }
806 0 : fits_read_key(fptr, TSTRING, "ORIGIN", orig, NULL, &status);
807 0 : if (status) {
808 0 : status = 0;
809 : }
810 0 : fits_read_key(fptr, TSTRING, "COMMENT", comm, NULL, &status);
811 0 : if (status) {
812 0 : status = 0;
813 : }
814 :
815 0 : fits_get_num_cols(fptr, &cnum, &status);
816 :
817 0 : BUN offset;
818 0 : if (store->storage_api.claim_tab(m->session->tr, fits_tbl, 1, &offset, NULL) != LOG_OK) {
819 0 : fits_close_file(fptr, &status);
820 0 : throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
821 : }
822 0 : if (!isNew(fits_tbl) && sql_trans_add_dependency_change(m->session->tr, fits_tbl->base.id, dml) != LOG_OK) {
823 0 : fits_close_file(fptr, &status);
824 0 : throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
825 : }
826 0 : store->storage_api.append_col(m->session->tr,
827 : mvc_bind_column(m, fits_tbl, "id"), offset, NULL, &tid, 1, TYPE_int);
828 0 : store->storage_api.append_col(m->session->tr,
829 : mvc_bind_column(m, fits_tbl, "name"), offset, NULL, &tname_low, 1, TYPE_str);
830 0 : store->storage_api.append_col(m->session->tr,
831 : mvc_bind_column(m, fits_tbl, "columns"), offset, NULL, &cnum, 1, TYPE_int);
832 0 : store->storage_api.append_col(m->session->tr,
833 : mvc_bind_column(m, fits_tbl, "file_id"), offset, NULL, &fid, 1, TYPE_int);
834 0 : store->storage_api.append_col(m->session->tr,
835 : mvc_bind_column(m, fits_tbl, "hdu"), offset, NULL, &i, 1, TYPE_int);
836 0 : char *vptr = tdate;
837 0 : store->storage_api.append_col(m->session->tr,
838 : mvc_bind_column(m, fits_tbl, "date"), offset, NULL, &vptr, 1, TYPE_str);
839 0 : vptr = orig;
840 0 : store->storage_api.append_col(m->session->tr,
841 : mvc_bind_column(m, fits_tbl, "origin"), offset, NULL, &vptr, 1, TYPE_str);
842 0 : vptr = comm;
843 0 : store->storage_api.append_col(m->session->tr,
844 : mvc_bind_column(m, fits_tbl, "comment"), offset, NULL, &vptr, 1, TYPE_str);
845 :
846 0 : if (store->storage_api.claim_tab(m->session->tr, fits_tp, 1, &offset, NULL) != LOG_OK) {
847 0 : fits_close_file(fptr, &status);
848 0 : throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
849 : }
850 0 : if (!isNew(fits_tp) && sql_trans_add_dependency_change(m->session->tr, fits_tp->base.id, dml) != LOG_OK) {
851 0 : fits_close_file(fptr, &status);
852 0 : throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
853 : }
854 0 : store->storage_api.append_col(m->session->tr,
855 : mvc_bind_column(m, fits_tp, "table_id"), offset, NULL, &tid, 1, TYPE_int);
856 0 : vptr = xtensionname;
857 0 : store->storage_api.append_col(m->session->tr,
858 : mvc_bind_column(m, fits_tp, "xtension"), offset, NULL, &vptr, 1, TYPE_str);
859 0 : store->storage_api.append_col(m->session->tr,
860 : mvc_bind_column(m, fits_tp, "bitpix"), offset, NULL, &bitpixnumber, 1, TYPE_int);
861 0 : vptr = stilversion;
862 0 : store->storage_api.append_col(m->session->tr,
863 : mvc_bind_column(m, fits_tp, "stilvers"), offset, NULL, &vptr, 1, TYPE_str);
864 0 : vptr = stilclass;
865 0 : store->storage_api.append_col(m->session->tr,
866 : mvc_bind_column(m, fits_tp, "stilclas"), offset, NULL, &vptr, 1, TYPE_str);
867 :
868 : /* read columns description */
869 0 : s = stmt;
870 0 : col = mvc_bind_column(m, fits_col, "id");
871 0 : cid = store->storage_api.count_col(tr, col, 0) + 1;
872 0 : for (j = 1; j <= cnum; j++, cid++) {
873 0 : fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status);
874 : /* escape the various strings to avoid SQL injection attacks */
875 0 : esc_cname = SQLescapeString(cname);
876 0 : if (!esc_cname) {
877 0 : fits_close_file(fptr, &status);
878 0 : throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
879 : }
880 0 : esc_tform = SQLescapeString(tform);
881 0 : if (!esc_tform) {
882 0 : GDKfree(esc_cname);
883 0 : fits_close_file(fptr, &status);
884 0 : throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
885 : }
886 0 : esc_tunit = SQLescapeString(tunit);
887 0 : if (!esc_tunit) {
888 0 : GDKfree(esc_tform);
889 0 : GDKfree(esc_cname);
890 0 : fits_close_file(fptr, &status);
891 0 : throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
892 : }
893 0 : snprintf(stmt, sizeof(stmt), FITS_INS_COL, (int)cid, esc_cname, esc_tform, esc_tunit, j, (int)tid);
894 0 : GDKfree(esc_tunit);
895 0 : GDKfree(esc_tform);
896 0 : GDKfree(esc_cname);
897 0 : msg = SQLstatementIntern(cntxt, s, "fits.attach", TRUE, FALSE, NULL);
898 0 : if (msg != MAL_SUCCEED) {
899 0 : fits_close_file(fptr, &status);
900 0 : return msg;
901 : }
902 : }
903 0 : tid++;
904 : }
905 0 : fits_close_file(fptr, &status);
906 :
907 0 : return MAL_SUCCEED;
908 : }
909 :
910 0 : str FITSloadTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
911 : {
912 0 : mvc *m = NULL;
913 0 : sql_schema *sch;
914 0 : sql_table *fits_fl, *fits_tbl, *tbl = NULL;
915 0 : sql_column *col;
916 0 : sql_subtype tpe;
917 0 : fitsfile *fptr;
918 0 : str tname = *getArgReference_str(stk, pci, 1);
919 0 : str fname;
920 0 : str msg = MAL_SUCCEED;
921 0 : oid rid = oid_nil, frid = oid_nil;
922 0 : int status = 0, cnum = 0, *fid, *hdu, hdutype, j, anynull = 0, mtype;
923 0 : int *tpcode = NULL;
924 0 : long *rep = NULL, *wid = NULL, rows; /* type long used by fits library */
925 0 : char keywrd[80], **cname, nm[FLEN_VALUE];
926 0 : const void *nilptr;
927 0 : BUN offset;
928 0 : BAT *pos = NULL;
929 :
930 0 : if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
931 : return msg;
932 0 : if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
933 : return msg;
934 0 : sch = mvc_bind_schema(m, "sys");
935 :
936 0 : fits_tbl = mvc_bind_table(m, sch, "fits_tables");
937 0 : if (fits_tbl == NULL) {
938 0 : msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "FITS catalog is missing.\n");
939 0 : return msg;
940 : }
941 :
942 0 : tbl = mvc_bind_table(m, sch, tname);
943 0 : if (tbl) {
944 0 : msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Table %s is already created.\n", tname);
945 0 : return msg;
946 : }
947 :
948 0 : col = mvc_bind_column(m, fits_tbl, "name");
949 0 : sqlstore *store = m->session->tr->store;
950 0 : rid = store->table_api.column_find_row(m->session->tr, col, tname, NULL);
951 0 : if (is_oid_nil(rid)) {
952 0 : msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname);
953 0 : return msg;
954 : }
955 :
956 : /* Open FITS file and move to the table HDU */
957 0 : col = mvc_bind_column(m, fits_tbl, "file_id");
958 0 : fid = (int*)store->table_api.column_find_value(m->session->tr, col, rid);
959 :
960 0 : fits_fl = mvc_bind_table(m, sch, "fits_files");
961 0 : col = mvc_bind_column(m, fits_fl, "id");
962 0 : frid = store->table_api.column_find_row(m->session->tr, col, (void *)fid, NULL);
963 0 : GDKfree(fid);
964 0 : if (is_oid_nil(frid)) {
965 0 : msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname);
966 0 : return msg;
967 : }
968 0 : col = mvc_bind_column(m, fits_fl, "name");
969 0 : fname = (char *)store->table_api.column_find_value(m->session->tr, col, frid);
970 0 : if (fits_open_file(&fptr, fname, READONLY, &status)) {
971 0 : msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Missing FITS file %s.\n", fname);
972 0 : GDKfree(fname);
973 0 : return msg;
974 : }
975 0 : GDKfree(fname);
976 :
977 0 : col = mvc_bind_column(m, fits_tbl, "hdu");
978 0 : hdu = (int*)store->table_api.column_find_value(m->session->tr, col, rid);
979 0 : fits_movabs_hdu(fptr, *hdu, &hdutype, &status);
980 0 : if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) {
981 0 : msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "HDU %d is not a table.\n", *hdu);
982 0 : GDKfree(hdu);
983 0 : fits_close_file(fptr, &status);
984 0 : return msg;
985 : }
986 0 : GDKfree(hdu);
987 :
988 : /* create a SQL table to hold the FITS table */
989 : /* col = mvc_bind_column(m, fits_tbl, "columns");
990 : cnum = *(int*) store->table_api.column_find_value(m->session->tr, col, rid); */
991 0 : fits_get_num_cols(fptr, &cnum, &status);
992 0 : mvc_create_table(&tbl, m, sch, tname, tt_table, 0, SQL_PERSIST, 0, cnum, 0);
993 :
994 : // TODO: Check that the allocations succeeded
995 0 : tpcode = (int *)GDKzalloc(sizeof(int) * cnum);
996 0 : rep = (long *)GDKzalloc(sizeof(long) * cnum);
997 0 : wid = (long *)GDKzalloc(sizeof(long) * cnum);
998 0 : cname = (char **)GDKzalloc(sizeof(char *) * cnum);
999 :
1000 0 : for (j = 1; j <= cnum; j++) {
1001 0 : sql_column *col = NULL;
1002 : /* fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status); */
1003 0 : snprintf(keywrd, 80, "TTYPE%d", j);
1004 0 : fits_read_key(fptr, TSTRING, keywrd, nm, NULL, &status);
1005 0 : if (status) {
1006 0 : snprintf(nm, FLEN_VALUE, "column_%d", j);
1007 0 : status = 0;
1008 : }
1009 0 : cname[j - 1] = toLower(nm);
1010 0 : fits_get_coltype(fptr, j, &tpcode[j - 1], &rep[j - 1], &wid[j - 1], &status);
1011 0 : fits2subtype(&tpe, tpcode[j - 1], rep[j - 1], wid[j - 1]);
1012 :
1013 0 : TRC_DEBUG(FITS, "%d %ld %ld - M: %s\n", tpcode[j-1], rep[j-1], wid[j-1], tpe.type->base.name);
1014 :
1015 0 : mvc_create_column(&col, m, tbl, cname[j - 1], &tpe);
1016 : }
1017 :
1018 : /* data load */
1019 0 : fits_get_num_rows(fptr, &rows, &status);
1020 : /* Nothing more to do */
1021 0 : if (rows == 0) {
1022 0 : goto bailout;
1023 : }
1024 :
1025 0 : TRC_INFO(FITS, "Loading %ld rows in table %s\n", rows, tname);
1026 :
1027 0 : if (store->storage_api.claim_tab(m->session->tr, tbl, rows, &offset, &pos) != LOG_OK) {
1028 0 : msg = createException(MAL, "fits.loadtable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
1029 0 : goto bailout;
1030 : }
1031 0 : for (j = 1; j <= cnum; j++) {
1032 0 : BAT *tmp = NULL;
1033 0 : lng time0 = GDKusec();
1034 0 : mtype = fits2mtype(tpcode[j - 1], rep[j - 1]);
1035 0 : nilptr = ATOMnilptr(mtype);
1036 0 : col = mvc_bind_column(m, tbl, cname[j - 1]);
1037 :
1038 0 : tmp = COLnew(0, mtype, rows, TRANSIENT);
1039 0 : if (tmp == NULL){
1040 0 : msg = createException(MAL,"fits.load", SQLSTATE(HY013) MAL_MALLOC_FAIL);
1041 0 : goto bailout;
1042 : }
1043 0 : if (mtype == TYPE_blob) {
1044 0 : long i;
1045 0 : unsigned long nbytes = rep[j - 1] * wid[j - 1];
1046 0 : blob **v = (blob **)GDKzalloc(sizeof(blob *) * rows);
1047 :
1048 0 : mtype = fits2mtype(tpcode[j - 1], 1);
1049 0 : nilptr = ATOMnilptr(mtype);
1050 :
1051 0 : if (v == NULL) {
1052 0 : BBPreclaim(tmp);
1053 0 : msg = createException(MAL,"fits.load", SQLSTATE(HY013) MAL_MALLOC_FAIL);
1054 0 : goto bailout;
1055 : }
1056 :
1057 0 : for(i = 0; i < rows; i++) {
1058 0 : v[i] = (blob *)GDKmalloc(offsetof(blob, data) + nbytes);
1059 0 : if (v[i] == NULL) {
1060 0 : BBPreclaim(tmp);
1061 0 : long k = 0;
1062 0 : for (k = 0; k < i; k++) {
1063 0 : GDKfree(v[k]);
1064 : }
1065 0 : GDKfree(v);
1066 0 : msg = createException(MAL,"fits.load", SQLSTATE(HY013) MAL_MALLOC_FAIL);
1067 0 : goto bailout;
1068 : }
1069 0 : fits_read_col(fptr, tpcode[j - 1], j, i + 1, 1, rep[j - 1], (void *)nilptr,
1070 0 : (void *)v[i]->data, &anynull, &status);
1071 0 : v[i]->nitems = nbytes;
1072 0 : if (BUNappend(tmp, v[i], false) != GDK_SUCCEED) {
1073 0 : BBPreclaim(tmp);
1074 0 : for (i = 0; i < rows; i++) {
1075 0 : GDKfree(v[i]);
1076 : }
1077 0 : GDKfree(v);
1078 0 : msg = createException(MAL,"fits.loadtable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
1079 0 : goto bailout;
1080 : }
1081 : }
1082 :
1083 0 : for(i = 0; i < rows; i++) {
1084 0 : GDKfree(v[i]);
1085 : }
1086 0 : GDKfree(v);
1087 : }
1088 0 : else if (mtype == TYPE_str) {
1089 : /* char *v = GDKzalloc(wid[j-1]);*/
1090 : /* type long demanded by "rows", i.e., by fits library */
1091 0 : long bsize = 50, batch = bsize, k, i;
1092 0 : lng tm0, tloadtm = 0, tattachtm = 0;
1093 0 : char **v = (char **) GDKzalloc(sizeof(char *) * bsize);
1094 0 : for(i = 0; i < bsize; i++)
1095 0 : v[i] = GDKzalloc(wid[j-1]);
1096 0 : for(i = 0; i < rows; i += batch) {
1097 0 : batch = rows - i < bsize ? rows - i: bsize;
1098 0 : tm0 = GDKusec();
1099 0 : fits_read_col(fptr, tpcode[j - 1], j, 1 + i, 1, batch, (void *) nilptr, (void *)v, &anynull, &status);
1100 0 : tloadtm += GDKusec() - tm0;
1101 0 : tm0 = GDKusec();
1102 0 : for(k = 0; k < batch ; k++)
1103 0 : if (BUNappend(tmp, v[k], false) != GDK_SUCCEED) {
1104 0 : BBPreclaim(tmp);
1105 0 : msg = createException(MAL, "fits.loadtable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
1106 0 : goto bailout;
1107 : }
1108 0 : tattachtm += GDKusec() - tm0;
1109 : }
1110 0 : for(i = 0; i < bsize ; i++)
1111 0 : GDKfree(v[i]);
1112 0 : GDKfree(v);
1113 0 : TRC_INFO(FITS, "String column load "LLFMT" usec, BUNappend "LLFMT" usec\n", tloadtm, tattachtm);
1114 : }
1115 : else {
1116 0 : BATiter bi = bat_iterator_nolock(tmp);
1117 0 : fits_read_col(fptr, tpcode[j - 1], j, 1, 1, rows, (void *) nilptr, (void *)BUNtloc(bi, 0), &anynull, &status);
1118 0 : BATsetcount(tmp, rows);
1119 0 : tmp->tsorted = false;
1120 0 : tmp->trevsorted = false;
1121 : }
1122 :
1123 0 : if (status) {
1124 0 : char buf[FLEN_ERRMSG + 1];
1125 0 : fits_read_errmsg(buf);
1126 0 : BBPreclaim(tmp);
1127 0 : msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Cannot load column %s of %s table: %s.\n", cname[j - 1], tname, buf);
1128 0 : break;
1129 : }
1130 :
1131 0 : TRC_INFO(FITS, "#Column %s loaded for "LLFMT" usec\t", cname[j-1], GDKusec() - time0);
1132 0 : if (store->storage_api.append_col(m->session->tr, col, offset, pos, tmp, BATcount(tmp), TYPE_bat) != LOG_OK) {
1133 0 : BBPreclaim(tmp);
1134 0 : msg = createException(MAL, "fits.loadtable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
1135 0 : break;
1136 : }
1137 0 : TRC_INFO(FITS, "Total "LLFMT" usec\n", GDKusec() - time0);
1138 0 : BBPreclaim(tmp);
1139 : }
1140 :
1141 0 : bailout:
1142 0 : bat_destroy(pos);
1143 0 : GDKfree(tpcode);
1144 0 : GDKfree(rep);
1145 0 : GDKfree(wid);
1146 0 : GDKfree(cname);
1147 :
1148 0 : fits_close_file(fptr, &status);
1149 0 : return msg;
1150 : }
1151 :
1152 : #include "mel.h"
1153 : static mel_func fits_init_funcs[] = {
1154 : pattern("fits", "listdir", FITSdir, true, "Attach all FITS files in the directory", args(1,2, arg("",void),arg("dirname",str))),
1155 : pattern("fits", "listdirpattern", FITSdirpat, true, "Attach all FITS file in the directory, giving a pattern", args(1,3, arg("",void),arg("dirname",str),arg("pattern",str))),
1156 : command("fits", "fitstest", FITStest, false, "Returns the type of first extension in the FITS file filename", args(1,2, arg("",int),arg("filename",str))),
1157 : pattern("fits", "attach", FITSattach, true, "Open a FITS file and return catalog of the table HDUs", args(1,2, arg("",void),arg("fname",str))),
1158 : pattern("fits", "load", FITSloadTable, true, "Load a FITS table from an attached file", args(1,2, arg("",void),arg("tablename",str))),
1159 : pattern("fits", "export", FITSexportTable, false, "Export a table to a FITS file", args(1,2, arg("",void),arg("tablename",str))),
1160 : { .imp=NULL }
1161 : };
1162 : #include "mal_import.h"
1163 : #ifdef _MSC_VER
1164 : #undef read
1165 : #pragma section(".CRT$XCU",read)
1166 : #endif
1167 324 : LIB_STARTUP_FUNC(init_fits_mal)
1168 324 : { mal_module("fits", NULL, fits_init_funcs); }
|
