LCOV - code coverage report
Current view: top level - sql/backends/monet5/vaults/fits - fits.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 2 756 0.3 %
Date: 2024-10-03 20:03:20 Functions: 1 9 11.1 %

          Line data    Source code
       1             : /*
       2             :  * SPDX-License-Identifier: MPL-2.0
       3             :  *
       4             :  * This Source Code Form is subject to the terms of the Mozilla Public
       5             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       6             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       7             :  *
       8             :  * Copyright 2024 MonetDB Foundation;
       9             :  * Copyright August 2008 - 2023 MonetDB B.V.;
      10             :  * Copyright 1997 - July 2008 CWI.
      11             :  */
      12             : 
      13             : /*
      14             :  * Authors: M. Ivanova, M. Kersten, N. Nes
      15             :  *
      16             :  * This module contains primitives for accessing data in FITS file format.
      17             :  */
      18             : 
      19             : #include "monetdb_config.h"
      20             : #include <glob.h>
      21             : 
      22             : /* clash with GDK? */
      23             : #undef ttype
      24             : #include <fitsio.h>
      25             : #include <fitsio2.h>
      26             : #include <longnam.h>
      27             : 
      28             : #include "fits.h"
      29             : #include "mutils.h"
      30             : #include "sql_mvc.h"
      31             : #include "sql_scenario.h"
      32             : #include "sql_execute.h"
      33             : #include "sql.h"
      34             : #include "mal_exception.h"
      35             : 
      36             : #define FITS_INS_COL "INSERT INTO sys.fits_columns(id, name, type, units, number, table_id) \
      37             :          VALUES(%d,'%s','%s','%s',%d,%d);"
      38             : #define ATTACHDIR "call sys.fitsattach('%s');"
      39             : 
      40             : static void
      41           0 : FITSinitCatalog(mvc *m)
      42             : {
      43           0 :         sql_schema *sch;
      44           0 :         sql_table *fits_tp, *fits_fl, *fits_tbl, *fits_col;
      45           0 :         sql_column *col = NULL;
      46             : 
      47           0 :         sch = mvc_bind_schema(m, "sys");
      48             : 
      49           0 :         fits_fl = mvc_bind_table(m, sch, "fits_files");
      50           0 :         if (fits_fl == NULL) {
      51           0 :                 mvc_create_table(&fits_fl, m, sch, "fits_files", tt_table, 0, SQL_PERSIST, 0, 2, 0);
      52           0 :                 mvc_create_column_(&col, m, fits_fl, "id", "int", 32);
      53           0 :                 mvc_create_column_(&col, m, fits_fl, "name", "varchar", 80);
      54             :         }
      55             : 
      56           0 :         fits_tbl = mvc_bind_table(m, sch, "fits_tables");
      57           0 :         if (fits_tbl == NULL) {
      58           0 :                 mvc_create_table(&fits_tbl, m, sch, "fits_tables", tt_table, 0, SQL_PERSIST, 0, 8, 0);
      59           0 :                 mvc_create_column_(&col, m, fits_tbl, "id", "int", 32);
      60           0 :                 mvc_create_column_(&col, m, fits_tbl, "name", "varchar", 80);
      61           0 :                 mvc_create_column_(&col, m, fits_tbl, "columns", "int", 32);
      62           0 :                 mvc_create_column_(&col, m, fits_tbl, "file_id", "int", 32);
      63           0 :                 mvc_create_column_(&col, m, fits_tbl, "hdu", "int", 32);
      64           0 :                 mvc_create_column_(&col, m, fits_tbl, "date", "varchar", 80);
      65           0 :                 mvc_create_column_(&col, m, fits_tbl, "origin", "varchar", 80);
      66           0 :                 mvc_create_column_(&col, m, fits_tbl, "comment", "varchar", 80);
      67             :         }
      68             : 
      69           0 :         fits_col = mvc_bind_table(m, sch, "fits_columns");
      70           0 :         if (fits_col == NULL) {
      71           0 :                 mvc_create_table(&fits_col, m, sch, "fits_columns", tt_table, 0, SQL_PERSIST, 0, 6, 0);
      72           0 :                 mvc_create_column_(&col, m, fits_col, "id", "int", 32);
      73           0 :                 mvc_create_column_(&col, m, fits_col, "name", "varchar", 80);
      74           0 :                 mvc_create_column_(&col, m, fits_col, "type", "varchar", 80);
      75           0 :                 mvc_create_column_(&col, m, fits_col, "units", "varchar", 80);
      76           0 :                 mvc_create_column_(&col, m, fits_col, "number", "int", 32);
      77           0 :                 mvc_create_column_(&col, m, fits_col, "table_id", "int", 32);
      78             :         }
      79             : 
      80           0 :         fits_tp = mvc_bind_table(m, sch, "fits_table_properties");
      81           0 :         if (fits_tp == NULL) {
      82           0 :                 mvc_create_table(&fits_tp, m, sch, "fits_table_properties", tt_table, 0, SQL_PERSIST, 0, 5, 0);
      83           0 :                 mvc_create_column_(&col, m, fits_tp, "table_id", "int", 32);
      84           0 :                 mvc_create_column_(&col, m, fits_tp, "xtension", "varchar", 80);
      85           0 :                 mvc_create_column_(&col, m, fits_tp, "bitpix", "int", 32);
      86           0 :                 mvc_create_column_(&col, m, fits_tp, "stilvers", "varchar", 80);
      87           0 :                 mvc_create_column_(&col, m, fits_tp, "stilclas", "varchar", 80);
      88             :         }
      89           0 : }
      90             : 
      91             : static int
      92           0 : fits2mtype(int t, int rep)
      93             : {
      94           0 :         if (rep > 1) {
      95             :                 return TYPE_blob;
      96             :         }
      97           0 :         switch (t) {
      98             :         case TBIT:
      99             :         case TLOGICAL:
     100             :                 return TYPE_bit;
     101             :         case TBYTE:
     102             :         case TSBYTE:
     103             :                 return TYPE_bte;
     104             :         case TSTRING:
     105             :                 return TYPE_str;
     106             :         case TUSHORT:
     107             :         case TSHORT:
     108             :                 return TYPE_sht;
     109             :         case TUINT:
     110             :         case TINT:
     111             :                 return TYPE_int;
     112             :         case TLONG:
     113             :         case TULONG:
     114             :         case TLONGLONG:
     115             :                 return TYPE_lng;
     116             :         case TFLOAT:
     117             :                 return TYPE_flt;
     118             :         case TDOUBLE:
     119             :                 return TYPE_dbl;
     120             :         /* missing */
     121             :         case TCOMPLEX:
     122             :         case TDBLCOMPLEX:
     123             :                 return -1;
     124             :         }
     125             :         return -1;
     126             : }
     127             : 
     128             : static int
     129           0 : fits2subtype(sql_subtype *tpe, int t, long rep, long wid) /* type long used by fits library */
     130             : {
     131           0 :         if (rep > 1) {
     132           0 :                 sql_find_subtype(tpe, "blob", (unsigned int)rep*wid, 0);
     133           0 :                 return 1;
     134             :         }
     135           0 :         switch (t) {
     136           0 :         case TBIT:
     137             :         case TLOGICAL:
     138           0 :                 sql_find_subtype(tpe, "boolean", 0, 0);
     139           0 :                 break;
     140           0 :         case TBYTE:
     141             :         case TSBYTE:
     142           0 :                 sql_find_subtype(tpe, "char", 1, 0);
     143           0 :                 break;
     144           0 :         case TSTRING:
     145           0 :                 sql_find_subtype(tpe, "varchar", (unsigned int)wid, 0);
     146           0 :                 break;
     147           0 :         case TUSHORT:
     148             :         case TSHORT:
     149           0 :                 sql_find_subtype(tpe, "smallint", 16, 0);
     150           0 :                 break;
     151           0 :         case TUINT:
     152             :         case TINT:
     153           0 :                 sql_find_subtype(tpe, "int", 32, 0);
     154           0 :                 break;
     155           0 :         case TULONG:
     156             :         case TLONG:
     157             :         case TLONGLONG:
     158           0 :                 sql_find_subtype(tpe, "bigint", 64, 0);
     159           0 :                 break;
     160           0 :         case TFLOAT:
     161           0 :                 sql_find_subtype(tpe, "real", 32, 0);
     162           0 :                 break;
     163           0 :         case TDOUBLE:
     164           0 :                 sql_find_subtype(tpe, "double", 51, 0);
     165           0 :                 break;
     166             :         /* missing */
     167             :         case TCOMPLEX:
     168             :         case TDBLCOMPLEX:
     169             :                 return -1;
     170             :         }
     171             :         return 1;
     172             : }
     173             : 
     174           0 : str FITSexportTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
     175             : {
     176           0 :         str msg = MAL_SUCCEED;
     177           0 :         str tname = *getArgReference_str(stk, pci, 1);
     178           0 :         mvc *m = NULL;
     179           0 :         sql_trans *tr;
     180           0 :         sql_schema *sch;
     181           0 :         sql_table *tbl, *column, *tables = NULL;
     182           0 :         sql_column *col;
     183           0 :         oid rid = oid_nil;
     184           0 :         str type, name, *colname, *tform;
     185           0 :         fitsfile *fptr;
     186           0 :         char filename[BUFSIZ];
     187           0 :         size_t nrows = 0;
     188           0 :         long optimal; /* type long used by fits library */
     189           0 :         rids * rs;
     190             : 
     191           0 :         lng tm0, texportboolean=0, texportchar=0, texportstring=0, texportshort=0, texportint=0, texportlng=0, texportfloat=0, texportdouble=0;
     192           0 :         size_t numberrow = 0, dimension = 0;
     193           0 :         int cc = 0, status = 0, j = 0, columns, *fid, block = 0;
     194           0 :         int boolcols = 0, charcols = 0, strcols = 0, shortcols = 0, intcols = 0, lngcols = 0, floatcols = 0, doublecols = 0;
     195           0 :         int hdutype;
     196             : 
     197           0 :         char *charvalue, *readcharrows;
     198           0 :         str strvalue; char **readstrrows;
     199           0 :         short *shortvalue, *readshortrows;
     200           0 :         int *intvalue, *readintrows;
     201           0 :         lng *lngvalue, *readlngrows;
     202           0 :         float *realvalue, *readfloatrows;
     203           0 :         double *doublevalue, *readdoublerows;
     204           0 :         _Bool *boolvalue, *readboolrows;
     205             : 
     206           0 :         if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
     207             :                 return msg;
     208           0 :         if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
     209             :                 return msg;
     210             : 
     211           0 :         tr = m->session->tr;
     212           0 :         sqlstore *store = tr->store;
     213           0 :         sch = mvc_bind_schema(m, "sys");
     214             : 
     215             :         /* First step: look if the table exists in the database. If the table is not in the database, the export function cannot continue */
     216             : 
     217           0 :         tbl = mvc_bind_table(m, sch, tname);
     218           0 :         if (tbl == NULL) {
     219           0 :                 msg = createException (MAL, "fits.exporttable", SQLSTATE(FI000) "Table %s is missing.\n", tname);
     220           0 :                 return msg;
     221             :         }
     222             : 
     223             : 
     224           0 :         columns = ol_length((*tbl).columns);
     225           0 :         colname = (str *) GDKmalloc(columns * sizeof(str));
     226           0 :         tform = (str *) GDKmalloc(columns * sizeof(str));
     227           0 :         if (colname == NULL || tform == NULL) {
     228           0 :                 GDKfree(colname);
     229           0 :                 GDKfree(tform);
     230           0 :                 throw(MAL, "fits.exporttable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     231             :         }
     232             : 
     233           0 :         TRC_DEBUG(FITS, "Number of columns: %d\n", columns);
     234             : 
     235           0 :         tables = mvc_bind_table(m, sch, "_tables");
     236           0 :         col = mvc_bind_column(m, tables, "name");
     237           0 :         rid = store->table_api.column_find_row(m->session->tr, col, tname, NULL);
     238             : 
     239           0 :         col = mvc_bind_column(m, tables, "id");
     240           0 :         fid = (int*) store->table_api.column_find_value(m->session->tr, col, rid);
     241             : 
     242           0 :         column =  mvc_bind_table(m, sch, "_columns");
     243           0 :         col = mvc_bind_column(m, column, "table_id");
     244             : 
     245           0 :         rs = store->table_api.rids_select(m->session->tr, col, (void *) fid, (void *) fid, NULL);
     246           0 :         if (!rs)
     247           0 :                 throw(MAL, "fits.exporttable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     248           0 :         GDKfree(fid);
     249             : 
     250           0 :         while ((rid = store->table_api.rids_next(rs)), !is_oid_nil(rid))
     251             :         {
     252           0 :                 col = mvc_bind_column(m, column, "name");
     253           0 :                 name = (char *) store->table_api.column_find_value(m->session->tr, col, rid);
     254           0 :                 colname[j] = toLower(name);
     255           0 :                 GDKfree(name);
     256             : 
     257           0 :                 col = mvc_bind_column(m, column, "type");
     258           0 :                 type = (char *) store->table_api.column_find_value(m->session->tr, col, rid);
     259             : 
     260           0 :                 if (strcmp(type,"boolean")==0) tform[j] = "1L";
     261             : 
     262           0 :                 if (strcmp(type,"char")==0) tform[j] = "1S";
     263             : 
     264           0 :                 if (strcmp(type,"varchar")==0) tform[j] = "8A";
     265             : 
     266           0 :                 if (strcmp(type,"smallint")==0) tform[j] = "1I";
     267             : 
     268           0 :                 if (strcmp(type,"int")==0) tform[j] = "1J";
     269             : 
     270           0 :                 if (strcmp(type,"bigint")==0) tform[j] = "1K";
     271             : 
     272           0 :                 if (strcmp(type,"real")==0) tform[j] = "1E";
     273             : 
     274           0 :                 if (strcmp(type,"double")==0) tform[j] = "1D";
     275           0 :                 GDKfree(type);
     276             : 
     277           0 :                 j++;
     278             :         }
     279             : 
     280           0 :         col = mvc_bind_column(m, tbl, colname[0]);
     281             : 
     282           0 :         nrows = store->storage_api.count_col(tr, col, 0);
     283           0 :         assert(nrows <= (size_t) GDK_oid_max);
     284             : 
     285           0 :         snprintf(filename,BUFSIZ,"\n%s.fit",tname);
     286           0 :         TRC_INFO(FITS, "Filename: %s\n", filename);
     287             : 
     288           0 :         MT_remove(filename);
     289             : 
     290           0 :         status=0;
     291             : 
     292           0 :         fits_create_file(&fptr, filename, &status);
     293           0 :         fits_create_img(fptr,  USHORT_IMG, 0, NULL, &status);
     294           0 :         fits_close_file(fptr, &status);
     295           0 :         fits_open_file(&fptr, filename, READWRITE, &status);
     296             : 
     297           0 :         fits_movabs_hdu(fptr, 1, &hdutype, &status);
     298           0 :         fits_create_tbl( fptr, BINARY_TBL, 0, columns, colname, tform, NULL, tname, &status);
     299             : 
     300           0 :         for (cc = 0; cc < columns; cc++)
     301             :         {
     302           0 :                 char * columntype;
     303           0 :                 col = mvc_bind_column(m, tbl, colname[cc]);
     304           0 :                 columntype = col -> type.type->base.name;
     305             : 
     306           0 :                 if (strcmp(columntype,"boolean")==0)
     307             :                 {
     308           0 :                         boolcols++; dimension = 0; block = 0;
     309           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     310           0 :                         readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal);
     311             : 
     312           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     313             :                         {
     314           0 :                                 boolvalue = (_Bool*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     315           0 :                                 readboolrows[dimension] = *boolvalue;
     316           0 :                                 GDKfree(boolvalue);
     317           0 :                                 dimension++;
     318             : 
     319           0 :                                 if (dimension == (size_t) optimal)
     320             :                                 {
     321           0 :                                         dimension = 0;
     322           0 :                                         tm0 = GDKusec();
     323           0 :                                         fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, optimal, readboolrows, &status);
     324           0 :                                         texportboolean += GDKusec() - tm0;
     325           0 :                                         GDKfree(readboolrows);
     326           0 :                                         readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal);
     327           0 :                                         block++;
     328             :                                 }
     329             :                         }
     330           0 :                         tm0 = GDKusec();
     331           0 :                         fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, dimension, readboolrows, &status);
     332           0 :                         texportboolean += GDKusec() - tm0;
     333           0 :                         GDKfree(readboolrows);
     334             :                 }
     335             : 
     336           0 :                 if (strcmp(columntype,"char")==0)
     337             :                 {
     338           0 :                         charcols++; dimension = 0; block = 0;
     339           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     340           0 :                         readcharrows = (char *) GDKmalloc (sizeof(char) * optimal);
     341             : 
     342           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     343             :                         {
     344           0 :                                 charvalue = (char*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     345           0 :                                 readcharrows[dimension] = *charvalue;
     346           0 :                                 GDKfree(charvalue);
     347           0 :                                 dimension++;
     348             : 
     349           0 :                                 if (dimension == (size_t) optimal)
     350             :                                 {
     351           0 :                                         dimension = 0;
     352           0 :                                         tm0 = GDKusec();
     353           0 :                                         fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, optimal, readcharrows, &status);
     354           0 :                                         texportchar += GDKusec() - tm0;
     355           0 :                                         GDKfree(readcharrows);
     356           0 :                                         readcharrows = (char *) GDKmalloc (sizeof(char) * optimal);
     357           0 :                                         block++;
     358             :                                 }
     359             :                         }
     360           0 :                         tm0 = GDKusec();
     361           0 :                         fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, dimension, readcharrows, &status);
     362           0 :                         texportchar += GDKusec() - tm0;
     363           0 :                         GDKfree(readcharrows);
     364             :                 }
     365             : 
     366           0 :                 if (strcmp(columntype,"varchar")==0)
     367             :                 {
     368           0 :                         strcols++; dimension=0; block=0;
     369           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     370           0 :                         readstrrows = (char **) GDKmalloc (sizeof(char *) * optimal);
     371             : 
     372           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     373             :                         {
     374           0 :                                 strvalue = (char *) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     375           0 :                                 readstrrows[dimension] = strvalue;
     376           0 :                                 dimension++;
     377             : 
     378           0 :                                 if (dimension == (size_t) optimal)
     379             :                                 {
     380           0 :                                         dimension = 0;
     381           0 :                                         tm0 = GDKusec();
     382           0 :                                         fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, optimal, readstrrows, &status);
     383           0 :                                         texportstring += GDKusec() - tm0;
     384           0 :                                         for (dimension = 0; dimension < (size_t) optimal; dimension++)
     385           0 :                                                 GDKfree(readstrrows[dimension]);
     386           0 :                                         dimension = 0;
     387           0 :                                         GDKfree(readstrrows);
     388           0 :                                         readstrrows = (char **) GDKmalloc(sizeof(char *) * optimal);
     389           0 :                                         block++;
     390             :                                 }
     391             :                         }
     392           0 :                         tm0 = GDKusec();
     393           0 :                         fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, dimension, readstrrows, &status);
     394           0 :                         texportstring += GDKusec() - tm0;
     395           0 :                         for (numberrow = 0; numberrow < dimension; numberrow++)
     396           0 :                                 GDKfree(readstrrows[numberrow]);
     397           0 :                         GDKfree(readstrrows);
     398             :                 }
     399             : 
     400           0 :                 if (strcmp(columntype,"smallint")==0)
     401             :                 {
     402           0 :                         shortcols++; dimension = 0; block = 0;
     403           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     404           0 :                         readshortrows = (short *) GDKmalloc (sizeof(short) * optimal);
     405             : 
     406           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     407             :                         {
     408           0 :                                 shortvalue = (short*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     409           0 :                                 readshortrows[dimension] = *shortvalue;
     410           0 :                                 GDKfree(shortvalue);
     411           0 :                                 dimension++;
     412             : 
     413           0 :                                 if (dimension == (size_t) optimal)
     414             :                                 {
     415           0 :                                         dimension = 0;
     416           0 :                                         tm0 = GDKusec();
     417           0 :                                         fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, optimal, readshortrows, &status);
     418           0 :                                         texportshort += GDKusec() - tm0;
     419           0 :                                         GDKfree(readshortrows);
     420           0 :                                         readshortrows = (short *) GDKmalloc (sizeof(short) * optimal);
     421           0 :                                         block++;
     422             :                                 }
     423             :                         }
     424           0 :                         tm0 = GDKusec();
     425           0 :                         fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, dimension, readshortrows, &status);
     426           0 :                         texportshort += GDKusec() - tm0;
     427           0 :                         GDKfree(readshortrows);
     428             :                 }
     429             : 
     430           0 :                 if (strcmp(columntype,"int")==0)
     431             :                 {
     432           0 :                         intcols++; dimension = 0; block = 0;
     433           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     434           0 :                         readintrows = (int *) GDKmalloc (sizeof(int) * optimal);
     435             : 
     436           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     437             :                         {
     438           0 :                                 intvalue = (int*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     439           0 :                                 readintrows[dimension] = *intvalue;
     440           0 :                                 GDKfree(intvalue);
     441           0 :                                 dimension++;
     442             : 
     443           0 :                                 if (dimension == (size_t) optimal)
     444             :                                 {
     445           0 :                                         dimension = 0;
     446           0 :                                         tm0 = GDKusec();
     447           0 :                                         fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, optimal, readintrows, &status);
     448           0 :                                         texportint += GDKusec() - tm0;
     449           0 :                                         GDKfree(readintrows);
     450           0 :                                         readintrows = (int *) GDKmalloc (sizeof(int) * optimal);
     451           0 :                                         block++;
     452             :                                 }
     453             :                         }
     454           0 :                         tm0 = GDKusec();
     455           0 :                         fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, dimension, readintrows, &status);
     456           0 :                         texportint += GDKusec() - tm0;
     457           0 :                         GDKfree(readintrows);
     458             :                 }
     459             : 
     460           0 :                 if (strcmp(columntype,"bigint")==0)
     461             :                 {
     462           0 :                         lngcols++; dimension = 0; block = 0;
     463           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     464           0 :                         readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal);
     465             : 
     466           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     467             :                         {
     468           0 :                                 lngvalue = (lng*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     469           0 :                                 readlngrows[dimension] = *lngvalue;
     470           0 :                                 GDKfree(lngvalue);
     471           0 :                                 dimension++;
     472             : 
     473           0 :                                 if (dimension == (size_t) optimal)
     474             :                                 {
     475           0 :                                         dimension = 0;
     476           0 :                                         tm0 = GDKusec();
     477           0 :                                         fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, optimal, readlngrows, &status);
     478           0 :                                         texportlng += GDKusec() - tm0;
     479           0 :                                         GDKfree(readlngrows);
     480           0 :                                         readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal);
     481           0 :                                         block++;
     482             :                                 }
     483             :                         }
     484           0 :                         tm0 = GDKusec();
     485           0 :                         fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, dimension, readlngrows, &status);
     486           0 :                         texportlng += GDKusec() - tm0;
     487           0 :                         GDKfree(readlngrows);
     488             :                 }
     489             : 
     490           0 :                 if (strcmp(columntype,"real")==0)
     491             :                 {
     492           0 :                         floatcols++; dimension = 0; block = 0;
     493           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     494           0 :                         readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal);
     495             : 
     496           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     497             :                         {
     498           0 :                                 realvalue = (float*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     499           0 :                                 readfloatrows[dimension] = *realvalue;
     500           0 :                                 GDKfree(realvalue);
     501           0 :                                 dimension++;
     502             : 
     503           0 :                                 if (dimension == (size_t) optimal)
     504             :                                 {
     505           0 :                                         dimension = 0;
     506           0 :                                         tm0 = GDKusec();
     507           0 :                                         fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, optimal, readfloatrows, &status);
     508           0 :                                         texportfloat += GDKusec() - tm0;
     509           0 :                                         GDKfree(readfloatrows);
     510           0 :                                         readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal);
     511           0 :                                         block++;
     512             :                                 }
     513             :                         }
     514           0 :                         tm0 = GDKusec();
     515           0 :                         fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, dimension, readfloatrows, &status);
     516           0 :                         texportfloat += GDKusec() - tm0;
     517           0 :                         GDKfree(readfloatrows);
     518             :                 }
     519             : 
     520           0 :                 if (strcmp(columntype,"double")==0)
     521             :                 {
     522           0 :                         doublecols++; dimension = 0; block = 0;
     523           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     524           0 :                         readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal);
     525             : 
     526           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     527             :                         {
     528           0 :                                 doublevalue = (double*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     529           0 :                                 readdoublerows[dimension] = *doublevalue;
     530           0 :                                 GDKfree(doublevalue);
     531           0 :                                 dimension++;
     532             : 
     533           0 :                                 if (dimension == (size_t) optimal)
     534             :                                 {
     535           0 :                                         dimension = 0;
     536           0 :                                         tm0 = GDKusec();
     537           0 :                                         fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status);
     538           0 :                                         texportdouble += GDKusec() - tm0;
     539           0 :                                         GDKfree(readdoublerows);
     540           0 :                                         readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal);
     541           0 :                                         block++;
     542             :                                 }
     543             :                         }
     544           0 :                         tm0 = GDKusec();
     545           0 :                         fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status);
     546           0 :                         texportdouble += GDKusec() - tm0;
     547           0 :                         GDKfree(readdoublerows);
     548             :                 }
     549             :         }
     550             : 
     551             : 
     552             :         // print all the times that were needed to export each one of the columns
     553           0 :         if (texportboolean > 0)              TRC_DEBUG(FITS, "%d Boolean\tcolumn(s) exported in "LLFMT" usec\n", boolcols, texportboolean);
     554           0 :         if (texportchar > 0)         TRC_DEBUG(FITS, "%d Char\t\tcolumn(s) exported in "LLFMT" usec\n", charcols, texportchar);
     555           0 :         if (texportstring > 0)               TRC_DEBUG(FITS, "%d String\tcolumn(s) exported in "LLFMT" usec\n", strcols, texportstring);
     556           0 :         if (texportshort > 0)                TRC_DEBUG(FITS, "%d Short\t\tcolumn(s) exported in "LLFMT" usec\n", shortcols, texportshort);
     557           0 :         if (texportint > 0)                  TRC_DEBUG(FITS, "%d Integer\tcolumn(s) exported in "LLFMT" usec\n", intcols, texportint);
     558           0 :         if (texportlng > 0)                  TRC_DEBUG(FITS, "%d Long\t\tcolumn(s) exported in "LLFMT" usec\n", lngcols, texportlng);
     559           0 :         if (texportfloat > 0)                TRC_DEBUG(FITS, "%d Float\t\tcolumn(s) exported in "LLFMT" usec\n", floatcols, texportfloat);
     560           0 :         if (texportdouble > 0)               TRC_DEBUG(FITS, "%d Double\tcolumn(s) exported in "LLFMT" usec\n", doublecols, texportdouble);
     561             : 
     562             : 
     563           0 :         fits_close_file(fptr, &status);
     564           0 :         return msg;
     565             : }
     566             : 
     567             : 
     568           0 : str FITSdir(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
     569             : {
     570           0 :         str msg = MAL_SUCCEED;
     571           0 :         str dir = *getArgReference_str(stk, pci, 1);
     572           0 :         DIR *dp;
     573           0 :         struct dirent *ep;
     574           0 :         fitsfile *fptr;
     575           0 :         char *s;
     576           0 :         int status = 0;
     577           0 :         (void)mb;
     578             : 
     579           0 :         dp = opendir(dir);
     580           0 :         if (dp != NULL) {
     581             :                 char stmt[BUFSIZ];
     582             :                 char fname[BUFSIZ];
     583             : 
     584             :                 s = stmt;
     585             : 
     586           0 :                 while ((ep = readdir(dp)) != NULL && !msg) {
     587           0 :                         snprintf(fname, sizeof(fname), "%s/%s", dir, ep->d_name);
     588           0 :                         status = 0;
     589           0 :                         fits_open_file(&fptr, fname, READONLY, &status);
     590           0 :                         if (status == 0) {
     591           0 :                                 char *filename = SQLescapeString(fname);
     592           0 :                                 if (!filename) {
     593           0 :                                         msg = createException(MAL, "fits.listdir", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     594           0 :                                         break;
     595             :                                 }
     596           0 :                                 snprintf(stmt, sizeof(stmt), ATTACHDIR, filename);
     597           0 :                                 GDKfree(filename);
     598           0 :                                 TRC_DEBUG(FITS, "Executing: %s\n", s);
     599           0 :                                 msg = SQLstatementIntern(cntxt, s, "fits.listofdir", TRUE, FALSE, NULL);
     600           0 :                                 fits_close_file(fptr, &status);
     601             :                         }
     602             : 
     603             :                 }
     604           0 :                 (void)closedir(dp);
     605             :         } else
     606           0 :                 msg = createException(MAL, "listdir", SQLSTATE(FI000) "Couldn't open the directory");
     607             : 
     608           0 :         return msg;
     609             : }
     610             : 
     611           0 : str FITSdirpat(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
     612             : {
     613           0 :         str msg = MAL_SUCCEED;
     614           0 :         str dir = *getArgReference_str(stk, pci, 1);
     615           0 :         str pat = *getArgReference_str(stk, pci, 2);
     616           0 :         fitsfile *fptr;
     617           0 :         char *s;
     618           0 :         int status = 0;
     619           0 :         glob_t globbuf;
     620           0 :         char fulldirectory[BUFSIZ];
     621           0 :         size_t j = 0;
     622             : 
     623           0 :         (void)mb;
     624             : 
     625           0 :         globbuf.gl_offs = 0;
     626           0 :         snprintf(fulldirectory, sizeof(fulldirectory), "%s/%s", dir, pat);
     627           0 :         glob(fulldirectory, GLOB_DOOFFS, NULL, &globbuf);
     628             : 
     629           0 :         TRC_DEBUG(FITS, "Fulldir: %s - Size: %zu\n", fulldirectory, globbuf.gl_pathc);
     630             : 
     631           0 :         if (globbuf.gl_pathc == 0)
     632           0 :                 throw(MAL, "fits.listdirpat", SQLSTATE(FI000) "Couldn't open the directory or there are no files that match the pattern");
     633             : 
     634           0 :         for (j = 0; j < globbuf.gl_pathc; j++) {
     635           0 :                 char stmt[BUFSIZ];
     636           0 :                 char fname[BUFSIZ];
     637             : 
     638           0 :                 s = stmt;
     639           0 :                 strcpy_len(fname, globbuf.gl_pathv[j], sizeof(fname));
     640           0 :                 status = 0;
     641           0 :                 fits_open_file(&fptr, fname, READONLY, &status);
     642           0 :                 if (status == 0) {
     643           0 :                         char *filename = SQLescapeString(fname);
     644           0 :                         if (!filename) {
     645           0 :                                 throw(MAL, "fits.listdirpat", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     646             :                         }
     647           0 :                         snprintf(stmt, sizeof(stmt), ATTACHDIR, filename);
     648           0 :                         GDKfree(filename);
     649           0 :                         TRC_DEBUG(FITS, "Executing: %s\n", s);
     650           0 :                         msg = SQLstatementIntern(cntxt, s, "fits.listofdirpat", TRUE, FALSE, NULL);
     651           0 :                         fits_close_file(fptr, &status);
     652             : 
     653           0 :                         break;
     654             :                 }
     655             :         }
     656             : 
     657             :         return msg;
     658             : }
     659             : 
     660             : 
     661             : str
     662           0 : FITStest(int *res, str *fname)
     663             : {
     664           0 :         fitsfile *fptr;       /* pointer to the FITS file, defined in fitsio.h */
     665           0 :         str msg = MAL_SUCCEED;
     666           0 :         int status = 0, hdutype;
     667             : 
     668           0 :         *res = 0;
     669           0 :         if (fits_open_file(&fptr, *fname, READONLY, &status))
     670           0 :                 msg = createException(MAL, "fits.test", SQLSTATE(FI000) "Missing FITS file %s", *fname);
     671             :         else {
     672           0 :                 fits_movabs_hdu(fptr, 2, &hdutype, &status);
     673           0 :                 *res = hdutype;
     674           0 :                 fits_close_file(fptr, &status);
     675             :         }
     676             : 
     677           0 :         return msg;
     678             : }
     679             : 
     680           0 : str FITSattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
     681             : {
     682           0 :         mvc *m = NULL;
     683           0 :         sql_trans *tr;
     684           0 :         sql_schema *sch;
     685           0 :         sql_table *fits_tp, *fits_fl, *fits_tbl, *fits_col, *tbl = NULL;
     686           0 :         sql_column *col;
     687           0 :         str msg = MAL_SUCCEED;
     688           0 :         str fname = *getArgReference_str(stk, pci, 1);
     689           0 :         fitsfile *fptr;  /* pointer to the FITS file */
     690           0 :         int status = 0, i, j, hdutype, hdunum = 1, cnum = 0, bitpixnumber = 0;
     691           0 :         oid fid, tid, cid, rid = oid_nil;
     692           0 :         char tname[BUFSIZ], *tname_low = NULL, *s, bname[BUFSIZ-100], stmt[BUFSIZ];
     693           0 :         long tbcol; /* type long used by fits library */
     694           0 :         char cname[BUFSIZ], tform[BUFSIZ], tunit[BUFSIZ], tnull[BUFSIZ], tdisp[BUFSIZ];
     695           0 :         char *esc_cname, *esc_tform, *esc_tunit;
     696           0 :         double tscal, tzero;
     697           0 :         char xtensionname[BUFSIZ] = "", stilversion[BUFSIZ] = "";
     698           0 :         char stilclass[BUFSIZ] = "", tdate[BUFSIZ] = "", orig[BUFSIZ] = "", comm[BUFSIZ] = "";
     699             : 
     700           0 :         if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
     701             :                 return msg;
     702           0 :         if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
     703             :                 return msg;
     704             : 
     705           0 :         if (fits_open_file(&fptr, fname, READONLY, &status)) {
     706           0 :                 msg = createException(MAL, "fits.attach", SQLSTATE(FI000) "Missing FITS file %s.\n", fname);
     707           0 :                 return msg;
     708             :         }
     709             : 
     710           0 :         tr = m->session->tr;
     711           0 :         sch = mvc_bind_schema(m, "sys");
     712           0 :         sqlstore *store = tr->store;
     713             : 
     714           0 :         fits_fl = mvc_bind_table(m, sch, "fits_files");
     715           0 :         if (fits_fl == NULL)
     716           0 :                 FITSinitCatalog(m);
     717             : 
     718           0 :         fits_fl = mvc_bind_table(m, sch, "fits_files");
     719           0 :         fits_tbl = mvc_bind_table(m, sch, "fits_tables");
     720           0 :         fits_col = mvc_bind_table(m, sch, "fits_columns");
     721           0 :         fits_tp = mvc_bind_table(m, sch, "fits_table_properties");
     722             : 
     723             :         /* check if the file is already attached */
     724           0 :         col = mvc_bind_column(m, fits_fl, "name");
     725           0 :         rid = store->table_api.column_find_row(m->session->tr, col, fname, NULL);
     726           0 :         if (!is_oid_nil(rid)) {
     727           0 :                 fits_close_file(fptr, &status);
     728           0 :                 throw(MAL, "fits.attach", SQLSTATE(FI000) "File %s already attached\n", fname);
     729             :         }
     730             : 
     731             :         /* add row in the fits_files catalog table */
     732           0 :         BUN offset;
     733           0 :         if (store->storage_api.claim_tab(m->session->tr, fits_fl, 1, &offset, NULL) != LOG_OK) {
     734           0 :                 fits_close_file(fptr, &status);
     735           0 :                 throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     736             :         }
     737           0 :         if (!isNew(fits_fl) && sql_trans_add_dependency_change(m->session->tr, fits_fl->base.id, dml) != LOG_OK) {
     738           0 :                 fits_close_file(fptr, &status);
     739           0 :                 throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     740             :         }
     741           0 :         col = mvc_bind_column(m, fits_fl, "id");
     742           0 :         fid = store->storage_api.count_col(tr, col, 0) + 1;
     743           0 :         store->storage_api.append_col(m->session->tr,
     744             :                 mvc_bind_column(m, fits_fl, "id"), offset, NULL, &fid, 1, false, TYPE_int);
     745           0 :         store->storage_api.append_col(m->session->tr,
     746             :                 mvc_bind_column(m, fits_fl, "name"), offset, NULL, &fname, 1, false, TYPE_str);
     747           0 :         col = mvc_bind_column(m, fits_tbl, "id");
     748           0 :         tid = store->storage_api.count_col(tr, col, 0) + 1;
     749             : 
     750           0 :         if ((s = strrchr(fname, DIR_SEP)) == NULL)
     751             :                 s = fname;
     752             :         else
     753           0 :                 s++;
     754           0 :         if (strcpy_len(bname, s, sizeof(bname)) >= sizeof(bname)) {
     755           0 :                 fits_close_file(fptr, &status);
     756           0 :                 throw(MAL, "fits.attach", SQLSTATE(FI000) "File name too long\n");
     757             :         }
     758           0 :         strcpy(bname, s);
     759           0 :         s = strrchr(bname, '.');
     760           0 :         if (s) *s = 0;
     761             : 
     762           0 :         fits_get_num_hdus(fptr, &hdunum, &status);
     763           0 :         for (i = 1; i <= hdunum; i++) {
     764           0 :                 fits_movabs_hdu(fptr, i, &hdutype, &status);
     765           0 :                 if (hdutype != ASCII_TBL && hdutype != BINARY_TBL)
     766           0 :                         continue;
     767             : 
     768             :                 /* SQL table name - the name of FITS extension */
     769           0 :                 fits_read_key(fptr, TSTRING, "EXTNAME", tname, NULL, &status);
     770           0 :                 if (status) {
     771           0 :                         snprintf(tname, sizeof(tname), "%s_%d", bname, i);
     772           0 :                         tname_low = toLower(tname);
     773           0 :                         status = 0;
     774             :                 }else  { /* check table name for existence in the fits catalog */
     775           0 :                         tname_low = toLower(tname);
     776           0 :                         col = mvc_bind_column(m, fits_tbl, "name");
     777           0 :                         rid = store->table_api.column_find_row(m->session->tr, col, tname_low, NULL);
     778             :                         /* or as regular SQL table */
     779           0 :                         tbl = mvc_bind_table(m, sch, tname_low);
     780           0 :                         if (!is_oid_nil(rid) || tbl) {
     781           0 :                                 snprintf(tname, sizeof(tname), "%s_%d", bname, i);
     782           0 :                                 tname_low = toLower(tname);
     783             :                         }
     784             :                 }
     785             : 
     786           0 :                 fits_read_key(fptr, TSTRING, "BITPIX", &bitpixnumber, NULL, &status);
     787           0 :                 if (status) {
     788           0 :                         status = 0;
     789             :                 }
     790           0 :                 fits_read_key(fptr, TSTRING, "DATE-HDU", tdate, NULL, &status);
     791           0 :                 if (status) {
     792           0 :                         status = 0;
     793             :                 }
     794           0 :                 fits_read_key(fptr, TSTRING, "XTENSION", xtensionname, NULL, &status);
     795           0 :                 if (status) {
     796           0 :                         status = 0;
     797             :                 }
     798           0 :                 fits_read_key(fptr, TSTRING, "STILVERS", stilversion, NULL, &status);
     799           0 :                 if (status) {
     800           0 :                         status = 0;
     801             :                 }
     802           0 :                 fits_read_key(fptr, TSTRING, "STILCLAS", stilclass, NULL, &status);
     803           0 :                 if (status) {
     804           0 :                         status = 0;
     805             :                 }
     806           0 :                 fits_read_key(fptr, TSTRING, "ORIGIN", orig, NULL, &status);
     807           0 :                 if (status) {
     808           0 :                         status = 0;
     809             :                 }
     810           0 :                 fits_read_key(fptr, TSTRING, "COMMENT", comm, NULL, &status);
     811           0 :                 if (status) {
     812           0 :                         status = 0;
     813             :                 }
     814             : 
     815           0 :                 fits_get_num_cols(fptr, &cnum, &status);
     816             : 
     817           0 :                 BUN offset;
     818           0 :                 if (store->storage_api.claim_tab(m->session->tr, fits_tbl, 1, &offset, NULL) != LOG_OK) {
     819           0 :                         fits_close_file(fptr, &status);
     820           0 :                         throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     821             :                 }
     822           0 :                 if (!isNew(fits_tbl) && sql_trans_add_dependency_change(m->session->tr, fits_tbl->base.id, dml) != LOG_OK) {
     823           0 :                         fits_close_file(fptr, &status);
     824           0 :                         throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     825             :                 }
     826           0 :                 store->storage_api.append_col(m->session->tr,
     827             :                         mvc_bind_column(m, fits_tbl, "id"), offset, NULL, &tid, 1, false, TYPE_int);
     828           0 :                 store->storage_api.append_col(m->session->tr,
     829             :                         mvc_bind_column(m, fits_tbl, "name"), offset, NULL, &tname_low, 1, false, TYPE_str);
     830           0 :                 store->storage_api.append_col(m->session->tr,
     831             :                         mvc_bind_column(m, fits_tbl, "columns"), offset, NULL, &cnum, 1, false, TYPE_int);
     832           0 :                 store->storage_api.append_col(m->session->tr,
     833             :                         mvc_bind_column(m, fits_tbl, "file_id"), offset, NULL, &fid, 1, false, TYPE_int);
     834           0 :                 store->storage_api.append_col(m->session->tr,
     835             :                         mvc_bind_column(m, fits_tbl, "hdu"), offset, NULL, &i, 1, false, TYPE_int);
     836           0 :                 char *vptr = tdate;
     837           0 :                 store->storage_api.append_col(m->session->tr,
     838             :                         mvc_bind_column(m, fits_tbl, "date"), offset, NULL, &vptr, 1, false, TYPE_str);
     839           0 :                 vptr = orig;
     840           0 :                 store->storage_api.append_col(m->session->tr,
     841             :                         mvc_bind_column(m, fits_tbl, "origin"), offset, NULL, &vptr, 1, false, TYPE_str);
     842           0 :                 vptr = comm;
     843           0 :                 store->storage_api.append_col(m->session->tr,
     844             :                         mvc_bind_column(m, fits_tbl, "comment"), offset, NULL, &vptr, 1, false, TYPE_str);
     845             : 
     846           0 :                 if (store->storage_api.claim_tab(m->session->tr, fits_tp, 1, &offset, NULL) != LOG_OK) {
     847           0 :                         fits_close_file(fptr, &status);
     848           0 :                         throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     849             :                 }
     850           0 :                 if (!isNew(fits_tp) && sql_trans_add_dependency_change(m->session->tr, fits_tp->base.id, dml) != LOG_OK) {
     851           0 :                         fits_close_file(fptr, &status);
     852           0 :                         throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     853             :                 }
     854           0 :                 store->storage_api.append_col(m->session->tr,
     855             :                         mvc_bind_column(m, fits_tp, "table_id"), offset, NULL, &tid, 1, false, TYPE_int);
     856           0 :                 vptr = xtensionname;
     857           0 :                 store->storage_api.append_col(m->session->tr,
     858             :                         mvc_bind_column(m, fits_tp, "xtension"), offset, NULL, &vptr, 1, false, TYPE_str);
     859           0 :                 store->storage_api.append_col(m->session->tr,
     860             :                         mvc_bind_column(m, fits_tp, "bitpix"), offset, NULL, &bitpixnumber, 1, false, TYPE_int);
     861           0 :                 vptr = stilversion;
     862           0 :                 store->storage_api.append_col(m->session->tr,
     863             :                         mvc_bind_column(m, fits_tp, "stilvers"), offset, NULL, &vptr, 1, false, TYPE_str);
     864           0 :                 vptr = stilclass;
     865           0 :                 store->storage_api.append_col(m->session->tr,
     866             :                         mvc_bind_column(m, fits_tp, "stilclas"), offset, NULL, &vptr, 1, false, TYPE_str);
     867             : 
     868             :                 /* read columns description */
     869           0 :                 s = stmt;
     870           0 :                 col = mvc_bind_column(m, fits_col, "id");
     871           0 :                 cid = store->storage_api.count_col(tr, col, 0) + 1;
     872           0 :                 for (j = 1; j <= cnum; j++, cid++) {
     873           0 :                         fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status);
     874             :                         /* escape the various strings to avoid SQL injection attacks */
     875           0 :                         esc_cname = SQLescapeString(cname);
     876           0 :                         if (!esc_cname) {
     877           0 :                                 fits_close_file(fptr, &status);
     878           0 :                                 throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     879             :                         }
     880           0 :                         esc_tform = SQLescapeString(tform);
     881           0 :                         if (!esc_tform) {
     882           0 :                                 GDKfree(esc_cname);
     883           0 :                                 fits_close_file(fptr, &status);
     884           0 :                                 throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     885             :                         }
     886           0 :                         esc_tunit = SQLescapeString(tunit);
     887           0 :                         if (!esc_tunit) {
     888           0 :                                 GDKfree(esc_tform);
     889           0 :                                 GDKfree(esc_cname);
     890           0 :                                 fits_close_file(fptr, &status);
     891           0 :                                 throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     892             :                         }
     893           0 :                         snprintf(stmt, sizeof(stmt), FITS_INS_COL, (int)cid, esc_cname, esc_tform, esc_tunit, j, (int)tid);
     894           0 :                         GDKfree(esc_tunit);
     895           0 :                         GDKfree(esc_tform);
     896           0 :                         GDKfree(esc_cname);
     897           0 :                         msg = SQLstatementIntern(cntxt, s, "fits.attach", TRUE, FALSE, NULL);
     898           0 :                         if (msg != MAL_SUCCEED) {
     899           0 :                                 fits_close_file(fptr, &status);
     900           0 :                                 return msg;
     901             :                         }
     902             :                 }
     903           0 :                 tid++;
     904             :         }
     905           0 :         fits_close_file(fptr, &status);
     906             : 
     907           0 :         return MAL_SUCCEED;
     908             : }
     909             : 
     910           0 : str FITSloadTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
     911             : {
     912           0 :         mvc *m = NULL;
     913           0 :         sql_schema *sch;
     914           0 :         sql_table *fits_fl, *fits_tbl, *tbl = NULL;
     915           0 :         sql_column *col;
     916           0 :         sql_subtype tpe;
     917           0 :         fitsfile *fptr;
     918           0 :         str tname = *getArgReference_str(stk, pci, 1);
     919           0 :         str fname;
     920           0 :         str msg = MAL_SUCCEED;
     921           0 :         oid rid = oid_nil, frid = oid_nil;
     922           0 :         int status = 0, cnum = 0, *fid, *hdu, hdutype, j, anynull = 0, mtype;
     923           0 :         int *tpcode = NULL;
     924           0 :         long *rep = NULL, *wid = NULL, rows; /* type long used by fits library */
     925           0 :         char keywrd[80], **cname, nm[FLEN_VALUE];
     926           0 :         const void *nilptr;
     927           0 :         BUN offset;
     928           0 :         BAT *pos = NULL;
     929             : 
     930           0 :         if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
     931             :                 return msg;
     932           0 :         if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
     933             :                 return msg;
     934           0 :         sch = mvc_bind_schema(m, "sys");
     935             : 
     936           0 :         fits_tbl = mvc_bind_table(m, sch, "fits_tables");
     937           0 :         if (fits_tbl == NULL) {
     938           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "FITS catalog is missing.\n");
     939           0 :                 return msg;
     940             :         }
     941             : 
     942           0 :         tbl = mvc_bind_table(m, sch, tname);
     943           0 :         if (tbl) {
     944           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Table %s is already created.\n", tname);
     945           0 :                 return msg;
     946             :         }
     947             : 
     948           0 :         col = mvc_bind_column(m, fits_tbl, "name");
     949           0 :         sqlstore *store = m->session->tr->store;
     950           0 :         rid = store->table_api.column_find_row(m->session->tr, col, tname, NULL);
     951           0 :         if (is_oid_nil(rid)) {
     952           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname);
     953           0 :                 return msg;
     954             :         }
     955             : 
     956             :         /* Open FITS file and move to the table HDU */
     957           0 :         col = mvc_bind_column(m, fits_tbl, "file_id");
     958           0 :         fid = (int*)store->table_api.column_find_value(m->session->tr, col, rid);
     959             : 
     960           0 :         fits_fl = mvc_bind_table(m, sch, "fits_files");
     961           0 :         col = mvc_bind_column(m, fits_fl, "id");
     962           0 :         frid = store->table_api.column_find_row(m->session->tr, col, (void *)fid, NULL);
     963           0 :         GDKfree(fid);
     964           0 :         if (is_oid_nil(frid)) {
     965           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname);
     966           0 :                 return msg;
     967             :         }
     968           0 :         col = mvc_bind_column(m, fits_fl, "name");
     969           0 :         fname = (char *)store->table_api.column_find_value(m->session->tr, col, frid);
     970           0 :         if (fits_open_file(&fptr, fname, READONLY, &status)) {
     971           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Missing FITS file %s.\n", fname);
     972           0 :                 GDKfree(fname);
     973           0 :                 return msg;
     974             :         }
     975           0 :         GDKfree(fname);
     976             : 
     977           0 :         col = mvc_bind_column(m, fits_tbl, "hdu");
     978           0 :         hdu = (int*)store->table_api.column_find_value(m->session->tr, col, rid);
     979           0 :         fits_movabs_hdu(fptr, *hdu, &hdutype, &status);
     980           0 :         if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) {
     981           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "HDU %d is not a table.\n", *hdu);
     982           0 :                 GDKfree(hdu);
     983           0 :                 fits_close_file(fptr, &status);
     984           0 :                 return msg;
     985             :         }
     986           0 :         GDKfree(hdu);
     987             : 
     988             :         /* create a SQL table to hold the FITS table */
     989             :         /*      col = mvc_bind_column(m, fits_tbl, "columns");
     990             :            cnum = *(int*) store->table_api.column_find_value(m->session->tr, col, rid); */
     991           0 :         fits_get_num_cols(fptr, &cnum, &status);
     992           0 :         mvc_create_table(&tbl, m, sch, tname, tt_table, 0, SQL_PERSIST, 0, cnum, 0);
     993             : 
     994             :         // TODO: Check that the allocations succeeded
     995           0 :         tpcode = (int *)GDKzalloc(sizeof(int) * cnum);
     996           0 :         rep = (long *)GDKzalloc(sizeof(long) * cnum);
     997           0 :         wid = (long *)GDKzalloc(sizeof(long) * cnum);
     998           0 :         cname = (char **)GDKzalloc(sizeof(char *) * cnum);
     999             : 
    1000           0 :         for (j = 1; j <= cnum; j++) {
    1001           0 :                 sql_column *col = NULL;
    1002             :                 /*              fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status); */
    1003           0 :                 snprintf(keywrd, 80, "TTYPE%d", j);
    1004           0 :                 fits_read_key(fptr, TSTRING, keywrd, nm, NULL, &status);
    1005           0 :                 if (status) {
    1006           0 :                         snprintf(nm, FLEN_VALUE, "column_%d", j);
    1007           0 :                         status = 0;
    1008             :                 }
    1009           0 :                 cname[j - 1] = toLower(nm);
    1010           0 :                 fits_get_coltype(fptr, j, &tpcode[j - 1], &rep[j - 1], &wid[j - 1], &status);
    1011           0 :                 fits2subtype(&tpe, tpcode[j - 1], rep[j - 1], wid[j - 1]);
    1012             : 
    1013           0 :                 TRC_DEBUG(FITS, "%d %ld %ld - M: %s\n", tpcode[j-1], rep[j-1], wid[j-1], tpe.type->base.name);
    1014             : 
    1015           0 :                 mvc_create_column(&col, m, tbl, cname[j - 1], &tpe);
    1016             :         }
    1017             : 
    1018             :         /* data load */
    1019           0 :         fits_get_num_rows(fptr, &rows, &status);
    1020             :         /* Nothing more to do */
    1021           0 :         if (rows == 0) {
    1022           0 :                 goto bailout;
    1023             :         }
    1024             : 
    1025           0 :         TRC_INFO(FITS, "Loading %ld rows in table %s\n", rows, tname);
    1026             : 
    1027           0 :         if (store->storage_api.claim_tab(m->session->tr, tbl, rows, &offset, &pos) != LOG_OK) {
    1028           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1029           0 :                 goto bailout;
    1030             :         }
    1031           0 :         for (j = 1; j <= cnum; j++) {
    1032           0 :                 BAT *tmp = NULL;
    1033           0 :                 lng time0 = GDKusec();
    1034           0 :                 mtype = fits2mtype(tpcode[j - 1], rep[j - 1]);
    1035           0 :                 nilptr = ATOMnilptr(mtype);
    1036           0 :                 col = mvc_bind_column(m, tbl, cname[j - 1]);
    1037             : 
    1038           0 :                 tmp = COLnew(0, mtype, rows, TRANSIENT);
    1039           0 :                 if (tmp == NULL){
    1040           0 :                         msg = createException(MAL,"fits.load", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1041           0 :                         goto bailout;
    1042             :                 }
    1043           0 :                 if (mtype == TYPE_blob) {
    1044           0 :                         long i;
    1045           0 :                         unsigned long nbytes = rep[j - 1] * wid[j - 1];
    1046           0 :                         blob **v = (blob **)GDKzalloc(sizeof(blob *) * rows);
    1047             : 
    1048           0 :                         mtype = fits2mtype(tpcode[j - 1], 1);
    1049           0 :                         nilptr = ATOMnilptr(mtype);
    1050             : 
    1051           0 :                         if (v == NULL) {
    1052           0 :                                 BBPreclaim(tmp);
    1053           0 :                                 msg = createException(MAL,"fits.load", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1054           0 :                                 goto bailout;
    1055             :                         }
    1056             : 
    1057           0 :                         for(i = 0; i < rows; i++) {
    1058           0 :                                 v[i] = (blob *)GDKmalloc(offsetof(blob, data) + nbytes);
    1059           0 :                                 if (v[i] == NULL) {
    1060           0 :                                         BBPreclaim(tmp);
    1061           0 :                                         long k = 0;
    1062           0 :                                         for (k = 0; k < i; k++) {
    1063           0 :                                                 GDKfree(v[k]);
    1064             :                                         }
    1065           0 :                                         GDKfree(v);
    1066           0 :                                         msg = createException(MAL,"fits.load", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1067           0 :                                         goto bailout;
    1068             :                                 }
    1069           0 :                                 fits_read_col(fptr, tpcode[j - 1], j, i + 1, 1, rep[j - 1], (void *)nilptr,
    1070           0 :                                               (void *)v[i]->data, &anynull, &status);
    1071           0 :                                 v[i]->nitems = nbytes;
    1072           0 :                                 if (BUNappend(tmp, v[i], false) != GDK_SUCCEED) {
    1073           0 :                                         BBPreclaim(tmp);
    1074           0 :                                         for (i = 0; i < rows; i++) {
    1075           0 :                                                 GDKfree(v[i]);
    1076             :                                         }
    1077           0 :                                         GDKfree(v);
    1078           0 :                                         msg = createException(MAL,"fits.loadtable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1079           0 :                                         goto bailout;
    1080             :                                 }
    1081             :                         }
    1082             : 
    1083           0 :                         for(i = 0; i < rows; i++) {
    1084           0 :                                 GDKfree(v[i]);
    1085             :                         }
    1086           0 :                         GDKfree(v);
    1087             :                 }
    1088           0 :                 else if (mtype == TYPE_str) {
    1089             : /*                      char *v = GDKzalloc(wid[j-1]);*/
    1090             :                         /* type long demanded by "rows", i.e., by fits library */
    1091           0 :                         long bsize = 50, batch = bsize, k, i;
    1092           0 :                         lng tm0, tloadtm = 0, tattachtm = 0;
    1093           0 :                         char **v = (char **) GDKzalloc(sizeof(char *) * bsize);
    1094           0 :                         for(i = 0; i < bsize; i++)
    1095           0 :                                 v[i] = GDKzalloc(wid[j-1]);
    1096           0 :                         for(i = 0; i < rows; i += batch) {
    1097           0 :                                 batch = rows - i < bsize ? rows - i: bsize;
    1098           0 :                                 tm0 = GDKusec();
    1099           0 :                                 fits_read_col(fptr, tpcode[j - 1], j, 1 + i, 1, batch, (void *) nilptr, (void *)v, &anynull, &status);
    1100           0 :                                 tloadtm += GDKusec() - tm0;
    1101           0 :                                 tm0 = GDKusec();
    1102           0 :                                 for(k = 0; k < batch ; k++)
    1103           0 :                                         if (BUNappend(tmp, v[k], false) != GDK_SUCCEED) {
    1104           0 :                                                 BBPreclaim(tmp);
    1105           0 :                                                 msg = createException(MAL, "fits.loadtable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1106           0 :                                                 goto bailout;
    1107             :                                         }
    1108           0 :                                 tattachtm += GDKusec() - tm0;
    1109             :                         }
    1110           0 :                         for(i = 0; i < bsize ; i++)
    1111           0 :                                 GDKfree(v[i]);
    1112           0 :                         GDKfree(v);
    1113           0 :                         TRC_INFO(FITS, "String column load "LLFMT" usec, BUNappend "LLFMT" usec\n", tloadtm, tattachtm);
    1114             :                 }
    1115             :                 else {
    1116           0 :                         BATiter bi = bat_iterator_nolock(tmp);
    1117           0 :                         fits_read_col(fptr, tpcode[j - 1], j, 1, 1, rows, (void *) nilptr, (void *)BUNtloc(bi, 0), &anynull, &status);
    1118           0 :                         BATsetcount(tmp, rows);
    1119           0 :                         tmp->tsorted = false;
    1120           0 :                         tmp->trevsorted = false;
    1121           0 :                         tmp->tkey = false;
    1122             :                 }
    1123             : 
    1124           0 :                 if (status) {
    1125           0 :                         char buf[FLEN_ERRMSG + 1];
    1126           0 :                         fits_read_errmsg(buf);
    1127           0 :                         BBPreclaim(tmp);
    1128           0 :                         msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Cannot load column %s of %s table: %s.\n", cname[j - 1], tname, buf);
    1129           0 :                         break;
    1130             :                 }
    1131             : 
    1132           0 :                 TRC_INFO(FITS, "#Column %s loaded for "LLFMT" usec\t", cname[j-1], GDKusec() - time0);
    1133           0 :                 if (store->storage_api.append_col(m->session->tr, col, offset, pos, tmp, BATcount(tmp), true, tmp->ttype) != LOG_OK) {
    1134           0 :                         BBPreclaim(tmp);
    1135           0 :                         msg = createException(MAL, "fits.loadtable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1136           0 :                         break;
    1137             :                 }
    1138           0 :                 TRC_INFO(FITS, "Total "LLFMT" usec\n", GDKusec() - time0);
    1139           0 :                 BBPreclaim(tmp);
    1140             :         }
    1141             : 
    1142           0 :   bailout:
    1143           0 :         bat_destroy(pos);
    1144           0 :         GDKfree(tpcode);
    1145           0 :         GDKfree(rep);
    1146           0 :         GDKfree(wid);
    1147           0 :         GDKfree(cname);
    1148             : 
    1149           0 :         fits_close_file(fptr, &status);
    1150           0 :         return msg;
    1151             : }
    1152             : 
    1153             : #include "mel.h"
    1154             : static mel_func fits_init_funcs[] = {
    1155             :  pattern("fits", "listdir", FITSdir, true, "Attach all FITS files in the directory", args(1,2, arg("",void),arg("dirname",str))),
    1156             :  pattern("fits", "listdirpattern", FITSdirpat, true, "Attach all FITS file in the directory, giving a pattern", args(1,3, arg("",void),arg("dirname",str),arg("pattern",str))),
    1157             :  command("fits", "fitstest", FITStest, false, "Returns the type of first extension in the FITS file filename", args(1,2, arg("",int),arg("filename",str))),
    1158             :  pattern("fits", "attach", FITSattach, true, "Open a FITS file and return catalog of the table HDUs", args(1,2, arg("",void),arg("fname",str))),
    1159             :  pattern("fits", "load", FITSloadTable, true, "Load a FITS table from an attached file", args(1,2, arg("",void),arg("tablename",str))),
    1160             :  pattern("fits", "export", FITSexportTable, false, "Export a table to a FITS file", args(1,2, arg("",void),arg("tablename",str))),
    1161             :  { .imp=NULL }
    1162             : };
    1163             : #include "mal_import.h"
    1164             : #ifdef _MSC_VER
    1165             : #undef read
    1166             : #pragma section(".CRT$XCU",read)
    1167             : #endif
    1168         315 : LIB_STARTUP_FUNC(init_fits_mal)
    1169         315 : { mal_module("fits", NULL, fits_init_funcs); }

Generated by: LCOV version 1.14