Line data Source code
1 : /*
2 : * SPDX-License-Identifier: MPL-2.0
3 : *
4 : * This Source Code Form is subject to the terms of the Mozilla Public
5 : * License, v. 2.0. If a copy of the MPL was not distributed with this
6 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 : *
8 : * Copyright 2024 MonetDB Foundation;
9 : * Copyright August 2008 - 2023 MonetDB B.V.;
10 : * Copyright 1997 - July 2008 CWI.
11 : */
12 :
13 : /* streams working on a gzip-compressed disk file */
14 :
15 : #include "monetdb_config.h"
16 : #include "stream.h"
17 : #include "stream_internal.h"
18 :
19 :
20 :
21 : /* ------------------------------------------------------------------ */
22 : /* streams working on a remote file using cURL */
23 :
24 : #ifdef HAVE_CURL
25 : #include <curl/curl.h>
26 :
27 : struct curl_data {
28 : CURL *handle;
29 : char *buffer; /* buffer to store incoming data */
30 : size_t maxsize; /* size of allocated buffer */
31 : size_t usesize; /* end of used data */
32 : size_t offset; /* start of unread data */
33 : int running; /* whether still transferring */
34 : char errbuf[CURL_ERROR_SIZE]; /* a place for error messages */
35 : };
36 :
37 : #define BLOCK_CURL ((size_t) 1 << 16)
38 :
39 : /* this function is called by libcurl when there is data for us */
40 : static size_t
41 0 : write_callback(char *buffer, size_t size, size_t nitems, void *userp)
42 : {
43 0 : stream *s = (stream *) userp;
44 0 : struct curl_data *c = (struct curl_data *) s->stream_data.p;
45 :
46 0 : size *= nitems;
47 0 : if (size == 0) /* unlikely */
48 : return 0;
49 : /* allocate a buffer if we don't have one yet */
50 0 : if (c->buffer == NULL) {
51 : /* BLOCK_CURL had better be a power of 2! */
52 0 : c->maxsize = (size + BLOCK_CURL - 1) & ~(BLOCK_CURL - 1);
53 0 : if ((c->buffer = malloc(c->maxsize)) == NULL)
54 : return 0;
55 0 : c->usesize = 0;
56 0 : c->offset = 0;
57 : }
58 : /* move data if we don't have enough space */
59 0 : if (c->maxsize - c->usesize < size && c->offset > 0) {
60 0 : memmove(c->buffer, c->buffer + c->offset, c->usesize - c->offset);
61 0 : c->usesize -= c->offset;
62 0 : c->offset = 0;
63 : }
64 : /* allocate more buffer space if we still don't have enough space */
65 0 : if (c->maxsize - c->usesize < size) {
66 0 : char *b;
67 0 : size_t maxsize;
68 :
69 0 : maxsize = (c->usesize + size + BLOCK_CURL - 1) & ~(BLOCK_CURL - 1);
70 0 : b = realloc(c->buffer, maxsize);
71 0 : if (b == NULL)
72 : return 0; /* indicate failure to library */
73 0 : c->buffer = b;
74 0 : c->maxsize = maxsize;
75 : }
76 : /* finally, store the data we received */
77 0 : memcpy(c->buffer + c->usesize, buffer, size);
78 0 : c->usesize += size;
79 0 : return size;
80 : }
81 :
82 : static void
83 0 : curl_destroy(stream *s)
84 : {
85 0 : struct curl_data *c;
86 :
87 0 : if ((c = (struct curl_data *) s->stream_data.p) != NULL) {
88 0 : s->stream_data.p = NULL;
89 0 : if (c->handle) {
90 0 : curl_easy_cleanup(c->handle);
91 : }
92 0 : if (c->buffer)
93 0 : free(c->buffer);
94 0 : free(c);
95 : }
96 0 : destroy_stream(s);
97 0 : }
98 :
99 : static ssize_t
100 0 : curl_read(stream *restrict s, void *restrict buf, size_t elmsize, size_t cnt)
101 : {
102 0 : struct curl_data *c = (struct curl_data *) s->stream_data.p;
103 0 : size_t size = cnt * elmsize;
104 :
105 0 : if (c == NULL) {
106 0 : mnstr_set_error(s, MNSTR_READ_ERROR, "stream already ended");
107 0 : return -1;
108 : }
109 :
110 0 : if (size == 0)
111 : return 0;
112 0 : if (c->usesize - c->offset >= elmsize || !c->running) {
113 : /* there is at least one element's worth of data
114 : * available, or we have reached the end: return as
115 : * much as we have, but no more than requested */
116 0 : if (size > c->usesize - c->offset) {
117 0 : cnt = (c->usesize - c->offset) / elmsize;
118 0 : size = cnt * elmsize;
119 : }
120 0 : memcpy(buf, c->buffer + c->offset, size);
121 0 : c->offset += size;
122 0 : if (c->offset == c->usesize)
123 0 : c->usesize = c->offset = 0;
124 0 : return (ssize_t) cnt;
125 : }
126 : /* not enough data, we must wait until we get some */
127 : return 0;
128 : }
129 :
130 : static ssize_t
131 0 : curl_write(stream *restrict s, const void *restrict buf, size_t elmsize, size_t cnt)
132 : {
133 0 : (void) s;
134 0 : (void) buf;
135 0 : (void) elmsize;
136 0 : (void) cnt;
137 0 : assert(0);
138 : return -1;
139 : }
140 :
141 : static void
142 0 : curl_close(stream *s)
143 : {
144 0 : (void) s;
145 0 : }
146 :
147 : stream *
148 0 : open_urlstream(const char *url)
149 : {
150 0 : stream *s;
151 0 : struct curl_data *c;
152 :
153 0 : if ((c = malloc(sizeof(*c))) == NULL) {
154 0 : mnstr_set_open_error(url, errno, NULL);
155 0 : return NULL;
156 : }
157 0 : *c = (struct curl_data) {
158 : .running = 1,
159 : .errbuf = {0},
160 : };
161 0 : if ((s = create_stream(url)) == NULL) {
162 0 : free(c);
163 0 : return NULL;
164 : }
165 0 : s->read = curl_read;
166 0 : s->write = curl_write;
167 0 : s->close = curl_close;
168 0 : s->destroy = curl_destroy;
169 0 : if ((c->handle = curl_easy_init()) == NULL) {
170 0 : free(c);
171 0 : destroy_stream(s);
172 0 : mnstr_set_open_error(url, 0, "curl_easy_init failed");
173 0 : return NULL;
174 : }
175 0 : s->stream_data.p = (void *) c;
176 0 : CURLcode ret;
177 0 : if ((ret = curl_easy_setopt(c->handle, CURLOPT_ERRORBUFFER, c->errbuf)) != CURLE_OK ||
178 0 : (ret = curl_easy_setopt(c->handle, CURLOPT_URL, s->name)) != CURLE_OK ||
179 0 : (ret = curl_easy_setopt(c->handle, CURLOPT_WRITEDATA, s)) != CURLE_OK ||
180 0 : (ret = curl_easy_setopt(c->handle, CURLOPT_VERBOSE, 0)) != CURLE_OK ||
181 0 : (ret = curl_easy_setopt(c->handle, CURLOPT_NOSIGNAL, 1)) != CURLE_OK ||
182 0 : (ret = curl_easy_setopt(c->handle, CURLOPT_FAILONERROR, 1)) != CURLE_OK ||
183 0 : (ret = curl_easy_setopt(c->handle, CURLOPT_WRITEFUNCTION, write_callback)) != CURLE_OK ||
184 0 : (ret = curl_easy_perform(c->handle)) != CURLE_OK) {
185 0 : if (c->errbuf[0] != 0)
186 0 : mnstr_set_open_error(url, 0, "%s", c->errbuf);
187 : else
188 0 : mnstr_set_open_error(url, 0, "curl_easy_perform: %s", curl_easy_strerror(ret));
189 0 : curl_destroy(s);
190 0 : return NULL;
191 : }
192 0 : curl_easy_cleanup(c->handle);
193 0 : c->handle = NULL;
194 0 : c->running = 0;
195 0 : return s;
196 : }
197 :
198 : #else
199 : stream *
200 : open_urlstream(const char *url)
201 : {
202 : if (url != NULL &&
203 : strncmp(url, "file://", sizeof("file://") - 1) == 0) {
204 : url +=sizeof("file://") - 1;
205 : #ifdef _MSC_VER
206 : /* file:///C:/... -- remove third / as well */
207 : if (url[0] == '/' && url[2] == ':')
208 : url++;
209 : #endif
210 : return open_rastream(url);
211 : }
212 : mnstr_set_open_error(url, 0, "Remote URL support has been left out of this MonetDB");
213 : return NULL;
214 : }
215 : #endif /* HAVE_CURL */
|
