|
6043
|
1 // urlwrite and urlread, a curl front-end for octave |
|
|
2 /* |
|
|
3 |
|
7017
|
4 Copyright (C) 2006, 2007 Alexander Barth |
|
6043
|
5 |
|
|
6 This file is part of Octave. |
|
|
7 |
|
|
8 Octave is free software; you can redistribute it and/or modify it |
|
|
9 under the terms of the GNU General Public License as published by the |
|
7016
|
10 Free Software Foundation; either version 3 of the License, or (at your |
|
|
11 option) any later version. |
|
6043
|
12 |
|
|
13 Octave is distributed in the hope that it will be useful, but WITHOUT |
|
|
14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
|
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
|
16 for more details. |
|
|
17 |
|
|
18 You should have received a copy of the GNU General Public License |
|
7016
|
19 along with Octave; see the file COPYING. If not, see |
|
|
20 <http://www.gnu.org/licenses/>. |
|
6043
|
21 |
|
|
22 */ |
|
|
23 |
|
|
24 // Author: Alexander Barth <abarth@marine.usf.edu> |
|
|
25 // Adapted-By: jwe |
|
|
26 |
|
|
27 #ifdef HAVE_CONFIG_H |
|
|
28 #include <config.h> |
|
|
29 #endif |
|
|
30 |
|
|
31 #include <string> |
|
|
32 #include <fstream> |
|
|
33 #include <iomanip> |
|
|
34 |
|
|
35 #include "oct-env.h" |
|
|
36 |
|
|
37 #include "defun-dld.h" |
|
|
38 #include "error.h" |
|
|
39 #include "oct-obj.h" |
|
|
40 #include "ov-cell.h" |
|
|
41 #include "pager.h" |
|
|
42 |
|
|
43 #if defined (HAVE_CURL) |
|
|
44 |
|
|
45 #include <curl/curl.h> |
|
|
46 #include <curl/types.h> |
|
|
47 #include <curl/easy.h> |
|
|
48 |
|
|
49 // Write callback function for curl. |
|
|
50 |
|
6986
|
51 static int |
|
6043
|
52 write_data (void *buffer, size_t size, size_t nmemb, void *streamp) |
|
|
53 { |
|
|
54 // *stream is actually an ostream object. |
|
|
55 std::ostream& stream = *(static_cast<std::ostream*> (streamp)); |
|
|
56 stream.write (static_cast<const char*> (buffer), size*nmemb); |
|
|
57 return (stream.fail () ? 0 : size * nmemb); |
|
|
58 } |
|
|
59 |
|
|
60 // Form the query string based on param. |
|
|
61 |
|
6986
|
62 static std::string |
|
6043
|
63 form_query_string (CURL *curl, const Cell& param) |
|
|
64 { |
|
|
65 std::ostringstream query; |
|
|
66 |
|
|
67 for (int i = 0; i < param.numel (); i += 2) |
|
|
68 { |
|
|
69 std::string name = param(i).string_value (); |
|
|
70 std::string text = param(i+1).string_value (); |
|
|
71 |
|
|
72 // Encode strings. |
|
|
73 char *enc_name = curl_easy_escape (curl, name.c_str (), name.length ()); |
|
|
74 char *enc_text = curl_easy_escape (curl, text.c_str (), text.length ()); |
|
|
75 |
|
|
76 query << enc_name << "=" << enc_text; |
|
|
77 |
|
|
78 curl_free (enc_name); |
|
|
79 curl_free (enc_text); |
|
|
80 |
|
|
81 if (i < param.numel()-1) |
|
|
82 query << "&"; |
|
|
83 } |
|
|
84 |
|
|
85 query.flush (); |
|
|
86 |
|
|
87 return query.str (); |
|
|
88 } |
|
|
89 |
|
|
90 // curl front-end |
|
|
91 |
|
6992
|
92 static void |
|
|
93 urlget_cleanup (CURL *curl) |
|
|
94 { |
|
|
95 curl_easy_cleanup (curl); |
|
|
96 curl_global_cleanup (); |
|
|
97 } |
|
|
98 |
|
6986
|
99 static CURLcode |
|
6043
|
100 urlget (const std::string& url, const std::string& method, |
|
|
101 const Cell& param, std::ostream& stream) |
|
|
102 { |
|
|
103 CURL *curl; |
|
|
104 |
|
|
105 curl_global_init(CURL_GLOBAL_DEFAULT); |
|
|
106 |
|
|
107 curl = curl_easy_init(); |
|
|
108 |
|
|
109 if (! curl) |
|
|
110 return CURLE_FAILED_INIT; |
|
|
111 |
|
|
112 // handle paramters of GET or POST request |
|
|
113 |
|
|
114 std::string query_string = form_query_string (curl,param); |
|
|
115 //octave_stdout << "query_string " << query_string << std::endl; |
|
|
116 |
|
|
117 if (method == "get") |
|
|
118 { |
|
|
119 query_string = url + "?" + query_string; |
|
|
120 curl_easy_setopt (curl, CURLOPT_URL, query_string.c_str ()); |
|
|
121 } |
|
|
122 else if (method == "post") |
|
|
123 { |
|
|
124 curl_easy_setopt (curl, CURLOPT_URL, url.c_str ()); |
|
|
125 curl_easy_setopt (curl, CURLOPT_POSTFIELDS, query_string.c_str ()); |
|
|
126 } |
|
|
127 else |
|
7013
|
128 curl_easy_setopt (curl, CURLOPT_URL, url.c_str()); |
|
6043
|
129 |
|
|
130 // Define our callback to get called when there's data to be written. |
|
|
131 curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_data); |
|
|
132 |
|
|
133 // Set a pointer to our struct to pass to the callback. |
|
|
134 curl_easy_setopt (curl, CURLOPT_WRITEDATA, static_cast<void*> (&stream)); |
|
|
135 |
|
6390
|
136 // Follow redirects. |
|
7013
|
137 curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, true); |
|
|
138 |
|
|
139 // Don't use EPSV since connecting to sites that don't support it |
|
|
140 // will hang for some time (3 minutes?) before moving on to try PASV |
|
|
141 // instead. |
|
|
142 curl_easy_setopt (curl, CURLOPT_FTP_USE_EPSV, false); |
|
6390
|
143 |
|
6992
|
144 curl_easy_setopt (curl, CURLOPT_NOPROGRESS, true); |
|
6043
|
145 curl_easy_setopt (curl, CURLOPT_PROGRESSDATA, url.c_str ()); |
|
|
146 curl_easy_setopt (curl, CURLOPT_FAILONERROR, true); |
|
|
147 |
|
7013
|
148 // Switch on full protocol/debug output. |
|
|
149 // curl_easy_setopt (curl, CURLOPT_VERBOSE, true); |
|
6043
|
150 |
|
6992
|
151 CURLcode res = CURLE_OK; |
|
|
152 |
|
|
153 // To understand the following, see the definitions of these macros |
|
|
154 // in libcruft/misc/quit.h. The idea is that we call sigsetjmp here |
|
|
155 // then the signal handler calls siglongjmp to get back here |
|
|
156 // immediately. Then we perform some cleanup and throw an interrupt |
|
|
157 // exception which will get us back to the top level, cleaning up |
|
|
158 // any local C++ objects on the stack as we go. |
|
|
159 |
|
|
160 BEGIN_INTERRUPT_IMMEDIATELY_IN_FOREIGN_CODE_1; |
|
|
161 |
|
|
162 // We were interrupted (this code is inside a block that is only |
|
|
163 // called when siglongjmp is called from a signal handler). |
|
6043
|
164 |
|
6992
|
165 // Is there a better error code to use? Maybe it doesn't matter |
|
|
166 // because we are about to throw an execption. |
|
|
167 |
|
|
168 res = CURLE_ABORTED_BY_CALLBACK; |
|
|
169 urlget_cleanup (curl); |
|
|
170 octave_throw_interrupt_exception (); |
|
|
171 |
|
|
172 BEGIN_INTERRUPT_IMMEDIATELY_IN_FOREIGN_CODE_2; |
|
6043
|
173 |
|
6992
|
174 res = curl_easy_perform (curl); |
|
|
175 |
|
|
176 END_INTERRUPT_IMMEDIATELY_IN_FOREIGN_CODE; |
|
|
177 |
|
|
178 // If we are not interuppted, we will end up here, so we still need |
|
|
179 // to clean up. |
|
|
180 |
|
|
181 urlget_cleanup (curl); |
|
6043
|
182 |
|
|
183 return res; |
|
|
184 } |
|
|
185 |
|
|
186 #endif |
|
|
187 |
|
|
188 DEFUN_DLD (urlwrite, args, nargout, |
|
|
189 "-*- texinfo -*-\n\ |
|
|
190 @deftypefn {Loadable Function} {} urlwrite (@var{URL}, @var{localfile})\n\ |
|
|
191 @deftypefnx {Loadable Function} {@var{f} =} urlwrite (@var{url}, @var{localfile})\n\ |
|
|
192 @deftypefnx {Loadable Function} {[@var{f}, @var{success}] =} urlwrite (@var{url}, @var{localfile})\n\ |
|
|
193 @deftypefnx {Loadable Function} {[@var{f}, @var{success}, @var{message}] =} urlwrite (@var{url}, @var{localfile})\n\ |
|
|
194 Download a remote file specified by its @var{URL} and save it as\n\ |
|
|
195 @var{localfile}. For example,\n\ |
|
|
196 \n\ |
|
|
197 @example\n\ |
|
6588
|
198 urlwrite (\"ftp://ftp.octave.org/pub/octave/README\", \"README.txt\");\n\ |
|
6043
|
199 @end example\n\ |
|
|
200 \n\ |
|
|
201 The full path of the downloaded file is returned in @var{f}. The\n\ |
|
|
202 variable @var{success} is 1 if the download was successful,\n\ |
|
|
203 otherwise it is 0 in which case @var{message} contains an error\n\ |
|
|
204 message. If no output argument is specified and if an error occurs,\n\ |
|
6588
|
205 then the error is signaled through Octave's error handling mechanism.\n\ |
|
6043
|
206 \n\ |
|
|
207 This function uses libcurl. Curl supports, among others, the HTTP,\n\ |
|
|
208 FTP and FILE protocols. Username and password may be specified in\n\ |
|
|
209 the URL, for example:\n\ |
|
|
210 \n\ |
|
|
211 @example\n\ |
|
6588
|
212 urlwrite (\"http://username:password@@example.com/file.txt\",\n\ |
|
|
213 \"file.txt\");\n\ |
|
6043
|
214 @end example\n\ |
|
|
215 \n\ |
|
|
216 GET and POST requests can be specified by @var{method} and @var{param}.\n\ |
|
6589
|
217 The parameter @var{method} is either @samp{get} or @samp{post}\n\ |
|
6588
|
218 and @var{param} is a cell array of parameter and value pairs.\n\ |
|
|
219 For example:\n\ |
|
6043
|
220 \n\ |
|
|
221 @example\n\ |
|
6588
|
222 urlwrite (\"http://www.google.com/search\", \"search.html\",\n\ |
|
|
223 \"get\", @{\"query\", \"octave\"@});\n\ |
|
6043
|
224 @end example\n\ |
|
|
225 @seealso{urlread}\n\ |
|
|
226 @end deftypefn") |
|
|
227 { |
|
|
228 octave_value_list retval; |
|
|
229 |
|
|
230 #if defined (HAVE_CURL) |
|
|
231 |
|
|
232 int nargin = args.length (); |
|
|
233 |
|
|
234 // verify arguments |
|
|
235 if (nargin != 2 && nargin != 4) |
|
|
236 { |
|
|
237 print_usage (); |
|
|
238 return retval; |
|
|
239 } |
|
|
240 |
|
|
241 std::string url = args(0).string_value(); |
|
|
242 |
|
|
243 if (error_state) |
|
|
244 { |
|
|
245 error ("urlwrite: url must be a character string"); |
|
|
246 return retval; |
|
|
247 } |
|
|
248 |
|
|
249 // name to store the file if download is succesful |
|
|
250 std::string filename = args(1).string_value(); |
|
|
251 |
|
|
252 if (error_state) |
|
|
253 { |
|
|
254 error ("urlwrite: localfile must be a character string"); |
|
|
255 return retval; |
|
|
256 } |
|
|
257 |
|
|
258 std::string method; |
|
|
259 Cell param; // empty cell array |
|
|
260 |
|
|
261 if (nargin == 4) |
|
|
262 { |
|
|
263 method = args(2).string_value(); |
|
|
264 |
|
|
265 if (error_state) |
|
|
266 { |
|
|
267 error ("urlwrite: method can only be \"get\" or \"post\""); |
|
|
268 return retval; |
|
|
269 } |
|
|
270 |
|
|
271 if (method != "get" && method != "post") |
|
|
272 { |
|
|
273 error ("urlwrite: method can only be \"get\" or \"post\""); |
|
|
274 return retval; |
|
|
275 } |
|
|
276 |
|
|
277 param = args(3).cell_value(); |
|
|
278 |
|
|
279 if (error_state) |
|
|
280 { |
|
|
281 error ("urlwrite: parameters for get and post requests must be given as a cell"); |
|
|
282 return retval; |
|
|
283 } |
|
|
284 |
|
|
285 |
|
|
286 if (param.numel () % 2 == 1 ) |
|
|
287 { |
|
|
288 error ("urlwrite: number of elements in param must be even"); |
|
|
289 return retval; |
|
|
290 } |
|
|
291 } |
|
|
292 |
|
6986
|
293 std::ofstream ofile (filename.c_str(), std::ios::out | std::ios::binary); |
|
6043
|
294 |
|
6986
|
295 if (! ofile.is_open ()) |
|
6043
|
296 { |
|
|
297 error ("urlwrite: unable to open file"); |
|
|
298 return retval; |
|
|
299 } |
|
|
300 |
|
6986
|
301 CURLcode res = urlget (url, method, param, ofile); |
|
6043
|
302 |
|
6986
|
303 ofile.close (); |
|
6043
|
304 |
|
|
305 if (nargout > 0) |
|
|
306 { |
|
|
307 retval(0) = octave_env::make_absolute (filename, octave_env::getcwd ()); |
|
|
308 retval(1) = res == CURLE_OK; |
|
|
309 retval(2) = std::string (res == CURLE_OK ? "" : curl_easy_strerror (res)); |
|
|
310 } |
|
|
311 |
|
6484
|
312 if (nargout < 2 && res != CURLE_OK) |
|
6043
|
313 error ("urlwrite: curl: %s", curl_easy_strerror (res)); |
|
|
314 |
|
|
315 #else |
|
6981
|
316 error ("urlwrite: not available in this version of Octave"); |
|
6043
|
317 #endif |
|
|
318 |
|
|
319 return retval; |
|
|
320 } |
|
|
321 |
|
|
322 DEFUN_DLD (urlread, args, nargout, |
|
|
323 "-*- texinfo -*-\n\ |
|
6549
|
324 @deftypefn {Loadable Function} {@var{s} =} urlread (@var{url})\n\ |
|
6043
|
325 @deftypefnx {Loadable Function} {[@var{s}, @var{success}] =} urlread (@var{url})\n\ |
|
6549
|
326 @deftypefnx {Loadable Function} {[@var{s}, @var{success}, @var{message}] =} urlread (@var{url})\n\ |
|
6547
|
327 @deftypefnx {Loadable Function} {[@dots{}] =} urlread (@var{url}, @var{method}, @var{param})\n\ |
|
6043
|
328 Download a remote file specified by its @var{URL} and return its content\n\ |
|
|
329 in string @var{s}. For example,\n\ |
|
|
330 \n\ |
|
|
331 @example\n\ |
|
6588
|
332 s = urlread (\"ftp://ftp.octave.org/pub/octave/README\");\n\ |
|
6043
|
333 @end example\n\ |
|
|
334 \n\ |
|
|
335 The variable @var{success} is 1 if the download was successful,\n\ |
|
|
336 otherwise it is 0 in which case @var{message} contains an error\n\ |
|
|
337 message. If no output argument is specified and if an error occurs,\n\ |
|
6588
|
338 then the error is signaled through Octave's error handling mechanism.\n\ |
|
6043
|
339 \n\ |
|
|
340 This function uses libcurl. Curl supports, among others, the HTTP,\n\ |
|
|
341 FTP and FILE protocols. Username and password may be specified in the\n\ |
|
|
342 URL. For example,\n\ |
|
|
343 \n\ |
|
|
344 @example\n\ |
|
6588
|
345 s = urlread (\"http://username:password@@example.com/file.txt\");\n\ |
|
6043
|
346 @end example\n\ |
|
|
347 \n\ |
|
|
348 GET and POST requests can be specified by @var{method} and @var{param}.\n\ |
|
6588
|
349 The parameter @var{method} is either @samp{get} or @samp{post}\n\ |
|
|
350 and @var{param} is a cell array of parameter and value pairs.\n\ |
|
6650
|
351 For example,\n\ |
|
6043
|
352 \n\ |
|
|
353 @example\n\ |
|
6588
|
354 s = urlread (\"http://www.google.com/search\", \"get\",\n\ |
|
|
355 @{\"query\", \"octave\"@});\n\ |
|
6043
|
356 @end example\n\ |
|
|
357 @seealso{urlwrite}\n\ |
|
|
358 @end deftypefn") |
|
|
359 { |
|
6588
|
360 // Octave's return value |
|
6043
|
361 octave_value_list retval; |
|
|
362 |
|
|
363 #if defined (HAVE_CURL) |
|
|
364 |
|
|
365 int nargin = args.length (); |
|
|
366 |
|
|
367 // verify arguments |
|
|
368 if (nargin != 1 && nargin != 3) |
|
|
369 { |
|
|
370 print_usage (); |
|
|
371 return retval; |
|
|
372 } |
|
|
373 |
|
|
374 std::string url = args(0).string_value(); |
|
|
375 |
|
|
376 if (error_state) |
|
|
377 { |
|
|
378 error ("urlread: url must be a character string"); |
|
|
379 return retval; |
|
|
380 } |
|
|
381 |
|
|
382 std::string method; |
|
|
383 Cell param; // empty cell array |
|
|
384 |
|
|
385 if (nargin == 3) |
|
|
386 { |
|
|
387 method = args(1).string_value(); |
|
|
388 |
|
|
389 if (error_state) |
|
|
390 { |
|
|
391 error ("urlread: method can only be \"get\" or \"post\""); |
|
|
392 return retval; |
|
|
393 } |
|
|
394 |
|
|
395 if (method != "get" && method != "post") |
|
|
396 { |
|
|
397 error ("urlread: method can only be \"get\" or \"post\""); |
|
|
398 return retval; |
|
|
399 } |
|
|
400 |
|
|
401 param = args(2).cell_value(); |
|
|
402 |
|
|
403 if (error_state) |
|
|
404 { |
|
|
405 error ("urlread: parameters for get and post requests must be given as a cell"); |
|
|
406 return retval; |
|
|
407 } |
|
|
408 |
|
|
409 if (param.numel () % 2 == 1 ) |
|
|
410 { |
|
|
411 error ("urlread: number of elements in param must be even"); |
|
|
412 return retval; |
|
|
413 } |
|
|
414 } |
|
|
415 |
|
6986
|
416 std::ostringstream buf; |
|
6043
|
417 |
|
6986
|
418 CURLcode res = urlget (url, method, param, buf); |
|
6043
|
419 |
|
|
420 if (nargout > 0) |
|
|
421 { |
|
6986
|
422 retval(0) = buf.str (); |
|
6043
|
423 retval(1) = res == CURLE_OK; |
|
6986
|
424 // Return empty string if no error occured. |
|
6043
|
425 retval(2) = std::string (res == CURLE_OK ? "" : curl_easy_strerror (res)); |
|
|
426 } |
|
|
427 |
|
6484
|
428 if (nargout < 2 && res != CURLE_OK) |
|
6043
|
429 error ("urlread: curl: %s", curl_easy_strerror (res)); |
|
|
430 |
|
|
431 #else |
|
6981
|
432 error ("urlread: not available in this version of Octave"); |
|
6043
|
433 #endif |
|
|
434 |
|
|
435 return retval; |
|
|
436 } |
