Mercurial > octave-nkf

comparison liboctave/util/url-transfer.cc @ 17555:0946b0e06544

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
move url_transfer classes to liboctave * liboctave/util/url-transfer.h, liboctave/util/url-transfer.cc: New files, extracted from libinterp/dldfcn/urlwrite.cc. * libinterp/corefcn/urlwrite.cc: Move here from libinterp/dldfcn/urlwrite.cc. * libinterp/corefcn/module.mk, libinterp/dldfcn/module-files, liboctave/link-deps.mk liboctave/util/module.mk: Update for new and renamed files.
author John W. Eaton <jwe@octave.org>
date Thu, 03 Oct 2013 15:52:49 -0400
parents
children 58039875767d
comparison
equal deleted inserted replaced
17554:f0d21e7d4653 17555:0946b0e06544
1 /*
2
3 Copyright (C) 2013 John W. Eaton
4 Copyright (C) 2006-2012 Alexander Barth
5 Copyright (C) 2009 David Bateman
6
7 This file is part of Octave.
8
9 Octave is free software; you can redistribute it and/or modify it
10 under the terms of the GNU General Public License as published by the
11 Free Software Foundation; either version 3 of the License, or (at your
12 option) any later version.
13
14 Octave is distributed in the hope that it will be useful, but WITHOUT
15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with Octave; see the file COPYING. If not, see
21 <http://www.gnu.org/licenses/>.
22
23 */
24
25 // Author: Alexander Barth <abarth@marine.usf.edu>
26 // Author: jwe
27
28 #ifdef HAVE_CONFIG_H
29 #include <config.h>
30 #endif
31
32 #include <fstream>
33 #include <iomanip>
34 #include <iostream>
35
36 #include "dir-ops.h"
37 #include "file-ops.h"
38 #include "file-stat.h"
39 #include "unwind-prot.h"
40 #include "url-transfer.h"
41
42 #ifdef HAVE_CURL
43 #include <curl/curl.h>
44 #include <curl/curlver.h>
45 #include <curl/easy.h>
46 #endif
47
48 void
49 base_url_transfer::mget_directory (const std::string& directory,
50 const std::string& target)
51 {
52 std::string sep = file_ops::dir_sep_str ();
53 file_stat fs (directory);
54
55 if (!fs || !fs.is_dir ())
56 {
57 std::string msg;
58 int status = octave_mkdir (directory, 0777, msg);
59
60 if (status < 0)
61 {
62 ok = false;
63 errmsg = "__ftp_mget__: can not create directory '"
64 + target + sep + directory + "': " + msg;
65 return;
66 }
67 }
68
69 cwd (directory);
70
71 if (good ())
72 {
73 unwind_protect_safe frame;
74
75 frame.add_fcn (reset_path, this);
76
77 string_vector sv = list ();
78
79 for (octave_idx_type i = 0; i < sv.length (); i++)
80 {
81 time_t ftime;
82 bool fisdir;
83 double fsize;
84
85 get_fileinfo (sv(i), fsize, ftime, fisdir);
86
87 if (fisdir)
88 mget_directory (sv(i), target + directory + sep);
89 else
90 {
91 std::string realfile = target + directory + sep + sv(i);
92
93 std::ofstream ofile (realfile.c_str (),
94 std::ios::out | std::ios::binary);
95
96 if (! ofile.is_open ())
97 {
98 ok = false;
99 errmsg = "__ftp_mget__: unable to open file";
100 break;
101 }
102
103 unwind_protect_safe frame2;
104
105 frame2.add_fcn (delete_file, realfile);
106
107 get (sv(i), ofile);
108
109 ofile.close ();
110
111 if (good ())
112 frame2.discard ();
113 }
114
115 if (! good ())
116 break;
117 }
118 }
119 }
120
121 string_vector
122 base_url_transfer::mput_directory (const std::string& base,
123 const std::string& directory)
124 {
125 string_vector file_list;
126
127 std::string realdir
128 = (base.length () == 0
129 ? directory : base + file_ops::dir_sep_str () + directory);
130
131 mkdir (directory);
132
133 if (! good ())
134 return file_list;
135
136 cwd (directory);
137
138 if (good ())
139 {
140 unwind_protect_safe frame;
141
142 frame.add_fcn (reset_path, this);
143
144 dir_entry dirlist (realdir);
145
146 if (dirlist)
147 {
148 string_vector files = dirlist.read ();
149
150 for (octave_idx_type i = 0; i < files.length (); i++)
151 {
152 std::string file = files (i);
153
154 if (file == "." || file == "..")
155 continue;
156
157 std::string realfile = realdir + file_ops::dir_sep_str () + file;
158 file_stat fs (realfile);
159
160 if (! fs.exists ())
161 {
162 ok = false;
163 errmsg = "__ftp__mput: file '" + realfile
164 + "' does not exist";
165 break;
166 }
167
168 if (fs.is_dir ())
169 {
170 file_list.append (mput_directory (realdir, file));
171
172 if (! good ())
173 break;
174 }
175 else
176 {
177 // FIXME Does ascii mode need to be flagged here?
178 std::ifstream ifile (realfile.c_str (), std::ios::in |
179 std::ios::binary);
180
181 if (! ifile.is_open ())
182 {
183 ok = false;
184 errmsg = "__ftp_mput__: unable to open file '"
185 + realfile + "'";
186 break;
187 }
188
189 put (file, ifile);
190
191 ifile.close ();
192
193 if (! good ())
194 break;
195
196 file_list.append (realfile);
197 }
198 }
199 }
200 else
201 {
202 ok = false;
203 errmsg = "__ftp_mput__: can not read the directory '"
204 + realdir + "'";
205 }
206 }
207 }
208
209 #if defined (HAVE_CURL)
210
211 static int
212 write_data (void *buffer, size_t size, size_t nmemb, void *streamp)
213 {
214 std::ostream& stream = *(static_cast<std::ostream*> (streamp));
215 stream.write (static_cast<const char*> (buffer), size*nmemb);
216 return (stream.fail () ? 0 : size * nmemb);
217 }
218
219 static int
220 read_data (void *buffer, size_t size, size_t nmemb, void *streamp)
221 {
222 std::istream& stream = *(static_cast<std::istream*> (streamp));
223 stream.read (static_cast<char*> (buffer), size*nmemb);
224 if (stream.eof ())
225 return stream.gcount ();
226 else
227 return (stream.fail () ? 0 : size * nmemb);
228 }
229
230 static size_t
231 throw_away (void *, size_t size, size_t nmemb, void *)
232 {
233 return static_cast<size_t>(size * nmemb);
234 }
235
236 // I'd love to rewrite this as a private method of the url_transfer
237 // class, but you can't pass the va_list from the wrapper SETOPT to
238 // the curl_easy_setopt function.
239 #define SETOPT(option, parameter) \
240 do \
241 { \
242 CURLcode res = curl_easy_setopt (curl, option, parameter); \
243 if (res != CURLE_OK) \
244 { \
245 ok = false; \
246 errmsg = curl_easy_strerror (res); \
247 return; \
248 } \
249 } \
250 while (0)
251
252 // Same as above but with a return value.
253 #define SETOPTR(option, parameter) \
254 do \
255 { \
256 CURLcode res = curl_easy_setopt (curl, option, parameter); \
257 if (res != CURLE_OK) \
258 { \
259 ok = false; \
260 errmsg = curl_easy_strerror (res); \
261 return retval; \
262 } \
263 } \
264 while (0)
265
266 class curl_transfer : public base_url_transfer
267 {
268 public:
269
270 curl_transfer (void)
271 : base_url_transfer (), curl (curl_easy_init ()), errnum ()
272 {
273 if (curl)
274 valid = true;
275 else
276 errmsg = "can not create curl object";
277 }
278
279 curl_transfer (const std::string& host_arg, const std::string& user_arg,
280 const std::string& passwd, std::ostream& os)
281 : base_url_transfer (host_arg, user_arg, passwd, os),
282 curl (curl_easy_init ()), errnum ()
283 {
284 if (curl)
285 valid = true;
286 else
287 {
288 errmsg = "can not create curl object";
289 return;
290 }
291
292 init (user_arg, passwd, std::cin, os);
293
294 std::string url ("ftp://" + host_arg);
295 SETOPT (CURLOPT_URL, url.c_str ());
296
297 // Setup the link, with no transfer.
298 perform ();
299 }
300
301 curl_transfer (const std::string& url, const std::string& method,
302 const Array<std::string>& param, std::ostream& os)
303 : base_url_transfer (url, method, param, os),
304 curl (curl_easy_init ()), errnum ()
305 {
306 if (curl)
307 valid = true;
308 else
309 {
310 errmsg = "can not create curl object";
311 return;
312 }
313
314 init ("", "", std::cin, os);
315
316 SETOPT (CURLOPT_NOBODY, 0);
317
318 // Restore the default HTTP request method to GET after setting
319 // NOBODY to true and back to false. This is needed for backward
320 // compatibility with versions of libcurl < 7.18.2.
321 SETOPT (CURLOPT_HTTPGET, 1);
322
323 // Don't need to store the parameters here as we can't change
324 // the URL after the object is created
325 std::string query_string = form_query_string (param);
326
327 if (method == "get")
328 {
329 query_string = url + "?" + query_string;
330 SETOPT (CURLOPT_URL, query_string.c_str ());
331 }
332 else if (method == "post")
333 {
334 SETOPT (CURLOPT_URL, url.c_str ());
335 SETOPT (CURLOPT_POSTFIELDS, query_string.c_str ());
336 }
337 else
338 SETOPT (CURLOPT_URL, url.c_str ());
339
340 perform ();
341 }
342
343 ~curl_transfer (void)
344 {
345 if (curl)
346 curl_easy_cleanup (curl);
347 }
348
349 void perform (void)
350 {
351 BEGIN_INTERRUPT_IMMEDIATELY_IN_FOREIGN_CODE;
352
353 errnum = curl_easy_perform (curl);
354
355 if (errnum != CURLE_OK)
356 {
357 ok = false;
358 errmsg = curl_easy_strerror (errnum);
359 }
360
361 END_INTERRUPT_IMMEDIATELY_IN_FOREIGN_CODE;
362 }
363
364 std::string lasterror (void) const
365 {
366 return std::string (curl_easy_strerror (errnum));
367 }
368
369 std::ostream& set_ostream (std::ostream& os)
370 {
371 std::ostream& retval = *curr_ostream;
372 curr_ostream = &os;
373 SETOPTR (CURLOPT_WRITEDATA, static_cast<void*> (curr_ostream));
374 return retval;
375 }
376
377 std::istream& set_istream (std::istream& is)
378 {
379 std::istream& retval = *curr_istream;
380 curr_istream = &is;
381 SETOPTR (CURLOPT_READDATA, static_cast<void*> (curr_istream));
382 return retval;
383 }
384
385 void ascii (void)
386 {
387 ascii_mode = true;
388 SETOPT (CURLOPT_TRANSFERTEXT, 1);
389 }
390
391 void binary (void)
392 {
393 ascii_mode = false;
394 SETOPT (CURLOPT_TRANSFERTEXT, 0);
395 }
396
397 void cwd (const std::string& path)
398 {
399 struct curl_slist *slist = 0;
400
401 unwind_protect frame;
402 frame.add_fcn (curl_slist_free_all, slist);
403
404 std::string cmd = "cwd " + path;
405 slist = curl_slist_append (slist, cmd.c_str ());
406 SETOPT (CURLOPT_POSTQUOTE, slist);
407
408 perform ();
409 if (! good ())
410 return;
411
412 SETOPT (CURLOPT_POSTQUOTE, 0);
413 }
414
415 void del (const std::string& file)
416 {
417 struct curl_slist *slist = 0;
418
419 unwind_protect frame;
420 frame.add_fcn (curl_slist_free_all, slist);
421
422 std::string cmd = "dele " + file;
423 slist = curl_slist_append (slist, cmd.c_str ());
424 SETOPT (CURLOPT_POSTQUOTE, slist);
425
426 perform ();
427 if (! good ())
428 return;
429
430 SETOPT (CURLOPT_POSTQUOTE, 0);
431 }
432
433 void rmdir (const std::string& path)
434 {
435 struct curl_slist *slist = 0;
436
437 unwind_protect frame;
438 frame.add_fcn (curl_slist_free_all, slist);
439
440 std::string cmd = "rmd " + path;
441 slist = curl_slist_append (slist, cmd.c_str ());
442 SETOPT (CURLOPT_POSTQUOTE, slist);
443
444 perform ();
445 if (! good ())
446 return;
447
448 SETOPT (CURLOPT_POSTQUOTE, 0);
449 }
450
451 void mkdir (const std::string& path)
452 {
453 struct curl_slist *slist = 0;
454
455 unwind_protect frame;
456 frame.add_fcn (curl_slist_free_all, slist);
457
458 std::string cmd = "mkd " + path;
459 slist = curl_slist_append (slist, cmd.c_str ());
460 SETOPT (CURLOPT_POSTQUOTE, slist);
461
462 perform ();
463 if (! good ())
464 return;
465
466 SETOPT (CURLOPT_POSTQUOTE, 0);
467 }
468
469 void rename (const std::string& oldname, const std::string& newname)
470 {
471 struct curl_slist *slist = 0;
472
473 unwind_protect frame;
474 frame.add_fcn (curl_slist_free_all, slist);
475
476 std::string cmd = "rnfr " + oldname;
477 slist = curl_slist_append (slist, cmd.c_str ());
478 cmd = "rnto " + newname;
479 slist = curl_slist_append (slist, cmd.c_str ());
480 SETOPT (CURLOPT_POSTQUOTE, slist);
481
482 perform ();
483 if (! good ())
484 return;
485
486 SETOPT (CURLOPT_POSTQUOTE, 0);
487 }
488
489 void put (const std::string& file, std::istream& is)
490 {
491 std::string url = "ftp://" + host + "/" + file;
492 SETOPT (CURLOPT_URL, url.c_str ());
493 SETOPT (CURLOPT_UPLOAD, 1);
494 SETOPT (CURLOPT_NOBODY, 0);
495 std::istream& old_is = set_istream (is);
496
497 perform ();
498 if (! good ())
499 return;
500
501 set_istream (old_is);
502 SETOPT (CURLOPT_NOBODY, 1);
503 SETOPT (CURLOPT_UPLOAD, 0);
504 url = "ftp://" + host;
505 SETOPT (CURLOPT_URL, url.c_str ());
506 }
507
508 void get (const std::string& file, std::ostream& os)
509 {
510 std::string url = "ftp://" + host + "/" + file;
511 SETOPT (CURLOPT_URL, url.c_str ());
512 SETOPT (CURLOPT_NOBODY, 0);
513 std::ostream& old_os = set_ostream (os);
514
515 perform ();
516 if (! good ())
517 return;
518
519 set_ostream (old_os);
520 SETOPT (CURLOPT_NOBODY, 1);
521 url = "ftp://" + host;
522 SETOPT (CURLOPT_URL, url.c_str ());
523 }
524
525 void dir (void)
526 {
527 std::string url = "ftp://" + host + "/";
528 SETOPT (CURLOPT_URL, url.c_str ());
529 SETOPT (CURLOPT_NOBODY, 0);
530
531 perform ();
532 if (! good ())
533 return;
534
535 SETOPT (CURLOPT_NOBODY, 1);
536 url = "ftp://" + host;
537 SETOPT (CURLOPT_URL, url.c_str ());
538 }
539
540 string_vector list (void)
541 {
542 string_vector retval;
543
544 std::ostringstream buf;
545 std::string url = "ftp://" + host + "/";
546 SETOPTR (CURLOPT_WRITEDATA, static_cast<void*> (&buf));
547 SETOPTR (CURLOPT_URL, url.c_str ());
548 SETOPTR (CURLOPT_DIRLISTONLY, 1);
549 SETOPTR (CURLOPT_NOBODY, 0);
550
551 perform ();
552 if (! good ())
553 return retval;
554
555 SETOPTR (CURLOPT_NOBODY, 1);
556 url = "ftp://" + host;
557 SETOPTR (CURLOPT_WRITEDATA, static_cast<void*> (curr_ostream));
558 SETOPTR (CURLOPT_DIRLISTONLY, 0);
559 SETOPTR (CURLOPT_URL, url.c_str ());
560
561 // Count number of directory entries
562 std::string str = buf.str ();
563 octave_idx_type n = 0;
564 size_t pos = 0;
565 while (true)
566 {
567 pos = str.find_first_of ('\n', pos);
568 if (pos == std::string::npos)
569 break;
570 pos++;
571 n++;
572 }
573 retval.resize (n);
574 pos = 0;
575 for (octave_idx_type i = 0; i < n; i++)
576 {
577 size_t newpos = str.find_first_of ('\n', pos);
578 if (newpos == std::string::npos)
579 break;
580
581 retval(i) = str.substr(pos, newpos - pos);
582 pos = newpos + 1;
583 }
584
585 return retval;
586 }
587
588 void get_fileinfo (const std::string& filename, double& filesize,
589 time_t& filetime, bool& fileisdir)
590 {
591 std::string path = pwd ();
592
593 std::string url = "ftp://" + host + "/" + path + "/" + filename;
594 SETOPT (CURLOPT_URL, url.c_str ());
595 SETOPT (CURLOPT_FILETIME, 1);
596 SETOPT (CURLOPT_HEADERFUNCTION, throw_away);
597 SETOPT (CURLOPT_WRITEFUNCTION, throw_away);
598
599 // FIXME
600 // The MDTM command fails for a directory on the servers I tested
601 // so this is a means of testing for directories. It also means
602 // I can't get the date of directories!
603
604 perform ();
605 if (! good ())
606 {
607 fileisdir = true;
608 filetime = -1;
609 filesize = 0;
610
611 return;
612 }
613
614 fileisdir = false;
615 time_t ft;
616 curl_easy_getinfo (curl, CURLINFO_FILETIME, &ft);
617 filetime = ft;
618 double fs;
619 curl_easy_getinfo (curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &fs);
620 filesize = fs;
621
622 SETOPT (CURLOPT_WRITEFUNCTION, write_data);
623 SETOPT (CURLOPT_HEADERFUNCTION, 0);
624 SETOPT (CURLOPT_FILETIME, 0);
625 url = "ftp://" + host;
626 SETOPT (CURLOPT_URL, url.c_str ());
627
628 // The MDTM command seems to reset the path to the root with the
629 // servers I tested with, so cd again into the correct path. Make
630 // the path absolute so that this will work even with servers that
631 // don't end up in the root after an MDTM command.
632 cwd ("/" + path);
633 }
634
635 std::string pwd (void)
636 {
637 std::string retval;
638
639 struct curl_slist *slist = 0;
640
641 unwind_protect frame;
642 frame.add_fcn (curl_slist_free_all, slist);
643
644 slist = curl_slist_append (slist, "pwd");
645 SETOPTR (CURLOPT_POSTQUOTE, slist);
646 SETOPTR (CURLOPT_HEADERFUNCTION, write_data);
647
648 std::ostringstream buf;
649 SETOPTR (CURLOPT_WRITEHEADER, static_cast<void *>(&buf));
650
651 perform ();
652 if (! good ())
653 return retval;
654
655 retval = buf.str ();
656
657 // Can I assume that the path is alway in "" on the last line
658 size_t pos2 = retval.rfind ('"');
659 size_t pos1 = retval.rfind ('"', pos2 - 1);
660 retval = retval.substr (pos1 + 1, pos2 - pos1 - 1);
661
662 SETOPTR (CURLOPT_HEADERFUNCTION, 0);
663 SETOPTR (CURLOPT_WRITEHEADER, 0);
664 SETOPTR (CURLOPT_POSTQUOTE, 0);
665
666 return retval;
667 }
668
669 private:
670
671 CURL *curl;
672 CURLcode errnum;
673
674 // No copying!
675
676 curl_transfer (const curl_transfer&);
677
678 curl_transfer& operator = (const curl_transfer&);
679
680 void init (const std::string& user, const std::string& passwd,
681 std::istream& is, std::ostream& os)
682 {
683 // No data transfer by default
684 SETOPT (CURLOPT_NOBODY, 1);
685
686 // Set the username and password
687 userpwd = user;
688 if (! passwd.empty ())
689 userpwd += ":" + passwd;
690 if (! userpwd.empty ())
691 SETOPT (CURLOPT_USERPWD, userpwd.c_str ());
692
693 // Define our callback to get called when there's data to be written.
694 SETOPT (CURLOPT_WRITEFUNCTION, write_data);
695
696 // Set a pointer to our struct to pass to the callback.
697 SETOPT (CURLOPT_WRITEDATA, static_cast<void*> (&os));
698
699 // Define our callback to get called when there's data to be read
700 SETOPT (CURLOPT_READFUNCTION, read_data);
701
702 // Set a pointer to our struct to pass to the callback.
703 SETOPT (CURLOPT_READDATA, static_cast<void*> (&is));
704
705 // Follow redirects.
706 SETOPT (CURLOPT_FOLLOWLOCATION, true);
707
708 // Don't use EPSV since connecting to sites that don't support it
709 // will hang for some time (3 minutes?) before moving on to try PASV
710 // instead.
711 SETOPT (CURLOPT_FTP_USE_EPSV, false);
712
713 SETOPT (CURLOPT_NOPROGRESS, true);
714 SETOPT (CURLOPT_FAILONERROR, true);
715
716 SETOPT (CURLOPT_POSTQUOTE, 0);
717 SETOPT (CURLOPT_QUOTE, 0);
718 }
719
720 std::string form_query_string (const Array<std::string>& param)
721 {
722 std::ostringstream query;
723
724 for (int i = 0; i < param.numel (); i += 2)
725 {
726 std::string name = param(i);
727 std::string text = param(i+1);
728
729 // Encode strings.
730 char *enc_name = curl_easy_escape (curl, name.c_str (),
731 name.length ());
732 char *enc_text = curl_easy_escape (curl, text.c_str (),
733 text.length ());
734
735 query << enc_name << "=" << enc_text;
736
737 curl_free (enc_name);
738 curl_free (enc_text);
739
740 if (i < param.numel ()-1)
741 query << "&";
742 }
743
744 query.flush ();
745
746 return query.str ();
747 }
748 };
749
750 #undef SETOPT
751
752 #else
753
754 static void
755 disabled_error (void)
756 {
757 error ("support for url transfers was disabled when Octave was built");
758 }
759
760 #endif
761
762 #if defined (HAVE_CURL)
763 # define REP_CLASS curl_transfer
764 #else
765 # define REP_CLASS base_url_transfer
766 #endif
767
768 url_transfer::url_transfer (void) : rep (new REP_CLASS ())
769 {
770 #if !defined (HAVE_CURL)
771 disabled_error ();
772 #endif
773 }
774
775 url_transfer::url_transfer (const std::string& host, const std::string& user,
776 const std::string& passwd, std::ostream& os)
777 : rep (new REP_CLASS (host, user, passwd, os))
778 {
779 #if !defined (HAVE_CURL)
780 disabled_error ();
781 #endif
782 }
783
784 url_transfer::url_transfer (const std::string& url, const std::string& method,
785 const Array<std::string>& param, std::ostream& os)
786 : rep (new REP_CLASS (url, method, param, os))
787 {
788 #if !defined (HAVE_CURL)
789 disabled_error ();
790 #endif
791 }
792
793 #undef REP_CLASS