WvStreams
wvftpstream.cc
1 /*
2  * Worldvisions Weaver Software:
3  * Copyright (C) 1997-2002 Net Integration Technologies, Inc.
4  *
5  * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
6  *
7  * See wvhttppool.h.
8  */
9 
10 #ifdef __GNUC__
11 # define alloca __builtin_alloca
12 #else
13 # ifdef _MSC_VER
14 # include <malloc.h>
15 # define alloca _alloca
16 # else
17 # if HAVE_ALLOCA_H
18 # include <alloca.h>
19 # else
20 # ifdef _AIX
21 #pragma alloca
22 # else
23 # ifndef alloca /* predefined by HP cc +Olibcalls */
24 char *alloca ();
25 # endif
26 # endif
27 # endif
28 # endif
29 #endif
30 
31 #include <ctype.h>
32 #include <time.h>
33 #include "wvhttppool.h"
34 #include "wvbufstream.h"
35 #include "wvtcp.h"
36 #include "wvsslstream.h"
37 #include "strutils.h"
38 #include <stdlib.h> // for alloca()... FIXME: which we shouldn't be using!
39 
40 WvFtpStream::WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
41  WvStringParm _password)
42  : WvUrlStream(_remaddr, _username, WvString("FTP %s", _remaddr)),
43  cont(wv::bind(&WvFtpStream::real_execute, this, _1))
44 {
45  data = NULL;
46  logged_in = false;
47  password = _password;
48  last_request_time = time(0);
49  alarm(60000); // timeout if no connection, or something goes wrong
50 }
51 
52 
53 void WvFtpStream::doneurl()
54 {
55  log("Done URL: %s\n", curl->url);
56 
57  curl->done();
58  curl = NULL;
59  WVRELEASE(data);
60  urls.unlink_first();
61  last_request_time = time(0);
62  alarm(60000);
63  request_next();
64  // We just processed the last url in the queue,
65  // so go away.
66  if (urls.isempty() && waiting_urls.isempty())
67  close();
68 }
69 
70 
71 void WvFtpStream::request_next()
72 {
73  // don't do a request if we've done too many already or we have none
74  // waiting.
75  if (request_count >= max_requests || waiting_urls.isempty())
76  return;
77 
78  if (!urls.isempty())
79  return;
80 
81  // okay then, we really do want to send a new request.
82  WvUrlRequest *url = waiting_urls.first();
83 
84  waiting_urls.unlink_first();
85 
86  request_count++;
87  log("Request #%s: %s\n", request_count, url->url);
88  urls.append(url, false, "request_url");
89  alarm(0);
90 }
91 
92 
94 {
95  if (isok())
96  log("Closing.\n");
98 
99  if (geterr())
100  {
101  // if there was an error, count the first URL as done. This prevents
102  // retrying indefinitely.
103  if (!curl && !urls.isempty())
104  curl = urls.first();
105  if (!curl && !waiting_urls.isempty())
106  curl = waiting_urls.first();
107  if (curl)
108  log("URL '%s' is FAILED\n", curl->url);
109  if (curl)
110  curl->done();
111  }
112 
113  if (curl)
114  curl->done();
115 }
116 
117 
118 char *WvFtpStream::get_important_line()
119 {
120  char *line;
121  do
122  {
123  line = blocking_getline(-1);
124  if (!line)
125  return NULL;
126  }
127  while (line[3] == '-');
128  log(WvLog::Debug5, ">> %s\n", line);
129  return line;
130 }
131 
132 
134 {
135  SelectRequest oldwant = si.wants;
136 
138 
139  if (data)
140  data->pre_select(si);
141 
142  if (curl && curl->putstream)
143  curl->putstream->pre_select(si);
144 
145  si.wants = oldwant;
146 }
147 
148 
150 {
151  SelectRequest oldwant = si.wants;
152 
153  if (WvUrlStream::post_select(si))
154  return true;
155 
156  if (data && data->post_select(si))
157  return true;
158 
159  if (curl && curl->putstream && curl->putstream->post_select(si))
160  return true;
161 
162  si.wants = oldwant;
163 
164  return false;
165 }
166 
167 
168 void *WvFtpStream::real_execute(void*)
169 {
170  WvString line;
172 
173  if (alarm_was_ticking && ((last_request_time + 60) <= time(0)))
174  {
175  log(WvLog::Debug4, "urls count: %s\n", urls.count());
176  if (urls.isempty())
177  close(); // timed out, but not really an error
178 
179  return 0;
180  }
181 
182  if (!logged_in)
183  {
184  line = get_important_line();
185  if (!line)
186  {
187  seterr("Server not reachable: %s\n",strerror(errno));
188  return 0;
189  }
190 
191  if (strncmp(line, "220", 3))
192  {
193  log("Server rejected connection: %s\n", line);
194  seterr("server rejected connection");
195  return 0;
196  }
197  print("USER %s\r\n", !target.username ? WvString("anonymous") :
198  target.username);
199  line = get_important_line();
200  if (!line)
201  return 0;
202 
203  if (!strncmp(line, "230", 3))
204  {
205  log(WvLog::Info, "Server doesn't need password.\n");
206  logged_in = true; // No password needed;
207  }
208  else if (!strncmp(line, "33", 2))
209  {
210  print("PASS %s\r\n", !password ? DEFAULT_ANON_PW : password);
211 
212  line = get_important_line();
213  if (!line)
214  return 0;
215 
216  if (line[0] == '2')
217  {
218  log(WvLog::Info, "Authenticated.\n");
219  logged_in = true;
220  }
221  else
222  {
223  log("Strange response to PASS command: %s\n", line);
224  seterr("strange response to PASS command");
225  return 0;
226  }
227  }
228  else
229  {
230  log("Strange response to USER command: %s\n", line);
231  seterr("strange response to USER command");
232  return 0;
233  }
234 
235  print("TYPE I\r\n");
236  log(WvLog::Debug5, "<< TYPE I\n");
237  line = get_important_line();
238  if (!line)
239  return 0;
240 
241  if (strncmp(line, "200", 3))
242  {
243  log("Strange response to TYPE I command: %s\n", line);
244  seterr("strange response to TYPE I command");
245  return 0;
246  }
247  }
248 
249  if (!curl && !urls.isempty())
250  {
251  curl = urls.first();
252 
253  print("CWD %s\r\n", curl->url.getfile());
254  line = get_important_line();
255  if (!line)
256  return 0;
257 
258  if (!strncmp(line, "250", 3))
259  {
260  log(WvLog::Debug5, "This is a directory.\n");
261  curl->is_dir = true;
262  }
263 
264  print("PASV\r\n");
265  line = get_important_line();
266  if (!line)
267  return 0;
268  WvIPPortAddr *dataip = parse_pasv_response(line.edit());
269 
270  if (!dataip)
271  return 0;
272 
273  log(WvLog::Debug4, "Data port is %s.\n", *dataip);
274  // Open data connection.
275  data = new WvTCPConn(*dataip);
276  if (!data)
277  {
278  log("Can't open data connection.\n");
279  seterr("can't open data connection");
280  return 0;
281  }
282 
283  if (curl->is_dir)
284  {
285  if (!curl->putstream)
286  {
287  print("LIST %s\r\n", curl->url.getfile());
288  if (curl->outstream)
289  {
290  WvString url_no_pw("ftp://%s%s%s%s", curl->url.getuser(),
291  !!curl->url.getuser() ? "@" : "",
292  curl->url.gethost(),
293  curl->url.getfile());
294  curl->outstream->print("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML "
295  "4.01//EN\">\n"
296  "<html>\n<head>\n<title>%s</title>\n"
297  "<meta http-equiv=\"Content-Type\" "
298  "content=\"text/html; "
299  "charset=ISO-8859-1\">\n"
300  "<base href=\"%s\"/>\n</head>\n"
301  "<style type=\"text/css\">\n"
302  "img { border: 0; padding: 0 2px; vertical-align: "
303  "text-bottom; }\n"
304  "td { font-family: monospace; padding: 2px 3px; "
305  "text-align: right; vertical-align: bottom; }\n"
306  "td:first-child { text-align: left; padding: "
307  "2px 10px 2px 3px; }\n"
308  "table { border: 0; }\n"
309  "a.symlink { font-style: italic; }\n"
310  "</style>\n<body>\n"
311  "<h1>Index of %s</h1>\n"
312  "<hr/><table>\n", url_no_pw, curl->url, url_no_pw
313  );
314  }
315  }
316  else
317  {
318  log("Target is a directory.\n");
319  seterr("target is a directory");
320  doneurl();
321  return 0;
322  }
323  }
324  else if (!curl->putstream)
325  print("RETR %s\r\n", curl->url.getfile());
326  else
327  {
328  if (curl->create_dirs)
329  {
330  print("CWD %s\r\n", getdirname(curl->url.getfile()));
331  line = get_important_line();
332  if (!line)
333  return 0;
334  if (strncmp(line, "250", 3))
335  {
336  log("Path doesn't exist; creating directories...\n");
337  // create missing directories.
338  WvString current_dir("");
339  WvStringList dirs;
340  dirs.split(getdirname(curl->url.getfile()), "/");
341  WvStringList::Iter i(dirs);
342  for (i.rewind(); i.next(); )
343  {
344  current_dir.append(WvString("/%s", i()));
345  print("MKD %s\r\n", current_dir);
346  line = get_important_line();
347  if (!line)
348  return 0;
349  }
350  }
351  }
352  print("STOR %s\r\n", curl->url.getfile());
353  }
354 
355  log(WvLog::Debug5, "Waiting for response to %s\n", curl->putstream ? "STOR" :
356  curl->is_dir ? "LIST" : "RETR");
357  line = get_important_line();
358 
359  if (!line)
360  doneurl();
361  else if (strncmp(line, "150", 3))
362  {
363  log("Strange response to %s command: %s\n",
364  curl->putstream ? "STOR" : "RETR", line);
365  seterr(WvString("strange response to %s command",
366  curl->putstream ? "STOR" : "RETR"));
367  doneurl();
368  }
369 
370  }
371 
372  if (curl)
373  {
374  if (curl->is_dir)
375  {
376  line = data->blocking_getline(-1);
377  if (line && curl->outstream)
378  {
379  WvString output_line(parse_for_links(line.edit()));
380  if (!!output_line)
381  curl->outstream->write(output_line);
382  else
383  curl->outstream->write("Unknown format of LIST "
384  "response\n");
385  }
386  }
387  else
388  {
389  char buf[1024];
390 
391  if (curl->putstream)
392  {
393  while (curl->putstream->isreadable())
394  {
395  int len = curl->putstream->read(buf, sizeof(buf));
396  log(WvLog::Debug5, "Read %s bytes.\n%s\n", len, hexdump_buffer(buf, len));
397 
398  if (len)
399  {
400  int wrote = data->write(buf, len);
401  log(WvLog::Debug5,"Wrote %s bytes\n", wrote);
402  data->flush(0);
403  }
404  }
405  curl->putstream->close();
406  }
407  else
408  {
409  while (data->isreadable() && curl->outstream->isok())
410  {
411  int len = data->read(buf, sizeof(buf));
412  log(WvLog::Debug5, "Read %s bytes from remote.\n", len);
413 
414  if (len && curl->outstream)
415  {
416  int wrote = curl->outstream->write(buf, len);
417  log(WvLog::Debug5, "Wrote %s bytes to local.\n", wrote);
418  }
419  }
420  }
421  }
422 
423  if (!data->isok() || (curl->putstream && !curl->putstream->isok()))
424  {
425  log("OK, we should have finished writing!\n");
426  if (curl->putstream && data->isok())
427  data->close();
428  line = get_important_line();
429  if (!line)
430  {
431  doneurl();
432  return 0;
433  }
434 
435  if (strncmp(line, "226", 3))
436  log("Unexpected message: %s\n", line);
437 
438  if (curl->is_dir)
439  {
440  if (curl->outstream)
441  curl->outstream->write("</table><hr/></body>\n"
442  "</html>\n");
443  write("CWD /\r\n");
444  log(WvLog::Debug5, "Waiting for response to CWD /\n");
445  line = get_important_line();
446  if (!line)
447  return 0;
448 
449  if (strncmp(line, "250", 3))
450  log("Strange resonse to \"CWD /\": %s\n", line);
451  // Don't bother failing here.
452  }
453  doneurl();
454  }
455  else
456  {
457  log("Why are we here??\n");
458  }
459  }
460 
461  return 0;
462 }
463 
464 
466 {
467  real_execute(0);
468 }
469 
470 
471 WvString WvFtpStream::parse_for_links(char *line)
472 {
473  WvString output_line("");
474  trim_string(line);
475 
476  if (curl->is_dir && curl->outstream)
477  {
478  struct ftpparse fp;
479  int res = ftpparse(&fp, line, strlen(line));
480  if (res)
481  {
482  char *linkname = (char *)alloca(fp.namelen+1);
483  int i;
484  for (i = 0; i < fp.namelen; i++)
485  {
486  if (fp.name[i] >= 32)
487  linkname[i] = fp.name[i];
488  else
489  {
490  linkname[i] = '?';
491  }
492  }
493  linkname[i] = 0;
494 
495  WvString linkurl(curl->url);
496  if (linkurl.cstr()[linkurl.len()-1] != '/')
497  linkurl.append("/");
498  linkurl.append(linkname);
499  WvUrlLink *link = new WvUrlLink(linkname, linkurl);
500  curl->outstream->links.append(link, true);
501 
502  output_line.append("<tr>\n");
503 
504  output_line.append(WvString(" <td>%s%s</td>\n", linkname,
505  fp.flagtrycwd ? "/" : ""));
506 
507  if (fp.flagtryretr)
508  {
509  if (!fp.sizetype)
510  output_line.append(" <td>? bytes</td>\n");
511  else
512  output_line.append(WvString(" <td>%s bytes</td>\n",
513  fp.size));
514  if (fp.mtimetype > 0)
515  output_line.append(WvString(" <td>%s</td>\n", (fp.mtime)));
516  else
517  output_line.append(" <td>?</td>\n");
518  }
519  else
520  output_line.append(" <td></td>\n");
521 
522  output_line.append("</tr>\n");
523  }
524  }
525  return output_line;
526 }
527 
528 
529 WvIPPortAddr *WvFtpStream::parse_pasv_response(char *line)
530 {
531  if (strncmp(line, "227 ", 4))
532  {
533  log("Strange response to PASV command: %s\n", line);
534  seterr("strange response to PASV command");
535  return NULL;
536  }
537 
538  char *p = &line[3];
539  while (!isdigit(*p))
540  {
541  if (*p == '\0' || *p == '\r' || *p == '\n')
542  {
543  log("Couldn't parse PASV response: %s\n", line);
544  seterr("couldn't parse response to PASV command");
545  return NULL;
546  }
547  p++;
548  }
549  char *ipstart = p;
550 
551  for (int i = 0; i < 4; i++)
552  {
553  p = strchr(p, ',');
554  if (!p)
555  {
556  log("Couldn't parse PASV IP: %s\n", line);
557  seterr("couldn't parse PASV IP");
558  return NULL;
559  }
560  *p = '.';
561  }
562  *p = '\0';
563  WvString pasvip(ipstart);
564  p++;
565  int pasvport;
566  pasvport = atoi(p)*256;
567  p = strchr(p, ',');
568  if (!p)
569  {
570  log("Couldn't parse PASV IP port: %s\n", line);
571  seterr("couldn't parse PASV IP port");
572  return NULL;
573  }
574  pasvport += atoi(++p);
575 
576  WvIPPortAddr *res = new WvIPPortAddr(pasvip.cstr(), pasvport);
577 
578  return res;
579 }
virtual bool isok() const
return true if the stream is actually usable right now
Definition: wvbufstream.cc:48
static WvString strerror(int errnum)
A replacement for the operating system ::strerror() function that can map more kinds of error strings...
Definition: wverror.cc:91
A WvFastString acts exactly like a WvString, but can take (const char *) strings without needing to a...
Definition: wvstring.h:94
virtual void close()
Closes the file descriptors.
Definition: wvfdstream.cc:117
virtual void close()
Close this stream.
Definition: wvftpstream.cc:93
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
Definition: wvftpstream.cc:465
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling ::select().
Definition: wvftpstream.cc:133
virtual bool post_select(SelectInfo &si)
post_select() is called after ::select(), and returns true if this object is now ready.
Definition: wvftpstream.cc:149
An IP+Port address also includes a port number, with the resulting form www.xxx.yyy....
Definition: wvaddr.h:394
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling ::select().
virtual void close()
Close this stream.
virtual bool isok() const
return true if the stream is actually usable right now
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
virtual bool post_select(SelectInfo &si)
post_select() is called after ::select(), and returns true if this object is now ready.
virtual int geterr() const
If isok() is false, return the system error number corresponding to the error, -1 for a special error...
virtual bool post_select(SelectInfo &si)
post_select() is called after ::select(), and returns true if this object is now ready.
Definition: wvstream.cc:875
virtual bool flush(time_t msec_timeout)
flush the output buffer, if we can do it without delaying more than msec_timeout milliseconds at a ti...
Definition: wvstream.cc:707
char * blocking_getline(time_t wait_msec, int separator='\n', int readahead=1024)
This is a version of getline() that allows you to block for more data to arrive.
Definition: wvstream.cc:602
virtual bool isreadable()
Returns true if the stream is readable.
Definition: wvstream.cc:590
virtual bool isok() const
return true if the stream is actually usable right now
Definition: wvstream.cc:445
void alarm(time_t msec_timeout)
set an alarm, ie.
Definition: wvstream.cc:1049
virtual size_t write(const void *buf, size_t count)
Write data to the stream.
Definition: wvstream.cc:532
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling ::select().
Definition: wvstream.cc:844
virtual size_t read(void *buf, size_t count)
read a data block on the stream.
Definition: wvstream.cc:490
virtual void close()
Close the stream if it is open; isok() becomes false from now on.
Definition: wvstream.cc:341
virtual void seterr(int _errnum)
Override seterr() from WvError so that it auto-closes the stream.
Definition: wvstream.cc:451
bool alarm_was_ticking
This will be true during callback execution if the callback was triggered by the alarm going off.
Definition: wvstream.h:54
This is a WvList of WvStrings, and is a really handy way to parse strings.
Definition: wvstringlist.h:28
void split(WvStringParm s, const char *splitchars=" \t\r\n", int limit=0)
split s and form a list ignoring splitchars (except at beginning and end) ie.
Definition: wvstringlist.cc:19
WvString is an implementation of a simple and efficient printable-string class.
Definition: wvstring.h:330
char * edit()
make the string editable, and return a non-const (char*)
Definition: wvstring.h:397
WvTCPConn tries to make all outgoing connections asynchronously (in the background).
Definition: wvtcp.h:40
virtual bool isok() const
Is this connection OK? Note: isok() will always be true if !resolved, even though fd==-1.
Definition: wvtcp.cc:375
virtual bool post_select(SelectInfo &si)
override post_select() to set the 'connected' variable as soon as we are connected.
Definition: wvtcp.cc:320
virtual void pre_select(SelectInfo &si)
override pre_select() to cause select() results when resolving names.
Definition: wvtcp.cc:291
the data structure used by pre_select()/post_select() and internally by select().
Definition: iwvstream.h:50
A SelectRequest is a convenient way to remember what we want to do to a particular stream: read from ...
Definition: iwvstream.h:34
WvString hexdump_buffer(const void *buf, size_t len, bool charRep=true)
Produce a hexadecimal dump of the data buffer in 'buf' of length 'len'.
Definition: strutils.cc:245
char * trim_string(char *string)
Trims whitespace from the beginning and end of the character string, including carriage return / line...
Definition: strutils.cc:59