ChangeLog

@@ -8,6 +8,8 @@
 	* main.cc: The value of --http-auth-scheme option is chagned from
 	'BASIC' to 'basic'
 	* main.cc: Added --timeout command-line option.
+	* main.cc: Added --min-segment-size command-line option.
+	* main.cc: Added --max-retries command-line option.
 	* prefs.h: option string constants are now defined in prefs.h
 	
 2006-02-19  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>

TODO

@@ -4,3 +4,4 @@
 * Add SSL client cert support
 * Better HTTP status handling
 * Download files listed in a specifed file.
+* check MD5 checksum

src/AbstractCommand.cc

@@ -102,7 +102,8 @@ bool AbstractCommand::execute() {
     delete(err);
     //req->resetUrl();
     req->addTryCount();
-    if(req->getTryCount() >= e->option->getAsInt(PREF_MAX_TRY)) {
+    if(e->option->getAsInt(PREF_MAX_TRIES) != 0 &&
+       req->getTryCount() >= e->option->getAsInt(PREF_MAX_TRIES)) {
       e->logger->error(MSG_MAX_TRY, cuid, req->getTryCount());
       return true;
     } else {

src/DownloadEngine.h

@@ -47,8 +47,8 @@ public:
   queue<Command*> commands;
   SegmentMan* segmentMan;
   DiskWriter* diskWriter;
-  Logger* logger;
-  Option* option;
+  const Logger* logger;
+  const Option* option;
 
   DownloadEngine();
   ~DownloadEngine();

src/FtpConnection.cc

@@ -110,8 +110,12 @@ void FtpConnection::sendRetr() const {
 
 int FtpConnection::getStatus(string response) const {
   int status;
-  // TODO we must handle when the response is not like "%d %*s"
-  // In this case, we return 0
+  // When the response is not like "%d %*s",
+  // we return 0.
+  if(response.find_first_not_of("0123456789") != 3
+     || response.find(" ") != 3) {
+    return 0;
+  }
   if(sscanf(response.c_str(), "%d %*s", &status) == 1) {
     return status;
   } else {

src/HttpResponseCommand.cc

@@ -65,10 +65,13 @@ bool HttpResponseCommand::executeInternal(Segment seg) {
       return handleDefaultEncoding(headers);
     }
   } else {
-    // TODO we must check headers["size"] == e->segmentMan->totalSize here
     if(req->getFile() != e->segmentMan->filename) {
       throw new DlAbortEx(EX_FILENAME_MISMATCH, req->getFile().c_str(), e->segmentMan->filename.c_str());
     }
+    long long int size = headers.getFirstAsLLInt("Content-Length");
+    if(e->segmentMan->totalSize != size) {
+      throw new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size);
+    }
     createHttpDownloadCommand();
     return true;
   }
@@ -99,9 +102,7 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) {
   bool segFileExists = e->segmentMan->segmentFileExists();
   e->segmentMan->downloadStarted = true;
   if(segFileExists) {
-
     e->diskWriter->openExistingFile(e->segmentMan->getFilePath());
-    // we must check headers["size"] == e->segmentMan->totalSize here
     if(e->segmentMan->totalSize != size) {
       return new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size);
     }

src/Option.cc

@@ -50,3 +50,12 @@ int Option::getAsInt(const string& name) const {
     return (int)strtol((*itr).second.c_str(), NULL, 10);
   }
 }
+
+long long int Option::getAsLLInt(const string& name) const {
+  map<string, string>::const_iterator itr = table.find(name);
+  if(itr == table.end()) {
+    return 0;
+  } else {
+    return (int)strtoll((*itr).second.c_str(), NULL, 10);
+  }
+}

src/Option.h

@@ -38,6 +38,7 @@ public:
   bool defined(const string& name) const;
   string get(const string& name) const;
   int getAsInt(const string& name) const;
+  long long int getAsLLInt(const string& name) const;
 };
 
 #endif // _D_OPTION_H_

src/Request.cc

@@ -79,14 +79,12 @@ bool Request::parseUrl(string url) {
   Util::split(hostAndPort, url.substr(hp, hep-hp), ':');
   host = hostAndPort.first;
   if(hostAndPort.second != "") {
-    // TODO rewrite this using strtoul function. If strtoul fails,
-    // return false.
     port = (int)strtol(hostAndPort.second.c_str(), NULL, 10);
     if(!(0 < port && port <= 65535)) {
       return false;
     }
   } else {
-    // If port is not specified, then we leave it 0.
+    // If port is not specified, then we set it to default port of its protocol..
     port = defPort;
   }
   string::size_type direp = url.find_last_of("/");

src/SegmentMan.cc

@@ -28,6 +28,7 @@
 #include "Util.h"
 #include "File.h"
 #include "message.h"
+#include "prefs.h"
 
 SegmentMan::SegmentMan():totalSize(0),isSplittable(true),downloadStarted(false),dir(".") {}
 
@@ -85,7 +86,7 @@ bool SegmentMan::getSegment(Segment& seg, int cuid) {
     if(s.finish) {
       continue;
     }
-    if(s.ep-(s.sp+s.ds) > 524288) {
+    if(s.ep-(s.sp+s.ds) > option->getAsLLInt(PREF_MIN_SEGMENT_SIZE)) {
       long long int nep = (s.ep-(s.sp+s.ds))/2+(s.sp+s.ds);
       //nseg = { cuid, nep+1, s.ep, 0, false };
       seg.cuid = cuid;

src/SegmentMan.h

@@ -26,6 +26,7 @@
 #include "common.h"
 #include "Logger.h"
 #include "Segment.h"
+#include "Option.h"
 
 using namespace std;
 
@@ -81,6 +82,7 @@ public:
   string ufilename;
 
   const Logger* logger;
+  const Option* option;
 
   SegmentMan();
   ~SegmentMan();

src/main.cc

@@ -101,40 +101,45 @@ void showUsage() {
   cout << "Usage: " << PACKAGE_NAME << " [options] URL ..." << endl;
   cout << endl;
   cout << "Options:" << endl;
-  cout << " -d, --dir=DIR              The directory to store downloaded file." << endl;
-  cout << " -o, --out=FILE             The file name for downloaded file." << endl;
-  cout << " -l, --log=LOG              The file path to store log. If '-' is specified," << endl;
-  cout << "                            log is written to stdout." << endl;
-  cout << " -D, --daemon               Run as daemon." << endl;
-  cout << " -s, --split=N              Download a file using s connections. s must be" << endl;
-  cout << "                            between 1 and 5. If this option is specified the" << endl;
-  cout << "                            first URL is used, and the other URLs are ignored." << endl;
-  cout << " --retry-wait=SEC           Set amount of time in second between requests" << endl;
-  cout << "                            for errors. Specify a value between 0 and 60." << endl;
-  cout << " -t, --timeout=SEC          Set timeout in second." << endl;
-  cout << " --http-proxy=HOST:PORT     Use HTTP proxy server. This affects to all" << endl;
-  cout << "                            URLs." << endl;
-  cout << " --http-user=USER           Set HTTP user. This affects to all URLs." << endl;
-  cout << " --http-passwd=PASSWD       Set HTTP password. This affects to all URLs." << endl;
-  cout << " --http-proxy-user=USER     Set HTTP proxy user. This affects to all URLs" << endl;
-  cout << " --http-proxy-passwd=PASSWD Set HTTP proxy password. This affects to all URLs." << endl;
-  cout << " --http-auth-scheme=SCHEME  Set HTTP authentication scheme. Currently, basic" << endl;
-  cout << "                            is the only supported scheme. You MUST specify" << endl;
-  cout << "                            this option in order to use HTTP authentication" << endl;
-  cout << "                            as well as --http-proxy option." << endl;
-  cout << " --referer=REFERER          Set Referer. This affects to all URLs." << endl;
-  cout << " --ftp-user=USER            Set FTP user. This affects to all URLs." << endl;
-  cout << "                            Default: anonymous" << endl;
-  cout << " --ftp-passwd=PASSWD        Set FTP password. This affects to all URLs." << endl;
-  cout << "                            Default: ARIA2USER@" << endl;
-  cout << " --ftp-type=TYPE            Set FTP transfer type. TYPE is either 'binary'" << endl;
-  cout << "                            or 'ascii'." << endl;
-  cout << "                            Default: binary" << endl;
-  cout << " -p, --ftp-pasv             Use passive mode in FTP." << endl;
-  cout << " --ftp-via-http-proxy=WAY   Use HTTP proxy in FTP. WAY is either 'get' or" << endl;
-  cout << "                            'tunnel'." << endl;
-  cout << " -v, --version              Print the version number and exit." << endl;
-  cout << " -h, --help                 Print this message and exit." << endl;
+  cout << " -d, --dir=DIR                The directory to store downloaded file." << endl;
+  cout << " -o, --out=FILE               The file name for downloaded file." << endl;
+  cout << " -l, --log=LOG                The file path to store log. If '-' is specified," << endl;
+  cout << "                              log is written to stdout." << endl;
+  cout << " -D, --daemon                 Run as daemon." << endl;
+  cout << " -s, --split=N                Download a file using s connections. s must be" << endl;
+  cout << "                              between 1 and 5. If this option is specified the" << endl;
+  cout << "                              first URL is used, and the other URLs are ignored." << endl;
+  cout << " --retry-wait=SEC             Set amount of time in second between requests" << endl;
+  cout << "                              for errors. Specify a value between 0 and 60." << endl;
+  cout << "                              Default: 5" << endl;
+  cout << " -t, --timeout=SEC            Set timeout in second. Default: 60" << endl;
+  cout << " -m, --max-tries=N            Set number of tries. 0 means unlimited." << endl;
+  cout << "                              Default: 5" << endl;
+  cout << " --min-segment-size=SIZE[K|M] Set minimum segment size. You can append" << endl;
+  cout << "                              K or M(1K = 1024, 1M = 1024K)." << endl;
+  cout << " --http-proxy=HOST:PORT       Use HTTP proxy server. This affects to all" << endl;
+  cout << "                              URLs." << endl;
+  cout << " --http-user=USER             Set HTTP user. This affects to all URLs." << endl;
+  cout << " --http-passwd=PASSWD         Set HTTP password. This affects to all URLs." << endl;
+  cout << " --http-proxy-user=USER       Set HTTP proxy user. This affects to all URLs" << endl;
+  cout << " --http-proxy-passwd=PASSWD   Set HTTP proxy password. This affects to all URLs." << endl;
+  cout << " --http-auth-scheme=SCHEME    Set HTTP authentication scheme. Currently, basic" << endl;
+  cout << "                              is the only supported scheme. You MUST specify" << endl;
+  cout << "                              this option in order to use HTTP authentication" << endl;
+  cout << "                              as well as --http-proxy option." << endl;
+  cout << " --referer=REFERER            Set Referer. This affects to all URLs." << endl;
+  cout << " --ftp-user=USER              Set FTP user. This affects to all URLs." << endl;
+  cout << "                              Default: anonymous" << endl;
+  cout << " --ftp-passwd=PASSWD          Set FTP password. This affects to all URLs." << endl;
+  cout << "                              Default: ARIA2USER@" << endl;
+  cout << " --ftp-type=TYPE              Set FTP transfer type. TYPE is either 'binary'" << endl;
+  cout << "                              or 'ascii'." << endl;
+  cout << "                              Default: binary" << endl;
+  cout << " -p, --ftp-pasv               Use passive mode in FTP." << endl;
+  cout << " --ftp-via-http-proxy=WAY     Use HTTP proxy in FTP. WAY is either 'get' or" << endl;
+  cout << "                              'tunnel'." << endl;
+  cout << " -v, --version                Print the version number and exit." << endl;
+  cout << " -h, --help                   Print this message and exit." << endl;
   cout << endl;
   cout << "URL:" << endl;
   cout << " You can specify multiple URLs. All URLs must point to the same file" << endl;
@@ -166,6 +171,8 @@ int main(int argc, char* argv[]) {
   Option* op = new Option();
   op->put(PREF_RETRY_WAIT, "5");
   op->put(PREF_TIMEOUT, "60");
+  op->put(PREF_MIN_SEGMENT_SIZE, "1048576");// 1M
+  op->put(PREF_MAX_TRIES, "5");
   op->put(PREF_FTP_USER, "anonymous");
   op->put(PREF_FTP_PASSWD, "ARIA2USER@");
   op->put(PREF_FTP_TYPE, V_BINARY);
@@ -182,6 +189,7 @@ int main(int argc, char* argv[]) {
       { "log", required_argument, NULL, 'l' },
       { "split", required_argument, NULL, 's' },
       { "timeout", required_argument, NULL, 't' },
+      { "max-retries", required_argument, NULL, 'm' },
       { "http-proxy", required_argument, &lopt, 1 },
       { "http-user", required_argument, &lopt, 2 },
       { "http-passwd", required_argument, &lopt, 3 },
@@ -195,11 +203,12 @@ int main(int argc, char* argv[]) {
       { "ftp-type", required_argument, &lopt, 11 },
       { "ftp-pasv", no_argument, NULL, 'p' },
       { "ftp-via-http-proxy", required_argument, &lopt, 12 },
+      { "min-segment-size", required_argument, &lopt, 13 },
       { "version", no_argument, NULL, 'v' },
       { "help", no_argument, NULL, 'h' },
       { 0, 0, 0, 0 }
     };
-    c = getopt_long(argc, argv, "Dd:o:l:s:pt:vh", longOpts, &optIndex);
+    c = getopt_long(argc, argv, "Dd:o:l:s:pt:m:vh", longOpts, &optIndex);
     if(c == -1) {
       break;
     }
@@ -278,6 +287,26 @@ int main(int argc, char* argv[]) {
 	  exit(1);
 	}
 	break;
+      case 13: {
+	string::size_type p = string(optarg).find_first_of("KM");
+	int mult = 1;
+	if(p != string::npos) {
+	  if(optarg[p] == 'K') {
+	    mult = 1024;
+	  } else if(optarg[p] == 'M') {
+	    mult = 1024*1024;
+	  }
+	  optarg[p] = '\0';
+	}
+	long long int size = strtoll(optarg, NULL, 10)*mult;
+	if(size <= 0) {
+	  cerr << "min-segment-size invalid" << endl;
+	  showUsage();
+	  exit(1);
+	}
+	op->put(PREF_MIN_SEGMENT_SIZE, Util::llitos(size));
+	break;
+      }
       }
       break;
     }
@@ -316,6 +345,16 @@ int main(int argc, char* argv[]) {
       }
       break;
     }
+    case 'm': {
+      int retries = (int)strtol(optarg, NULL, 10);
+      if(retries < 0) {
+	cerr << "max-retires invalid" << endl;
+	showUsage();
+	exit(1);
+      }
+      op->put(PREF_MAX_TRIES, Util::itos(retries));
+      break;
+    }
     case 'p':
       op->put(PREF_FTP_PASV_ENABLED, V_TRUE);
       break;
@@ -363,6 +402,7 @@ int main(int argc, char* argv[]) {
   e->segmentMan->dir = dir;
   e->segmentMan->ufilename = ufilename;
   e->segmentMan->logger = logger;
+  e->segmentMan->option = op;
   vector<Request*> requests;
   if(split > 0) {
     for(int i = 1; i <= split; i++) {

src/prefs.h

@@ -36,7 +36,9 @@
 // values: 1*digit
 #define PREF_TIMEOUT "timeout"
 // values: 1*digit
-#define PREF_MAX_TRY "max_try"
+#define PREF_MAX_TRIES "max_try"
+// values: 1*digit
+#define PREF_MIN_SEGMENT_SIZE "min_segment_size"
 
 /**
  * FTP related preferences