2008-09-10 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

	Added --max-file-not-found=NUM option. If aria2 receives `file 
not
	found' error from HTTP/FTP server NUM times without getting 
single
	bytes of data, then force the download to fail.
	Basically aria2 tris all URIs available. So if long list of URIs 
are
	specified in Metalink, then aria2 will take a long time to fail
	eventually if all URIs are outdated. This option eases this 
situation
	to limit the number of servers to try.
	* src/FtpNegotiationCommand.cc
	* src/HelpItemFactory.cc
	* src/HttpResponseCommand.cc
	* src/OptionHandlerFactory.cc
	* src/PeerStat.h
	* src/RequestGroup.cc
	* src/RequestGroup.h
	* src/SegmentMan.cc
	* src/SegmentMan.h
	* src/option_processing.cc
	* src/prefs.cc
	* src/prefs.h
	* src/usage_text.h

ChangeLog

@@ -1,3 +1,26 @@
+2008-09-10  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Added --max-file-not-found=NUM option. If aria2 receives `file not
+	found' error from HTTP/FTP server NUM times without getting single
+	bytes of data, then force the download to fail.
+	Basically aria2 tris all URIs available. So if long list of URIs are
+	specified in Metalink, then aria2 will take a long time to fail
+	eventually if all URIs are outdated. This option eases this situation
+	to limit the number of servers to try.
+	* src/FtpNegotiationCommand.cc
+	* src/HelpItemFactory.cc
+	* src/HttpResponseCommand.cc
+	* src/OptionHandlerFactory.cc
+	* src/PeerStat.h
+	* src/RequestGroup.cc
+	* src/RequestGroup.h
+	* src/SegmentMan.cc
+	* src/SegmentMan.h
+	* src/option_processing.cc
+	* src/prefs.cc
+	* src/prefs.h
+	* src/usage_text.h
+
 2008-09-10  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	Fixed the bug that bulkReceiveResponse() reads all received data as a

src/FtpNegotiationCommand.cc

@@ -204,6 +204,7 @@ bool FtpNegotiationCommand::recvCwd() {
   }
   if(status != 250) {
     poolConnection();
+    _requestGroup->increaseAndValidateFileNotFoundCount();
     throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str());
   }
   if(e->option->getAsBool(PREF_REMOTE_TIME)) {
@@ -444,6 +445,7 @@ bool FtpNegotiationCommand::recvRetr() {
     return false;
   }
   if(status != 150 && status != 125) {
+    _requestGroup->increaseAndValidateFileNotFoundCount();
     throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str());
   }
   if(e->option->getAsBool(PREF_FTP_PASV)) {

src/HelpItemFactory.cc

@@ -526,6 +526,14 @@ TagContainerHandle HelpItemFactory::createHelpItems(const Option* op)
     item->addTag(TAG_ADVANCED);
     tc->addItem(item);
   }
+  {
+    HelpItemHandle item(new HelpItem(PREF_MAX_FILE_NOT_FOUND,
+				     TEXT_MAX_FILE_NOT_FOUND,
+				     op->get(PREF_MAX_FILE_NOT_FOUND)));
+    item->addTag(TAG_HTTP);
+    item->addTag(TAG_FTP);
+    tc->addItem(item);	     
+  }
   {
     HelpItemHandle item(new HelpItem("help", TEXT_HELP, TAG_BASIC));
     item->setAvailableValues

src/HttpResponseCommand.cc

@@ -91,9 +91,11 @@ bool HttpResponseCommand::executeInternal()
   httpResponse->retrieveCookie();
 
   if(httpResponse->getResponseStatus() >= HttpHeader::S300) {
+    if(httpResponse->getResponseStatus() == HttpHeader::S404) {
+      _requestGroup->increaseAndValidateFileNotFoundCount();
+    }
     return skipResponseBody(httpResponse);
   }
-
   if(!_requestGroup->isSingleHostMultiConnectionEnabled()) {
     // Query by hostname. Searching by CUID may returns NULL.
     // In case when resuming download, ServerHost is registered with CUID A.

src/OptionHandlerFactory.cc

@@ -156,6 +156,7 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
   handlers.push_back(SH(new DefaultOptionHandler(PREF_SERVER_STAT_OF)));
   handlers.push_back(SH(new BooleanOptionHandler(PREF_REMOTE_TIME)));
   handlers.push_back(SH(new NumberOptionHandler(PREF_CONNECT_TIMEOUT, 1, 600)));
+  handlers.push_back(SH(new NumberOptionHandler(PREF_MAX_FILE_NOT_FOUND, 0)));
 
   return handlers;
 }

src/PeerStat.h

@@ -59,6 +59,8 @@ private:
   PeerStat::STATUS status;
   unsigned int _avgDownloadSpeed;
   unsigned int _avgUploadSpeed;
+  uint64_t _sessionDownloadLength;
+  uint64_t _sessionUploadLength;
 public:
 
   PeerStat(int32_t cuid, const std::string& hostname,
@@ -68,11 +70,15 @@ public:
     _protocol(protocol),
     status(PeerStat::IDLE),
     _avgDownloadSpeed(0),
-    _avgUploadSpeed(0) {}
+    _avgUploadSpeed(0),
+    _sessionDownloadLength(0),
+    _sessionUploadLength(0) {}
 
   PeerStat(int32_t cuid = 0):cuid(cuid), status(PeerStat::IDLE),
 			     _avgDownloadSpeed(0),
-			     _avgUploadSpeed(0) {}
+			     _avgUploadSpeed(0),
+			     _sessionDownloadLength(0),
+			     _sessionUploadLength(0) {}
 
   ~PeerStat() {}
 
@@ -107,10 +113,12 @@ public:
 
   void updateDownloadLength(size_t bytes) {
     downloadSpeed.update(bytes);
+    _sessionDownloadLength += bytes;
   }
 
   void updateUploadLength(size_t bytes) {
     uploadSpeed.update(bytes);
+    _sessionUploadLength += bytes;
   }
 
   unsigned int getMaxDownloadSpeed() const {
@@ -172,6 +180,16 @@ public:
   {
     return _protocol;
   }
+
+  uint64_t getSessionDownloadLength() const
+  {
+    return _sessionDownloadLength;
+  }
+
+  uint64_t getSessionUploadLength() const
+  {
+    return _sessionUploadLength;
+  }
 };
 
 typedef SharedHandle<PeerStat> PeerStatHandle;

src/RequestGroup.cc

@@ -124,6 +124,7 @@ RequestGroup::RequestGroup(const Option* option,
   _singleHostMultiConnectionEnabled(true),
   _uriSelector(new InOrderURISelector()),
   _lastModifiedTime(-1),
+  _fileNotFoundCount(0),
   _option(option),
   _logger(LogFactory::getInstance())
 {
@@ -1037,6 +1038,7 @@ void RequestGroup::applyLastModifiedTimeToLocalFiles()
     _logger->info("Last-Modified attrs of %zu files were updated.", n);
   }
 }
+
 void RequestGroup::updateLastModifiedTime(const Time& time)
 {
   if(time.good() && _lastModifiedTime < time) {
@@ -1044,4 +1046,15 @@ void RequestGroup::updateLastModifiedTime(const Time& time)
   }
 }
 
+void RequestGroup::increaseAndValidateFileNotFoundCount()
+{
+  ++_fileNotFoundCount;
+  const unsigned int maxCount = _option->getAsInt(PREF_MAX_FILE_NOT_FOUND);
+  if(maxCount > 0 && _fileNotFoundCount >= maxCount &&
+     _segmentMan->calculateSessionDownloadLength() == 0) {
+    throw DownloadFailureException
+      (StringFormat("Reached max-file-not-found count=%u", maxCount).str());
+  }
+}
+
 } // namespace aria2

src/RequestGroup.h

@@ -118,6 +118,8 @@ private:
 
   Time _lastModifiedTime;
 
+  unsigned int _fileNotFoundCount;
+
   const Option* _option;
 
   Logger* _logger;
@@ -368,6 +370,8 @@ public:
   void applyLastModifiedTimeToLocalFiles();
 
   void updateLastModifiedTime(const Time& time);
+
+  void increaseAndValidateFileNotFoundCount();
 };
 
 typedef SharedHandle<RequestGroup> RequestGroupHandle;

src/SegmentMan.cc

@@ -46,6 +46,7 @@
 #include "DownloadContext.h"
 #include "Piece.h"
 #include <algorithm>
+#include <numeric>
 #include <cassert>
 
 namespace aria2 {
@@ -306,6 +307,21 @@ unsigned int SegmentMan::calculateDownloadSpeed() const {
   return speed;
 }
 
+class PeerStatDownloadLengthOperator {
+public:
+  uint64_t operator()(uint64_t total, const SharedHandle<PeerStat>& ps)
+  {
+    return ps->getSessionDownloadLength()+total;
+  }
+};
+
+uint64_t SegmentMan::calculateSessionDownloadLength() const
+{
+  return std::accumulate(peerStats.begin(), peerStats.end(), 0LL,
+			 PeerStatDownloadLengthOperator());
+ 
+}
+
 size_t SegmentMan::countFreePieceFrom(size_t index) const
 {
   size_t numPieces = _downloadContext->getNumPieces();

src/SegmentMan.h

@@ -179,6 +179,11 @@ public:
    */
   unsigned int calculateDownloadSpeed() const;
 
+  /**
+   * Returns the downloaded bytes in this session.
+   */
+  uint64_t calculateSessionDownloadLength() const;
+
   size_t countFreePieceFrom(size_t index) const;
 };
 

src/option_processing.cc

@@ -160,6 +160,7 @@ Option* createDefaultOption()
   op->put(PREF_URI_SELECTOR, V_INORDER);
   op->put(PREF_SERVER_STAT_TIMEOUT, "86400");// 1day
   op->put(PREF_REMOTE_TIME, V_FALSE);
+  op->put(PREF_MAX_FILE_NOT_FOUND, "0");
   return op;
 }
 
@@ -243,6 +244,7 @@ Option* option_processing(int argc, char* const argv[])
       { PREF_SERVER_STAT_TIMEOUT.c_str(), required_argument, &lopt, 223 },
       { PREF_REMOTE_TIME.c_str(), optional_argument, 0, 'R' },
       { PREF_CONNECT_TIMEOUT.c_str(), required_argument, &lopt, 224 },
+      { PREF_MAX_FILE_NOT_FOUND.c_str(), required_argument, &lopt, 225 },
 #if defined ENABLE_BITTORRENT || defined ENABLE_METALINK
       { PREF_SHOW_FILES.c_str(), no_argument, NULL, 'S' },
       { PREF_SELECT_FILE.c_str(), required_argument, &lopt, 21 },
@@ -488,6 +490,9 @@ Option* option_processing(int argc, char* const argv[])
       case 224:
 	cmdstream << PREF_CONNECT_TIMEOUT << "=" << optarg << "\n";
 	break;
+      case 225:
+	cmdstream << PREF_MAX_FILE_NOT_FOUND << "=" << optarg << "\n";
+	break;
       }
       break;
     }

src/prefs.cc

@@ -149,6 +149,8 @@ const std::string PREF_SERVER_STAT_IF("server-stat-if");
 const std::string PREF_SERVER_STAT_OF("server-stat-of");
 // value: true | false
 const std::string PREF_REMOTE_TIME("remote-time");
+// value: 1*digit
+const std::string PREF_MAX_FILE_NOT_FOUND("max-file-not-found");
 
 /**
  * FTP related preferences

src/prefs.h

@@ -153,6 +153,8 @@ extern const std::string PREF_SERVER_STAT_IF;
 extern const std::string PREF_SERVER_STAT_OF;
 // value: true | false
 extern const std::string PREF_REMOTE_TIME;
+// value: 1*digit
+extern const std::string PREF_MAX_FILE_NOT_FOUND;
 
 /**
  * FTP related preferences

src/usage_text.h

@@ -365,3 +365,11 @@ _(" --connect-timeout=SEC        Set the connect timeout in seconds to establish
   "                              connection to HTTP/FTP/proxy server. After the\n"\
   "                              connection is established, this option makes no\n"\
   "                              effect and --timeout option is used instead.")
+#define TEXT_MAX_FILE_NOT_FOUND \
+_(" --max-file-not-found=NUM     If aria2 recieves `file not found' status from the\n"\
+  "                              remote HTTP/FTP servers NUM times without getting\n"\
+  "                              a single byte, then force the download to fail.\n"\
+  "                              Specify 0 to disable this option.\n"\
+  "                              This options is only effective only when using\n"\
+  "                              HTTP/FTP servers.")
+