2010-09-10 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

	Added --max-download-result=NUM option. This option sets maximum
	number of download result kept in memory. The download results are
	completed/error/ removed downloads. The download results are
	stored in FIFO queue and it can store at most NUM download
	results. When queue is full and new download result is created,
	oldest download result is removed from the front of the queue and
	new one is pushed to the back. Setting big number in this option
	may result high memory consumption after thousands of
	downloads. Specifying 0 means no download result is kept.  Default
	value is 1000.
	* src/OptionHandlerFactory.cc
	* src/RequestGroupMan.cc
	* src/RequestGroupMan.h
	* src/prefs.cc
	* src/prefs.h
	* src/usage_text.h

ChangeLog

@@ -1,3 +1,22 @@
+2010-09-10  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
+
+	Added --max-download-result=NUM option. This option sets maximum
+	number of download result kept in memory. The download results are
+	completed/error/ removed downloads. The download results are
+	stored in FIFO queue and it can store at most NUM download
+	results. When queue is full and new download result is created,
+	oldest download result is removed from the front of the queue and
+	new one is pushed to the back. Setting big number in this option
+	may result high memory consumption after thousands of
+	downloads. Specifying 0 means no download result is kept.  Default
+	value is 1000.
+	* src/OptionHandlerFactory.cc
+	* src/RequestGroupMan.cc
+	* src/RequestGroupMan.h
+	* src/prefs.cc
+	* src/prefs.h
+	* src/usage_text.h
+
 2010-09-10  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
 
 	Check hash(hash for entire file, not piece hash) if

src/OptionHandlerFactory.cc

@@ -365,6 +365,15 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
     op->addTag(TAG_HTTP);
     handlers.push_back(op);
   }
+  {
+    SharedHandle<OptionHandler> op(new NumberOptionHandler
+                                   (PREF_MAX_DOWNLOAD_RESULT,
+                                    TEXT_MAX_DOWNLOAD_RESULT,
+                                    "1000",
+                                    0));
+    op->addTag(TAG_ADVANCED);
+    handlers.push_back(op);
+  }
   {
     SharedHandle<OptionHandler> op(new UnitNumberOptionHandler
                                    (PREF_MAX_OVERALL_DOWNLOAD_LIMIT,

src/RequestGroupMan.cc

@@ -87,7 +87,10 @@ RequestGroupMan::RequestGroupMan
   (option->getAsInt(PREF_MAX_OVERALL_DOWNLOAD_LIMIT)),
   maxOverallUploadSpeedLimit_(option->getAsInt(PREF_MAX_OVERALL_UPLOAD_LIMIT)),
   xmlRpc_(option->getAsBool(PREF_ENABLE_XML_RPC)),
-  queueCheck_(true)
+  queueCheck_(true),
+  removedErrorResult_(0),
+  removedLastErrorResult_(downloadresultcode::FINISHED),
+  maxDownloadResult_(option->getAsInt(PREF_MAX_DOWNLOAD_RESULT))
 {}
 
 bool RequestGroupMan::downloadFinished()
@@ -344,18 +347,17 @@ public:
       }
       if(group->isPauseRequested()) {
         reservedGroups_.push_front(group);
-      } else {
-        downloadResults_.push_back(group->createDownloadResult());
-      }
-      group->releaseRuntimeResource(e_);
-      if(group->isPauseRequested()) {
+        group->releaseRuntimeResource(e_);
         group->setForceHaltRequested(false);
         util::executeHookByOptName
           (group, e_->getOption(), PREF_ON_DOWNLOAD_PAUSE);
         // TODO Should we have to prepend spend uris to remaining uris
         // in case PREF_REUSE_URI is disabed?
       } else {
-        executeStopHook(downloadResults_.back(), e_->getOption());
+        SharedHandle<DownloadResult> dr = group->createDownloadResult();
+        e_->getRequestGroupMan()->addDownloadResult(dr);
+        group->releaseRuntimeResource(e_);
+        executeStopHook(dr, e_->getOption());
       }
     }
   }
@@ -766,6 +768,23 @@ RequestGroupMan::findDownloadResult(gid_t gid) const
   return SharedHandle<DownloadResult>();
 }
 
+void RequestGroupMan::addDownloadResult(const SharedHandle<DownloadResult>& dr)
+{
+  if(maxDownloadResult_ == 0) {
+    if(!downloadResults_.empty()) {
+      downloadResults_.clear();
+    }
+  } else {
+    size_t curSize = downloadResults_.size();
+    if(curSize >= maxDownloadResult_) {
+      downloadResults_.erase
+        (downloadResults_.begin(),
+         downloadResults_.begin()+curSize-maxDownloadResult_+1);
+    }
+    downloadResults_.push_back(dr);
+  }
+}
+
 void RequestGroupMan::purgeDownloadResult()
 {
   downloadResults_.clear();

src/RequestGroupMan.h

@@ -78,6 +78,15 @@ private:
 
   bool queueCheck_;
 
+  // The number of error DownloadResult removed because of upper limit
+  // of the queue
+  size_t removedErrorResult_;
+
+  // The last error of removed DownloadResult
+  downloadresultcode::RESULT removedLastErrorResult_;
+
+  size_t maxDownloadResult_;
+
   std::string
   formatDownloadResult(const std::string& status,
                        const SharedHandle<DownloadResult>& downloadResult) const;
@@ -205,6 +214,8 @@ public:
   // Removes all download results.
   void purgeDownloadResult();
 
+  void addDownloadResult(const SharedHandle<DownloadResult>& downloadResult);
+
   SharedHandle<ServerStat> findServerStat(const std::string& hostname,
                                           const std::string& protocol) const;
 

src/prefs.cc

@@ -200,6 +200,8 @@ const std::string PREF_CONDITIONAL_GET("conditional-get");
 const std::string PREF_SELECT_LEAST_USED_HOST("select-least-used-host");
 // value: true | false
 const std::string PREF_ENABLE_ASYNC_DNS6("enable-async-dns6");
+// value: 1*digit
+const std::string PREF_MAX_DOWNLOAD_RESULT("max-download-result");
 
 /**
  * FTP related preferences

src/prefs.h

@@ -204,6 +204,8 @@ extern const std::string PREF_CONDITIONAL_GET;
 extern const std::string PREF_SELECT_LEAST_USED_HOST;
 // value: true | false
 extern const std::string PREF_ENABLE_ASYNC_DNS6;
+// value: 1*digit
+extern const std::string PREF_MAX_DOWNLOAD_RESULT;
 
 /**
  * FTP related preferences

src/usage_text.h

@@ -741,3 +741,15 @@
     "                              announce URIs. When specifying '*' in shell\n" \
     "                              command-line, don't forget to escape or quote it.\n" \
     "                              See also --bt-tracker option.")
+#define TEXT_MAX_DOWNLOAD_RESULT                \
+  _(" --max-download-result=NUM    Set maximum number of download result kept in\n" \
+    "                              memory. The download results are completed/error/\n" \
+    "                              removed downloads. The download results are stored\n" \
+    "                              in FIFO queue and it can store at most NUM\n" \
+    "                              download results. When queue is full and new\n" \
+    "                              download result is created, oldest download result\n" \
+    "                              is removed from the front of the queue and new one\n" \
+    "                              is pushed to the back. Setting big number in this\n" \
+    "                              option may result high memory consumption after\n" \
+    "                              thousands of downloads. Specifying 0 means no\n" \
+    "                              download result is kept.")