فهرست منبع

Added --deferred-input option.

If true is given to --deferred-input option, aria2 does not read all
URIs and options from file specified by -i option at startup, but it
reads one by one when it needs later. This may reduce memory usage if
input file contains a lot of URIs to download. If false is given,
aria2 reads all URIs and options at startup. This option defaults to
false.
Tatsuhiro Tsujikawa 13 سال پیش
والد
کامیت
f2d55f0edc
12فایلهای تغییر یافته به همراه140 افزوده شده و 26 حذف شده
  1. 11 0
      doc/aria2c.1.asciidoc
  2. 8 2
      src/MultiUrlRequestInfo.cc
  3. 5 1
      src/MultiUrlRequestInfo.h
  4. 9 0
      src/OptionHandlerFactory.cc
  5. 23 1
      src/RequestGroupMan.cc
  6. 6 0
      src/RequestGroupMan.h
  7. 33 18
      src/download_helper.cc
  8. 21 0
      src/download_helper.h
  9. 11 3
      src/main.cc
  10. 2 0
      src/prefs.cc
  11. 2 0
      src/prefs.h
  12. 9 1
      src/usage_text.h

+ 11 - 0
doc/aria2c.1.asciidoc

@@ -42,6 +42,7 @@ Basic Options
   URI. This optional line must start with one or more white spaces and have
   one option per single line.
   See *<<_input_file, Input File>>* subsection for details.
+  See also *<<aria2_optref_deferred_input, --deferred-input>>* option.
 
 [[aria2_optref_log]]*-l*, *--log*=LOG::
   The file name of the log file. If '-' is specified, log is written to
@@ -916,6 +917,16 @@ Advanced Options
   and standard input, standard output and standard error will be
   redirected to '/dev/null'. Default: 'false'
 
+[[aria2_optref_deferred_input]]*--deferred-input*[='true'|'false']::
+
+  If 'true' is given, aria2 does not read all URIs and options from file
+  specified by *<<aria2_optref_input_file, -i>>* option at startup,
+  but it reads one by one when it
+  needs later. This may reduce memory usage if input file contains a
+  lot of URIs to download.  If 'false' is given, aria2 reads all URIs
+  and options at startup.
+  Default: 'false'
+
 [[aria2_optref_disable_ipv6]]*--disable-ipv6*[='true'|'false']::
 
   Disable IPv6. This is useful if you have to use broken DNS and want

+ 8 - 2
src/MultiUrlRequestInfo.cc

@@ -60,6 +60,7 @@
 #include "fmt.h"
 #include "SocketCore.h"
 #include "OutputFile.h"
+#include "UriListParser.h"
 #ifdef ENABLE_SSL
 # include "TLSContext.h"
 #endif // ENABLE_SSL
@@ -143,11 +144,13 @@ MultiUrlRequestInfo::MultiUrlRequestInfo
 (const std::vector<SharedHandle<RequestGroup> >& requestGroups,
  const SharedHandle<Option>& op,
  const SharedHandle<StatCalc>& statCalc,
- const SharedHandle<OutputFile>& summaryOut)
+ const SharedHandle<OutputFile>& summaryOut,
+ const SharedHandle<UriListParser>& uriListParser)
   : requestGroups_(requestGroups),
     option_(op),
     statCalc_(statCalc),
-    summaryOut_(summaryOut)
+    summaryOut_(summaryOut),
+    uriListParser_(uriListParser)
 {}
 
 MultiUrlRequestInfo::~MultiUrlRequestInfo() {}
@@ -235,6 +238,9 @@ error_code::Value MultiUrlRequestInfo::execute()
         (option_->getAsInt(PREF_SERVER_STAT_TIMEOUT));
     }
     e->setStatCalc(statCalc_);
+    if(uriListParser_) {
+      e->getRequestGroupMan()->setUriListParser(uriListParser_);
+    }
 #ifdef SIGHUP
     util::setGlobalSignalHandler(SIGHUP, handler, 0);
 #endif // SIGHUP

+ 5 - 1
src/MultiUrlRequestInfo.h

@@ -48,6 +48,7 @@ class RequestGroup;
 class Option;
 class StatCalc;
 class OutputFile;
+class UriListParser;
 
 class MultiUrlRequestInfo {
 private:
@@ -59,13 +60,16 @@ private:
 
   SharedHandle<OutputFile> summaryOut_;
 
+  SharedHandle<UriListParser> uriListParser_;
+
   void printMessageForContinue();
 public:
   MultiUrlRequestInfo
   (const std::vector<SharedHandle<RequestGroup> >& requestGroups,
    const SharedHandle<Option>& op,
    const SharedHandle<StatCalc>& statCalc,
-   const SharedHandle<OutputFile>& summaryOut);
+   const SharedHandle<OutputFile>& summaryOut,
+   const SharedHandle<UriListParser>& uriListParser);
   
   virtual ~MultiUrlRequestInfo();
 

+ 9 - 0
src/OptionHandlerFactory.cc

@@ -201,6 +201,15 @@ OptionHandlerFactory::createOptionHandlers()
     op->addTag(TAG_ADVANCED);
     handlers.push_back(op);
   }
+  {
+    SharedHandle<OptionHandler> op(new BooleanOptionHandler
+                                   (PREF_DEFERRED_INPUT,
+                                    TEXT_DEFERRED_INPUT,
+                                    A2_V_FALSE,
+                                    OptionHandler::OPT_ARG));
+    op->addTag(TAG_ADVANCED);
+    handlers.push_back(op);
+  }
   {
     SharedHandle<OptionHandler> op(new DefaultOptionHandler
                                    (PREF_DIR,

+ 23 - 1
src/RequestGroupMan.cc

@@ -74,6 +74,8 @@
 #include "Triplet.h"
 #include "Signature.h"
 #include "OutputFile.h"
+#include "download_helper.h"
+#include "UriListParser.h"
 
 namespace aria2 {
 
@@ -485,7 +487,21 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e)
   std::vector<SharedHandle<RequestGroup> > temp;
   int count = 0;
   int num = maxSimultaneousDownloads_-requestGroups_.size();
-  while(count < num && !reservedGroups_.empty()) {
+  while(count < num && (uriListParser_ || !reservedGroups_.empty())) {
+    if(uriListParser_ && reservedGroups_.empty()) {
+      std::vector<SharedHandle<RequestGroup> > groups;
+      bool ok = createRequestGroupFromUriListParser(groups, option_,
+                                                    uriListParser_.get());
+      if(ok) {
+        reservedGroups_.insert(reservedGroups_.end(), groups.begin(),
+                               groups.end());
+      } else {
+        uriListParser_.reset();
+        if(reservedGroups_.empty()) {
+          break;
+        }
+      }
+    }
     SharedHandle<RequestGroup> groupToAdd = reservedGroups_.front();
     reservedGroups_.pop_front();
     std::vector<Command*> commands;
@@ -962,4 +978,10 @@ void RequestGroupMan::getUsedHosts
                  std::back_inserter(usedHosts), Tuple2Pair<1, 3>());
 }
 
+void RequestGroupMan::setUriListParser
+(const SharedHandle<UriListParser>& uriListParser)
+{
+  uriListParser_ = uriListParser;
+}
+
 } // namespace aria2

+ 6 - 0
src/RequestGroupMan.h

@@ -55,6 +55,7 @@ class ServerStatMan;
 class ServerStat;
 class Option;
 class OutputFile;
+class UriListParser;
 
 class RequestGroupMan {
 private:
@@ -85,6 +86,9 @@ private:
 
   int maxDownloadResult_;
 
+  // UriListParser for deferred input.
+  SharedHandle<UriListParser> uriListParser_;
+
   void formatDownloadResultFull
   (OutputFile& out,
    const std::string& status,
@@ -309,6 +313,8 @@ public:
   {
     maxDownloadResult_ = v;
   }
+
+  void setUriListParser(const SharedHandle<UriListParser>& uriListParser);
 };
 
 typedef SharedHandle<RequestGroupMan> RequestGroupManHandle;

+ 33 - 18
src/download_helper.cc

@@ -412,21 +412,23 @@ void createRequestGroupForUri
   }
 }
 
-namespace {
-void createRequestGroupForUriList
+bool createRequestGroupFromUriListParser
 (std::vector<SharedHandle<RequestGroup> >& result,
- const SharedHandle<Option>& option,
- const std::string& filename)
+ const Option* option,
+ UriListParser* uriListParser)
 {
-  UriListParser p(filename);
-  while(p.hasNext()) {
+  // Since result already contains some entries, we cache the size of
+  // it. Later, we use this value to determine RequestGroup is
+  // actually created.
+  size_t num = result.size();
+  while(uriListParser->hasNext()) {
     std::vector<std::string> uris;
     Option tempOption;
-    p.parseNext(uris, tempOption);
+    uriListParser->parseNext(uris, tempOption);
     if(uris.empty()) {
       continue;
     }
-    SharedHandle<Option> requestOption(new Option(*option.get()));
+    SharedHandle<Option> requestOption(new Option(*option));
     requestOption->remove(PREF_OUT);
     const SharedHandle<OptionParser>& oparser = OptionParser::getInstance();
     for(size_t i = 1, len = option::countOption(); i < len; ++i) {
@@ -436,25 +438,38 @@ void createRequestGroupForUriList
         requestOption->put(pref, tempOption.get(pref));
       }
     }
+    // This does not throw exception because throwOnError = false.
     createRequestGroupForUri(result, requestOption, uris);
+    if(num < result.size()) {
+      return true;
+    }
   }
+  return false;
 }
-} // namespace
 
-void createRequestGroupForUriList
-(std::vector<SharedHandle<RequestGroup> >& result,
- const SharedHandle<Option>& option)
+SharedHandle<UriListParser> openUriListParser(const std::string& filename)
 {
-  if(option->get(PREF_INPUT_FILE) == "-") {
-    createRequestGroupForUriList(result, option, DEV_STDIN);
+  std::string listPath;
+  if(filename == "-") {
+    listPath = DEV_STDIN;
   } else {
-    if(!File(option->get(PREF_INPUT_FILE)).isFile()) {
+    if(!File(filename).isFile()) {
       throw DL_ABORT_EX
-        (fmt(EX_FILE_OPEN, option->get(PREF_INPUT_FILE).c_str(),
-             "No such file"));
+        (fmt(EX_FILE_OPEN, filename.c_str(), "No such file"));
     }
-    createRequestGroupForUriList(result, option, option->get(PREF_INPUT_FILE));
+    listPath = filename;
   }
+  return SharedHandle<UriListParser>(new UriListParser(listPath));
+}
+
+void createRequestGroupForUriList
+(std::vector<SharedHandle<RequestGroup> >& result,
+ const SharedHandle<Option>& option)
+{
+  SharedHandle<UriListParser> uriListParser = openUriListParser
+    (option->get(PREF_INPUT_FILE));
+  while(createRequestGroupFromUriListParser(result, option.get(),
+                                            uriListParser.get()));
 }
 
 SharedHandle<MetadataInfo>

+ 21 - 0
src/download_helper.h

@@ -49,6 +49,7 @@ class RequestGroup;
 class Option;
 class MetadataInfo;
 class DownloadContext;
+class UriListParser;
 
 #ifdef ENABLE_BITTORRENT
 // Create RequestGroup object using torrent file specified by
@@ -75,6 +76,26 @@ void createRequestGroupForMetalink
  const std::string& metalinkData = "");
 #endif // ENABLE_METALINK
 
+// Reads one entry from uriListParser and creates RequestGroups from
+// it and store them in result. If the bad entry is found, this
+// function just skips it and reads next entry. If at least one
+// RequestGroup is created successfully, this function returns true
+// and created RequestGroups are stored in result. If no RequestGroup
+// is created and uriListParser reads all input, this function returns
+// false. The option is used as a option template.
+bool createRequestGroupFromUriListParser
+(std::vector<SharedHandle<RequestGroup> >& result,
+ const Option* option,
+ UriListParser* uriListParser);
+
+// Creates UriListParser using given filename.  If filename is "-",
+// then UriListParser is configured to read from standard input.
+// Otherwise, this function first checks file denoted by filename
+// exists.  If it does not exist, this function throws exception.
+// This function returns SharedHandle<UriListParser> object if it
+// succeeds.
+SharedHandle<UriListParser> openUriListParser(const std::string& filename);
+
 // Create RequestGroup objects from reading file specified by input-file option.
 // If the value of input-file option is "-", stdin is used as a input source.
 // Each line is treated as if it is provided in command-line argument.

+ 11 - 3
src/main.cc

@@ -70,6 +70,7 @@
 #include "fmt.h"
 #include "NullOutputFile.h"
 #include "console.h"
+#include "UriListParser.h"
 #ifdef ENABLE_BITTORRENT
 # include "bittorrent_helper.h"
 #endif // ENABLE_BITTORRENT
@@ -225,6 +226,7 @@ error_code::Value main(int argc, char* argv[])
   util::setGlobalSignalHandler(SIGCHLD, SIG_IGN, 0);
 #endif // SIGCHILD
   std::vector<SharedHandle<RequestGroup> > requestGroups;
+  SharedHandle<UriListParser> uriListParser;
 #ifdef ENABLE_BITTORRENT
   if(!op->blank(PREF_TORRENT_FILE)) {
     if(op->get(PREF_SHOW_FILES) == A2_V_TRUE) {
@@ -248,7 +250,11 @@ error_code::Value main(int argc, char* argv[])
     else
 #endif // ENABLE_METALINK
       if(!op->blank(PREF_INPUT_FILE)) {
-        createRequestGroupForUriList(requestGroups, op);
+        if(op->getAsBool(PREF_DEFERRED_INPUT)) {
+          uriListParser = openUriListParser(op->get(PREF_INPUT_FILE));
+        } else {
+          createRequestGroupForUriList(requestGroups, op);
+        }
 #if defined ENABLE_BITTORRENT || defined ENABLE_METALINK
       } else if(op->get(PREF_SHOW_FILES) == A2_V_TRUE) {
         showFiles(args, op);
@@ -269,11 +275,13 @@ error_code::Value main(int argc, char* argv[])
   op->remove(PREF_SELECT_FILE);
   op->remove(PREF_PAUSE);
   op->remove(PREF_CHECKSUM);
-  if(!op->getAsBool(PREF_ENABLE_RPC) && requestGroups.empty()) {
+  if(!op->getAsBool(PREF_ENABLE_RPC) && requestGroups.empty() &&
+     !uriListParser) {
     global::cout()->printf("%s\n", MSG_NO_FILES_TO_DOWNLOAD);
   } else {
     exitStatus = MultiUrlRequestInfo(requestGroups, op, getStatCalc(op),
-                                     getSummaryOut(op)).execute();
+                                     getSummaryOut(op),
+                                     uriListParser).execute();
   }
   return exitStatus;
 }

+ 2 - 0
src/prefs.cc

@@ -213,6 +213,8 @@ const Pref* PREF_NO_NETRC = makePref("no-netrc");
 const Pref* PREF_MAX_DOWNLOADS = makePref("max-downloads");
 // value: string that your file system recognizes as a file name.
 const Pref* PREF_INPUT_FILE = makePref("input-file");
+// value: true | false
+const Pref* PREF_DEFERRED_INPUT = makePref("deferred-input");
 // value: 1*digit
 const Pref* PREF_MAX_CONCURRENT_DOWNLOADS = makePref("max-concurrent-downloads");
 // value: true | false

+ 2 - 0
src/prefs.h

@@ -156,6 +156,8 @@ extern const Pref* PREF_MAX_OVERALL_DOWNLOAD_LIMIT;
 extern const Pref* PREF_MAX_DOWNLOADS;
 // value: string that your file system recognizes as a file name.
 extern const Pref* PREF_INPUT_FILE;
+// value: true | false
+extern const Pref* PREF_DEFERRED_INPUT;
 // value: 1*digit
 extern const Pref* PREF_MAX_CONCURRENT_DOWNLOADS;
 // value: true | false

+ 9 - 1
src/usage_text.h

@@ -226,7 +226,7 @@
     "                              line of URI. This optional line must start with\n" \
     "                              one or more white spaces and have one option per\n" \
     "                              single line. See INPUT FILE section of man page\n" \
-    "                              for details.")
+    "                              for details. See also --deferred-input option.")
 #define TEXT_MAX_CONCURRENT_DOWNLOADS                                   \
   _(" -j, --max-concurrent-downloads=N Set maximum number of parallel downloads for\n" \
     "                              every static (HTTP/FTP) URL, torrent and metalink.\n" \
@@ -859,3 +859,11 @@
     "                              with its own pid and when parent process exits\n" \
     "                              for some reason, aria2 can detect it and shutdown\n" \
     "                              itself.")
+#define TEXT_DEFERRED_INPUT                     \
+  _(" --deferred-input[=true|false] If true is given, aria2 does not read all URIs\n" \
+    "                              and options from file specified by -i option at\n" \
+    "                              startup, but it reads one by one when it needs\n" \
+    "                              later. This may reduce memory usage if input\n" \
+    "                              file contains a lot of URIs to download.\n" \
+    "                              If false is given, aria2 reads all URIs and\n" \
+    "                              options at startup.")