Added --deferred-input option.

If true is given to --deferred-input option, aria2 does not read all
URIs and options from file specified by -i option at startup, but it
reads one by one when it needs later. This may reduce memory usage if
input file contains a lot of URIs to download. If false is given,
aria2 reads all URIs and options at startup. This option defaults to
false.

doc/aria2c.1.asciidoc

@@ -42,6 +42,7 @@ Basic Options
   URI. This optional line must start with one or more white spaces and have
   one option per single line.
   See *<<_input_file, Input File>>* subsection for details.
+  See also *<<aria2_optref_deferred_input, --deferred-input>>* option.
 
 [[aria2_optref_log]]*-l*, *--log*=LOG::
   The file name of the log file. If '-' is specified, log is written to
@@ -916,6 +917,16 @@ Advanced Options
   and standard input, standard output and standard error will be
   redirected to '/dev/null'. Default: 'false'
 
+[[aria2_optref_deferred_input]]*--deferred-input*[='true'|'false']::
+
+  If 'true' is given, aria2 does not read all URIs and options from file
+  specified by *<<aria2_optref_input_file, -i>>* option at startup,
+  but it reads one by one when it
+  needs later. This may reduce memory usage if input file contains a
+  lot of URIs to download.  If 'false' is given, aria2 reads all URIs
+  and options at startup.
+  Default: 'false'
+
 [[aria2_optref_disable_ipv6]]*--disable-ipv6*[='true'|'false']::
 
   Disable IPv6. This is useful if you have to use broken DNS and want

src/MultiUrlRequestInfo.cc

@@ -60,6 +60,7 @@
 #include "fmt.h"
 #include "SocketCore.h"
 #include "OutputFile.h"
+#include "UriListParser.h"
 #ifdef ENABLE_SSL
 # include "TLSContext.h"
 #endif // ENABLE_SSL
@@ -143,11 +144,13 @@ MultiUrlRequestInfo::MultiUrlRequestInfo
 (const std::vector<SharedHandle<RequestGroup> >& requestGroups,
  const SharedHandle<Option>& op,
  const SharedHandle<StatCalc>& statCalc,
- const SharedHandle<OutputFile>& summaryOut)
+ const SharedHandle<OutputFile>& summaryOut,
+ const SharedHandle<UriListParser>& uriListParser)
   : requestGroups_(requestGroups),
     option_(op),
     statCalc_(statCalc),
-    summaryOut_(summaryOut)
+    summaryOut_(summaryOut),
+    uriListParser_(uriListParser)
 {}
 
 MultiUrlRequestInfo::~MultiUrlRequestInfo() {}
@@ -235,6 +238,9 @@ error_code::Value MultiUrlRequestInfo::execute()
         (option_->getAsInt(PREF_SERVER_STAT_TIMEOUT));
     }
     e->setStatCalc(statCalc_);
+    if(uriListParser_) {
+      e->getRequestGroupMan()->setUriListParser(uriListParser_);
+    }
 #ifdef SIGHUP
     util::setGlobalSignalHandler(SIGHUP, handler, 0);
 #endif // SIGHUP

src/MultiUrlRequestInfo.h

@@ -48,6 +48,7 @@ class RequestGroup;
 class Option;
 class StatCalc;
 class OutputFile;
+class UriListParser;
 
 class MultiUrlRequestInfo {
 private:
@@ -59,13 +60,16 @@ private:
 
   SharedHandle<OutputFile> summaryOut_;
 
+  SharedHandle<UriListParser> uriListParser_;
+
   void printMessageForContinue();
 public:
   MultiUrlRequestInfo
   (const std::vector<SharedHandle<RequestGroup> >& requestGroups,
    const SharedHandle<Option>& op,
    const SharedHandle<StatCalc>& statCalc,
-   const SharedHandle<OutputFile>& summaryOut);
+   const SharedHandle<OutputFile>& summaryOut,
+   const SharedHandle<UriListParser>& uriListParser);
   
   virtual ~MultiUrlRequestInfo();
 

src/OptionHandlerFactory.cc

@@ -201,6 +201,15 @@ OptionHandlerFactory::createOptionHandlers()
     op->addTag(TAG_ADVANCED);
     handlers.push_back(op);
   }
+  {
+    SharedHandle<OptionHandler> op(new BooleanOptionHandler
+                                   (PREF_DEFERRED_INPUT,
+                                    TEXT_DEFERRED_INPUT,
+                                    A2_V_FALSE,
+                                    OptionHandler::OPT_ARG));
+    op->addTag(TAG_ADVANCED);
+    handlers.push_back(op);
+  }
   {
     SharedHandle<OptionHandler> op(new DefaultOptionHandler
                                    (PREF_DIR,

src/RequestGroupMan.cc

@@ -74,6 +74,8 @@
 #include "Triplet.h"
 #include "Signature.h"
 #include "OutputFile.h"
+#include "download_helper.h"
+#include "UriListParser.h"
 
 namespace aria2 {
 
@@ -485,7 +487,21 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e)
   std::vector<SharedHandle<RequestGroup> > temp;
   int count = 0;
   int num = maxSimultaneousDownloads_-requestGroups_.size();
-  while(count < num && !reservedGroups_.empty()) {
+  while(count < num && (uriListParser_ || !reservedGroups_.empty())) {
+    if(uriListParser_ && reservedGroups_.empty()) {
+      std::vector<SharedHandle<RequestGroup> > groups;
+      bool ok = createRequestGroupFromUriListParser(groups, option_,
+                                                    uriListParser_.get());
+      if(ok) {
+        reservedGroups_.insert(reservedGroups_.end(), groups.begin(),
+                               groups.end());
+      } else {
+        uriListParser_.reset();
+        if(reservedGroups_.empty()) {
+          break;
+        }
+      }
+    }
     SharedHandle<RequestGroup> groupToAdd = reservedGroups_.front();
     reservedGroups_.pop_front();
     std::vector<Command*> commands;
@@ -962,4 +978,10 @@ void RequestGroupMan::getUsedHosts
                  std::back_inserter(usedHosts), Tuple2Pair<1, 3>());
 }
 
+void RequestGroupMan::setUriListParser
+(const SharedHandle<UriListParser>& uriListParser)
+{
+  uriListParser_ = uriListParser;
+}
+
 } // namespace aria2

src/RequestGroupMan.h

@@ -55,6 +55,7 @@ class ServerStatMan;
 class ServerStat;
 class Option;
 class OutputFile;
+class UriListParser;
 
 class RequestGroupMan {
 private:
@@ -85,6 +86,9 @@ private:
 
   int maxDownloadResult_;
 
+  // UriListParser for deferred input.
+  SharedHandle<UriListParser> uriListParser_;
+
   void formatDownloadResultFull
   (OutputFile& out,
    const std::string& status,
@@ -309,6 +313,8 @@ public:
   {
     maxDownloadResult_ = v;
   }
+
+  void setUriListParser(const SharedHandle<UriListParser>& uriListParser);
 };
 
 typedef SharedHandle<RequestGroupMan> RequestGroupManHandle;

src/download_helper.cc

@@ -412,21 +412,23 @@ void createRequestGroupForUri
   }
 }
 
-namespace {
-void createRequestGroupForUriList
+bool createRequestGroupFromUriListParser
 (std::vector<SharedHandle<RequestGroup> >& result,
- const SharedHandle<Option>& option,
- const std::string& filename)
+ const Option* option,
+ UriListParser* uriListParser)
 {
-  UriListParser p(filename);
-  while(p.hasNext()) {
+  // Since result already contains some entries, we cache the size of
+  // it. Later, we use this value to determine RequestGroup is
+  // actually created.
+  size_t num = result.size();
+  while(uriListParser->hasNext()) {
     std::vector<std::string> uris;
     Option tempOption;
-    p.parseNext(uris, tempOption);
+    uriListParser->parseNext(uris, tempOption);
     if(uris.empty()) {
       continue;
     }
-    SharedHandle<Option> requestOption(new Option(*option.get()));
+    SharedHandle<Option> requestOption(new Option(*option));
     requestOption->remove(PREF_OUT);
     const SharedHandle<OptionParser>& oparser = OptionParser::getInstance();
     for(size_t i = 1, len = option::countOption(); i < len; ++i) {
@@ -436,25 +438,38 @@ void createRequestGroupForUriList
         requestOption->put(pref, tempOption.get(pref));
       }
     }
+    // This does not throw exception because throwOnError = false.
     createRequestGroupForUri(result, requestOption, uris);
+    if(num < result.size()) {
+      return true;
+    }
   }
+  return false;
 }
-} // namespace
 
-void createRequestGroupForUriList
-(std::vector<SharedHandle<RequestGroup> >& result,
- const SharedHandle<Option>& option)
+SharedHandle<UriListParser> openUriListParser(const std::string& filename)
 {
-  if(option->get(PREF_INPUT_FILE) == "-") {
-    createRequestGroupForUriList(result, option, DEV_STDIN);
+  std::string listPath;
+  if(filename == "-") {
+    listPath = DEV_STDIN;
   } else {
-    if(!File(option->get(PREF_INPUT_FILE)).isFile()) {
+    if(!File(filename).isFile()) {
       throw DL_ABORT_EX
-        (fmt(EX_FILE_OPEN, option->get(PREF_INPUT_FILE).c_str(),
-             "No such file"));
+        (fmt(EX_FILE_OPEN, filename.c_str(), "No such file"));
     }
-    createRequestGroupForUriList(result, option, option->get(PREF_INPUT_FILE));
+    listPath = filename;
   }
+  return SharedHandle<UriListParser>(new UriListParser(listPath));
+}
+
+void createRequestGroupForUriList
+(std::vector<SharedHandle<RequestGroup> >& result,
+ const SharedHandle<Option>& option)
+{
+  SharedHandle<UriListParser> uriListParser = openUriListParser
+    (option->get(PREF_INPUT_FILE));
+  while(createRequestGroupFromUriListParser(result, option.get(),
+                                            uriListParser.get()));
 }
 
 SharedHandle<MetadataInfo>

src/download_helper.h

@@ -49,6 +49,7 @@ class RequestGroup;
 class Option;
 class MetadataInfo;
 class DownloadContext;
+class UriListParser;
 
 #ifdef ENABLE_BITTORRENT
 // Create RequestGroup object using torrent file specified by
@@ -75,6 +76,26 @@ void createRequestGroupForMetalink
  const std::string& metalinkData = "");
 #endif // ENABLE_METALINK
 
+// Reads one entry from uriListParser and creates RequestGroups from
+// it and store them in result. If the bad entry is found, this
+// function just skips it and reads next entry. If at least one
+// RequestGroup is created successfully, this function returns true
+// and created RequestGroups are stored in result. If no RequestGroup
+// is created and uriListParser reads all input, this function returns
+// false. The option is used as a option template.
+bool createRequestGroupFromUriListParser
+(std::vector<SharedHandle<RequestGroup> >& result,
+ const Option* option,
+ UriListParser* uriListParser);
+
+// Creates UriListParser using given filename.  If filename is "-",
+// then UriListParser is configured to read from standard input.
+// Otherwise, this function first checks file denoted by filename
+// exists.  If it does not exist, this function throws exception.
+// This function returns SharedHandle<UriListParser> object if it
+// succeeds.
+SharedHandle<UriListParser> openUriListParser(const std::string& filename);
+
 // Create RequestGroup objects from reading file specified by input-file option.
 // If the value of input-file option is "-", stdin is used as a input source.
 // Each line is treated as if it is provided in command-line argument.

src/main.cc

@@ -70,6 +70,7 @@
 #include "fmt.h"
 #include "NullOutputFile.h"
 #include "console.h"
+#include "UriListParser.h"
 #ifdef ENABLE_BITTORRENT
 # include "bittorrent_helper.h"
 #endif // ENABLE_BITTORRENT
@@ -225,6 +226,7 @@ error_code::Value main(int argc, char* argv[])
   util::setGlobalSignalHandler(SIGCHLD, SIG_IGN, 0);
 #endif // SIGCHILD
   std::vector<SharedHandle<RequestGroup> > requestGroups;
+  SharedHandle<UriListParser> uriListParser;
 #ifdef ENABLE_BITTORRENT
   if(!op->blank(PREF_TORRENT_FILE)) {
     if(op->get(PREF_SHOW_FILES) == A2_V_TRUE) {
@@ -248,7 +250,11 @@ error_code::Value main(int argc, char* argv[])
     else
 #endif // ENABLE_METALINK
       if(!op->blank(PREF_INPUT_FILE)) {
-        createRequestGroupForUriList(requestGroups, op);
+        if(op->getAsBool(PREF_DEFERRED_INPUT)) {
+          uriListParser = openUriListParser(op->get(PREF_INPUT_FILE));
+        } else {
+          createRequestGroupForUriList(requestGroups, op);
+        }
 #if defined ENABLE_BITTORRENT || defined ENABLE_METALINK
       } else if(op->get(PREF_SHOW_FILES) == A2_V_TRUE) {
         showFiles(args, op);
@@ -269,11 +275,13 @@ error_code::Value main(int argc, char* argv[])
   op->remove(PREF_SELECT_FILE);
   op->remove(PREF_PAUSE);
   op->remove(PREF_CHECKSUM);
-  if(!op->getAsBool(PREF_ENABLE_RPC) && requestGroups.empty()) {
+  if(!op->getAsBool(PREF_ENABLE_RPC) && requestGroups.empty() &&
+     !uriListParser) {
     global::cout()->printf("%s\n", MSG_NO_FILES_TO_DOWNLOAD);
   } else {
     exitStatus = MultiUrlRequestInfo(requestGroups, op, getStatCalc(op),
-                                     getSummaryOut(op)).execute();
+                                     getSummaryOut(op),
+                                     uriListParser).execute();
   }
   return exitStatus;
 }

src/prefs.cc

@@ -213,6 +213,8 @@ const Pref* PREF_NO_NETRC = makePref("no-netrc");
 const Pref* PREF_MAX_DOWNLOADS = makePref("max-downloads");
 // value: string that your file system recognizes as a file name.
 const Pref* PREF_INPUT_FILE = makePref("input-file");
+// value: true | false
+const Pref* PREF_DEFERRED_INPUT = makePref("deferred-input");
 // value: 1*digit
 const Pref* PREF_MAX_CONCURRENT_DOWNLOADS = makePref("max-concurrent-downloads");
 // value: true | false

src/prefs.h

@@ -156,6 +156,8 @@ extern const Pref* PREF_MAX_OVERALL_DOWNLOAD_LIMIT;
 extern const Pref* PREF_MAX_DOWNLOADS;
 // value: string that your file system recognizes as a file name.
 extern const Pref* PREF_INPUT_FILE;
+// value: true | false
+extern const Pref* PREF_DEFERRED_INPUT;
 // value: 1*digit
 extern const Pref* PREF_MAX_CONCURRENT_DOWNLOADS;
 // value: true | false

src/usage_text.h

@@ -226,7 +226,7 @@
     "                              line of URI. This optional line must start with\n" \
     "                              one or more white spaces and have one option per\n" \
     "                              single line. See INPUT FILE section of man page\n" \
-    "                              for details.")
+    "                              for details. See also --deferred-input option.")
 #define TEXT_MAX_CONCURRENT_DOWNLOADS                                   \
   _(" -j, --max-concurrent-downloads=N Set maximum number of parallel downloads for\n" \
     "                              every static (HTTP/FTP) URL, torrent and metalink.\n" \
@@ -859,3 +859,11 @@
     "                              with its own pid and when parent process exits\n" \
     "                              for some reason, aria2 can detect it and shutdown\n" \
     "                              itself.")
+#define TEXT_DEFERRED_INPUT                     \
+  _(" --deferred-input[=true|false] If true is given, aria2 does not read all URIs\n" \
+    "                              and options from file specified by -i option at\n" \
+    "                              startup, but it reads one by one when it needs\n" \
+    "                              later. This may reduce memory usage if input\n" \
+    "                              file contains a lot of URIs to download.\n" \
+    "                              If false is given, aria2 reads all URIs and\n" \
+    "                              options at startup.")