Limit number of opened file globally with --bt-max-open-files option

This change changes the behavior of --bt-max-open-files. Previously,
it specifies the maximum number of opened files for each multi-file
download. Since it is more useful to limit the number globally, the
option now specifies the global limit. This change suggests that
aria2.changeOption() method now ignores --bt-max-open-files and
aria2.changeGlobalOption now reads it and dynamically change the
limit.

doc/manual-src/en/aria2c.rst

@@ -620,7 +620,8 @@ BitTorrent Specific Options
 
 .. option:: --bt-max-open-files=<NUM>
 
-  Specify maximum number of files to open in each BitTorrent download.
+  Specify maximum number of files to open in multi-file
+  BitTorrent/Metalink download globally.
   Default: ``100``
 
 .. option:: --bt-max-peers=<NUM>
@@ -1806,7 +1807,6 @@ of URIs. These optional lines must start with white space(s).
   * :option:`bt-exclude-tracker <--bt-exclude-tracker>`
   * :option:`bt-external-ip <--bt-external-ip>`
   * :option:`bt-hash-check-seed <--bt-hash-check-seed>`
-  * :option:`bt-max-open-files <--bt-max-open-files>`
   * :option:`bt-max-peers <--bt-max-peers>`
   * :option:`bt-metadata-only <--bt-metadata-only>`
   * :option:`bt-min-crypto-level <--bt-min-crypto-level>`
@@ -2942,6 +2942,7 @@ All code examples come from Python2.7 interpreter.
   struct.
   The following options are available:
 
+  * :option:`bt-max-open-files <--bt-max-open-files>`
   * :option:`download-result <--download-result>`
   * :option:`log <-l>`
   * :option:`log-level <--log-level>`

src/DefaultPieceStorage.cc

@@ -646,8 +646,6 @@ void DefaultPieceStorage::initStorage()
     multiDiskAdaptor->setFileEntries(downloadContext_->getFileEntries().begin(),
                                      downloadContext_->getFileEntries().end());
     multiDiskAdaptor->setPieceLength(downloadContext_->getPieceLength());
-    multiDiskAdaptor->setMaxOpenFiles
-      (option_->getAsInt(PREF_BT_MAX_OPEN_FILES));
     diskAdaptor_ = std::move(multiDiskAdaptor);
   }
   if(option_->get(PREF_FILE_ALLOCATION) == V_FALLOC) {

src/DiskAdaptor.cc

@@ -38,7 +38,8 @@
 namespace aria2 {
 
 DiskAdaptor::DiskAdaptor()
-  : fileAllocationMethod_(FILE_ALLOC_ADAPTIVE)
+  : fileAllocationMethod_(FILE_ALLOC_ADAPTIVE),
+    requestGroupMan_(nullptr)
 {}
 
 DiskAdaptor::~DiskAdaptor() {}

src/DiskAdaptor.h

@@ -48,6 +48,7 @@ namespace aria2 {
 class FileEntry;
 class FileAllocationIterator;
 class WrDiskCacheEntry;
+class RequestGroupMan;
 
 class DiskAdaptor:public BinaryStream {
 public:
@@ -119,10 +120,26 @@ public:
     return fileAllocationMethod_;
   }
 
+  // Closes at most |numClose| files if possible. This method is used to
+  // ensure that global number of open file stays under certain limit.
+  // Returns the number of closed files.
+  virtual size_t tryCloseFile(size_t numClose) { return 0; }
+
+  void setRequestGroupMan(RequestGroupMan* rgman)
+  {
+    requestGroupMan_ = rgman;
+  }
+
+  RequestGroupMan* getRequestGroupMan() const
+  {
+    return requestGroupMan_;
+  }
 private:
   std::vector<std::shared_ptr<FileEntry> > fileEntries_;
 
   FileAllocationMethod fileAllocationMethod_;
+
+  RequestGroupMan* requestGroupMan_;
 };
 
 } // namespace aria2

src/MultiDiskAdaptor.cc

@@ -51,6 +51,7 @@
 #include "LogFactory.h"
 #include "SimpleRandomizer.h"
 #include "WrDiskCacheEntry.h"
+#include "RequestGroupMan.h"
 
 namespace aria2 {
 
@@ -120,7 +121,6 @@ bool DiskWriterEntry::operator<(const DiskWriterEntry& entry) const
 
 MultiDiskAdaptor::MultiDiskAdaptor()
   : pieceLength_{0},
-    maxOpenFiles_{DEFAULT_MAX_OPEN_FILES},
     readOnly_{false}
 {}
 
@@ -200,29 +200,37 @@ void MultiDiskAdaptor::resetDiskWriterEntries()
   }
 }
 
+size_t MultiDiskAdaptor::tryCloseFile(size_t numClose)
+{
+  size_t left = numClose;
+  for(; !openedDiskWriterEntries_.empty() && left > 0; --left) {
+    // Choose one DiskWriterEntry randomly and close it.
+    size_t index =
+      SimpleRandomizer::getInstance()->getRandomNumber
+      (openedDiskWriterEntries_.size());
+    auto i = std::begin(openedDiskWriterEntries_);
+    std::advance(i, index);
+    (*i)->closeFile();
+    (*i) = openedDiskWriterEntries_.back();
+    openedDiskWriterEntries_.pop_back();
+  }
+  return numClose - left;
+}
+
 void MultiDiskAdaptor::openIfNot
 (DiskWriterEntry* entry, void (DiskWriterEntry::*open)())
 {
   if(!entry->isOpen()) {
-    //     A2_LOG_DEBUG(fmt("DiskWriterEntry: Cache MISS. offset=%s",
-    //            util::itos(entry->getFileEntry()->getOffset()).c_str()));
-    int numOpened = openedDiskWriterEntries_.size();
-    (entry->*open)();
-    if(numOpened >= maxOpenFiles_) {
-      // Cache is full.
-      // Choose one DiskWriterEntry randomly and close it.
-      size_t index =
-        SimpleRandomizer::getInstance()->getRandomNumber(numOpened);
-      auto i = std::begin(openedDiskWriterEntries_);
-      std::advance(i, index);
-      (*i)->closeFile();
-      *i = entry;
-    } else {
-      openedDiskWriterEntries_.push_back(entry);
+        // A2_LOG_NOTICE(fmt("DiskWriterEntry: Cache MISS. offset=%s",
+        //        util::itos(entry->getFileEntry()->getOffset()).c_str()));
+    if(getRequestGroupMan()) {
+      getRequestGroupMan()->ensureMaxOpenFileLimit(1);
     }
+    (entry->*open)();
+    openedDiskWriterEntries_.push_back(entry);
   } else {
-    //     A2_LOG_DEBUG(fmt("DiskWriterEntry: Cache HIT. offset=%s",
-    //            util::itos(entry->getFileEntry()->getOffset()).c_str()));
+        // A2_LOG_NOTICE(fmt("DiskWriterEntry: Cache HIT. offset=%s",
+        //        util::itos(entry->getFileEntry()->getOffset()).c_str()));
   }
 }
 
@@ -519,11 +527,6 @@ void MultiDiskAdaptor::cutTrailingGarbage()
   }
 }
 
-void MultiDiskAdaptor::setMaxOpenFiles(int maxOpenFiles)
-{
-  maxOpenFiles_ = maxOpenFiles;
-}
-
 size_t MultiDiskAdaptor::utime(const Time& actime, const Time& modtime)
 {
   size_t numOK = 0;

src/MultiDiskAdaptor.h

@@ -117,8 +117,6 @@ private:
 
   std::vector<DiskWriterEntry*> openedDiskWriterEntries_;
 
-  int maxOpenFiles_;
-
   bool readOnly_;
 
   void resetDiskWriterEntries();
@@ -175,8 +173,6 @@ public:
 
   virtual void cutTrailingGarbage() CXX11_OVERRIDE;
 
-  void setMaxOpenFiles(int maxOpenFiles);
-
   virtual size_t utime(const Time& actime, const Time& modtime) CXX11_OVERRIDE;
 
   const DiskWriterEntries& getDiskWriterEntries() const
@@ -184,6 +180,7 @@ public:
     return diskWriterEntries_;
   }
 
+  virtual size_t tryCloseFile(size_t numClose) CXX11_OVERRIDE;
 };
 
 } // namespace aria2

src/OptionHandlerFactory.cc

@@ -1720,9 +1720,7 @@ std::vector<OptionHandler*> OptionHandlerFactory::createOptionHandlers()
                        "100",
                        1));
     op->addTag(TAG_BITTORRENT);
-    op->setInitialOption(true);
     op->setChangeGlobalOption(true);
-    op->setChangeOptionForReserved(true);
     handlers.push_back(op);
   }
   {

src/RequestGroup.cc

@@ -614,6 +614,7 @@ void RequestGroup::initPieceStorage()
     tempPieceStorage = ps;
   }
   tempPieceStorage->initStorage();
+  tempPieceStorage->getDiskAdaptor()->setRequestGroupMan(requestGroupMan_);
   segmentMan_ = std::make_shared<SegmentMan>(downloadContext_, tempPieceStorage);
   pieceStorage_ = tempPieceStorage;
 }

src/RequestGroupMan.cc

@@ -78,6 +78,9 @@
 #include "Notifier.h"
 #include "PeerStat.h"
 #include "WrDiskCache.h"
+#include "PieceStorage.h"
+#include "DiskAdaptor.h"
+#include "SimpleRandomizer.h"
 #ifdef ENABLE_BITTORRENT
 #  include "bittorrent_helper.h"
 #endif // ENABLE_BITTORRENT
@@ -111,7 +114,8 @@ RequestGroupMan::RequestGroupMan
     removedErrorResult_(0),
     removedLastErrorResult_(error_code::FINISHED),
     maxDownloadResult_(option->getAsInt(PREF_MAX_DOWNLOAD_RESULT)),
-    wrDiskCache_(nullptr)
+    wrDiskCache_(nullptr),
+    numOpenFile_(0)
 {
   appendReservedGroup(reservedGroups_,
                       requestGroups.begin(), requestGroups.end());
@@ -480,6 +484,8 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e)
     groupToAdd->dropPieceStorage();
     configureRequestGroup(groupToAdd);
     groupToAdd->setRequestGroupMan(this);
+    groupToAdd->setState(RequestGroup::STATE_ACTIVE);
+    requestGroups_.push_back(groupToAdd->getGID(), groupToAdd);
     try {
       auto res = createInitialCommand(groupToAdd, e);
       ++count;
@@ -496,8 +502,6 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e)
       // removeStoppedGroup().
       requestQueueCheck();
     }
-    groupToAdd->setState(RequestGroup::STATE_ACTIVE);
-    requestGroups_.push_back(groupToAdd->getGID(), groupToAdd);
 
     util::executeHookByOptName(groupToAdd, e->getOption(),
                                PREF_ON_DOWNLOAD_START);
@@ -957,4 +961,28 @@ void RequestGroupMan::initWrDiskCache()
   }
 }
 
+void RequestGroupMan::ensureMaxOpenFileLimit(size_t numNewFile)
+{
+  size_t max = option_->getAsInt(PREF_BT_MAX_OPEN_FILES);
+  if(numOpenFile_ + numNewFile <= max) {
+    numOpenFile_ += numNewFile;
+    return;
+  }
+  assert(numNewFile <= max);
+  size_t numClose = numOpenFile_ + numNewFile - max;
+  size_t left = numClose;
+  auto mark = std::begin(requestGroups_);
+  std::advance(mark,
+               SimpleRandomizer::getInstance()->
+               getRandomNumber(requestGroups_.size()));
+  for(auto i = mark; i != std::end(requestGroups_) && left > 0; ++i) {
+    left -= (*i)->getPieceStorage()->getDiskAdaptor()->tryCloseFile(left);
+  }
+  for(auto i = std::begin(requestGroups_); i != mark && left > 0; ++i) {
+    left -= (*i)->getPieceStorage()->getDiskAdaptor()->tryCloseFile(left);
+  }
+  assert(left == 0);
+  numOpenFile_ += numNewFile - numClose;
+}
+
 } // namespace aria2

src/RequestGroupMan.h

@@ -104,6 +104,8 @@ private:
 
   WrDiskCache* wrDiskCache_;
 
+  size_t numOpenFile_;
+
   void formatDownloadResultFull
   (OutputFile& out,
    const char* status,
@@ -346,6 +348,15 @@ public:
   {
     return keepRunning_;
   }
+
+  // Keeps the number of open files under the global limit specified
+  // in the option. The caller requests that |numNewFile| files are
+  // going to be opened. This function requires that |numNewFile| is
+  // less than or equal to the limit.
+  //
+  // Currently the only download using MultiDiskAdaptor is affected by
+  // the global limit.
+  void ensureMaxOpenFileLimit(size_t numNewFile);
 };
 
 } // namespace aria2

src/usage_text.h

@@ -344,8 +344,9 @@
     "                              download speed in some cases.\n"     \
     "                              You can append K or M(1K = 1024, 1M = 1024K).")
 #define TEXT_BT_MAX_OPEN_FILES                                          \
-  _(" --bt-max-open-files=NUM      Specify maximum number of files to open in each\n" \
-    "                              BitTorrent download.")
+  _(" --bt-max-open-files=NUM      Specify maximum number of files to open in\n" \
+    "                              multi-file BitTorrent/Metalink downloads\n" \
+    "                              globally.")
 #define TEXT_BT_SEED_UNVERIFIED                                         \
   _(" --bt-seed-unverified[=true|false] Seed previously downloaded files without\n" \
     "                              verifying piece hashes.")

test/MultiDiskAdaptorTest.cc

@@ -369,7 +369,6 @@ void MultiDiskAdaptorTest::testCutTrailingGarbage()
 
   MultiDiskAdaptor adaptor;
   adaptor.setFileEntries(std::begin(fileEntries), std::end(fileEntries));
-  adaptor.setMaxOpenFiles(1);
   adaptor.setPieceLength(1);
 
   adaptor.openFile();
@@ -396,7 +395,6 @@ void MultiDiskAdaptorTest::testSize()
 
   MultiDiskAdaptor adaptor;
   adaptor.setFileEntries(std::begin(fileEntries), std::end(fileEntries));
-  adaptor.setMaxOpenFiles(1);
   adaptor.setPieceLength(1);
 
   adaptor.openFile();