2010-07-15 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

	Added --min-split-size=SIZE option.  aria2 does not split less
	than 2*SIZE byte range.  For example, let's consider downloading
	20MiB file. If SIZE is 10M, aria2 can split file into 2 range
	[0-10MiB) and [10MiB-20MiB) and download it using 2 sources(if
	--split >= 2, of course).  If SIZE is 15M, since 2*15M > 20MiB,
	aria2 does not split file and download it using 1 source.
	* src/BitfieldMan.cc
	* src/BitfieldMan.h
	* src/DefaultPieceStorage.cc
	* src/DefaultPieceStorage.h
	* src/OptionHandlerFactory.cc
	* src/RequestGroup.cc
	* src/prefs.cc
	* src/prefs.h
	* src/usage_text.h
	* test/BitfieldManTest.cc
	* test/SegmentManTest.cc

ChangeLog

@@ -1,3 +1,23 @@
+2010-07-15  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
+
+	Added --min-split-size=SIZE option.  aria2 does not split less
+	than 2*SIZE byte range.  For example, let's consider downloading
+	20MiB file. If SIZE is 10M, aria2 can split file into 2 range
+	[0-10MiB) and [10MiB-20MiB) and download it using 2 sources(if
+	--split >= 2, of course).  If SIZE is 15M, since 2*15M > 20MiB,
+	aria2 does not split file and download it using 1 source.
+	* src/BitfieldMan.cc
+	* src/BitfieldMan.h
+	* src/DefaultPieceStorage.cc
+	* src/DefaultPieceStorage.h
+	* src/OptionHandlerFactory.cc
+	* src/RequestGroup.cc
+	* src/prefs.cc
+	* src/prefs.h
+	* src/usage_text.h
+	* test/BitfieldManTest.cc
+	* test/SegmentManTest.cc
+
 2010-07-14  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
 
 	FeedbackURISelector now tries to chooses URI which is not used in

src/BitfieldMan.cc

@@ -221,37 +221,59 @@ static size_t getEndIndex(size_t index, const Array& bitfield, size_t blocks) {
 template<typename Array>
 static bool getSparseMissingUnusedIndex
 (size_t& index,
+ size_t minSplitSize,
  const Array& bitfield,
  const unsigned char* useBitfield,
+ size_t blockLength_,
  size_t blocks)
 {
   BitfieldMan::Range maxRange;
   BitfieldMan::Range currentRange;
-  {
-    size_t nextIndex = 0;
-    while(nextIndex < blocks) {
-      currentRange.startIndex =
-        getStartIndex(nextIndex, bitfield, blocks);
-      if(currentRange.startIndex == blocks) {
-        break;
-      }
-      currentRange.endIndex =
-        getEndIndex(currentRange.startIndex, bitfield, blocks);
-      if(maxRange < currentRange) {
-        maxRange = currentRange;
+  size_t nextIndex = 0;
+  while(nextIndex < blocks) {
+    currentRange.startIndex =
+      getStartIndex(nextIndex, bitfield, blocks);
+    if(currentRange.startIndex == blocks) {
+      break;
+    }
+    currentRange.endIndex =
+      getEndIndex(currentRange.startIndex, bitfield, blocks);
+
+    if(currentRange.startIndex > 0) {
+      if(bitfield::test(useBitfield, blocks, currentRange.startIndex-1)) {
+        currentRange.startIndex = currentRange.getMidIndex();
       }
-      nextIndex = currentRange.endIndex;
     }
+    // If range is equal, choose a range where its startIndex-1 is
+    // set.
+    if(maxRange < currentRange ||
+       (maxRange == currentRange &&
+        maxRange.startIndex > 0 && currentRange.startIndex > 0 &&
+        (!bitfield::test(bitfield, blocks, maxRange.startIndex-1) ||
+         bitfield::test(useBitfield, blocks, maxRange.startIndex-1))
+        &&
+        bitfield::test(bitfield, blocks, currentRange.startIndex-1) &&
+        !bitfield::test(useBitfield, blocks, currentRange.startIndex-1))) {
+      maxRange = currentRange;
+    }
+    nextIndex = currentRange.endIndex;
+      
   }
   if(maxRange.getSize()) {
     if(maxRange.startIndex == 0) {
       index = 0;
-    } else if(bitfield::test(useBitfield, blocks, maxRange.startIndex-1)) {
-      index = maxRange.getMidIndex();
+      return true;
     } else {
-      index = maxRange.startIndex;
+      if((!bitfield::test(useBitfield, blocks, maxRange.startIndex-1) &&
+          bitfield::test(bitfield, blocks, maxRange.startIndex-1)) ||
+         ((uint64_t)(maxRange.endIndex-maxRange.startIndex)*blockLength_
+          >= minSplitSize)) {
+        index = maxRange.startIndex;
+        return true;
+      } else {
+        return false;
+      }
     }
-    return true;
   } else {
     return false;
   }
@@ -259,17 +281,21 @@ static bool getSparseMissingUnusedIndex
 
 bool BitfieldMan::getSparseMissingUnusedIndex
 (size_t& index,
+ size_t minSplitSize,
  const unsigned char* ignoreBitfield,
  size_t ignoreBitfieldLength) const
 {
   if(filterEnabled_) {
     return aria2::getSparseMissingUnusedIndex
-      (index, array(ignoreBitfield)|~array(filterBitfield_)|array(bitfield_)|array(useBitfield_),
-       useBitfield_, blocks_);
+      (index, minSplitSize,
+       array(ignoreBitfield)|~array(filterBitfield_)|
+       array(bitfield_)|array(useBitfield_),
+       useBitfield_, blockLength_, blocks_);
   } else {
     return aria2::getSparseMissingUnusedIndex
-      (index, array(ignoreBitfield)|array(bitfield_)|array(useBitfield_),
-       useBitfield_, blocks_);
+      (index, minSplitSize,
+       array(ignoreBitfield)|array(bitfield_)|array(useBitfield_),
+       useBitfield_, blockLength_, blocks_);
   }
 }
 

src/BitfieldMan.h

@@ -92,6 +92,11 @@ public:
     bool operator<(const Range& range) const {
       return getSize() < range.getSize();
     }
+    
+    bool operator==(const Range& range) const
+    {
+      return getSize() == range.getSize();
+    }
   };
 public:
   BitfieldMan(size_t blockLength, uint64_t totalLength);
@@ -144,6 +149,7 @@ public:
   // affected by filter
   bool getSparseMissingUnusedIndex
   (size_t& index,
+   size_t minSplitSize,
    const unsigned char* ignoreBitfield,
    size_t ignoreBitfieldLength) const;
 

src/DefaultPieceStorage.cc

@@ -72,7 +72,8 @@ DefaultPieceStorage::DefaultPieceStorage
   logger_(LogFactory::getInstance()),
   option_(option),
   pieceStatMan_(new PieceStatMan(downloadContext->getNumPieces(), true)),
-  pieceSelector_(new RarestPieceSelector(pieceStatMan_))
+  pieceSelector_(new RarestPieceSelector(pieceStatMan_)),
+  minSplitSize_(1024*1024)
 {}
 
 DefaultPieceStorage::~DefaultPieceStorage() {
@@ -272,7 +273,8 @@ SharedHandle<Piece> DefaultPieceStorage::getSparseMissingUnusedPiece
 (const unsigned char* ignoreBitfield, size_t length)
 {
   size_t index;
-  if(bitfieldMan_->getSparseMissingUnusedIndex(index, ignoreBitfield, length)) {
+  if(bitfieldMan_->getSparseMissingUnusedIndex
+     (index, minSplitSize_, ignoreBitfield, length)) {
     return checkOutPiece(index);
   } else {
     return SharedHandle<Piece>();

src/DefaultPieceStorage.h

@@ -87,6 +87,8 @@ private:
 
   SharedHandle<PieceSelector> pieceSelector_;
 
+  size_t minSplitSize_;
+
   bool getMissingPieceIndex(size_t& index,
                             const unsigned char* bitfield, size_t length);
 
@@ -251,7 +253,11 @@ public:
   {
     return pieceSelector_;
   }
-    
+
+  void setMinSplitSize(size_t minSplitSize)
+  {
+    minSplitSize_ = minSplitSize;
+  }
 };
 
 typedef SharedHandle<DefaultPieceStorage> DefaultPieceStorageHandle;

src/OptionHandlerFactory.cc

@@ -366,6 +366,17 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
     op->addTag(TAG_HTTP);
     handlers.push_back(op);    
   }
+  {
+    SharedHandle<OptionHandler> op(new UnitNumberOptionHandler
+                                   (PREF_MIN_SPLIT_SIZE,
+                                    TEXT_MIN_SPLIT_SIZE,
+                                    "20M",
+                                    1024*1024, 1024*1024*1024));
+    op->addTag(TAG_ADVANCED);
+    op->addTag(TAG_FTP);
+    op->addTag(TAG_HTTP);
+    handlers.push_back(op);    
+  }
   {
     SharedHandle<OptionHandler> op(new BooleanOptionHandler
                                    (PREF_NO_CONF,

src/RequestGroup.cc

@@ -534,6 +534,7 @@ void RequestGroup::initPieceStorage()
     if(!diskWriterFactory_.isNull()) {
       ps->setDiskWriterFactory(diskWriterFactory_);
     }
+    ps->setMinSplitSize(option_->getAsInt(PREF_MIN_SPLIT_SIZE));
     tempPieceStorage = ps;
   } else {
     UnknownLengthPieceStorageHandle ps

src/prefs.cc

@@ -192,6 +192,8 @@ const std::string PREF_MAX_RESUME_FAILURE_TRIES("max-resume-failure-tries");
 const std::string PREF_SAVE_SESSION("save-session");
 // value: 1*digit
 const std::string PREF_MAX_CONNECTION_PER_SERVER("max-connection-per-server");
+// value: 1*digit
+const std::string PREF_MIN_SPLIT_SIZE("min-split-size");
 
 /**
  * FTP related preferences

src/prefs.h

@@ -196,6 +196,8 @@ extern const std::string PREF_MAX_RESUME_FAILURE_TRIES;
 extern const std::string PREF_SAVE_SESSION;
 // value: 1*digit
 extern const std::string PREF_MAX_CONNECTION_PER_SERVER;
+// value: 1*digit
+extern const std::string PREF_MIN_SPLIT_SIZE;
 
 /**
  * FTP related preferences

src/usage_text.h

@@ -686,3 +686,11 @@
 #define TEXT_MAX_CONNECTION_PER_SERVER          \
   _(" --max-connection-per-server=NUM The maximum number of connections to one server\n"\
     "                              for each download.")
+#define TEXT_MIN_SPLIT_SIZE                     \
+  _(" --min-split-size=SIZE        aria2 does not split less than 2*SIZE byte range.\n" \
+    "                              For example, let's consider downloading 20MiB\n" \
+    "                              file. If SIZE is 10M, aria2 can split file into 2\n" \
+    "                              range [0-10MiB) and [10MiB-20MiB) and download it\n" \
+    "                              using 2 sources(if --split >= 2, of course).\n" \
+    "                              If SIZE is 15M, since 2*15M > 20MiB, aria2 does\n" \
+    "                              not split file and download it using 1 source.")

test/BitfieldManTest.cc

@@ -24,6 +24,7 @@ class BitfieldManTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testAddNotFilter_overflow);
   CPPUNIT_TEST(testGetSparceMissingUnusedIndex);
   CPPUNIT_TEST(testGetSparceMissingUnusedIndex_setBit);
+  CPPUNIT_TEST(testGetSparceMissingUnusedIndex_withMinSplitSize);
   CPPUNIT_TEST(testIsBitSetOffsetRange);
   CPPUNIT_TEST(testGetMissingUnusedLength);
   CPPUNIT_TEST(testSetBitRange);
@@ -53,6 +54,7 @@ public:
   void testAddNotFilter_overflow();
   void testGetSparceMissingUnusedIndex();
   void testGetSparceMissingUnusedIndex_setBit();
+  void testGetSparceMissingUnusedIndex_withMinSplitSize();
   void testIsBitSetOffsetRange();
   void testGetMissingUnusedLength();
   void testSetBitRange();
@@ -280,48 +282,49 @@ void BitfieldManTest::testGetSparceMissingUnusedIndex() {
   const size_t length = 2;
   unsigned char ignoreBitfield[length];
   memset(ignoreBitfield, 0, sizeof(ignoreBitfield));
+  size_t minSplitSize = 1024*1024;
   size_t index;
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)0, index);
   bitfield.setUseBit(0);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)5, index);
   bitfield.setUseBit(5);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)3, index);
   bitfield.setUseBit(3);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)8, index);
   bitfield.setUseBit(8);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)2, index);
   bitfield.setUseBit(2);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
-                                                      ignoreBitfield, length));
-  CPPUNIT_ASSERT_EQUAL((size_t)7, index);
-  bitfield.setUseBit(7);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)1, index);
   bitfield.setUseBit(1);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)4, index);
   bitfield.setUseBit(4);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
+                                                      ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)7, index);
+  bitfield.setUseBit(7);
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)6, index);
   bitfield.setUseBit(6);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)9, index);
   bitfield.setUseBit(9);
-  CPPUNIT_ASSERT(!bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(!bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                        ignoreBitfield, length));
 }
 
@@ -330,48 +333,82 @@ void BitfieldManTest::testGetSparceMissingUnusedIndex_setBit() {
   const size_t length = 2;
   unsigned char ignoreBitfield[length];
   memset(ignoreBitfield, 0, sizeof(ignoreBitfield));
+  size_t minSplitSize = 1024*1024;
   size_t index;
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)0, index);
   bitfield.setBit(0);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)1, index);
   bitfield.setBit(1);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)2, index);
   bitfield.setBit(2);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)3, index);
   bitfield.setBit(3);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)4, index);
   bitfield.setBit(4);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)5, index);
   bitfield.setBit(5);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)6, index);
   bitfield.setBit(6);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)7, index);
   bitfield.setBit(7);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)8, index);
   bitfield.setBit(8);
-  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                       ignoreBitfield, length));
   CPPUNIT_ASSERT_EQUAL((size_t)9, index);
   bitfield.setBit(9);
-  CPPUNIT_ASSERT(!bitfield.getSparseMissingUnusedIndex(index,
+  CPPUNIT_ASSERT(!bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
+                                                       ignoreBitfield, length));
+}
+
+void BitfieldManTest::testGetSparceMissingUnusedIndex_withMinSplitSize()
+{
+  BitfieldMan bitfield(1024*1024, 10*1024*1024);
+  const size_t length = 2;
+  unsigned char ignoreBitfield[length];
+  memset(ignoreBitfield, 0, sizeof(ignoreBitfield));
+  size_t minSplitSize = 2*1024*1024;
+  size_t index;
+  bitfield.setUseBit(1);
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
+                                                      ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)6, index);
+  bitfield.setBit(6);
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
+                                                      ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)7, index);
+  bitfield.setUseBit(7);
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
+                                                      ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)4, index);
+  bitfield.setBit(4);
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
+                                                      ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)0, index);
+  bitfield.setBit(0);
+  CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
+                                                      ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)5, index);
+  bitfield.setBit(5);  
+  CPPUNIT_ASSERT(!bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
                                                        ignoreBitfield, length));
 }
 

test/SegmentManTest.cc

@@ -112,6 +112,7 @@ void SegmentManTest::testGetSegment_sameFileEntry()
   };
   dctx->setFileEntries(&fileEntries[0], &fileEntries[3]);
   SharedHandle<DefaultPieceStorage> ps(new DefaultPieceStorage(dctx, &op));
+  ps->setMinSplitSize(dctx->getPieceLength());
   SegmentMan segman(&op, dctx, ps);
 
   std::vector<SharedHandle<Segment> > segments;