2008-05-17 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

	Put piece selection strategy algorithm to RarestPieceSelector 
class,
	Added a switch to choose whether randomized selection so that 
unit
	tests emit same results in, possibly, win32.
	* src/DefaultPieceStorage.cc
	* src/DefaultPieceStorage.h
	* src/RarestPieceSelector.cc
	* src/RarestPieceSelector.h
	* test/DefaultPieceStorageTest.cc
	* test/RarestPieceSelectorTest.cc

ChangeLog

@@ -1,3 +1,15 @@
+2008-05-17  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Put piece selection strategy algorithm to RarestPieceSelector class,
+	Added a switch to choose whether randomized selection so that unit
+	tests emit same results in, possibly, win32.
+	* src/DefaultPieceStorage.cc
+	* src/DefaultPieceStorage.h
+	* src/RarestPieceSelector.cc
+	* src/RarestPieceSelector.h
+	* test/DefaultPieceStorageTest.cc
+	* test/RarestPieceSelectorTest.cc
+
 2008-05-17  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	Don't pool connection if HTTP proxy is enabled.

src/DefaultPieceStorage.cc

@@ -53,63 +53,24 @@
 #include "a2functional.h"
 #include "Option.h"
 #include "StringFormat.h"
+#include "RarestPieceSelector.h"
 #include <numeric>
 #include <algorithm>
 
 namespace aria2 {
 
-class GenPieceStat {
-private:
-  size_t _index;
-public:
-  GenPieceStat():_index(0) {}
-
-  SharedHandle<PieceStat> operator()()
-  {
-    return SharedHandle<PieceStat>(new PieceStat(_index++));
-  }
-};
-
-class PieceRarer
-{
-public:
-  bool operator()(const SharedHandle<PieceStat>& left,
-		  const SharedHandle<PieceStat>& right)
-  {
-    if(left->getCount() == right->getCount()) {
-      return left->getOrder() < right->getOrder();
-    } else {
-      return left->getCount() < right->getCount();
-    }
-  }
-};
-
-DefaultPieceStorage::DefaultPieceStorage(const DownloadContextHandle& downloadContext, const Option* option):
+DefaultPieceStorage::DefaultPieceStorage(const DownloadContextHandle& downloadContext, const Option* option, bool randomPieceStatsOrdering):
   downloadContext(downloadContext),
+  bitfieldMan(BitfieldManFactory::getFactoryInstance()->
+	      createBitfieldMan(downloadContext->getPieceLength(),
+				downloadContext->getTotalLength())),
   _diskWriterFactory(new DefaultDiskWriterFactory()),
   endGamePieceNum(END_GAME_PIECE_NUM),
+  logger(LogFactory::getInstance()),
   option(option),
-  _pieceStats(downloadContext->getNumPieces())
-{
-  bitfieldMan =
-    BitfieldManFactory::getFactoryInstance()->
-    createBitfieldMan(downloadContext->getPieceLength(),
-		      downloadContext->getTotalLength());
-
-  std::generate(_pieceStats.begin(), _pieceStats.end(), GenPieceStat());
-  _sortedPieceStats = _pieceStats;
-  // we need some randomness in ordering.
-  std::random_shuffle(_sortedPieceStats.begin(), _sortedPieceStats.end());
-  {
-    size_t order = 0;
-    for(std::deque<SharedHandle<PieceStat> >::iterator i = _sortedPieceStats.begin();
-	i != _sortedPieceStats.end(); ++i) {
-      (*i)->setOrder(order++);
-    }
-  }
-
-  logger = LogFactory::getInstance();
-}
+  _pieceSelector(new RarestPieceSelector(downloadContext->getNumPieces(),
+					 randomPieceStatsOrdering))
+{}
 
 DefaultPieceStorage::~DefaultPieceStorage() {
   delete bitfieldMan;
@@ -126,19 +87,6 @@ bool DefaultPieceStorage::isEndGame()
   return bitfieldMan->countMissingBlock() <= endGamePieceNum;
 }
 
-class FindRarestPiece
-{
-private:
-  const std::deque<size_t>& _indexes;
-public:
-  FindRarestPiece(const std::deque<size_t>& indexes):_indexes(indexes) {}
-
-  bool operator()(const SharedHandle<PieceStat>& pieceStat)
-  {
-    return std::binary_search(_indexes.begin(), _indexes.end(), pieceStat->getIndex());
-  }
-};
-
 bool DefaultPieceStorage::getMissingPieceIndex(size_t& index, const PeerHandle& peer)
 {
   std::deque<size_t> indexes;
@@ -153,11 +101,7 @@ bool DefaultPieceStorage::getMissingPieceIndex(size_t& index, const PeerHandle&
   }
   if(r) {
     // We assume indexes is sorted using comparator less.
-    //std::sort(indexes.begin(), indexes.end());
-    std::deque<SharedHandle<PieceStat> >::const_iterator i =
-      std::find_if(_sortedPieceStats.begin(), _sortedPieceStats.end(),
-		   FindRarestPiece(indexes));
-    index = (*i)->getIndex();
+    _pieceSelector->select(index, indexes);
     return true;
   } else {
     return false;
@@ -647,113 +591,26 @@ void DefaultPieceStorage::setDiskWriterFactory(const DiskWriterFactoryHandle& di
 void DefaultPieceStorage::addPieceStats(const unsigned char* bitfield,
 					size_t bitfieldLength)
 {
-  size_t index = 0;
-  for(size_t bi = 0; bi < bitfieldLength; ++bi) {
-    
-    for(size_t i = 0; i < 8; ++i, ++index) {
-      unsigned char mask = 128 >> i;
-      if(bitfield[bi]&mask) {
-	_pieceStats[index]->addCount();
-      }
-    }
-
-  }
-  std::sort(_sortedPieceStats.begin(), _sortedPieceStats.end(), PieceRarer());
+  _pieceSelector->addPieceStats(bitfield, bitfieldLength);
 }
 
 void DefaultPieceStorage::subtractPieceStats(const unsigned char* bitfield,
 					     size_t bitfieldLength)
 {
-  size_t index = 0;
-  for(size_t bi = 0; bi < bitfieldLength; ++bi) {
-    
-    for(size_t i = 0; i < 8; ++i, ++index) {
-      unsigned char mask = 128 >> i;
-      if(bitfield[bi]&mask) {
-	_pieceStats[index]->subCount();
-      }
-    }
-
-  }
-  std::sort(_sortedPieceStats.begin(), _sortedPieceStats.end(), PieceRarer());
+  _pieceSelector->subtractPieceStats(bitfield, bitfieldLength);
 }
 
 void DefaultPieceStorage::updatePieceStats(const unsigned char* newBitfield,
 					   size_t newBitfieldLength,
 					   const unsigned char* oldBitfield)
 {
-  size_t index = 0;
-  for(size_t bi = 0; bi < newBitfieldLength; ++bi) {
-    
-    for(size_t i = 0; i < 8; ++i, ++index) {
-      unsigned char mask = 128 >> i;
-      if((newBitfield[bi]&mask) && !(oldBitfield[bi]&mask)) {
-	_pieceStats[index]->addCount();
-      } else if(!(newBitfield[bi]&mask) && (oldBitfield[bi]&mask)) {
-	_pieceStats[index]->subCount();
-      }
-    }
-
-  }
-  std::sort(_sortedPieceStats.begin(), _sortedPieceStats.end(), PieceRarer());
+  _pieceSelector->updatePieceStats(newBitfield, newBitfieldLength,
+				   oldBitfield);
 }
 
 void DefaultPieceStorage::addPieceStats(size_t index)
 {
-  SharedHandle<PieceStat> pieceStat(_pieceStats[index]);
-  {
-    std::deque<SharedHandle<PieceStat> >::iterator cur =
-      std::lower_bound(_sortedPieceStats.begin(), _sortedPieceStats.end(),
-		       pieceStat, PieceRarer());
-    _sortedPieceStats.erase(cur);
-  }
-  pieceStat->addCount();
-
-  std::deque<SharedHandle<PieceStat> >::iterator to =
-    std::lower_bound(_sortedPieceStats.begin(), _sortedPieceStats.end(),
-		     pieceStat, PieceRarer());
-
-  _sortedPieceStats.insert(to, pieceStat);
-
-//    for(std::deque<SharedHandle<PieceStat> >::const_iterator i = _sortedPieceStats.begin(); i != _sortedPieceStats.end(); ++i) {
-//      logger->debug("index = %u, count = %u", (*i)->getIndex(), (*i)->getCount());
-//    }
-}
-
-PieceStat::PieceStat(size_t index):_order(0), _index(index), _count(0) {}
-
-void PieceStat::addCount()
-{
-  if(_count < SIZE_MAX) {
-    ++_count;
-  }
-}
-
-void PieceStat::subCount()
-{
-  if(_count > 0) {
-    --_count;
-  }
-}
-
-size_t PieceStat::getIndex() const
-{
-  return _index;
-}
-
-size_t PieceStat::getCount() const
-{
-  return _count;
-}
-
-void PieceStat::setOrder(size_t order)
-{
-  _order = order;
-}
-
-size_t PieceStat::getOrder() const
-{
-  return _order;
+  _pieceSelector->addPieceStats(index);
 }
 
 } // namespace aria2

src/DefaultPieceStorage.h

@@ -45,6 +45,7 @@ class Logger;
 class Option;
 class DiskWriterFactory;
 class FileEntry;
+class RarestPieceSelector;
 
 #define END_GAME_PIECE_NUM 20
 
@@ -67,23 +68,6 @@ public:
 
 typedef std::deque<HaveEntry> Haves;
 
-class PieceStat {
-private:
-  size_t _order;
-  size_t _index;
-  size_t _count;
-public:
-  PieceStat(size_t index);
-
-  void addCount();
-  void subCount();
-
-  size_t getOrder() const;
-  void setOrder(size_t order);
-  size_t getIndex() const;
-  size_t getCount() const;
-};
-
 class DefaultPieceStorage : public PieceStorage {
 private:
   SharedHandle<DownloadContext> downloadContext;
@@ -97,9 +81,8 @@ private:
   const Option* option;
   Haves haves;
 
-  std::deque<SharedHandle<PieceStat> > _pieceStats;
-  std::deque<SharedHandle<PieceStat> > _sortedPieceStats;
-  
+  SharedHandle<RarestPieceSelector> _pieceSelector;
+
   bool getMissingPieceIndex(size_t& index, const SharedHandle<Peer>& peer);
   bool getMissingFastPieceIndex(size_t& index, const SharedHandle<Peer>& peer);
   SharedHandle<Piece> checkOutPiece(size_t index);
@@ -111,7 +94,13 @@ private:
   size_t getInFlightPieceCompletedLength() const;
 
 public:
-  DefaultPieceStorage(const SharedHandle<DownloadContext>& downloadContext, const Option* option);
+  // Setting randomPieceStatsOrdering to true means a piece is chosen in
+  // random when more than 2 pieces has the same rarity.
+  // If it is set to false, a piece whose index is smallest has the highest
+  // priority.
+  DefaultPieceStorage(const SharedHandle<DownloadContext>& downloadContext,
+		      const Option* option,
+		      bool randomPieceStatsOrdering = true);
   virtual ~DefaultPieceStorage();
 
   virtual bool hasMissingPiece(const SharedHandle<Peer>& peer);

src/Makefile.am

@@ -187,7 +187,8 @@ SRCS =  Socket.h\
 	HttpSkipResponseCommand.cc HttpSkipResponseCommand.h\
 	InitiateConnectionCommand.cc InitiateConnectionCommand.h\
 	FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h\
-	A2STR.cc A2STR.h
+	A2STR.cc A2STR.h\
+	RarestPieceSelector.cc RarestPieceSelector.h
 
 if ENABLE_ASYNC_DNS
 SRCS += AsyncNameResolver.cc AsyncNameResolver.h

src/Makefile.in

@@ -409,7 +409,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
 	HttpSkipResponseCommand.cc HttpSkipResponseCommand.h \
 	InitiateConnectionCommand.cc InitiateConnectionCommand.h \
 	FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h \
-	A2STR.cc A2STR.h AsyncNameResolver.cc AsyncNameResolver.h \
+	A2STR.cc A2STR.h RarestPieceSelector.cc RarestPieceSelector.h \
+	AsyncNameResolver.cc AsyncNameResolver.h \
 	IteratableChunkChecksumValidator.cc \
 	IteratableChunkChecksumValidator.h \
 	IteratableChecksumValidator.cc IteratableChecksumValidator.h \
@@ -793,11 +794,12 @@ am__objects_15 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \
 	StringFormat.$(OBJEXT) HttpSkipResponseCommand.$(OBJEXT) \
 	InitiateConnectionCommand.$(OBJEXT) \
 	FtpFinishDownloadCommand.$(OBJEXT) A2STR.$(OBJEXT) \
-	$(am__objects_1) $(am__objects_2) $(am__objects_3) \
-	$(am__objects_4) $(am__objects_5) $(am__objects_6) \
-	$(am__objects_7) $(am__objects_8) $(am__objects_9) \
-	$(am__objects_10) $(am__objects_11) $(am__objects_12) \
-	$(am__objects_13) $(am__objects_14)
+	RarestPieceSelector.$(OBJEXT) $(am__objects_1) \
+	$(am__objects_2) $(am__objects_3) $(am__objects_4) \
+	$(am__objects_5) $(am__objects_6) $(am__objects_7) \
+	$(am__objects_8) $(am__objects_9) $(am__objects_10) \
+	$(am__objects_11) $(am__objects_12) $(am__objects_13) \
+	$(am__objects_14)
 am_libaria2c_a_OBJECTS = $(am__objects_15)
 libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS)
 am__installdirs = "$(DESTDIR)$(bindir)"
@@ -1136,11 +1138,12 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \
 	HttpSkipResponseCommand.cc HttpSkipResponseCommand.h \
 	InitiateConnectionCommand.cc InitiateConnectionCommand.h \
 	FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h \
-	A2STR.cc A2STR.h $(am__append_1) $(am__append_2) \
-	$(am__append_3) $(am__append_4) $(am__append_5) \
-	$(am__append_6) $(am__append_7) $(am__append_8) \
-	$(am__append_9) $(am__append_10) $(am__append_11) \
-	$(am__append_12) $(am__append_13) $(am__append_14)
+	A2STR.cc A2STR.h RarestPieceSelector.cc RarestPieceSelector.h \
+	$(am__append_1) $(am__append_2) $(am__append_3) \
+	$(am__append_4) $(am__append_5) $(am__append_6) \
+	$(am__append_7) $(am__append_8) $(am__append_9) \
+	$(am__append_10) $(am__append_11) $(am__append_12) \
+	$(am__append_13) $(am__append_14)
 noinst_LIBRARIES = libaria2c.a
 libaria2c_a_SOURCES = $(SRCS)
 aria2c_LDADD = libaria2c.a @LIBINTL@ @ALLOCA@ @LIBGNUTLS_LIBS@\
@@ -1459,6 +1462,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PiecesMetalinkParserState.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Platform.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ProtocolDetector.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RarestPieceSelector.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RealtimeCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ReceiverMSEHandshakeCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Request.Po@am__quote@

src/RarestPieceSelector.cc

@@ -0,0 +1,221 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#include "RarestPieceSelector.h"
+#include <algorithm>
+
+namespace aria2 {
+
+PieceStat::PieceStat(size_t index):_order(0), _index(index), _count(0) {}
+
+bool PieceStat::operator<(const PieceStat& pieceStat) const
+{
+  if(_count == pieceStat._count) {
+    return _order < pieceStat._order;
+  } else {
+    return _count < pieceStat._count;
+  }
+}
+
+void PieceStat::addCount()
+{
+  if(_count < SIZE_MAX) {
+    ++_count;
+  }
+}
+
+void PieceStat::subCount()
+{
+  if(_count > 0) {
+    --_count;
+  }
+}
+
+size_t PieceStat::getIndex() const
+{
+  return _index;
+}
+
+size_t PieceStat::getCount() const
+{
+  return _count;
+}
+
+void PieceStat::setOrder(size_t order)
+{
+  _order = order;
+}
+
+size_t PieceStat::getOrder() const
+{
+  return _order;
+}
+
+class GenPieceStat {
+private:
+  size_t _index;
+public:
+  GenPieceStat():_index(0) {}
+
+  SharedHandle<PieceStat> operator()()
+  {
+    return SharedHandle<PieceStat>(new PieceStat(_index++));
+  }
+};
+
+RarestPieceSelector::RarestPieceSelector(size_t pieceNum, bool randomShuffle):
+  _pieceStats(pieceNum)
+{
+  std::generate(_pieceStats.begin(), _pieceStats.end(), GenPieceStat());
+  _sortedPieceStats = _pieceStats;
+  // we need some randomness in ordering.
+  if(randomShuffle) {
+    std::random_shuffle(_sortedPieceStats.begin(), _sortedPieceStats.end());
+  }
+  {
+    size_t order = 0;
+    for(std::deque<SharedHandle<PieceStat> >::iterator i = _sortedPieceStats.begin();
+	i != _sortedPieceStats.end(); ++i) {
+      (*i)->setOrder(order++);
+    }
+  }
+}
+
+class FindRarestPiece
+{
+private:
+  const std::deque<size_t>& _indexes;
+public:
+  FindRarestPiece(const std::deque<size_t>& indexes):_indexes(indexes) {}
+
+  bool operator()(const SharedHandle<PieceStat>& pieceStat)
+  {
+    return std::binary_search(_indexes.begin(), _indexes.end(), pieceStat->getIndex());
+  }
+};
+
+bool RarestPieceSelector::select
+(size_t& index,
+ const std::deque<size_t>& candidateIndexes) const
+{
+  std::deque<SharedHandle<PieceStat> >::const_iterator i =
+    std::find_if(_sortedPieceStats.begin(), _sortedPieceStats.end(),
+		 FindRarestPiece(candidateIndexes));
+  if(i == _sortedPieceStats.end()) {
+    return false;
+  } else {
+    index = (*i)->getIndex();
+    return true;
+  }
+}
+
+void RarestPieceSelector::addPieceStats(const unsigned char* bitfield,
+					size_t bitfieldLength)
+{
+  size_t index = 0;
+  for(size_t bi = 0; bi < bitfieldLength; ++bi) {
+    
+    for(size_t i = 0; i < 8; ++i, ++index) {
+      unsigned char mask = 128 >> i;
+      if(bitfield[bi]&mask) {
+	_pieceStats[index]->addCount();
+      }
+    }
+
+  }
+  std::sort(_sortedPieceStats.begin(), _sortedPieceStats.end());
+}
+
+void RarestPieceSelector::subtractPieceStats(const unsigned char* bitfield,
+					     size_t bitfieldLength)
+{
+  size_t index = 0;
+  for(size_t bi = 0; bi < bitfieldLength; ++bi) {
+    
+    for(size_t i = 0; i < 8; ++i, ++index) {
+      unsigned char mask = 128 >> i;
+      if(bitfield[bi]&mask) {
+	_pieceStats[index]->subCount();
+      }
+    }
+
+  }
+  std::sort(_sortedPieceStats.begin(), _sortedPieceStats.end());
+}
+
+void RarestPieceSelector::updatePieceStats(const unsigned char* newBitfield,
+					   size_t newBitfieldLength,
+					   const unsigned char* oldBitfield)
+{
+  size_t index = 0;
+  for(size_t bi = 0; bi < newBitfieldLength; ++bi) {
+    
+    for(size_t i = 0; i < 8; ++i, ++index) {
+      unsigned char mask = 128 >> i;
+      if((newBitfield[bi]&mask) && !(oldBitfield[bi]&mask)) {
+	_pieceStats[index]->addCount();
+      } else if(!(newBitfield[bi]&mask) && (oldBitfield[bi]&mask)) {
+	_pieceStats[index]->subCount();
+      }
+    }
+
+  }
+  std::sort(_sortedPieceStats.begin(), _sortedPieceStats.end());
+}
+
+void RarestPieceSelector::addPieceStats(size_t index)
+{
+  SharedHandle<PieceStat> pieceStat(_pieceStats[index]);
+  {
+    std::deque<SharedHandle<PieceStat> >::iterator cur =
+      std::lower_bound(_sortedPieceStats.begin(), _sortedPieceStats.end(),
+		       pieceStat);
+    _sortedPieceStats.erase(cur);
+  }
+  pieceStat->addCount();
+
+  std::deque<SharedHandle<PieceStat> >::iterator to =
+    std::lower_bound(_sortedPieceStats.begin(), _sortedPieceStats.end(),
+		     pieceStat);
+
+  _sortedPieceStats.insert(to, pieceStat);
+}
+
+const std::deque<SharedHandle<PieceStat> >&
+RarestPieceSelector::getSortedPieceStats() const
+{
+  return _sortedPieceStats;
+}
+
+} // namespace aria2

src/RarestPieceSelector.h

@@ -0,0 +1,91 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef _D_RAREST_PIECE_SELECTOR_H_
+#define _D_RAREST_PIECE_SELECTOR_H_
+
+#include "common.h"
+#include "SharedHandle.h"
+#include <deque>
+
+namespace aria2 {
+
+class PieceStat {
+private:
+  size_t _order;
+  size_t _index;
+  size_t _count;
+public:
+  PieceStat(size_t index);
+
+  bool operator<(const PieceStat& pieceStat) const;
+
+  void addCount();
+  void subCount();
+
+  size_t getOrder() const;
+  void setOrder(size_t order);
+  size_t getIndex() const;
+  size_t getCount() const;
+};
+
+class RarestPieceSelector {
+private:
+  std::deque<SharedHandle<PieceStat> > _pieceStats;
+
+  std::deque<SharedHandle<PieceStat> > _sortedPieceStats;
+public:
+  RarestPieceSelector(size_t pieceNum, bool randomShuffle);
+
+  bool select(size_t& index, const std::deque<size_t>& candidateIndexes) const;
+
+  void addPieceStats(size_t index);
+
+  void addPieceStats(const unsigned char* bitfield,
+		     size_t bitfieldLength);
+  
+  void subtractPieceStats(const unsigned char* bitfield,
+			  size_t bitfieldLength);
+
+  void updatePieceStats(const unsigned char* newBitfield,
+			size_t newBitfieldLength,
+			const unsigned char* oldBitfield);
+
+  const std::deque<SharedHandle<PieceStat> >& getSortedPieceStats() const;
+};
+
+} // namespace aria2
+
+#endif // _D_RAREST_PIECE_SELECTOR_H_
+

test/DefaultPieceStorageTest.cc

@@ -76,20 +76,19 @@ void DefaultPieceStorageTest::testGetTotalLength() {
 }
 
 void DefaultPieceStorageTest::testGetMissingPiece() {
-  DefaultPieceStorage pss(btContext, option);
+  DefaultPieceStorage pss(btContext, option, false);
   pss.setEndGamePieceNum(0);
 
   peer->setAllBitfield();
-  // TODO the ordering of piece may vary depending on the system, so the test
-  // may fail.
+
   SharedHandle<Piece> piece = pss.getMissingPiece(peer);
-  CPPUNIT_ASSERT_EQUAL(std::string("piece: index=2, length=128"),
+  CPPUNIT_ASSERT_EQUAL(std::string("piece: index=0, length=128"),
 		       piece->toString());
   piece = pss.getMissingPiece(peer);
   CPPUNIT_ASSERT_EQUAL(std::string("piece: index=1, length=128"),
 		       piece->toString());
   piece = pss.getMissingPiece(peer);
-  CPPUNIT_ASSERT_EQUAL(std::string("piece: index=0, length=128"),
+  CPPUNIT_ASSERT_EQUAL(std::string("piece: index=2, length=128"),
 		       piece->toString());
   piece = pss.getMissingPiece(peer);
   CPPUNIT_ASSERT(piece.isNull());
@@ -119,15 +118,13 @@ void DefaultPieceStorageTest::testHasMissingPiece() {
 }
 
 void DefaultPieceStorageTest::testCompletePiece() {
-  DefaultPieceStorage pss(btContext, option);
+  DefaultPieceStorage pss(btContext, option, true);
   pss.setEndGamePieceNum(0);
 
   peer->setAllBitfield();
 
-  // TODO the ordering of piece may vary depending on the system, so the test
-  // may fail.
   SharedHandle<Piece> piece = pss.getMissingPiece(peer);
-  CPPUNIT_ASSERT_EQUAL(std::string("piece: index=2, length=128"),
+  CPPUNIT_ASSERT_EQUAL(std::string("piece: index=0, length=128"),
 		       piece->toString());
 
   CPPUNIT_ASSERT_EQUAL(0ULL, pss.getCompletedLength());

test/RarestPieceSelectorTest.cc

@@ -0,0 +1,179 @@
+#include "RarestPieceSelector.h"
+#include "Exception.h"
+#include "Util.h"
+#include <cstring>
+#include <iostream>
+#include <cppunit/extensions/HelperMacros.h>
+
+namespace aria2 {
+
+class RarestPieceSelectorTest:public CppUnit::TestFixture {
+
+  CPPUNIT_TEST_SUITE(RarestPieceSelectorTest);
+  CPPUNIT_TEST(testAddPieceStats_index);
+  CPPUNIT_TEST(testAddPieceStats_bitfield);
+  CPPUNIT_TEST(testUpdatePieceStats);
+  CPPUNIT_TEST(testSubtractPieceStats);
+  CPPUNIT_TEST_SUITE_END();
+public:
+  void setUp() {}
+
+  void tearDown() {}
+
+  void testAddPieceStats_index();
+  void testAddPieceStats_bitfield();
+  void testUpdatePieceStats();
+  void testSubtractPieceStats();
+};
+
+
+CPPUNIT_TEST_SUITE_REGISTRATION(RarestPieceSelectorTest);
+
+void RarestPieceSelectorTest::testAddPieceStats_index()
+{
+  RarestPieceSelector selector(10, false);
+  selector.addPieceStats(1);
+  {
+    size_t indexes[] = { 0, 2, 3, 4, 5, 6, 7, 8, 9, 1 };
+    size_t counts[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 };
+    
+    const std::deque<SharedHandle<PieceStat> >& stats(selector.getSortedPieceStats());
+    CPPUNIT_ASSERT_EQUAL((size_t)10, stats.size());
+    
+    for(size_t i = 0; i < 10; ++i) {
+      CPPUNIT_ASSERT_EQUAL(indexes[i], stats[i]->getIndex());
+      CPPUNIT_ASSERT_EQUAL(counts[i], stats[i]->getCount());
+    }
+  }
+
+  selector.addPieceStats(1);
+
+  {
+    size_t indexes[] = { 0, 2, 3, 4, 5, 6, 7, 8, 9, 1 };
+    size_t counts[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 };
+
+    const std::deque<SharedHandle<PieceStat> >& stats(selector.getSortedPieceStats());
+
+    for(size_t i = 0; i < 10; ++i) {
+      CPPUNIT_ASSERT_EQUAL(indexes[i], stats[i]->getIndex());
+      CPPUNIT_ASSERT_EQUAL(counts[i], stats[i]->getCount());
+    }
+  }
+
+  selector.addPieceStats(3);
+  selector.addPieceStats(9);
+  selector.addPieceStats(3);
+  selector.addPieceStats(0);
+
+  {
+    size_t indexes[] = { 2, 4, 5, 6, 7, 8, 0, 9, 1, 3 };
+    size_t counts[] = {  0, 0, 0, 0, 0, 0, 1, 1, 2, 2 };
+
+    const std::deque<SharedHandle<PieceStat> >& stats(selector.getSortedPieceStats());
+
+    for(size_t i = 0; i < 10; ++i) {
+      CPPUNIT_ASSERT_EQUAL(indexes[i], stats[i]->getIndex());
+      CPPUNIT_ASSERT_EQUAL(counts[i], stats[i]->getCount());
+    }
+  }
+
+}
+
+void RarestPieceSelectorTest::testAddPieceStats_bitfield()
+{
+  RarestPieceSelector selector(10, false);
+  const unsigned char bitfield[] = { 0xaa, 0x80 };
+  selector.addPieceStats(bitfield, sizeof(bitfield));
+  {
+    size_t indexes[] = { 1, 3, 5, 7, 9, 0, 2, 4, 6, 8 };
+    size_t counts[] = { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 };
+
+    const std::deque<SharedHandle<PieceStat> >& stats(selector.getSortedPieceStats());
+    CPPUNIT_ASSERT_EQUAL((size_t)10, stats.size());
+    
+    for(size_t i = 0; i < 10; ++i) {
+      CPPUNIT_ASSERT_EQUAL(indexes[i], stats[i]->getIndex());
+      CPPUNIT_ASSERT_EQUAL(counts[i], stats[i]->getCount());
+    }
+  }
+
+  selector.addPieceStats(bitfield, sizeof(bitfield));
+
+  {
+    size_t indexes[] = { 1, 3, 5, 7, 9, 0, 2, 4, 6, 8 };
+    size_t counts[] = { 0, 0, 0, 0, 0, 2, 2, 2, 2, 2 };
+
+    const std::deque<SharedHandle<PieceStat> >& stats(selector.getSortedPieceStats());
+    CPPUNIT_ASSERT_EQUAL((size_t)10, stats.size());
+    
+    for(size_t i = 0; i < 10; ++i) {
+      CPPUNIT_ASSERT_EQUAL(indexes[i], stats[i]->getIndex());
+      CPPUNIT_ASSERT_EQUAL(counts[i], stats[i]->getCount());
+    }
+  }
+}
+
+void RarestPieceSelectorTest::testUpdatePieceStats()
+{
+  RarestPieceSelector selector(10, false);
+
+  const unsigned char bitfield[] = { 0xff, 0xc0 };
+  selector.addPieceStats(bitfield, sizeof(bitfield));
+
+  const unsigned char oldBitfield[] = { 0xf0, 0x00 };
+  const unsigned char newBitfield[] = { 0x1f, 0x00 };
+
+  selector.updatePieceStats(newBitfield, sizeof(newBitfield), oldBitfield);
+  {
+    // idx: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
+    // bf : 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
+    // old: 1, 1, 1, 1, 0, 0, 0, 0, 0, 0
+    // new: 0, 0, 0, 1, 1, 1, 1, 1, 0, 0
+    // ---------------------------------
+    // res: 0, 0, 0, 1, 2, 2, 2, 2, 1, 1
+
+    size_t indexes[] = { 0, 1, 2, 3, 8, 9, 4, 5, 6, 7 };
+    size_t counts[] =  { 0, 0, 0, 1, 1, 1, 2, 2, 2, 2 };
+
+    const std::deque<SharedHandle<PieceStat> >& stats(selector.getSortedPieceStats());
+    CPPUNIT_ASSERT_EQUAL((size_t)10, stats.size());
+    
+    for(size_t i = 0; i < 10; ++i) {
+      CPPUNIT_ASSERT_EQUAL(indexes[i], stats[i]->getIndex());
+      CPPUNIT_ASSERT_EQUAL(counts[i], stats[i]->getCount());
+    }
+  }
+}
+
+void RarestPieceSelectorTest::testSubtractPieceStats()
+{
+  RarestPieceSelector selector(10, false);
+
+  const unsigned char bitfield[] = { 0xf0, 0x00 };
+  selector.addPieceStats(bitfield, sizeof(bitfield));
+
+  const unsigned char newBitfield[] = { 0x3f, 0x00 };
+
+  selector.subtractPieceStats(newBitfield, sizeof(newBitfield));
+  {
+    // idx: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
+    // bf : 1, 1, 1, 1, 0, 0, 0, 0, 0, 0
+    // new: 0, 0, 1, 1, 1, 1, 1, 1, 0, 0
+    // ---------------------------------
+    // res: 1, 1, 0, 0, 0, 0, 0, 0, 0, 0
+
+    size_t indexes[] = { 2, 3, 4, 5, 6, 7, 8, 9, 0, 1 };
+    size_t counts[] =  { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1 };
+
+    const std::deque<SharedHandle<PieceStat> >& stats(selector.getSortedPieceStats());
+    CPPUNIT_ASSERT_EQUAL((size_t)10, stats.size());
+    
+    for(size_t i = 0; i < 10; ++i) {
+      CPPUNIT_ASSERT_EQUAL(indexes[i], stats[i]->getIndex());
+      CPPUNIT_ASSERT_EQUAL(counts[i], stats[i]->getCount());
+    }
+  }
+}
+
+
+} // namespace aria2