2009-06-24 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

	Added experimental support of WEB-Seeding for multi-file torrent.
	Due to fundamental changes in file handling in HTTP/FTP code, many
	functions are not working: PeerStat, ServerHost, proxy..etc
	* src/AbstractCommand.cc
	* src/AbstractCommand.h
	* src/BitfieldMan.cc
	* src/BitfieldMan.h
	* src/CreateRequestCommand.cc
	* src/CreateRequestCommand.h
	* src/DefaultPieceStorage.cc
	* src/DefaultPieceStorage.h
	* src/DownloadCommand.cc
	* src/DownloadCommand.h
	* src/DownloadContext.cc
	* src/DownloadContext.h
	* src/FileEntry.cc
	* src/FileEntry.h
	* src/FtpDownloadCommand.cc
	* src/FtpDownloadCommand.h
	* src/FtpFinishDownloadCommand.cc
	* src/FtpFinishDownloadCommand.h
	* src/FtpInitiateConnectionCommand.cc
	* src/FtpInitiateConnectionCommand.h
	* src/FtpNegotiationCommand.cc
	* src/FtpNegotiationCommand.h
	* src/FtpTunnelResponseCommand.cc
	* src/HttpDownloadCommand.cc
	* src/HttpDownloadCommand.h
	* src/HttpInitiateConnectionCommand.cc
	* src/HttpInitiateConnectionCommand.h
	* src/HttpProxyResponseCommand.cc
	* src/HttpRequest.cc
	* src/HttpRequest.h
	* src/HttpRequestCommand.cc
	* src/HttpRequestCommand.h
	* src/HttpResponseCommand.cc
	* src/HttpResponseCommand.h
	* src/HttpSkipResponseCommand.cc
	* src/HttpSkipResponseCommand.h
	* src/InitiateConnectionCommand.cc
	* src/InitiateConnectionCommand.h
	* src/InitiateConnectionCommandFactory.cc
	* src/InitiateConnectionCommandFactory.h
	* src/Makefile.am
	* src/PieceStorage.h
	* src/RequestGroup.cc
	* src/RequestGroup.h
	* src/RequestGroupMan.cc
	* src/SegmentMan.cc
	* src/SegmentMan.h
	* src/SingleFileDownloadContext.h
	* src/StreamFileAllocationEntry.cc
	* src/TrackerWatcherCommand.cc
	* src/UnknownLengthPieceStorage.cc
	* src/UnknownLengthPieceStorage.h
	* src/array_fun.h
	* src/bitfield.h
	* src/download_helper.cc
	* test/DownloadContextTest.cc
	* test/Makefile.am
	* test/MockDownloadContext.h

ChangeLog

@@ -1,3 +1,67 @@
+2009-06-24  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
+
+	Added experimental support of WEB-Seeding for multi-file torrent.
+	Due to fundamental changes in file handling in HTTP/FTP code, many
+	functions are not working: PeerStat, ServerHost, proxy..etc
+	* src/AbstractCommand.cc
+	* src/AbstractCommand.h
+	* src/BitfieldMan.cc
+	* src/BitfieldMan.h
+	* src/CreateRequestCommand.cc
+	* src/CreateRequestCommand.h
+	* src/DefaultPieceStorage.cc
+	* src/DefaultPieceStorage.h
+	* src/DownloadCommand.cc
+	* src/DownloadCommand.h
+	* src/DownloadContext.cc
+	* src/DownloadContext.h
+	* src/FileEntry.cc
+	* src/FileEntry.h
+	* src/FtpDownloadCommand.cc
+	* src/FtpDownloadCommand.h
+	* src/FtpFinishDownloadCommand.cc
+	* src/FtpFinishDownloadCommand.h
+	* src/FtpInitiateConnectionCommand.cc
+	* src/FtpInitiateConnectionCommand.h
+	* src/FtpNegotiationCommand.cc
+	* src/FtpNegotiationCommand.h
+	* src/FtpTunnelResponseCommand.cc
+	* src/HttpDownloadCommand.cc
+	* src/HttpDownloadCommand.h
+	* src/HttpInitiateConnectionCommand.cc
+	* src/HttpInitiateConnectionCommand.h
+	* src/HttpProxyResponseCommand.cc
+	* src/HttpRequest.cc
+	* src/HttpRequest.h
+	* src/HttpRequestCommand.cc
+	* src/HttpRequestCommand.h
+	* src/HttpResponseCommand.cc
+	* src/HttpResponseCommand.h
+	* src/HttpSkipResponseCommand.cc
+	* src/HttpSkipResponseCommand.h
+	* src/InitiateConnectionCommand.cc
+	* src/InitiateConnectionCommand.h
+	* src/InitiateConnectionCommandFactory.cc
+	* src/InitiateConnectionCommandFactory.h
+	* src/Makefile.am
+	* src/PieceStorage.h
+	* src/RequestGroup.cc
+	* src/RequestGroup.h
+	* src/RequestGroupMan.cc
+	* src/SegmentMan.cc
+	* src/SegmentMan.h
+	* src/SingleFileDownloadContext.h
+	* src/StreamFileAllocationEntry.cc
+	* src/TrackerWatcherCommand.cc
+	* src/UnknownLengthPieceStorage.cc
+	* src/UnknownLengthPieceStorage.h
+	* src/array_fun.h
+	* src/bitfield.h
+	* src/download_helper.cc
+	* test/DownloadContextTest.cc
+	* test/Makefile.am
+	* test/MockDownloadContext.h
+
 2009-06-24  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
 
 	Added tellWaiting XML-RPC method.

src/AbstractCommand.cc

@@ -47,7 +47,7 @@
 #include "DlAbortEx.h"
 #include "DlRetryEx.h"
 #include "DownloadFailureException.h"
-#include "InitiateConnectionCommandFactory.h"
+#include "CreateRequestCommand.h"
 #include "SleepCommand.h"
 #ifdef ENABLE_ASYNC_DNS
 #include "AsyncNameResolver.h"
@@ -62,9 +62,12 @@
 #include "RequestGroupMan.h"
 #include "A2STR.h"
 #include "Util.h"
+#include "LogFactory.h"
+#include "DownloadContext.h"
 
 namespace aria2 {
 
+// TODO1.5 Remove this
 AbstractCommand::AbstractCommand(int32_t cuid,
 				 const SharedHandle<Request>& req,
 				 RequestGroup* requestGroup,
@@ -83,6 +86,25 @@ AbstractCommand::AbstractCommand(int32_t cuid,
   _requestGroup->increaseNumCommand();
 }
 
+AbstractCommand::AbstractCommand(int32_t cuid,
+				 const SharedHandle<Request>& req,
+				 const SharedHandle<FileEntry>& fileEntry,
+				 RequestGroup* requestGroup,
+				 DownloadEngine* e,
+				 const SocketHandle& s):
+  Command(cuid), _requestGroup(requestGroup),
+  req(req), _fileEntry(fileEntry), e(e), socket(s),
+  checkSocketIsReadable(false), checkSocketIsWritable(false),
+  nameResolverCheck(false)
+{
+  if(!socket.isNull() && socket->isOpen()) {
+    setReadCheckSocket(socket);
+  }
+  timeout = _requestGroup->getTimeout();
+  _requestGroup->increaseStreamConnection();
+  _requestGroup->increaseNumCommand();
+}
+
 AbstractCommand::~AbstractCommand() {
   disableReadCheckSocket();
   disableWriteCheckSocket();
@@ -125,7 +147,8 @@ bool AbstractCommand::execute() {
       if(!_requestGroup->getPieceStorage().isNull()) {
 	_segments.clear();
 	_requestGroup->getSegmentMan()->getInFlightSegment(_segments, cuid);
-	while(_segments.size() < req->getMaxPipelinedRequest()) {
+	size_t maxSegments = req.isNull()?1:req->getMaxPipelinedRequest();
+	while(_segments.size() < maxSegments) {
 	  SegmentHandle segment = _requestGroup->getSegmentMan()->getSegment(cuid);
 	  if(segment.isNull()) {
 	    break;
@@ -135,7 +158,7 @@ bool AbstractCommand::execute() {
 	if(_segments.empty()) {
 	  // TODO socket could be pooled here if pipelining is enabled...
 	  logger->info(MSG_NO_SEGMENT_AVAILABLE, cuid);
-	  return prepareForRetry(1);
+	  return true;
 	}
       }
       return executeInternal();
@@ -146,6 +169,7 @@ bool AbstractCommand::execute() {
     } else {
       if(checkPoint.elapsed(timeout)) {
 	// timeout triggers ServerStat error state.
+
 	SharedHandle<ServerStat> ss =
 	  e->_requestGroupMan->getOrCreateServerStat(req->getHost(),
 						     req->getProtocol());
@@ -157,13 +181,18 @@ bool AbstractCommand::execute() {
       return false;
     }
   } catch(DlAbortEx& err) {
-    logger->error(MSG_DOWNLOAD_ABORTED,
-		  DL_ABORT_EX2(StringFormat
-			      ("URI=%s", req->getCurrentUrl().c_str()).str(),err),
-		  cuid, req->getUrl().c_str());
-    _requestGroup->addURIResult(req->getUrl(), err.getCode());
+    if(req.isNull()) {
+      logger->debug(EX_EXCEPTION_CAUGHT, err);
+    } else {
+      logger->error(MSG_DOWNLOAD_ABORTED,
+		    DL_ABORT_EX2(StringFormat
+				 ("URI=%s", req->getCurrentUrl().c_str()).str(),err),
+		    cuid, req->getUrl().c_str());
+      _requestGroup->addURIResult(req->getUrl(), err.getCode());
+    }
     onAbort();
-    req->resetUrl();
+    // TODO Do we need this?
+    //req->resetUrl();
     tryReserved();
     return true;
   } catch(DlRetryEx& err) {
@@ -202,6 +231,16 @@ bool AbstractCommand::execute() {
 
 void AbstractCommand::tryReserved() {
   _requestGroup->removeServerHost(cuid);
+  if(_requestGroup->getDownloadContext()->getFileMode() == DownloadContext::SINGLE) {
+    const SharedHandle<FileEntry>& entry =
+      _requestGroup->getDownloadContext()->getFileEntries().front();
+    // Don't create new command if currently file length is unknown
+    // and there are no URI left. Because file length is unknown, we
+    // can assume that there are no in-flight request object.
+    if(entry->getLength() == 0 && entry->getRemainingUris().size() == 0) {
+      return;
+    }
+  }
   Commands commands;
   _requestGroup->createNextCommand(commands, e, 1);
   e->setNoWait(true);
@@ -212,7 +251,18 @@ bool AbstractCommand::prepareForRetry(time_t wait) {
   if(!_requestGroup->getPieceStorage().isNull()) {
     _requestGroup->getSegmentMan()->cancelSegment(cuid);
   }
-  Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, req, _requestGroup, e);
+  if(!req.isNull()) {
+    _fileEntry->poolRequest(req);
+  }
+  if(!_segments.empty()) {
+    // TODO1.5 subtract 1 from getPositionToWrite()
+    SharedHandle<FileEntry> fileEntry = _requestGroup->getDownloadContext()->findFileEntryByOffset(_segments.front()->getPositionToWrite()-1);
+    logger->debug("CUID#%d - Pooling request URI=%s",
+		  cuid, req->getUrl().c_str());
+    _requestGroup->getSegmentMan()->recognizeSegmentFor(_fileEntry);
+  }
+
+  Command* command = new CreateRequestCommand(cuid, _requestGroup, e);
   if(wait == 0) {
     e->setNoWait(true);
     e->commands.push_back(command);
@@ -225,16 +275,22 @@ bool AbstractCommand::prepareForRetry(time_t wait) {
 }
 
 void AbstractCommand::onAbort() {
-  // TODO This might be a problem if the failure is caused by proxy.
-  e->_requestGroupMan->getOrCreateServerStat(req->getHost(),
-					     req->getProtocol())->setError();
+  if(!req.isNull()) {
+    logger->debug(req->getCurrentUrl().c_str());
+    // TODO This might be a problem if the failure is caused by proxy.
+    e->_requestGroupMan->getOrCreateServerStat(req->getHost(),
+					       req->getProtocol())->setError();
+    _requestGroup->removeIdenticalURI(req->getUrl());
+    _fileEntry->removeRequest(req);
+  }
 
   logger->debug(MSG_UNREGISTER_CUID, cuid);
   //_segmentMan->unregisterId(cuid);
   if(!_requestGroup->getPieceStorage().isNull()) {
     _requestGroup->getSegmentMan()->cancelSegment(cuid);
   }
-  _requestGroup->removeIdenticalURI(req->getUrl());
+  // TODO1.5 Should be moved to FileEntry
+  // _requestGroup->removeIdenticalURI(req->getUrl());
 }
 
 void AbstractCommand::disableReadCheckSocket() {

src/AbstractCommand.h

@@ -38,6 +38,7 @@
 #include "Command.h"
 #include "SharedHandle.h"
 #include "TimeA2.h"
+#include "FileEntry.h"
 
 namespace aria2 {
 
@@ -59,6 +60,7 @@ private:
 protected:
   RequestGroup* _requestGroup;
   SharedHandle<Request> req;
+  SharedHandle<FileEntry> _fileEntry;
   DownloadEngine* e;
   SharedHandle<SocketCore> socket;
   std::deque<SharedHandle<Segment> > _segments;
@@ -140,6 +142,11 @@ public:
 		  RequestGroup* requestGroup, DownloadEngine* e,
 		  const SharedHandle<SocketCore>& s = SharedHandle<SocketCore>());
 
+  AbstractCommand(int32_t cuid, const SharedHandle<Request>& req,
+		  const SharedHandle<FileEntry>& fileEntry,
+		  RequestGroup* requestGroup, DownloadEngine* e,
+		  const SharedHandle<SocketCore>& s = SharedHandle<SocketCore>());
+
   virtual ~AbstractCommand();
   bool execute();
 };

src/BitfieldMan.cc

@@ -308,9 +308,10 @@ bool BitfieldMan::getMissingUnusedIndex(size_t& index) const
   }
 }
 
-size_t BitfieldMan::getStartIndex(size_t index) const {
-  while(index < blocks && (isUseBitSet(index) || isBitSet(index))) {
-    index++;
+template<typename Array>
+static size_t getStartIndex(size_t index, const Array& bitfield, const unsigned char* useBitfield, size_t blocks) {
+  while(index < blocks && (bitfield::test(bitfield, blocks, index) || bitfield::test(useBitfield, blocks, index))) {
+    ++index;
   }
   if(blocks <= index) {
     return blocks;
@@ -319,24 +320,33 @@ size_t BitfieldMan::getStartIndex(size_t index) const {
   }
 }
 
-size_t BitfieldMan::getEndIndex(size_t index) const {
-  while(index < blocks && (!isUseBitSet(index) && !isBitSet(index))) {
-    index++;
+template<typename Array>
+static size_t getEndIndex(size_t index, const Array& bitfield, const unsigned char* useBitfield, size_t blocks) {
+  while(index < blocks && (!bitfield::test(bitfield, blocks, index) && !bitfield::test(useBitfield, blocks, index))) {
+    ++index;
   }
   return index;
 }
 
-bool BitfieldMan::getSparseMissingUnusedIndex(size_t& index) const {
-  Range maxRange;
-  Range currentRange;
+template<typename Array>
+static bool getSparseMissingUnusedIndex
+(size_t& index,
+ const Array& bitfield,
+ const unsigned char* useBitfield,
+ size_t blocks)
+{
+  BitfieldMan::Range maxRange;
+  BitfieldMan::Range currentRange;
   {
     size_t nextIndex = 0;
     while(nextIndex < blocks) {
-      currentRange.startIndex = getStartIndex(nextIndex);
+      currentRange.startIndex =
+	getStartIndex(nextIndex, bitfield, useBitfield, blocks);
       if(currentRange.startIndex == blocks) {
 	break;
       }
-      currentRange.endIndex = getEndIndex(currentRange.startIndex);
+      currentRange.endIndex =
+	getEndIndex(currentRange.startIndex, bitfield, useBitfield, blocks);
       if(maxRange < currentRange) {
 	maxRange = currentRange;
       }
@@ -346,7 +356,7 @@ bool BitfieldMan::getSparseMissingUnusedIndex(size_t& index) const {
   if(maxRange.getSize()) {
     if(maxRange.startIndex == 0) {
       index = 0;
-    } else if(isUseBitSet(maxRange.startIndex-1)) {
+    } else if(bitfield::test(useBitfield, blocks, maxRange.startIndex-1)) {
       index = maxRange.getMidIndex();
     } else {
       index = maxRange.startIndex;
@@ -357,6 +367,22 @@ bool BitfieldMan::getSparseMissingUnusedIndex(size_t& index) const {
   }
 }
 
+bool BitfieldMan::getSparseMissingUnusedIndex
+(size_t& index,
+ const unsigned char* ignoreBitfield,
+ size_t ignoreBitfieldLength) const
+{
+  if(filterEnabled) {
+    return aria2::getSparseMissingUnusedIndex
+      (index, array(ignoreBitfield)|~array(filterBitfield)|array(bitfield),
+       useBitfield, blocks);
+  } else {
+    return aria2::getSparseMissingUnusedIndex
+      (index, array(ignoreBitfield)|array(bitfield),
+       useBitfield, blocks);
+  }
+}
+
 template<typename Array>
 static bool copyBitfield(unsigned char* dst, const Array& src, size_t blocks)
 {
@@ -571,6 +597,21 @@ void BitfieldMan::addFilter(uint64_t offset, uint64_t length) {
   updateCache();
 }
 
+void BitfieldMan::removeFilter(uint64_t offset, uint64_t length) {
+  if(!filterBitfield) {
+    filterBitfield = new unsigned char[bitfieldLength];
+    memset(filterBitfield, 0, bitfieldLength);
+  }
+  if(length > 0) {
+    size_t startBlock = offset/blockLength;
+    size_t endBlock = (offset+length-1)/blockLength;
+    for(size_t i = startBlock; i <= endBlock && i < blocks; i++) {
+      setBitInternal(filterBitfield, i, false);
+    }
+  }
+  updateCache();
+}
+
 void BitfieldMan::enableFilter() {
   if(!filterBitfield) {
     filterBitfield = new unsigned char[bitfieldLength];

src/BitfieldMan.h

@@ -83,7 +83,7 @@ private:
   size_t getEndIndex(size_t index) const;
 
   uint64_t getCompletedLength(bool useFilter) const;
-
+public:
   // [startIndex, endIndex)
   class Range {
   public:
@@ -160,7 +160,11 @@ public:
   /**
    * affected by filter
    */
-  bool getSparseMissingUnusedIndex(size_t& index) const;
+  bool getSparseMissingUnusedIndex
+  (size_t& index,
+   const unsigned char* ignoreBitfield,
+   size_t ignoreBitfieldLength) const;
+
   /**
    * affected by filter
    */
@@ -243,6 +247,7 @@ public:
   void setAllUseBit();
 
   void addFilter(uint64_t offset, uint64_t length);
+  void removeFilter(uint64_t offset, uint64_t length);
   /**
    * Clears filter and disables filter
    */
@@ -306,6 +311,10 @@ public:
 
   uint64_t getMissingUnusedLength(size_t startingIndex) const;
 
+  const unsigned char* getFilterBitfield() const
+  {
+    return filterBitfield;
+  }
 };
 
 } // namespace aria2

src/CreateRequestCommand.cc

@@ -0,0 +1,85 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2009 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#include "CreateRequestCommand.h"
+
+#include "InitiateConnectionCommandFactory.h"
+#include "RequestGroup.h"
+#include "Segment.h"
+#include "DownloadContext.h"
+#include "DlAbortEx.h"
+#include "DownloadEngine.h"
+#include "SocketCore.h"
+#include "SegmentMan.h"
+
+namespace aria2 {
+
+CreateRequestCommand::CreateRequestCommand(int32_t cuid,
+					   RequestGroup* requestGroup,
+					   DownloadEngine* e):
+  AbstractCommand(cuid, SharedHandle<Request>(), requestGroup, e)
+{
+  setStatus(Command::STATUS_ONESHOT_REALTIME);
+  disableReadCheckSocket();
+  disableWriteCheckSocket();
+}
+		  
+bool CreateRequestCommand::executeInternal()
+{
+  if(_segments.empty()) {
+    _fileEntry = _requestGroup->getDownloadContext()->findFileEntryByOffset(0);
+  } else {
+    // We assume all segments belongs to same file.
+    _fileEntry = _requestGroup->getDownloadContext()->findFileEntryByOffset
+      (_segments.front()->getPositionToWrite());
+  }
+  req = _fileEntry->getRequest(_requestGroup->getURISelector());
+  if(req.isNull()) {
+    if(!_requestGroup->getSegmentMan().isNull()) {
+      _requestGroup->getSegmentMan()->ignoreSegmentFor(_fileEntry);
+    }
+    throw DL_ABORT_EX("No URI available.");
+  }
+
+  Command* command =
+    InitiateConnectionCommandFactory::createInitiateConnectionCommand
+    (cuid, req, _fileEntry, _requestGroup, e);
+  //ServerHostHandle sv(new ServerHost(command->getCuid(), req->getHost()));
+  //registerServerHost(sv);
+  e->setNoWait(true);
+  e->commands.push_back(command);
+  return true;
+}
+
+} // namespace aria2

src/CreateRequestCommand.h

@@ -0,0 +1,53 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2009 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef _D_CREATE_REQUEST_COMMAND_H_
+#define _D_CREATE_REQUEST_COMMAND_H_
+
+#include "AbstractCommand.h"
+
+namespace aria2 {
+
+class CreateRequestCommand:public AbstractCommand {
+public:
+  CreateRequestCommand(int32_t cuid,
+		       RequestGroup* requestGroup,
+		       DownloadEngine* e);
+protected:
+  virtual bool executeInternal();
+};
+
+} // namespace aria2
+
+#endif // _D_CREATE_REQUEST_COMMAND_H_

src/DefaultPieceStorage.cc

@@ -261,10 +261,11 @@ SharedHandle<Piece> DefaultPieceStorage::getMissingFastPiece
 
 #endif // ENABLE_BITTORRENT
 
-PieceHandle DefaultPieceStorage::getMissingPiece()
+PieceHandle DefaultPieceStorage::getSparseMissingUnusedPiece
+(const unsigned char* ignoreBitfield, size_t length)
 {
   size_t index;
-  if(bitfieldMan->getSparseMissingUnusedIndex(index)) {
+  if(bitfieldMan->getSparseMissingUnusedIndex(index, ignoreBitfield, length)) {
     return checkOutPiece(index);
   } else {
     return SharedHandle<Piece>();

src/DefaultPieceStorage.h

@@ -133,7 +133,8 @@ public:
 
 #endif // ENABLE_BITTORRENT
 
-  virtual SharedHandle<Piece> getMissingPiece();
+  virtual SharedHandle<Piece> getSparseMissingUnusedPiece
+  (const unsigned char* ignoreBitfield, size_t length);
 
   virtual SharedHandle<Piece> getMissingPiece(size_t index);
 

src/DownloadCommand.cc

@@ -66,10 +66,11 @@ namespace aria2 {
 
 DownloadCommand::DownloadCommand(int cuid,
 				 const RequestHandle& req,
+				 const SharedHandle<FileEntry>& fileEntry,
 				 RequestGroup* requestGroup,
 				 DownloadEngine* e,
 				 const SocketHandle& s):
-  AbstractCommand(cuid, req, requestGroup, e, s)
+  AbstractCommand(cuid, req, fileEntry, requestGroup, e, s)
 #ifdef ENABLE_MESSAGE_DIGEST
   , _pieceHashValidationEnabled(false)
 #endif // ENABLE_MESSAGE_DIGEST
@@ -114,8 +115,13 @@ bool DownloadCommand::executeInternal() {
   size_t BUFSIZE = 16*1024;
   unsigned char buf[BUFSIZE];
   size_t bufSize;
-  if(segment->getLength() > 0 && segment->getLength()-segment->getWrittenLength() < BUFSIZE) {
-    bufSize = segment->getLength()-segment->getWrittenLength();
+  if(segment->getLength() > 0) {
+    if(segment->getPosition()+segment->getLength() <= static_cast<uint64_t>(_fileEntry->getLastOffset())) {
+      bufSize = std::min(segment->getLength()-segment->getWrittenLength(),
+			 BUFSIZE);
+    } else {
+      bufSize = std::min(static_cast<size_t>(_fileEntry->getLastOffset()-_fileEntry->gtoloff(segment->getPositionToWrite())), BUFSIZE);
+    }
   } else {
     bufSize = BUFSIZE;
   }
@@ -167,13 +173,15 @@ bool DownloadCommand::executeInternal() {
   bool segmentComplete = false;
   // Note that GrowSegment::complete() always returns false.
   if(_transferEncodingDecoder.isNull() && _contentEncodingDecoder.isNull()) {
-    if(segment->complete()) {
+    if(segment->complete() ||
+       segment->getPositionToWrite() == _fileEntry->getLastOffset()) {
       segmentComplete = true;
     } else if(segment->getLength() == 0 && bufSize == 0 &&
 	      !socket->wantRead() && !socket->wantWrite()) {
       segmentComplete = true;
     }
-  } else if(!_transferEncodingDecoder.isNull() && segment->complete()) {
+  } else if(!_transferEncodingDecoder.isNull() &&
+	    (segment->complete() || segment->getPositionToWrite() == _fileEntry->getLastOffset())) {
     segmentComplete = true;
   } else if((_transferEncodingDecoder.isNull() ||
 	     _transferEncodingDecoder->finished()) &&
@@ -188,36 +196,40 @@ bool DownloadCommand::executeInternal() {
   }
 
   if(segmentComplete) {
-    logger->info(MSG_SEGMENT_DOWNLOAD_COMPLETED, cuid);
+    if(segment->complete() || segment->getLength() == 0) {
+      // If segment->getLength() == 0, the server doesn't provide
+      // content length, but the client detected that download
+      // completed.
+      logger->info(MSG_SEGMENT_DOWNLOAD_COMPLETED, cuid);
 #ifdef ENABLE_MESSAGE_DIGEST
 
-    {
-      std::string expectedPieceHash =
-	_requestGroup->getDownloadContext()->getPieceHash(segment->getIndex());
-      if(_pieceHashValidationEnabled && !expectedPieceHash.empty()) {
-	if(segment->isHashCalculated()) {
-	  logger->debug("Hash is available! index=%lu",
-			static_cast<unsigned long>(segment->getIndex()));
-	  validatePieceHash(segment, expectedPieceHash, segment->getHashString());
+      {
+	std::string expectedPieceHash =
+	  _requestGroup->getDownloadContext()->getPieceHash(segment->getIndex());
+	if(_pieceHashValidationEnabled && !expectedPieceHash.empty()) {
+	  if(segment->isHashCalculated()) {
+	    logger->debug("Hash is available! index=%lu",
+			  static_cast<unsigned long>(segment->getIndex()));
+	    validatePieceHash(segment, expectedPieceHash, segment->getHashString());
+	  } else {
+	    _messageDigestContext->digestReset();
+	    validatePieceHash(segment, expectedPieceHash,
+			      MessageDigestHelper::digest
+			      (_messageDigestContext.get(),
+			       _requestGroup->getPieceStorage()->getDiskAdaptor(),
+			       segment->getPosition(),
+			       segment->getLength()));
+	  }
 	} else {
-	  _messageDigestContext->digestReset();
-	  validatePieceHash(segment, expectedPieceHash,
-			    MessageDigestHelper::digest
-			    (_messageDigestContext.get(),
-			     _requestGroup->getPieceStorage()->getDiskAdaptor(),
-			     segment->getPosition(),
-			     segment->getLength()));
+	  _requestGroup->getSegmentMan()->completeSegment(cuid, segment);
 	}
-      } else {
-	_requestGroup->getSegmentMan()->completeSegment(cuid, segment);
       }
-    }
 
 #else // !ENABLE_MESSAGE_DIGEST
-
-    _requestGroup->getSegmentMan()->completeSegment(cuid, segment);
-
+      _requestGroup->getSegmentMan()->completeSegment(cuid, segment);
 #endif // !ENABLE_MESSAGE_DIGEST
+    }
+
 
     checkLowestDownloadSpeed();
     // this unit is going to download another segment.
@@ -263,6 +275,10 @@ bool DownloadCommand::prepareForNextSegment() {
     // segment.
     if(_segments.size() == 1) {
       SegmentHandle tempSegment = _segments.front();
+      if(!tempSegment->complete()) {
+	return prepareForRetry(0);
+      }
+      // TODO1.5 get segment for the same file only
       SegmentHandle nextSegment =
 	_requestGroup->getSegmentMan()->getSegment(cuid,
 						   tempSegment->getIndex()+1);

src/DownloadCommand.h

@@ -76,6 +76,7 @@ protected:
 public:
   DownloadCommand(int cuid,
 		  const SharedHandle<Request>& req,
+		  const SharedHandle<FileEntry>& fileEntry,
 		  RequestGroup* requestGroup,
 		  DownloadEngine* e,
 		  const SharedHandle<SocketCore>& s);

src/DownloadContext.cc

@@ -33,6 +33,9 @@
  */
 /* copyright --> */
 #include "DownloadContext.h"
+
+#include <algorithm>
+
 #include "FileEntry.h"
 
 namespace aria2 {
@@ -79,4 +82,25 @@ int64_t DownloadContext::calculateSessionTime() const
   }
 }
 
+SharedHandle<FileEntry>
+DownloadContext::findFileEntryByOffset(off_t offset) const
+{
+  if(_fileEntries.empty() ||
+     (offset > 0 &&
+      _fileEntries.back()->getOffset()+_fileEntries.back()->getLength() <=
+      static_cast<uint64_t>(offset))){
+    return SharedHandle<FileEntry>();
+  }
+
+  SharedHandle<FileEntry> obj(new FileEntry());
+  obj->setOffset(offset);
+  std::deque<SharedHandle<FileEntry> >::const_iterator i =
+    std::upper_bound(_fileEntries.begin(), _fileEntries.end(), obj);
+  if(i != _fileEntries.end() && (*i)->getOffset() == offset) {
+    return *i;
+  } else {
+    return *(--i);
+  }
+}
+
 } // namespace aria2

src/DownloadContext.h

@@ -117,6 +117,10 @@ public:
   void resetDownloadStopTime();
 
   int64_t calculateSessionTime() const;
+  
+  // Returns FileEntry at given offset. SharedHandle<FileEntry>() is
+  // returned if no such FileEntry is found.
+  SharedHandle<FileEntry> findFileEntryByOffset(off_t offset) const;
 };
 
 typedef SharedHandle<DownloadContext> DownloadContextHandle;

src/FileEntry.cc

@@ -33,8 +33,11 @@
  */
 /* copyright --> */
 #include "FileEntry.h"
-#include "File.h"
+
+#include <cassert>
+
 #include "Util.h"
+#include "URISelector.h"
 
 namespace aria2 {
 
@@ -74,4 +77,66 @@ bool FileEntry::exists() const
   return File(getPath()).exists();
 }
 
+off_t FileEntry::gtoloff(off_t goff) const
+{
+  assert(offset <= goff);
+  return goff-offset;
+}
+
+void FileEntry::getUris(std::deque<std::string>& uris) const
+{
+  uris.insert(uris.end(), _spentUris.begin(), _spentUris.end());
+  uris.insert(uris.end(), _uris.begin(), _uris.end());
+}
+
+std::string FileEntry::selectUri(const SharedHandle<URISelector>& uriSelector)
+{
+  return uriSelector->select(_uris);
+}
+
+SharedHandle<Request>
+FileEntry::getRequest(const SharedHandle<URISelector>& selector)
+{
+  SharedHandle<Request> req;
+  if(_requestPool.empty()) {
+    while(1) {
+      std::string uri = selector->select(_uris);
+      if(uri.empty()) {
+	return req;
+      }
+      req.reset(new Request());
+      if(req->setUrl(uri)) {
+	_spentUris.push_back(uri);
+	_inFlightRequests.push_back(req);
+	return req;
+      } else {
+	req.reset();
+      }
+    }
+  } else {
+    req = _requestPool.back();
+    _requestPool.pop_back();
+    _inFlightRequests.push_back(req);
+    return req;
+  }
+}
+
+void FileEntry::poolRequest(const SharedHandle<Request>& request)
+{
+  removeRequest(request);
+  _requestPool.push_back(request);
+}
+
+bool FileEntry::removeRequest(const SharedHandle<Request>& request)
+{
+  for(std::deque<SharedHandle<Request> >::iterator i =
+	_inFlightRequests.begin(); i != _inFlightRequests.end(); ++i) {
+    if((*i).get() == request.get()) {
+      _inFlightRequests.erase(i);
+      return true;
+    }
+  }
+  return false;
+}
+
 } // namespace aria2

src/FileEntry.h

@@ -43,17 +43,23 @@
 
 #include "SharedHandle.h"
 #include "File.h"
+#include "Request.h"
 
 namespace aria2 {
 
+class URISelector;
+
 class FileEntry {
 private:
   std::string path;
   std::deque<std::string> _uris;
+  std::deque<std::string> _spentUris;
   uint64_t length;
   off_t offset;
   bool extracted;
   bool requested;
+  std::deque<SharedHandle<Request> > _requestPool;
+  std::deque<SharedHandle<Request> > _inFlightRequests;
 public:
   FileEntry():length(0), offset(0), extracted(false), requested(false) {}
 
@@ -86,6 +92,8 @@ public:
 
   void setOffset(off_t offset) { this->offset = offset; }
 
+  off_t getLastOffset() { return offset+length; }
+
   bool isExtracted() const { return extracted; }
 
   void setExtracted(bool flag) { this->extracted = flag; }
@@ -96,14 +104,53 @@ public:
 
   void setupDir();
 
+  // TODO1.5 remove this in favor of getRemainingUris()
   const std::deque<std::string>& getAssociatedUris() const
   {
     return _uris;
   }
 
+  const std::deque<std::string>& getRemainingUris() const
+  {
+    return _uris;
+  }
+
+  const std::deque<std::string>& getSpentUris() const
+  {
+    return _spentUris;
+  }
+
+  void setUris(const std::deque<std::string>& uris)
+  {
+    _uris = uris;
+  }
+
+  // Inserts _uris and _spentUris into uris.
+  void getUris(std::deque<std::string>& uris) const;
+
+  std::string selectUri(const SharedHandle<URISelector>& uriSelector);
+
+  // If pooled Request object is available, one of them is removed
+  // from the pool and returned.  If pool is empty, then select URI
+  // using selectUri(selector) and construct Request object using it
+  // and return the Request object.
+  SharedHandle<Request> getRequest(const SharedHandle<URISelector>& selector);
+
+  void poolRequest(const SharedHandle<Request>& request);
+
+  bool removeRequest(const SharedHandle<Request>& request);
+
+  size_t countInFlightRequest() const
+  {
+    return _inFlightRequests.size();
+  }
+
   bool operator<(const FileEntry& fileEntry) const;
 
   bool exists() const;
+
+  // Translate global offset goff to file local offset.
+  off_t gtoloff(off_t goff) const;
 };
 
 typedef SharedHandle<FileEntry> FileEntryHandle;

src/FtpDownloadCommand.cc

@@ -49,12 +49,13 @@ namespace aria2 {
 FtpDownloadCommand::FtpDownloadCommand
 (int cuid,
  const RequestHandle& req,
+ const SharedHandle<FileEntry>& fileEntry,
  RequestGroup* requestGroup,
  const SharedHandle<FtpConnection>& ftpConnection,
  DownloadEngine* e,
  const SocketHandle& dataSocket,
  const SocketHandle& ctrlSocket)
-  :DownloadCommand(cuid, req, requestGroup, e, dataSocket),
+  :DownloadCommand(cuid, req, fileEntry, requestGroup, e, dataSocket),
    _ftpConnection(ftpConnection),
    ctrlSocket(ctrlSocket) {}
 
@@ -64,8 +65,9 @@ FtpDownloadCommand::~FtpDownloadCommand() {}
 bool FtpDownloadCommand::prepareForNextSegment()
 {
   if(getOption()->getAsBool(PREF_FTP_REUSE_CONNECTION) &&
-     (uint64_t)_segments.front()->getPositionToWrite() == _requestGroup->getTotalLength()) {
-    Command* command = new FtpFinishDownloadCommand(cuid, req, _requestGroup, _ftpConnection, e, ctrlSocket);
+     static_cast<uint64_t>(_fileEntry->gtoloff(_segments.front()->getPositionToWrite())) == _fileEntry->getLength()) {
+    Command* command = new FtpFinishDownloadCommand
+      (cuid, req, _fileEntry, _requestGroup, _ftpConnection, e, ctrlSocket);
     e->commands.push_back(command);
 
     if(_requestGroup->downloadFinished()) {

src/FtpDownloadCommand.h

@@ -51,6 +51,7 @@ protected:
 public:
   FtpDownloadCommand(int cuid,
 		     const SharedHandle<Request>& req,
+		     const SharedHandle<FileEntry>& fileEntry,
 		     RequestGroup* requestGroup,
 		     const SharedHandle<FtpConnection>& ftpConnection,
 		     DownloadEngine* e,

src/FtpFinishDownloadCommand.cc

@@ -53,11 +53,12 @@ namespace aria2 {
 FtpFinishDownloadCommand::FtpFinishDownloadCommand
 (int cuid,
  const RequestHandle& req,
+ const SharedHandle<FileEntry>& fileEntry,
  RequestGroup* requestGroup,
  const SharedHandle<FtpConnection>& ftpConnection,
  DownloadEngine* e,
  const SharedHandle<SocketCore>& socket)
-  :AbstractCommand(cuid, req, requestGroup, e, socket),
+  :AbstractCommand(cuid, req, fileEntry, requestGroup, e, socket),
    _ftpConnection(ftpConnection)
 {
   e->addSocketForReadCheck(socket, this);

src/FtpFinishDownloadCommand.h

@@ -51,6 +51,7 @@ protected:
 public:
   FtpFinishDownloadCommand(int cuid,
 			   const SharedHandle<Request>& req,
+			   const SharedHandle<FileEntry>& fileEntry,
 			   RequestGroup* requestGroup,
 			   const SharedHandle<FtpConnection>& ftpConnection,
 			   DownloadEngine* e,

src/FtpInitiateConnectionCommand.cc

@@ -56,9 +56,10 @@ namespace aria2 {
 FtpInitiateConnectionCommand::FtpInitiateConnectionCommand
 (int cuid,
  const RequestHandle& req,
+ const SharedHandle<FileEntry>& fileEntry,
  RequestGroup* requestGroup,
  DownloadEngine* e)
-  :InitiateConnectionCommand(cuid, req, requestGroup, e) {}
+  :InitiateConnectionCommand(cuid, req, fileEntry, requestGroup, e) {}
 
 FtpInitiateConnectionCommand::~FtpInitiateConnectionCommand() {}
 
@@ -86,7 +87,8 @@ Command* FtpInitiateConnectionCommand::createNextCommand
 	  (new HttpConnection(cuid, socket, getOption().get()));
 	
 	HttpRequestCommand* c =
-	  new HttpRequestCommand(cuid, req, _requestGroup, hc, e, socket);
+	  new HttpRequestCommand(cuid, req, _fileEntry,
+				 _requestGroup, hc, e, socket);
 	c->setProxyRequest(proxyRequest);
 	command = c;
       } else if(proxyMethod == V_TUNNEL) {
@@ -99,7 +101,8 @@ Command* FtpInitiateConnectionCommand::createNextCommand
     } else {
       if(proxyMethod == V_TUNNEL) {
 	command =
-	  new FtpNegotiationCommand(cuid, req, _requestGroup, e, pooledSocket,
+	  new FtpNegotiationCommand(cuid, req, _fileEntry,
+				    _requestGroup, e, pooledSocket,
 				    FtpNegotiationCommand::SEQ_SEND_CWD,
 				    options["baseWorkingDir"]);
       } else if(proxyMethod == V_GET) {
@@ -109,7 +112,8 @@ Command* FtpInitiateConnectionCommand::createNextCommand
 	  (new HttpConnection(cuid, pooledSocket, getOption().get()));
 	
 	HttpRequestCommand* c =
-	  new HttpRequestCommand(cuid, req, _requestGroup, hc, e, pooledSocket);
+	  new HttpRequestCommand(cuid, req, _fileEntry,
+				 _requestGroup, hc, e, pooledSocket);
 	c->setProxyRequest(proxyRequest);
 	command = c;
       } else {
@@ -126,10 +130,12 @@ Command* FtpInitiateConnectionCommand::createNextCommand
 		   req->getPort());
       socket.reset(new SocketCore());
       socket->establishConnection(resolvedAddresses.front(), req->getPort());
-      command = new FtpNegotiationCommand(cuid, req, _requestGroup, e, socket);
+      command = new FtpNegotiationCommand(cuid, req, _fileEntry,
+					  _requestGroup, e, socket);
     } else {
       command =
-	new FtpNegotiationCommand(cuid, req, _requestGroup, e, pooledSocket,
+	new FtpNegotiationCommand(cuid, req, _fileEntry,
+				  _requestGroup, e, pooledSocket,
 				  FtpNegotiationCommand::SEQ_SEND_CWD,
 				  options["baseWorkingDir"]);
     }

src/FtpInitiateConnectionCommand.h

@@ -46,6 +46,7 @@ protected:
    const SharedHandle<Request>& proxyRequest);
 public:
   FtpInitiateConnectionCommand(int cuid, const SharedHandle<Request>& req,
+			       const SharedHandle<FileEntry>& fileEntry,
 			       RequestGroup* requestGroup, DownloadEngine* e);
 
   virtual ~FtpInitiateConnectionCommand();

src/FtpNegotiationCommand.cc

@@ -68,14 +68,16 @@
 
 namespace aria2 {
 
-FtpNegotiationCommand::FtpNegotiationCommand(int32_t cuid,
-					     const RequestHandle& req,
-					     RequestGroup* requestGroup,
-					     DownloadEngine* e,
-					     const SocketHandle& s,
-					     Seq seq,
-					     const std::string& baseWorkingDir):
-  AbstractCommand(cuid, req, requestGroup, e, s), sequence(seq),
+FtpNegotiationCommand::FtpNegotiationCommand
+(int32_t cuid,
+ const RequestHandle& req,
+ const SharedHandle<FileEntry>& fileEntry,
+ RequestGroup* requestGroup,
+ DownloadEngine* e,
+ const SocketHandle& s,
+ Seq seq,
+ const std::string& baseWorkingDir):
+  AbstractCommand(cuid, req, fileEntry, requestGroup, e, s), sequence(seq),
   ftp(new FtpConnection(cuid, socket, req,
 			e->getAuthConfigFactory()->createAuthConfig(req),
 			getOption().get()))
@@ -96,7 +98,8 @@ bool FtpNegotiationCommand::executeInternal() {
     return prepareForRetry(0);
   } else if(sequence == SEQ_NEGOTIATION_COMPLETED) {
     FtpDownloadCommand* command =
-      new FtpDownloadCommand(cuid, req, _requestGroup, ftp, e, dataSocket, socket);
+      new FtpDownloadCommand
+      (cuid, req, _fileEntry, _requestGroup, ftp, e, dataSocket, socket);
     command->setStartupIdleTime(getOption()->getAsInt(PREF_STARTUP_IDLE_TIME));
     command->setLowestDownloadSpeedLimit(getOption()->getAsInt(PREF_LOWEST_SPEED_LIMIT));
     if(!_requestGroup->isSingleHostMultiConnectionEnabled()) {
@@ -328,6 +331,7 @@ bool FtpNegotiationCommand::onFileSizeDetermined(uint64_t totalLength)
   SingleFileDownloadContextHandle dctx =
     dynamic_pointer_cast<SingleFileDownloadContext>(_requestGroup->getDownloadContext());
   dctx->setTotalLength(totalLength);
+  _fileEntry->setLength(totalLength);
   dctx->setFilename
     (strconcat(dctx->getDir(), "/", Util::urldecode(req->getFile())));
   _requestGroup->preDownloadProcessing();
@@ -429,7 +433,7 @@ bool FtpNegotiationCommand::recvSize() {
       return onFileSizeDetermined(size);
 
     } else {
-      _requestGroup->validateTotalLength(size);
+      _requestGroup->validateTotalLength(_fileEntry->getLength(), size);
     }
 
   } else {

src/FtpNegotiationCommand.h

@@ -123,6 +123,7 @@ protected:
 public:
   FtpNegotiationCommand(int32_t cuid,
 			const SharedHandle<Request>& req,
+			const SharedHandle<FileEntry>& fileEntry,
 			RequestGroup* requestGroup,
 			DownloadEngine* e,
 			const SharedHandle<SocketCore>& s,

src/FtpTunnelResponseCommand.cc

@@ -54,7 +54,8 @@ FtpTunnelResponseCommand::~FtpTunnelResponseCommand() {}
 
 Command* FtpTunnelResponseCommand::getNextCommand()
 {
-  return new FtpNegotiationCommand(cuid, req, _requestGroup, e, socket);
+  return new FtpNegotiationCommand(cuid, req, _fileEntry,
+				   _requestGroup, e, socket);
 }
 
 } // namespace aria2

src/HttpDownloadCommand.cc

@@ -52,12 +52,13 @@ namespace aria2 {
 HttpDownloadCommand::HttpDownloadCommand
 (int cuid,
  const RequestHandle& req,
+ const SharedHandle<FileEntry>& fileEntry,
  RequestGroup* requestGroup,
  const SharedHandle<HttpResponse>& httpResponse,
  const HttpConnectionHandle& httpConnection,
  DownloadEngine* e,
  const SocketHandle& socket)
-  :DownloadCommand(cuid, req, requestGroup, e, socket),
+  :DownloadCommand(cuid, req, fileEntry, requestGroup, e, socket),
    _httpResponse(httpResponse),
    _httpConnection(httpConnection) {}
 
@@ -67,7 +68,8 @@ bool HttpDownloadCommand::prepareForNextSegment() {
   bool downloadFinished = _requestGroup->downloadFinished();
   if(req->isPipeliningEnabled() && !downloadFinished) {
     HttpRequestCommand* command =
-      new HttpRequestCommand(cuid, req, _requestGroup, _httpConnection, e,
+      new HttpRequestCommand(cuid, req, _fileEntry,
+			     _requestGroup, _httpConnection, e,
 			     socket);
     // Set proxy request here. aria2 sends the HTTP request specialized for
     // proxy.
@@ -81,8 +83,7 @@ bool HttpDownloadCommand::prepareForNextSegment() {
        (req->isKeepAliveEnabled() &&
 	((!_transferEncodingDecoder.isNull() &&
 	  _requestGroup->downloadFinished()) ||
-	 (uint64_t)_segments.front()->getPositionToWrite() ==
-	 _requestGroup->getTotalLength()))) {
+	 static_cast<uint64_t>(_fileEntry->gtoloff(_segments.front()->getPositionToWrite())) == _fileEntry->getLength()))) {
       e->poolSocket(req, isProxyDefined(), socket);
     }
     // The request was sent assuming that server supported pipelining, but

src/HttpDownloadCommand.h

@@ -51,6 +51,7 @@ protected:
 public:
   HttpDownloadCommand(int cuid,
 		      const SharedHandle<Request>& req,
+		      const SharedHandle<FileEntry>& fileEntry,
 		      RequestGroup* requestGroup,
 		      const SharedHandle<HttpResponse>& httpResponse,
 		      const SharedHandle<HttpConnection>& httpConnection,

src/HttpInitiateConnectionCommand.cc

@@ -53,9 +53,10 @@ namespace aria2 {
 HttpInitiateConnectionCommand::HttpInitiateConnectionCommand
 (int cuid,
  const RequestHandle& req,
+ const SharedHandle<FileEntry>& fileEntry,
  RequestGroup* requestGroup,
  DownloadEngine* e):
-  InitiateConnectionCommand(cuid, req, requestGroup, e) {}
+  InitiateConnectionCommand(cuid, req, fileEntry, requestGroup, e) {}
 
 HttpInitiateConnectionCommand::~HttpInitiateConnectionCommand() {}
 
@@ -81,7 +82,9 @@ Command* HttpInitiateConnectionCommand::createNextCommand
       } else if(proxyMethod == V_GET) {
 	SharedHandle<HttpConnection> httpConnection
 	  (new HttpConnection(cuid, socket, getOption().get()));
-	HttpRequestCommand* c = new HttpRequestCommand(cuid, req, _requestGroup,
+	HttpRequestCommand* c = new HttpRequestCommand(cuid, req,
+						       _fileEntry,
+						       _requestGroup,
 						       httpConnection, e,
 						       socket);
 	c->setProxyRequest(proxyRequest);
@@ -93,7 +96,9 @@ Command* HttpInitiateConnectionCommand::createNextCommand
     } else {
       SharedHandle<HttpConnection> httpConnection
 	(new HttpConnection(cuid, pooledSocket, getOption().get()));
-      HttpRequestCommand* c = new HttpRequestCommand(cuid, req, _requestGroup,
+      HttpRequestCommand* c = new HttpRequestCommand(cuid, req,
+						     _fileEntry,
+						     _requestGroup,
 						     httpConnection, e,
 						     pooledSocket);
       if(proxyMethod == V_GET) {
@@ -113,8 +118,8 @@ Command* HttpInitiateConnectionCommand::createNextCommand
       socket = pooledSocket;
     }
     SharedHandle<HttpConnection> httpConnection(new HttpConnection(cuid, socket, getOption().get()));
-    command = new HttpRequestCommand(cuid, req, _requestGroup, httpConnection,
-				     e, socket);
+    command = new HttpRequestCommand(cuid, req, _fileEntry, _requestGroup,
+				     httpConnection, e, socket);
   }
   return command;
 }

src/HttpInitiateConnectionCommand.h

@@ -46,6 +46,7 @@ protected:
    const SharedHandle<Request>& proxyRequest);
 public:
   HttpInitiateConnectionCommand(int cuid, const SharedHandle<Request>& req,
+				const SharedHandle<FileEntry>& fileEntry,
 				RequestGroup* requestGroup,
 				DownloadEngine* e);
 

src/HttpProxyResponseCommand.cc

@@ -54,7 +54,8 @@ HttpProxyResponseCommand::~HttpProxyResponseCommand() {}
 
 Command* HttpProxyResponseCommand::getNextCommand()
 {
-  return new HttpRequestCommand(cuid, req, _requestGroup, httpConnection, e, socket);
+  return new HttpRequestCommand(cuid, req, _fileEntry,
+				_requestGroup, httpConnection, e, socket);
 }
 
 } // namespace aria2

src/HttpRequest.cc

@@ -73,7 +73,7 @@ off_t HttpRequest::getStartByte() const
   if(segment.isNull()) {
     return 0;
   } else {
-    return segment->getPositionToWrite();
+    return _fileEntry->gtoloff(segment->getPositionToWrite());
   }
 }
 
@@ -83,7 +83,7 @@ off_t HttpRequest::getEndByte() const
     return 0;
   } else {
     if(request->isPipeliningEnabled()) {
-      return segment->getPosition()+segment->getLength()-1;
+      return _fileEntry->gtoloff(segment->getPosition()+segment->getLength()-1);
     } else {
       return 0;
     }

src/HttpRequest.h

@@ -42,6 +42,7 @@
 
 #include "SharedHandle.h"
 #include "Request.h"
+#include "FileEntry.h"
 
 namespace aria2 {
 
@@ -59,6 +60,8 @@ private:
 
   SharedHandle<Request> request;
 
+  SharedHandle<FileEntry> _fileEntry;
+
   SharedHandle<Segment> segment;
 
   uint64_t entityLength;
@@ -232,6 +235,16 @@ public:
   // Returns AuthConfig used in the last invocation of
   // createRequest().
   const SharedHandle<AuthConfig>& getAuthConfig() const;
+
+  void setFileEntry(const SharedHandle<FileEntry>& fileEntry)
+  {
+    _fileEntry = fileEntry;
+  }
+
+  const SharedHandle<FileEntry>& getFileEntry() const
+  {
+    return _fileEntry;
+  }
 };
 
 typedef SharedHandle<HttpRequest> HttpRequestHandle;

src/HttpRequestCommand.cc

@@ -58,11 +58,12 @@ namespace aria2 {
 HttpRequestCommand::HttpRequestCommand
 (int cuid,
  const RequestHandle& req,
+ const SharedHandle<FileEntry>& fileEntry,
  RequestGroup* requestGroup,
  const HttpConnectionHandle& httpConnection,
  DownloadEngine* e,
  const SocketHandle& s)
-  :AbstractCommand(cuid, req, requestGroup, e, s),
+  :AbstractCommand(cuid, req, fileEntry, requestGroup, e, s),
    _httpConnection(httpConnection)
 {
   setTimeout(getOption()->getAsInt(PREF_CONNECT_TIMEOUT));
@@ -74,6 +75,7 @@ HttpRequestCommand::~HttpRequestCommand() {}
 
 static SharedHandle<HttpRequest>
 createHttpRequest(const SharedHandle<Request>& req,
+		  const SharedHandle<FileEntry>& fileEntry,
 		  const SharedHandle<Segment>& segment,
 		  uint64_t totalLength,
 		  const SharedHandle<Option>& option,
@@ -85,8 +87,9 @@ createHttpRequest(const SharedHandle<Request>& req,
   HttpRequestHandle httpRequest(new HttpRequest());
   httpRequest->setUserAgent(option->get(PREF_USER_AGENT));
   httpRequest->setRequest(req);
+  httpRequest->setFileEntry(fileEntry);
   httpRequest->setSegment(segment);
-  httpRequest->setEntityLength(totalLength);
+  httpRequest->setEntityLength(fileEntry->getLength());
   httpRequest->addHeader(option->get(PREF_HEADER));
   httpRequest->setCookieStorage(cookieStorage);
   httpRequest->setAuthConfigFactory(authConfigFactory);
@@ -113,7 +116,9 @@ bool HttpRequestCommand::executeInternal() {
 
     if(_segments.empty()) {
       HttpRequestHandle httpRequest
-	(createHttpRequest(req, SharedHandle<Segment>(),
+	(createHttpRequest(req,
+			   _fileEntry,
+			   SharedHandle<Segment>(),
 			   _requestGroup->getTotalLength(),
 			   getOption(),
 			   _requestGroup,
@@ -126,7 +131,9 @@ bool HttpRequestCommand::executeInternal() {
 	const SegmentHandle& segment = *itr;
 	if(!_httpConnection->isIssued(segment)) {
 	  HttpRequestHandle httpRequest
-	    (createHttpRequest(req, segment,
+	    (createHttpRequest(req,
+			       _fileEntry,
+			       segment,
 			       _requestGroup->getTotalLength(),
 			       getOption(),
 			       _requestGroup,
@@ -141,7 +148,8 @@ bool HttpRequestCommand::executeInternal() {
     _httpConnection->sendPendingData();
   }
   if(_httpConnection->sendBufferIsEmpty()) {
-    Command* command = new HttpResponseCommand(cuid, req, _requestGroup,
+    Command* command = new HttpResponseCommand(cuid, req, _fileEntry,
+					       _requestGroup,
 					       _httpConnection, e, socket);
     e->commands.push_back(command);
     return true;

src/HttpRequestCommand.h

@@ -52,6 +52,7 @@ protected:
 public:
   HttpRequestCommand(int cuid,
 		     const SharedHandle<Request>& req,
+		     const SharedHandle<FileEntry>& fileEntry,
 		     RequestGroup* requestGroup,
 		     const SharedHandle<HttpConnection>& httpConnection,
 		     DownloadEngine* e,

src/HttpResponseCommand.cc

@@ -75,14 +75,17 @@ static SharedHandle<Decoder> getTransferEncodingDecoder
 static SharedHandle<Decoder> getContentEncodingDecoder
 (const SharedHandle<HttpResponse>& httpResponse);
 
-HttpResponseCommand::HttpResponseCommand(int32_t cuid,
-					 const RequestHandle& req,
-					 RequestGroup* requestGroup,
-					 const HttpConnectionHandle& httpConnection,
-					 DownloadEngine* e,
-					 const SocketHandle& s)
-  :AbstractCommand(cuid, req, requestGroup, e, s),
-   httpConnection(httpConnection) {}
+HttpResponseCommand::HttpResponseCommand
+(int32_t cuid,
+ const RequestHandle& req,
+ const SharedHandle<FileEntry>& fileEntry,
+ RequestGroup* requestGroup,
+ const HttpConnectionHandle& httpConnection,
+ DownloadEngine* e,
+ const SocketHandle& s)
+  :AbstractCommand(cuid, req, fileEntry, requestGroup, e, s),
+   httpConnection(httpConnection)
+{}
 
 HttpResponseCommand::~HttpResponseCommand() {}
 
@@ -136,6 +139,7 @@ bool HttpResponseCommand::executeInternal()
     SingleFileDownloadContextHandle dctx =
       dynamic_pointer_cast<SingleFileDownloadContext>(_requestGroup->getDownloadContext());
     dctx->setTotalLength(totalLength);
+    _fileEntry->setLength(totalLength);
     dctx->setFilename
       (strconcat(dctx->getDir(), "/", httpResponse->determinFilename()));
     dctx->setContentType(httpResponse->getContentType());
@@ -166,7 +170,8 @@ bool HttpResponseCommand::executeInternal()
     }
   } else {
     // validate totalsize
-    _requestGroup->validateTotalLength(httpResponse->getEntityLength());
+    _requestGroup->validateTotalLength(_fileEntry->getLength(),
+				       httpResponse->getEntityLength());
     // update last modified time
     updateLastModifiedTime(httpResponse->getLastModifiedTime());
     if(_requestGroup->getTotalLength() == 0) {
@@ -258,6 +263,7 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpResponseHandle& httpRe
       command = createHttpDownloadCommand(httpResponse);
     } else {
       _requestGroup->getSegmentMan()->cancelSegment(cuid);
+      _fileEntry->poolRequest(req);
     }
     prepareForNextAction(command);
     if(req->getMethod() == Request::METHOD_HEAD) {
@@ -361,7 +367,7 @@ bool HttpResponseCommand::skipResponseBody
   // thrown away.
 
   HttpSkipResponseCommand* command = new HttpSkipResponseCommand
-    (cuid, req, _requestGroup, httpConnection, httpResponse, e, socket);
+    (cuid, req, _fileEntry, _requestGroup, httpConnection, httpResponse, e, socket);
   command->setTransferEncodingDecoder(decoder);
 
   // If request method is HEAD or the response body is zero-length,
@@ -386,7 +392,7 @@ HttpDownloadCommand* HttpResponseCommand::createHttpDownloadCommand
 {
 
   HttpDownloadCommand* command =
-    new HttpDownloadCommand(cuid, req, _requestGroup,
+    new HttpDownloadCommand(cuid, req, _fileEntry, _requestGroup,
 			    httpResponse, httpConnection, e, socket);
   command->setStartupIdleTime(getOption()->getAsInt(PREF_STARTUP_IDLE_TIME));
   command->setLowestDownloadSpeedLimit

src/HttpResponseCommand.h

@@ -75,6 +75,7 @@ protected:
 public:
   HttpResponseCommand(int32_t cuid,
 		      const SharedHandle<Request>& req,
+		      const SharedHandle<FileEntry>& fileEntry,
 		      RequestGroup* requestGroup,
 		      const SharedHandle<HttpConnection>& httpConnection,
 		      DownloadEngine* e,

src/HttpSkipResponseCommand.cc

@@ -59,12 +59,13 @@ namespace aria2 {
 HttpSkipResponseCommand::HttpSkipResponseCommand
 (int cuid,
  const SharedHandle<Request>& req,
+ const SharedHandle<FileEntry>& fileEntry,
  RequestGroup* requestGroup,
  const SharedHandle<HttpConnection>& httpConnection,
  const SharedHandle<HttpResponse>& httpResponse,
  DownloadEngine* e,
  const SharedHandle<SocketCore>& s):
-  AbstractCommand(cuid, req, requestGroup, e, s),
+  AbstractCommand(cuid, req, fileEntry, requestGroup, e, s),
   _httpConnection(httpConnection),
   _httpResponse(httpResponse),
   _totalLength(_httpResponse->getEntityLength()),

src/HttpSkipResponseCommand.h

@@ -63,6 +63,7 @@ protected:
 public:
   HttpSkipResponseCommand(int cuid,
 			  const SharedHandle<Request>& req,
+			  const SharedHandle<FileEntry>& fileEntry,
 			  RequestGroup* requestGroup,
 			  const SharedHandle<HttpConnection>& httpConnection,
 			  const SharedHandle<HttpResponse>& httpResponse,

src/InitiateConnectionCommand.cc

@@ -42,15 +42,20 @@
 #include "NameResolver.h"
 #include "DNSCache.h"
 #include "SocketCore.h"
+#include "FileEntry.h"
+#include "RequestGroup.h"
+#include "DownloadContext.h"
+#include "Segment.h"
 
 namespace aria2 {
 
 InitiateConnectionCommand::InitiateConnectionCommand
 (int cuid,
  const RequestHandle& req,
+ const SharedHandle<FileEntry>& fileEntry,
  RequestGroup* requestGroup,
  DownloadEngine* e):
-  AbstractCommand(cuid, req, requestGroup, e)
+  AbstractCommand(cuid, req, fileEntry, requestGroup, e)
 {
   setTimeout(getOption()->getAsInt(PREF_DNS_TIMEOUT));
   // give a chance to be executed in the next loop in DownloadEngine

src/InitiateConnectionCommand.h

@@ -54,6 +54,7 @@ protected:
    const SharedHandle<Request>& proxyRequest) = 0;
 public:
   InitiateConnectionCommand(int cuid, const SharedHandle<Request>& req,
+			    const SharedHandle<FileEntry>& fileEntry,
 			    RequestGroup* requestGroup,
 			    DownloadEngine* e);
 

src/InitiateConnectionCommandFactory.cc

@@ -47,7 +47,13 @@
 namespace aria2 {
 
 Command*
-InitiateConnectionCommandFactory::createInitiateConnectionCommand(int32_t cuid, const RequestHandle& req, RequestGroup* requestGroup, DownloadEngine* e) {
+InitiateConnectionCommandFactory::createInitiateConnectionCommand
+(int32_t cuid,
+ const RequestHandle& req,
+ const SharedHandle<FileEntry>& fileEntry,
+ RequestGroup* requestGroup,
+ DownloadEngine* e)
+{
   if(req->getProtocol() == Request::PROTO_HTTP
 #ifdef ENABLE_SSL
      // for SSL
@@ -62,9 +68,11 @@ InitiateConnectionCommandFactory::createInitiateConnectionCommand(int32_t cuid,
       req->setPipeliningHint(true);
     }
 
-    return new HttpInitiateConnectionCommand(cuid, req, requestGroup, e);
+    return
+      new HttpInitiateConnectionCommand(cuid, req, fileEntry, requestGroup, e);
   } else if(req->getProtocol() == Request::PROTO_FTP) {
-    return new FtpInitiateConnectionCommand(cuid, req, requestGroup, e);
+    return
+      new FtpInitiateConnectionCommand(cuid, req, fileEntry, requestGroup, e);
   } else {
     // these protocols are not supported yet
     throw DL_ABORT_EX

src/InitiateConnectionCommandFactory.h

@@ -44,12 +44,14 @@ class Request;
 class RequestGroup;
 class DownloadEngine;
 class Command;
+class FileEntry;
 
 class InitiateConnectionCommandFactory {
 public:
   static Command*
   createInitiateConnectionCommand(int32_t cuid,
 				  const SharedHandle<Request>& req,
+				  const SharedHandle<FileEntry>& fileEntry,
 				  RequestGroup* requestGroup,
 				  DownloadEngine* e);
 };

src/Makefile.am

@@ -201,7 +201,8 @@ SRCS =  Socket.h\
 	PieceSelector.h\
 	LongestSequencePieceSelector.cc LongestSequencePieceSelector.h\
 	bitfield.h\
-	BDE.cc BDE.h
+	BDE.cc BDE.h\
+	CreateRequestCommand.cc CreateRequestCommand.h
 
 if ENABLE_XML_RPC
 SRCS += XmlRpcRequestParserController.cc XmlRpcRequestParserController.h\

src/Makefile.in

@@ -425,7 +425,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
 	SelectEventPoll.cc SelectEventPoll.h SequentialPicker.h \
 	SequentialDispatcherCommand.h PieceSelector.h \
 	LongestSequencePieceSelector.cc LongestSequencePieceSelector.h \
-	bitfield.h BDE.cc BDE.h XmlRpcRequestParserController.cc \
+	bitfield.h BDE.cc BDE.h CreateRequestCommand.cc \
+	CreateRequestCommand.h XmlRpcRequestParserController.cc \
 	XmlRpcRequestParserController.h \
 	XmlRpcRequestParserStateMachine.cc \
 	XmlRpcRequestParserStateMachine.h XmlRpcRequestParserState.h \
@@ -846,15 +847,15 @@ am__objects_26 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \
 	SocketBuffer.$(OBJEXT) OptionHandlerException.$(OBJEXT) \
 	URIResult.$(OBJEXT) SelectEventPoll.$(OBJEXT) \
 	LongestSequencePieceSelector.$(OBJEXT) BDE.$(OBJEXT) \
-	$(am__objects_1) $(am__objects_2) $(am__objects_3) \
-	$(am__objects_4) $(am__objects_5) $(am__objects_6) \
-	$(am__objects_7) $(am__objects_8) $(am__objects_9) \
-	$(am__objects_10) $(am__objects_11) $(am__objects_12) \
-	$(am__objects_13) $(am__objects_14) $(am__objects_15) \
-	$(am__objects_16) $(am__objects_17) $(am__objects_18) \
-	$(am__objects_19) $(am__objects_20) $(am__objects_21) \
-	$(am__objects_22) $(am__objects_23) $(am__objects_24) \
-	$(am__objects_25)
+	CreateRequestCommand.$(OBJEXT) $(am__objects_1) \
+	$(am__objects_2) $(am__objects_3) $(am__objects_4) \
+	$(am__objects_5) $(am__objects_6) $(am__objects_7) \
+	$(am__objects_8) $(am__objects_9) $(am__objects_10) \
+	$(am__objects_11) $(am__objects_12) $(am__objects_13) \
+	$(am__objects_14) $(am__objects_15) $(am__objects_16) \
+	$(am__objects_17) $(am__objects_18) $(am__objects_19) \
+	$(am__objects_20) $(am__objects_21) $(am__objects_22) \
+	$(am__objects_23) $(am__objects_24) $(am__objects_25)
 am_libaria2c_a_OBJECTS = $(am__objects_26)
 libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS)
 am__installdirs = "$(DESTDIR)$(bindir)"
@@ -1181,7 +1182,8 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \
 	SelectEventPoll.cc SelectEventPoll.h SequentialPicker.h \
 	SequentialDispatcherCommand.h PieceSelector.h \
 	LongestSequencePieceSelector.cc LongestSequencePieceSelector.h \
-	bitfield.h BDE.cc BDE.h $(am__append_1) $(am__append_2) \
+	bitfield.h BDE.cc BDE.h CreateRequestCommand.cc \
+	CreateRequestCommand.h $(am__append_1) $(am__append_2) \
 	$(am__append_3) $(am__append_4) $(am__append_5) \
 	$(am__append_6) $(am__append_7) $(am__append_8) \
 	$(am__append_9) $(am__append_10) $(am__append_11) \
@@ -1339,6 +1341,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Cookie.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CookieParser.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CookieStorage.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CreateRequestCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DHTAbstractMessage.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DHTAbstractNodeLookupTask.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DHTAbstractTask.Po@am__quote@

src/PieceStorage.h

@@ -101,8 +101,10 @@ public:
 
   /**
    * Returns a missing piece if available. Otherwise returns 0;
+   * If ignoreBitfield is set, indexes of true bit are excluded.
    */
-  virtual SharedHandle<Piece> getMissingPiece() = 0;
+  virtual SharedHandle<Piece> getSparseMissingUnusedPiece
+  (const unsigned char* ignoreBitfield, size_t length) = 0;
 
   /**
    * Returns a missing piece whose index is index.

src/RequestGroup.cc

@@ -44,7 +44,7 @@
 #include "NullProgressInfoFile.h"
 #include "Dependency.h"
 #include "prefs.h"
-#include "InitiateConnectionCommandFactory.h"
+#include "CreateRequestCommand.h"
 #include "File.h"
 #include "message.h"
 #include "Util.h"
@@ -78,6 +78,7 @@
 #include "InOrderURISelector.h"
 #include "PieceSelector.h"
 #include "a2functional.h"
+#include "SocketCore.h"
 #ifdef ENABLE_MESSAGE_DIGEST
 # include "CheckIntegrityCommand.h"
 #endif // ENABLE_MESSAGE_DIGEST
@@ -604,6 +605,7 @@ void RequestGroup::createNextCommand(std::deque<Command*>& commands,
 				     unsigned int numCommand,
 				     const std::string& method)
 {
+  // TODO1.5 The following block should be moved into FileEntry
   if(_option->getAsBool(PREF_REUSE_URI) && _uris.empty()) {
     std::deque<std::string> uris = _spentUris;
     std::sort(uris.begin(), uris.end());
@@ -639,35 +641,45 @@ void RequestGroup::createNextCommand(std::deque<Command*>& commands,
   }
 
   std::deque<std::string> pendingURIs;
-  for(; numCommand--; ) {    
-    std::string uri = _uriSelector->select(_uris);
-    if(uri.empty())
-      continue;
-    RequestHandle req(new Request());
-    if(req->setUrl(uri)) {
-      ServerHostHandle sv;
-      if(!_singleHostMultiConnectionEnabled){
-	sv = searchServerHost(req->getHost());
-      }
-      if(sv.isNull()) {
-	_spentUris.push_back(uri);
-	req->setReferer(_option->get(PREF_REFERER));
-	req->setMethod(method);
-
-	Command* command =
-	  InitiateConnectionCommandFactory::createInitiateConnectionCommand
-	  (e->newCUID(), req, this, e);
-	ServerHostHandle sv(new ServerHost(command->getCuid(), req->getHost()));
-	registerServerHost(sv);
-	commands.push_back(command);
-      } else {
-	pendingURIs.push_back(uri);
-      }
-    } else {
-      _logger->error(MSG_UNRECOGNIZED_URI, req->getUrl().c_str());
-    }
+  for(; numCommand--; ) {
+    Command* command = new CreateRequestCommand(e->newCUID(), this, e);
+    _logger->debug("filePath=%s", _downloadContext->getFileEntries().front()->getPath().c_str());
+    commands.push_back(command);
+
+    // TODO1.5 ServerHost stuff should be moved into FileEntry or
+    // CreateRequestCommand
+
+//     std::string uri = _uriSelector->select(_uris);
+//     if(uri.empty())
+//       continue;
+//     RequestHandle req(new Request());
+//     if(req->setUrl(uri)) {
+//       ServerHostHandle sv;
+//       if(!_singleHostMultiConnectionEnabled){
+// 	sv = searchServerHost(req->getHost());
+//       }
+//       if(sv.isNull()) {
+// 	_spentUris.push_back(uri);
+// 	req->setReferer(_option->get(PREF_REFERER));
+// 	req->setMethod(method);
+
+// 	Command* command =
+// 	  InitiateConnectionCommandFactory::createInitiateConnectionCommand
+// 	  (e->newCUID(), req, this, e);
+// 	ServerHostHandle sv(new ServerHost(command->getCuid(), req->getHost()));
+// 	registerServerHost(sv);
+// 	// give a chance to be executed in the next loop in DownloadEngine
+// 	command->setStatus(Command::STATUS_ONESHOT_REALTIME);
+// 	commands.push_back(command);
+//       } else {
+// 	pendingURIs.push_back(uri);
+//       }
+//     } else {
+//       _logger->error(MSG_UNRECOGNIZED_URI, req->getUrl().c_str());
+//     }
+//  }
   }
-  _uris.insert(_uris.begin(), pendingURIs.begin(), pendingURIs.end());
+//  _uris.insert(_uris.begin(), pendingURIs.begin(), pendingURIs.end());
   if(!commands.empty()) {
     e->setNoWait(true);
   }

src/RequestGroup.h

@@ -164,9 +164,6 @@ private:
   void validateFilename(const std::string& expectedFilename,
 			const std::string& actualFilename) const;
 
-  void validateTotalLength(uint64_t expectedTotalLength,
-			   uint64_t actualTotalLength) const;
-
   void initializePreDownloadHandler();
 
   void initializePostDownloadHandler();
@@ -249,6 +246,9 @@ public:
    */
   void validateFilename(const std::string& actualFilename) const;
 
+  void validateTotalLength(uint64_t expectedTotalLength,
+			   uint64_t actualTotalLength) const;
+
   void validateTotalLength(uint64_t actualTotalLength) const;
 
   void setSegmentManFactory(const SharedHandle<SegmentManFactory>& segmentManFactory);
@@ -449,6 +449,11 @@ public:
 
   void setURISelector(const SharedHandle<URISelector>& uriSelector);
 
+  const SharedHandle<URISelector>& getURISelector() const
+  {
+    return _uriSelector;
+  }
+
   void applyLastModifiedTimeToLocalFiles();
 
   void updateLastModifiedTime(const Time& time);

src/RequestGroupMan.cc

@@ -486,6 +486,7 @@ void RequestGroupMan::getInitialCommands(std::deque<Command*>& commands,
       if((*itr)->isDependencyResolved()) {
 	configureRequestGroup(*itr);
 	createInitialCommand(*itr, commands, e,
+			     // TODO1.5 We need PREF_DRY_RUN here?
 			     _option->getAsBool(PREF_USE_HEAD));
 	executeStartHook(*itr, e->option);
 	++itr;

src/SegmentMan.cc

@@ -50,6 +50,7 @@
 #include "Option.h"
 #include "DownloadContext.h"
 #include "Piece.h"
+#include "FileEntry.h"
 
 namespace aria2 {
 
@@ -66,8 +67,12 @@ SegmentMan::SegmentMan(const Option* option,
   _downloadContext(downloadContext),
   _pieceStorage(pieceStorage),
   _lastPeerStatDlspdMapUpdated(0),
-  _cachedDlspd(0)
-{}
+  _cachedDlspd(0),
+  _ignoreBitfield(downloadContext->getPieceLength(),
+		  downloadContext->getTotalLength())
+{
+  _ignoreBitfield.enableFilter();
+}
 
 SegmentMan::~SegmentMan() {}
 
@@ -126,6 +131,22 @@ SegmentHandle SegmentMan::checkoutSegment(cuid_t cuid,
 		segment->getLength(),
 		segment->getSegmentLength(),
 		segment->getWrittenLength());
+  if(piece->getLength() > 0) {
+    std::map<size_t, size_t>::iterator positr =
+      _segmentWrittenLengthMemo.find(segment->getIndex());
+    if(positr != _segmentWrittenLengthMemo.end()) {
+      const size_t writtenLength = (*positr).second;
+      logger->debug("writtenLength(in memo)=%d, writtenLength=%d",
+		    writtenLength, segment->getWrittenLength());
+      //  If the difference between cached writtenLength and segment's
+      //  writtenLength is less than one block, we assume that these
+      //  missing bytes are already downloaded.
+      if(segment->getWrittenLength() < writtenLength &&
+	 writtenLength-segment->getWrittenLength() < piece->getBlockLength()) {
+	segment->updateWrittenLength(writtenLength-segment->getWrittenLength());
+      }
+    }
+  }
   return segment;
 }
 
@@ -178,7 +199,9 @@ void SegmentMan::getInFlightSegment(std::deque<SharedHandle<Segment> >& segments
 }
 
 SegmentHandle SegmentMan::getSegment(cuid_t cuid) {
-  PieceHandle piece = _pieceStorage->getMissingPiece();
+  PieceHandle piece =
+    _pieceStorage->getSparseMissingUnusedPiece
+    (_ignoreBitfield.getFilterBitfield(),_ignoreBitfield.getBitfieldLength());
   if(piece.isNull()) {
     PeerStatHandle myPeerStat = getPeerStat(cuid);
     if(myPeerStat.isNull()) {
@@ -219,6 +242,11 @@ void SegmentMan::cancelSegment(cuid_t cuid) {
       itr != usedSegmentEntries.end();) {
     if((*itr)->cuid == cuid) {
       _pieceStorage->cancelPiece((*itr)->segment->getPiece());
+      _segmentWrittenLengthMemo[(*itr)->segment->getIndex()] =
+	(*itr)->segment->getWrittenLength();
+      logger->debug("Memorized segment index=%u, writtenLength=%u",
+		    (*itr)->segment->getIndex(),
+		    (*itr)->segment->getWrittenLength());
       itr = usedSegmentEntries.erase(itr);
     } else {
       ++itr;
@@ -356,4 +384,14 @@ size_t SegmentMan::countFreePieceFrom(size_t index) const
   return _downloadContext->getNumPieces()-index;
 }
 
+void SegmentMan::ignoreSegmentFor(const SharedHandle<FileEntry>& fileEntry)
+{
+  _ignoreBitfield.addFilter(fileEntry->getOffset(), fileEntry->getLength());
+}
+
+void SegmentMan::recognizeSegmentFor(const SharedHandle<FileEntry>& fileEntry)
+{
+  _ignoreBitfield.removeFilter(fileEntry->getOffset(), fileEntry->getLength());
+}
+
 } // namespace aria2

src/SegmentMan.h

@@ -43,6 +43,7 @@
 #include "SharedHandle.h"
 #include "TimeA2.h"
 #include "Command.h"
+#include "BitfieldMan.h"
 
 namespace aria2 {
 
@@ -53,6 +54,7 @@ class PeerStat;
 class DownloadContext;
 class PieceStorage;
 class Piece;
+class FileEntry;
 
 class SegmentEntry {
 public:
@@ -82,6 +84,10 @@ private:
 
   SegmentEntries usedSegmentEntries;
 
+  // Remember writtenLength for each segment. The key is an index of a
+  // segment. The value is writtenLength for that segment.
+  std::map<size_t, size_t> _segmentWrittenLengthMemo;
+
   std::deque<SharedHandle<PeerStat> > peerStats;
 
   // key: PeerStat's cuid, value: its download speed
@@ -91,6 +97,8 @@ private:
 
   unsigned int _cachedDlspd;
 
+  BitfieldMan _ignoreBitfield;
+
   SharedHandle<Segment> checkoutSegment(cuid_t cuid,
 					const SharedHandle<Piece>& piece);
 
@@ -201,6 +209,12 @@ public:
   uint64_t calculateSessionDownloadLength() const;
 
   size_t countFreePieceFrom(size_t index) const;
+
+  // Excludes segments that fileEntry covers from segment selection.
+  void ignoreSegmentFor(const SharedHandle<FileEntry>& fileEntry);
+
+  // Includes segments that fileEntry covers in segment selection.
+  void recognizeSegmentFor(const SharedHandle<FileEntry>& fileEntry);
 };
 
 typedef SharedHandle<SegmentMan> SegmentManHandle;

src/SingleFileDownloadContext.h

@@ -93,7 +93,11 @@ public:
 
   virtual FILE_MODE getFileMode() const
   {
-    return SINGLE;
+    if(_fileEntries.size() == 1) {
+      return SINGLE;
+    } else {
+      return MULTI;
+    }
   }
 
   virtual size_t getPieceLength() const

src/StreamFileAllocationEntry.cc

@@ -41,7 +41,6 @@
 #include "Request.h"
 #include "prefs.h"
 #include "RequestGroup.h"
-#include "InitiateConnectionCommandFactory.h"
 #include "DownloadContext.h"
 #include "Command.h"
 
@@ -68,16 +67,7 @@ void StreamFileAllocationEntry::prepareForNextAction(std::deque<Command*>& comma
     // try remaining uris
     _requestGroup->createNextCommandWithAdj(commands, e, -1);
   } else {
-    if(_currentRequest.isNull()) {
-      _requestGroup->createNextCommandWithAdj(commands, e, 0);
-    } else {
-      Command* command =
-	InitiateConnectionCommandFactory::createInitiateConnectionCommand
-	(e->newCUID(), _currentRequest, _requestGroup, e);
-      e->setNoWait(true);
-      commands.push_back(command);
-      _requestGroup->createNextCommandWithAdj(commands, e, -1);
-    }
+    _requestGroup->createNextCommandWithAdj(commands, e, 0);
   }
 }
 

src/TrackerWatcherCommand.cc

@@ -223,6 +223,7 @@ TrackerWatcherCommand::createRequestGroup(const std::string& uri)
 				   A2STR::NIL,
 				   TRACKER_ANNOUNCE_FILE));
   dctx->setDir(A2STR::NIL);
+  dctx->getFileEntries().front()->setUris(uris);
   rg->setDownloadContext(dctx);
   SharedHandle<DiskWriterFactory> dwf(new ByteArrayDiskWriterFactory());
   rg->setDiskWriterFactory(dwf);

src/UnknownLengthPieceStorage.cc

@@ -100,7 +100,8 @@ SharedHandle<Piece> UnknownLengthPieceStorage::getMissingFastPiece
 
 #endif // ENABLE_BITTORRENT
 
-PieceHandle UnknownLengthPieceStorage::getMissingPiece()
+SharedHandle<Piece> UnknownLengthPieceStorage::getSparseMissingUnusedPiece
+(const unsigned char* ignoreBitfield, size_t length)
 {
   if(_downloadFinished) {
     return SharedHandle<Piece>();
@@ -116,7 +117,7 @@ PieceHandle UnknownLengthPieceStorage::getMissingPiece()
 PieceHandle UnknownLengthPieceStorage::getMissingPiece(size_t index)
 {
   if(index == 0) {
-    return getMissingPiece();
+    return getSparseMissingUnusedPiece(0, 0);
   } else {
     return SharedHandle<Piece>();
   }

src/UnknownLengthPieceStorage.h

@@ -101,7 +101,8 @@ public:
   /**
    * Returns a missing piece if available. Otherwise returns 0;
    */
-  virtual SharedHandle<Piece> getMissingPiece();
+  virtual SharedHandle<Piece> getSparseMissingUnusedPiece
+  (const unsigned char* ignoreBitfield, size_t length);
 
   /**
    * Returns a missing piece whose index is index.

src/array_fun.h

@@ -154,6 +154,13 @@ struct And
   static inline returnType apply(T lhs, T rhs) { return lhs&rhs; }
 };
 
+template<typename T>
+struct Or
+{
+  typedef T returnType;
+  static inline returnType apply(T lhs, T rhs) { return lhs|rhs; }
+};
+
 template<typename T>
 struct Negate
 {
@@ -184,6 +191,13 @@ operator&(const L& l, const R& r)
   return BinExpr<L, And<typename L::returnType>, R>(l, r);
 }
 
+template<typename L, typename R>
+BinExpr<L, Or<typename L::returnType>, R>
+operator|(const L& l, const R& r)
+{
+  return BinExpr<L, Or<typename L::returnType>, R>(l, r);
+}
+
 template<typename A>
 UnExpr<Negate<typename A::returnType>, A>
 operator~(const A& a)

src/bitfield.h

@@ -54,7 +54,8 @@ inline unsigned char lastByteMask(size_t nbits)
 }
 
 // Returns true if index-th bits is set. Otherwise returns false.
-inline bool test(const unsigned char* bitfield, size_t nbits, size_t index)
+template<typename Array>
+inline bool test(const Array& bitfield, size_t nbits, size_t index)
 {
   assert(index < nbits);
   unsigned char mask = 128 >> (index%8);

src/download_helper.cc

@@ -187,6 +187,7 @@ static SharedHandle<RequestGroup> createRequestGroup
       strconcat(option->get(PREF_DIR), "/", option->get(PREF_OUT)):A2STR::NIL));
 
   dctx->setDir(option->get(PREF_DIR));
+  dctx->getFileEntries().front()->setUris(uris);
   rg->setDownloadContext(dctx);
   return rg;
 }

test/DownloadContextTest.cc

@@ -0,0 +1,44 @@
+#include "DownloadContext.h"
+
+#include <cppunit/extensions/HelperMacros.h>
+
+#include "FileEntry.h"
+#include "MockDownloadContext.h"
+
+namespace aria2 {
+
+class DownloadContextTest:public CppUnit::TestFixture {
+
+  CPPUNIT_TEST_SUITE(DownloadContextTest);
+  CPPUNIT_TEST(testFindFileEntryByOffset);
+  CPPUNIT_TEST_SUITE_END();
+public:
+  void testFindFileEntryByOffset();
+};
+
+
+CPPUNIT_TEST_SUITE_REGISTRATION(DownloadContextTest);
+
+void DownloadContextTest::testFindFileEntryByOffset()
+{
+  MockDownloadContext ctx;
+
+  CPPUNIT_ASSERT(ctx.findFileEntryByOffset(0).isNull());
+  
+  ctx.addFileEntry(SharedHandle<FileEntry>(new FileEntry("file1",1000,0)));
+  ctx.addFileEntry(SharedHandle<FileEntry>(new FileEntry("file2",0,1000)));
+  ctx.addFileEntry(SharedHandle<FileEntry>(new FileEntry("file3",0,1000)));
+  ctx.addFileEntry(SharedHandle<FileEntry>(new FileEntry("file4",2000,1000)));
+  ctx.addFileEntry(SharedHandle<FileEntry>(new FileEntry("file5",3000,3000)));
+  ctx.addFileEntry(SharedHandle<FileEntry>(new FileEntry("file6",0,6000)));
+
+  CPPUNIT_ASSERT_EQUAL(std::string("file1"),
+		       ctx.findFileEntryByOffset(0)->getPath());
+  CPPUNIT_ASSERT_EQUAL(std::string("file4"),
+		       ctx.findFileEntryByOffset(1500)->getPath());
+  CPPUNIT_ASSERT_EQUAL(std::string("file5"),
+		       ctx.findFileEntryByOffset(5999)->getPath());
+  CPPUNIT_ASSERT(ctx.findFileEntryByOffset(6000).isNull());
+}
+
+} // namespace aria2

test/Makefile.am

@@ -70,7 +70,9 @@ aria2c_SOURCES = AllTest.cc\
 	LongestSequencePieceSelectorTest.cc\
 	a2algoTest.cc\
 	bitfieldTest.cc\
-	BDETest.cc
+	BDETest.cc\
+	DownloadContextTest.cc\
+	MockDownloadContext.h
 
 if ENABLE_XML_RPC
 aria2c_SOURCES += XmlRpcRequestParserControllerTest.cc\

test/Makefile.in

@@ -200,8 +200,8 @@ am__aria2c_SOURCES_DIST = AllTest.cc TestUtil.cc TestUtil.h \
 	SequentialPickerTest.cc RarestPieceSelectorTest.cc \
 	PieceStatManTest.cc InOrderPieceSelector.h \
 	LongestSequencePieceSelectorTest.cc a2algoTest.cc \
-	bitfieldTest.cc BDETest.cc \
-	XmlRpcRequestParserControllerTest.cc \
+	bitfieldTest.cc BDETest.cc DownloadContextTest.cc \
+	MockDownloadContext.h XmlRpcRequestParserControllerTest.cc \
 	XmlRpcRequestProcessorTest.cc XmlRpcMethodTest.cc \
 	FallocFileAllocationIteratorTest.cc GZipDecoderTest.cc \
 	Sqlite3MozCookieParserTest.cc MessageDigestHelperTest.cc \
@@ -377,9 +377,9 @@ am_aria2c_OBJECTS = AllTest.$(OBJEXT) TestUtil.$(OBJEXT) \
 	RarestPieceSelectorTest.$(OBJEXT) PieceStatManTest.$(OBJEXT) \
 	LongestSequencePieceSelectorTest.$(OBJEXT) \
 	a2algoTest.$(OBJEXT) bitfieldTest.$(OBJEXT) BDETest.$(OBJEXT) \
-	$(am__objects_1) $(am__objects_2) $(am__objects_3) \
-	$(am__objects_4) $(am__objects_5) $(am__objects_6) \
-	$(am__objects_7)
+	DownloadContextTest.$(OBJEXT) $(am__objects_1) \
+	$(am__objects_2) $(am__objects_3) $(am__objects_4) \
+	$(am__objects_5) $(am__objects_6) $(am__objects_7)
 aria2c_OBJECTS = $(am_aria2c_OBJECTS)
 am__DEPENDENCIES_1 =
 aria2c_DEPENDENCIES = ../src/libaria2c.a ../src/download_helper.o \
@@ -607,7 +607,8 @@ aria2c_SOURCES = AllTest.cc TestUtil.cc TestUtil.h SocketCoreTest.cc \
 	SequentialPickerTest.cc RarestPieceSelectorTest.cc \
 	PieceStatManTest.cc InOrderPieceSelector.h \
 	LongestSequencePieceSelectorTest.cc a2algoTest.cc \
-	bitfieldTest.cc BDETest.cc $(am__append_1) $(am__append_2) \
+	bitfieldTest.cc BDETest.cc DownloadContextTest.cc \
+	MockDownloadContext.h $(am__append_1) $(am__append_2) \
 	$(am__append_3) $(am__append_4) $(am__append_5) \
 	$(am__append_6) $(am__append_7)
 
@@ -773,6 +774,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPeerStorageTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPieceStorageTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DirectDiskAdaptorTest.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadContextTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadHandlerFactoryTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadHelperTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ExceptionTest.Po@am__quote@

test/MockDownloadContext.h

@@ -0,0 +1,67 @@
+#ifndef _D_MOCK_DOWNLOAD_CONTEXT_H_
+#define _D_MOCK_DOWNLOAD_CONTEXT_H_
+
+#include "DownloadContext.h"
+#include "A2STR.h"
+
+namespace aria2 {
+
+class MockDownloadContext:public DownloadContext
+{
+private:
+  std::deque<std::string> _pieceHashes;
+public:
+  virtual const std::string& getPieceHash(size_t index) const
+  {
+    return A2STR::NIL;
+  }
+  
+  virtual const std::deque<std::string>& getPieceHashes() const
+  {
+    return _pieceHashes;
+  }
+
+  virtual uint64_t getTotalLength() const
+  {
+    return 0;
+  }
+
+  virtual bool knowsTotalLength() const
+  {
+    return false;
+  }
+
+  virtual FILE_MODE getFileMode() const
+  {
+    return MULTI;
+  }
+
+  virtual size_t getPieceLength() const
+  {
+    return 0;
+  }
+
+  virtual size_t getNumPieces() const
+  {
+    return 0;
+  }
+
+  virtual const std::string& getPieceHashAlgo() const
+  {
+    return A2STR::NIL;
+  }
+
+  virtual std::string getActualBasePath() const
+  {
+    return A2STR::NIL;
+  }
+
+  void addFileEntry(const SharedHandle<FileEntry>& fileEntry)
+  {
+    _fileEntries.push_back(fileEntry);
+  }
+};
+
+} // namespace aria2
+
+#endif // _D_MOCK_DOWNLOAD_CONTEXT_H_