2010-07-12 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

	Set end byte in Range header if start byte > 0 to get more chance
	to pool socket.
	* src/DefaultPieceStorage.cc
	* src/DefaultPieceStorage.h
	* src/DownloadCommand.cc
	* src/HttpDownloadCommand.cc
	* src/HttpRequest.cc
	* src/HttpRequest.h
	* src/HttpRequestCommand.cc
	* src/PieceStorage.h
	* src/UnknownLengthPieceStorage.h
	* test/DefaultPieceStorageTest.cc
	* test/HttpRequestTest.cc
	* test/MockPieceStorage.h

ChangeLog

@@ -1,3 +1,20 @@
+2010-07-12  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
+
+	Set end byte in Range header if start byte > 0 to get more chance
+	to pool socket.
+	* src/DefaultPieceStorage.cc
+	* src/DefaultPieceStorage.h
+	* src/DownloadCommand.cc
+	* src/HttpDownloadCommand.cc
+	* src/HttpRequest.cc
+	* src/HttpRequest.h
+	* src/HttpRequestCommand.cc
+	* src/PieceStorage.h
+	* src/UnknownLengthPieceStorage.h
+	* test/DefaultPieceStorageTest.cc
+	* test/HttpRequestTest.cc
+	* test/MockPieceStorage.h
+
 2010-07-11  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
 
 	Moved segment empty check before socket check.

src/DefaultPieceStorage.cc

@@ -684,4 +684,14 @@ void DefaultPieceStorage::addPieceStats(size_t index)
   pieceStatMan_->addPieceStats(index);
 }
 
+size_t DefaultPieceStorage::getNextUsedIndex(size_t index)
+{
+  for(size_t i = index+1; i < bitfieldMan_->countBlock(); ++i) {
+    if(bitfieldMan_->isUseBitSet(i) || bitfieldMan_->isBitSet(i)) {
+      return i;
+    }
+  }
+  return bitfieldMan_->countBlock();
+}
+
 } // namespace aria2

src/DefaultPieceStorage.h

@@ -227,6 +227,8 @@ public:
                                 size_t newBitfieldLength,
                                 const unsigned char* oldBitfield);
 
+  virtual size_t getNextUsedIndex(size_t index);
+
   /**
    * This method is made private for test purpose only.
    */

src/DownloadCommand.cc

@@ -336,6 +336,11 @@ bool DownloadCommand::prepareForNextSegment() {
       if(!tempSegment->complete()) {
         return prepareForRetry(0);
       }
+      if(getRequestEndOffset() ==
+         getFileEntry()->gtoloff
+         (tempSegment->getPosition()+tempSegment->getLength())) {
+        return prepareForRetry(0);
+      }
       SharedHandle<Segment> nextSegment = getSegmentMan()->getSegment
         (getCuid(), tempSegment->getIndex()+1);
       if(nextSegment.isNull()) {

src/HttpDownloadCommand.cc

@@ -52,7 +52,7 @@
 #include "FileAllocationEntry.h"
 #include "CheckIntegrityEntry.h"
 #include "ServerStatMan.h"
-
+#include "Logger.h"
 namespace aria2 {
 
 HttpDownloadCommand::HttpDownloadCommand
@@ -88,12 +88,13 @@ bool HttpDownloadCommand::prepareForNextSegment() {
     if(getRequest()->isPipeliningEnabled() ||
        (getRequest()->isKeepAliveEnabled() &&
         (
+         // TODO make sure that all decoder is finished to pool socket
          ((!getTransferEncodingDecoder().isNull() &&
            getTransferEncodingDecoder()->finished()) ||
           (!getContentEncodingDecoder().isNull() &&
            getContentEncodingDecoder()->finished())) ||
-         getFileEntry()->getLastOffset() ==
-         getSegments().front()->getPositionToWrite()
+         getRequestEndOffset() ==
+         getFileEntry()->gtoloff(getSegments().front()->getPositionToWrite())
          )
         )
        ) {

src/HttpRequest.cc

@@ -58,7 +58,8 @@ const std::string HttpRequest::USER_AGENT("aria2");
 HttpRequest::HttpRequest():contentEncodingEnabled_(true),
                            userAgent_(USER_AGENT),
                            noCache_(true),
-                           acceptGzip_(false)
+                           acceptGzip_(false),
+                           endOffsetOverride_(false)
 {}
 
 void HttpRequest::setSegment(const SharedHandle<Segment>& segment)
@@ -196,6 +197,9 @@ std::string HttpRequest::createRequest()
     rangeHeader += "-";
     if(request_->isPipeliningEnabled()) {
       rangeHeader += util::itos(getEndByte());
+    } else if(getProtocol() != Request::PROTO_FTP && endOffsetOverride_ > 0) {
+      // FTP via http proxy does not support endbytes 
+      rangeHeader += util::itos(endOffsetOverride_-1);
     }
     builtinHds.push_back(std::make_pair("Range:", rangeHeader));
   }

src/HttpRequest.h

@@ -87,6 +87,8 @@ private:
 
   bool acceptGzip_;
 
+  off_t endOffsetOverride_;
+
   std::pair<std::string, std::string> getProxyAuthString() const;
 public:
   HttpRequest();
@@ -272,6 +274,11 @@ public:
   {
     return acceptGzip_;
   }
+
+  void setEndOffsetOverride(off_t offset)
+  {
+    endOffsetOverride_ = offset;
+  }
 };
 
 } // namespace aria2

src/HttpRequestCommand.cc

@@ -57,6 +57,7 @@
 #include "FileAllocationEntry.h"
 #include "CheckIntegrityEntry.h"
 #include "ServerStatMan.h"
+#include "PieceStorage.h"
 
 namespace aria2 {
 
@@ -87,7 +88,8 @@ createHttpRequest(const SharedHandle<Request>& req,
                   const RequestGroup* rg,
                   const SharedHandle<CookieStorage>& cookieStorage,
                   const SharedHandle<AuthConfigFactory>& authConfigFactory,
-                  const SharedHandle<Request>& proxyRequest)
+                  const SharedHandle<Request>& proxyRequest,
+                  off_t endOffset = 0)
 {
   SharedHandle<HttpRequest> httpRequest(new HttpRequest());
   httpRequest->setUserAgent(option->get(PREF_USER_AGENT));
@@ -110,6 +112,7 @@ createHttpRequest(const SharedHandle<Request>& req,
   } else {
     httpRequest->disableNoCache();
   }
+  httpRequest->setEndOffsetOverride(endOffset);
   return httpRequest;
 }
 
@@ -148,6 +151,15 @@ bool HttpRequestCommand::executeInternal() {
           itr != eoi; ++itr) {
         const SharedHandle<Segment>& segment = *itr;
         if(!httpConnection_->isIssued(segment)) {
+          off_t endOffset = 0;
+          if(getRequestGroup()->getTotalLength() > 0 &&
+             !getPieceStorage().isNull()) {
+            size_t nextIndex =
+              getPieceStorage()->getNextUsedIndex(segment->getIndex());
+            endOffset = std::min
+              (static_cast<off_t>(getFileEntry()->getLength()),
+               getFileEntry()->gtoloff(segment->getSegmentLength()*nextIndex));
+          }
           SharedHandle<HttpRequest> httpRequest
             (createHttpRequest(getRequest(),
                                getFileEntry(),
@@ -157,7 +169,8 @@ bool HttpRequestCommand::executeInternal() {
                                getRequestGroup(),
                                getDownloadEngine()->getCookieStorage(),
                                getDownloadEngine()->getAuthConfigFactory(),
-                               proxyRequest_));
+                               proxyRequest_,
+                               endOffset));
           httpConnection_->sendRequest(httpRequest);
         }
       }

src/PieceStorage.h

@@ -244,6 +244,10 @@ public:
                                 size_t newBitfieldLength,
                                 const unsigned char* oldBitfield) = 0;
 
+  // Returns index x where all pieces in [index+1, x-1], inclusive,
+  // are not used and not completed. If all pieces after index+1 are
+  // used or completed, returns the number of pieces.
+  virtual size_t getNextUsedIndex(size_t index) = 0;
 };
 
 typedef SharedHandle<PieceStorage> PieceStorageHandle;

src/UnknownLengthPieceStorage.h

@@ -273,6 +273,7 @@ public:
                                 size_t newBitfieldLength,
                                 const unsigned char* oldBitfield) {}
 
+  virtual size_t getNextUsedIndex(size_t index) { return 0; }
 
   void setDiskWriterFactory(const SharedHandle<DiskWriterFactory>& diskWriterFactory);
 };

test/DefaultPieceStorageTest.cc

@@ -31,6 +31,7 @@ class DefaultPieceStorageTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testCancelPiece);
   CPPUNIT_TEST(testMarkPiecesDone);
   CPPUNIT_TEST(testGetCompletedLength);
+  CPPUNIT_TEST(testGetNextUsedIndex);
   CPPUNIT_TEST_SUITE_END();
 private:
   SharedHandle<DownloadContext> dctx_;
@@ -67,6 +68,7 @@ public:
   void testCancelPiece();
   void testMarkPiecesDone();
   void testGetCompletedLength();
+  void testGetNextUsedIndex();
 };
 
 
@@ -302,4 +304,16 @@ void DefaultPieceStorageTest::testGetCompletedLength()
   CPPUNIT_ASSERT_EQUAL((uint64_t)256*1024*1024, ps.getCompletedLength());
 }
 
+void DefaultPieceStorageTest::testGetNextUsedIndex()
+{
+  DefaultPieceStorage pss(dctx_, option);
+  CPPUNIT_ASSERT_EQUAL((size_t)3, pss.getNextUsedIndex(0));
+  SharedHandle<Piece> piece = pss.getMissingPiece(2);
+  CPPUNIT_ASSERT_EQUAL((size_t)2, pss.getNextUsedIndex(0));
+  pss.completePiece(piece);
+  CPPUNIT_ASSERT_EQUAL((size_t)2, pss.getNextUsedIndex(0));
+  piece = pss.getMissingPiece(0);
+  CPPUNIT_ASSERT_EQUAL((size_t)2, pss.getNextUsedIndex(0));
+}
+
 } // namespace aria2

test/HttpRequestTest.cc

@@ -29,6 +29,7 @@ class HttpRequestTest : public CppUnit::TestFixture {
   CPPUNIT_TEST(testCreateRequest_query);
   CPPUNIT_TEST(testCreateRequest_head);
   CPPUNIT_TEST(testCreateRequest_ipv6LiteralAddr);
+  CPPUNIT_TEST(testCreateRequest_endOffsetOverride);
   CPPUNIT_TEST(testCreateProxyRequest);
   CPPUNIT_TEST(testIsRangeSatisfied);
   CPPUNIT_TEST(testUserAgent);
@@ -55,6 +56,7 @@ public:
   void testCreateRequest_query();
   void testCreateRequest_head();
   void testCreateRequest_ipv6LiteralAddr();
+  void testCreateRequest_endOffsetOverride();
   void testCreateProxyRequest();
   void testIsRangeSatisfied();
   void testUserAgent();
@@ -494,6 +496,50 @@ void HttpRequestTest::testCreateRequest_head()
   CPPUNIT_ASSERT_EQUAL(std::string("HEAD /aria2-1.0.0.tar.bz2 HTTP/1.1"), line);
 }
 
+void HttpRequestTest::testCreateRequest_endOffsetOverride()
+{
+  SharedHandle<Request> request(new Request());
+  request->setUri("http://localhost/myfile");
+  HttpRequest httpRequest;
+  httpRequest.disableContentEncoding();
+  httpRequest.setRequest(request);
+  httpRequest.setAuthConfigFactory(authConfigFactory_, option_.get());
+  SharedHandle<Piece> p(new Piece(0, 1024*1024));
+  SharedHandle<Segment> segment(new PiecedSegment(1024*1024, p));
+  httpRequest.setSegment(segment);
+  httpRequest.setEndOffsetOverride(1024*1024*1024);
+  SharedHandle<FileEntry> fileEntry
+    (new FileEntry("file", 1024*1024*1024*10LL, 0));
+  httpRequest.setFileEntry(fileEntry);
+  // End byte is not passed if start byte is 0.
+  std::string expectedText =
+    "GET /myfile HTTP/1.1\r\n"
+    "User-Agent: aria2\r\n"
+    "Accept: */*\r\n"
+    "Host: localhost\r\n"
+    "Pragma: no-cache\r\n"
+    "Cache-Control: no-cache\r\n"
+    "Connection: close\r\n"
+    "\r\n";
+  
+  CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest());
+
+  segment->updateWrittenLength(1);
+
+  expectedText =
+    "GET /myfile HTTP/1.1\r\n"
+    "User-Agent: aria2\r\n"
+    "Accept: */*\r\n"
+    "Host: localhost\r\n"
+    "Pragma: no-cache\r\n"
+    "Cache-Control: no-cache\r\n"
+    "Connection: close\r\n"
+    "Range: bytes=1-1073741823\r\n"
+    "\r\n";
+  
+  CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest());
+}
+
 void HttpRequestTest::testCreateProxyRequest()
 {
   SharedHandle<Request> request(new Request());

test/MockPieceStorage.h

@@ -247,6 +247,11 @@ public:
   virtual void updatePieceStats(const unsigned char* newBitfield,
                                 size_t newBitfieldLength,
                                 const unsigned char* oldBitfield) {}
+
+  virtual size_t getNextUsedIndex(size_t index)
+  {
+    return 0;
+  }
 };
 
 } // namespace aria2