Fix (unknown length) downloads larger than 2GiB

Closes #215

src/AbstractDiskWriter.cc

@@ -310,6 +310,7 @@ ssize_t AbstractDiskWriter::readDataInternal(unsigned char* data, size_t len,
 
 void AbstractDiskWriter::seek(int64_t offset)
 {
+  assert(offset >= 0);
 #ifdef __MINGW32__
   LARGE_INTEGER fileLength;
   fileLength.QuadPart = offset;

src/GrowSegment.cc

@@ -43,7 +43,7 @@ GrowSegment::GrowSegment(const std::shared_ptr<Piece>& piece):
 
 GrowSegment::~GrowSegment() {}
 
-void GrowSegment::updateWrittenLength(int32_t bytes)
+void GrowSegment::updateWrittenLength(int64_t bytes)
 {
   writtenLength_ += bytes;
   piece_->reconfigure(writtenLength_);

src/GrowSegment.h

@@ -42,7 +42,7 @@ namespace aria2 {
 class GrowSegment:public Segment {
 private:
   std::shared_ptr<Piece> piece_;
-  int32_t writtenLength_;
+  int64_t writtenLength_;
 public:
   GrowSegment(const std::shared_ptr<Piece>& piece);
 
@@ -68,25 +68,25 @@ public:
     return writtenLength_;
   }
 
-  virtual int32_t getLength() const CXX11_OVERRIDE
+  virtual int64_t getLength() const CXX11_OVERRIDE
   {
     return 0;
   }
 
-  virtual int32_t getSegmentLength() const CXX11_OVERRIDE
+  virtual int64_t getSegmentLength() const CXX11_OVERRIDE
   {
     return 0;
   }
 
-  virtual int32_t getWrittenLength() const CXX11_OVERRIDE
+  virtual int64_t getWrittenLength() const CXX11_OVERRIDE
   {
     return writtenLength_;
   }
 
-  virtual void updateWrittenLength(int32_t bytes) CXX11_OVERRIDE;
+  virtual void updateWrittenLength(int64_t bytes) CXX11_OVERRIDE;
 
   virtual bool updateHash
-  (int32_t begin,
+  (int64_t begin,
    const unsigned char* data,
    size_t dataLength) CXX11_OVERRIDE
   {

src/PiecedSegment.cc

@@ -71,16 +71,16 @@ int64_t PiecedSegment::getPositionToWrite() const
   return getPosition()+writtenLength_;
 }
 
-int32_t PiecedSegment::getLength() const
+int64_t PiecedSegment::getLength() const
 {
   return piece_->getLength();
 }
 
-void PiecedSegment::updateWrittenLength(int32_t bytes)
+void PiecedSegment::updateWrittenLength(int64_t bytes)
 {
-  int32_t newWrittenLength = writtenLength_+bytes;
+  auto newWrittenLength = writtenLength_+bytes;
   assert(newWrittenLength <= piece_->getLength());
-  for(size_t i = writtenLength_/piece_->getBlockLength(),
+  for(auto i = writtenLength_/piece_->getBlockLength(),
         end = newWrittenLength/piece_->getBlockLength(); i < end; ++i) {
     piece_->completeBlock(i);
   }
@@ -91,7 +91,7 @@ void PiecedSegment::updateWrittenLength(int32_t bytes)
 }
 
 bool PiecedSegment::updateHash
-(int32_t begin,
+(int64_t begin,
  const unsigned char* data,
  size_t dataLength)
 {

src/PiecedSegment.h

@@ -47,7 +47,7 @@ private:
    * The last piece likely have shorter length than the other length.
    */
   int32_t pieceLength_;
-  int32_t writtenLength_;
+  int64_t writtenLength_;
 
 public:
   PiecedSegment(int32_t pieceLength, const std::shared_ptr<Piece>& piece);
@@ -62,23 +62,23 @@ public:
 
   virtual int64_t getPositionToWrite() const CXX11_OVERRIDE;
 
-  virtual int32_t getLength() const CXX11_OVERRIDE;
+  virtual int64_t getLength() const CXX11_OVERRIDE;
 
-  virtual int32_t getSegmentLength() const CXX11_OVERRIDE
+  virtual int64_t getSegmentLength() const CXX11_OVERRIDE
   {
     return pieceLength_;
   }
 
-  virtual int32_t getWrittenLength() const CXX11_OVERRIDE
+  virtual int64_t getWrittenLength() const CXX11_OVERRIDE
   {
     return writtenLength_;
   }
 
-  virtual void updateWrittenLength(int32_t bytes) CXX11_OVERRIDE;
+  virtual void updateWrittenLength(int64_t bytes) CXX11_OVERRIDE;
 
   // `begin' is a offset inside this segment.
   virtual bool updateHash
-  (int32_t begin,
+  (int64_t begin,
    const unsigned char* data,
    size_t dataLength) CXX11_OVERRIDE;
 

src/Segment.h

@@ -58,17 +58,17 @@ public:
 
   virtual int64_t getPositionToWrite() const = 0;
 
-  virtual int32_t getLength() const = 0;
+  virtual int64_t getLength() const = 0;
 
-  virtual int32_t getSegmentLength() const = 0;
+  virtual int64_t getSegmentLength() const = 0;
 
-  virtual int32_t getWrittenLength() const = 0;
+  virtual int64_t getWrittenLength() const = 0;
 
-  virtual void updateWrittenLength(int32_t bytes) = 0;
+  virtual void updateWrittenLength(int64_t bytes) = 0;
 
   // `begin' is a offset inside this segment.
   virtual bool updateHash
-  (int32_t begin,
+  (int64_t begin,
    const unsigned char* data,
    size_t dataLength) = 0;
 

src/SegmentMan.cc

@@ -156,8 +156,8 @@ std::shared_ptr<Segment> SegmentMan::checkoutSegment
   }
   std::shared_ptr<SegmentEntry> entry(new SegmentEntry(cuid, segment));
   usedSegmentEntries_.push_back(entry);
-  A2_LOG_DEBUG(fmt("index=%lu, length=%d, segmentLength=%d,"
-                   " writtenLength=%d",
+  A2_LOG_DEBUG(fmt("index=%lu, length=%" PRId64 ", segmentLength=%" PRId64 ","
+                   " writtenLength=%" PRId64,
                    static_cast<unsigned long>(segment->getIndex()),
                    segment->getLength(),
                    segment->getSegmentLength(),
@@ -166,8 +166,8 @@ std::shared_ptr<Segment> SegmentMan::checkoutSegment
   if(piece->getLength() > 0) {
     auto positr = segmentWrittenLengthMemo_.find(segment->getIndex());
     if(positr != segmentWrittenLengthMemo_.end()) {
-      const int32_t writtenLength = (*positr).second;
-      A2_LOG_DEBUG(fmt("writtenLength(in memo)=%d, writtenLength=%d",
+      const auto writtenLength = (*positr).second;
+      A2_LOG_DEBUG(fmt("writtenLength(in memo)=%" PRId64 ", writtenLength=%" PRId64,
                        writtenLength, segment->getWrittenLength()));
       //  If the difference between cached writtenLength and segment's
       //  writtenLength is less than one block, we assume that these
@@ -296,7 +296,7 @@ void SegmentMan::cancelSegmentInternal
   piece->setUsedBySegment(false);
   pieceStorage_->cancelPiece(piece, cuid);
   segmentWrittenLengthMemo_[segment->getIndex()] = segment->getWrittenLength();
-  A2_LOG_DEBUG(fmt("Memorized segment index=%lu, writtenLength=%d",
+  A2_LOG_DEBUG(fmt("Memorized segment index=%lu, writtenLength=%" PRId64,
                    static_cast<unsigned long>(segment->getIndex()),
                    segment->getWrittenLength()));
 }

src/SegmentMan.h

@@ -83,7 +83,7 @@ private:
 
   // Remember writtenLength for each segment. The key is an index of a
   // segment. The value is writtenLength for that segment.
-  std::map<size_t, int32_t> segmentWrittenLengthMemo_;
+  std::map<size_t, int64_t> segmentWrittenLengthMemo_;
 
   // Used for calculating download speed.
   std::vector<std::shared_ptr<PeerStat> > peerStats_;

test/GZipDecodingStreamFilterTest.cc

@@ -28,7 +28,7 @@ class GZipDecodingStreamFilterTest:public CppUnit::TestFixture {
   public:
     MockSegment2():positionToWrite_(0) {}
 
-    virtual void updateWrittenLength(int32_t bytes) CXX11_OVERRIDE
+    virtual void updateWrittenLength(int64_t bytes) CXX11_OVERRIDE
     {
       positionToWrite_ += bytes;
     }

test/GrowSegmentTest.cc

@@ -36,9 +36,9 @@ void GrowSegmentTest::testClear()
 {
   GrowSegment segment(std::shared_ptr<Piece>(new Piece()));
   segment.updateWrittenLength(32*1024);
-  CPPUNIT_ASSERT_EQUAL(32*1024, segment.getWrittenLength());
+  CPPUNIT_ASSERT_EQUAL((int64_t)32*1024, segment.getWrittenLength());
   segment.clear(nullptr);
-  CPPUNIT_ASSERT_EQUAL(0, segment.getWrittenLength());
+  CPPUNIT_ASSERT_EQUAL((int64_t)0, segment.getWrittenLength());
 }
 
 } // namespace aria2

test/MockSegment.h

@@ -29,26 +29,26 @@ public:
     return 0;
   }
 
-  virtual int32_t getLength() const CXX11_OVERRIDE
+  virtual int64_t getLength() const CXX11_OVERRIDE
   {
     return 0;
   }
 
-  virtual int32_t getSegmentLength() const CXX11_OVERRIDE
+  virtual int64_t getSegmentLength() const CXX11_OVERRIDE
   {
     return 0;
   }
 
-  virtual int32_t getWrittenLength() const CXX11_OVERRIDE
+  virtual int64_t getWrittenLength() const CXX11_OVERRIDE
   {
     return 0;
   }
 
-  virtual void updateWrittenLength(int32_t bytes) CXX11_OVERRIDE {}
+  virtual void updateWrittenLength(int64_t bytes) CXX11_OVERRIDE {}
 
   // `begin' is a offset inside this segment.
   virtual bool updateHash
-  (int32_t begin, const unsigned char* data, size_t dataLength) CXX11_OVERRIDE
+  (int64_t begin, const unsigned char* data, size_t dataLength) CXX11_OVERRIDE
   {
     return false;
   }

test/SegmentManTest.cc

@@ -66,9 +66,9 @@ void SegmentManTest::testNullBitfield()
   std::shared_ptr<Segment> segment = segmentMan.getSegment(1, minSplitSize);
   CPPUNIT_ASSERT(segment);
   CPPUNIT_ASSERT_EQUAL((size_t)0, segment->getIndex());
-  CPPUNIT_ASSERT_EQUAL(0, segment->getLength());
-  CPPUNIT_ASSERT_EQUAL(0, segment->getSegmentLength());
-  CPPUNIT_ASSERT_EQUAL(0, segment->getWrittenLength());
+  CPPUNIT_ASSERT_EQUAL((int64_t)0, segment->getLength());
+  CPPUNIT_ASSERT_EQUAL((int64_t)0, segment->getSegmentLength());
+  CPPUNIT_ASSERT_EQUAL((int64_t)0, segment->getWrittenLength());
 
   std::shared_ptr<Segment> segment2 = segmentMan.getSegment(2, minSplitSize);
   CPPUNIT_ASSERT(!segment2);

test/SegmentTest.cc

@@ -32,7 +32,7 @@ void SegmentTest::testUpdateWrittenLength()
 {
   std::shared_ptr<Piece> p(new Piece(0, 16*1024*10));
   PiecedSegment s(16*1024*10, p);
-  CPPUNIT_ASSERT_EQUAL(0, s.getWrittenLength());
+  CPPUNIT_ASSERT_EQUAL((int64_t)0, s.getWrittenLength());
 
   s.updateWrittenLength(16*1024);
   CPPUNIT_ASSERT(p->hasBlock(0));
@@ -67,9 +67,9 @@ void SegmentTest::testClear()
   std::shared_ptr<Piece> p(new Piece(0, 16*1024*10));
   PiecedSegment s(16*1024*10, p);
   s.updateWrittenLength(16*1024*10);
-  CPPUNIT_ASSERT_EQUAL(16*1024*10, s.getWrittenLength());
+  CPPUNIT_ASSERT_EQUAL((int64_t)16*1024*10, s.getWrittenLength());
   s.clear(nullptr);
-  CPPUNIT_ASSERT_EQUAL(0, s.getWrittenLength());
+  CPPUNIT_ASSERT_EQUAL((int64_t)0, s.getWrittenLength());
 }
 
 } // namespace aria2

test/SinkStreamFilterTest.cc

@@ -22,23 +22,23 @@ class SinkStreamFilterTest:public CppUnit::TestFixture {
   public:
     MockSegment2(int32_t length):length(length), writtenLength(0) {}
 
-    virtual int32_t getLength() const CXX11_OVERRIDE
+    virtual int64_t getLength() const CXX11_OVERRIDE
     {
       return length;
     }
 
-    virtual int32_t getWrittenLength() const CXX11_OVERRIDE
+    virtual int64_t getWrittenLength() const CXX11_OVERRIDE
     {
       return writtenLength;
     }
 
-    virtual void updateWrittenLength(int32_t bytes) CXX11_OVERRIDE
+    virtual void updateWrittenLength(int64_t bytes) CXX11_OVERRIDE
     {
       writtenLength += bytes;
     }
 
-    int32_t length;
-    int32_t writtenLength;
+    int64_t length;
+    int64_t writtenLength;
   };
 
   std::shared_ptr<SinkStreamFilter> filter_;