2008-06-16 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

	Create only requested files and files which shares a piece with
	requested file in multi-torrent downloads. Directory structures 
are
	also created in the same rule.
	MultiDiskAdaptor::fileExists() now uses FileEntry instead of
	DiskWriterEntry as in the previous implementation.
	* src/MultiDiskWriter.cc
	* src/MultiFileAllocationIterator.cc
	* src/MultiFileAllocationIterator.h
	* test/MultiFileAllocationIteratorTest.cc

ChangeLog

@@ -1,3 +1,15 @@
+2008-06-16  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Create only requested files and files which shares a piece with
+	requested file in multi-torrent downloads. Directory structures are
+	also created in the same rule.
+	MultiDiskAdaptor::fileExists() now uses FileEntry instead of
+	DiskWriterEntry as in the previous implementation.
+	* src/MultiDiskWriter.cc
+	* src/MultiFileAllocationIterator.cc
+	* src/MultiFileAllocationIterator.h
+	* test/MultiFileAllocationIteratorTest.cc
+
 2008-06-14  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	Commented out -lprofiler

src/MultiDiskAdaptor.cc

@@ -149,20 +149,77 @@ MultiDiskAdaptor::MultiDiskAdaptor():
 
 MultiDiskAdaptor::~MultiDiskAdaptor() {}
 
+static SharedHandle<DiskWriterEntry> createDiskWriterEntry
+(const SharedHandle<FileEntry>& fileEntry,
+ DiskWriterFactory& dwFactory,
+ bool directIOAllowed)
+{
+  SharedHandle<DiskWriterEntry> entry(new DiskWriterEntry(fileEntry));
+  entry->setDiskWriter(dwFactory.newDiskWriter());
+  entry->getDiskWriter()->setDirectIOAllowed(directIOAllowed);
+  
+  return entry;
+}
+ 
+
 void MultiDiskAdaptor::resetDiskWriterEntries()
 {
   diskWriterEntries.clear();
-  for(FileEntries::const_iterator itr = fileEntries.begin();
-      itr != fileEntries.end(); itr++) {
-    DiskWriterEntryHandle entry(new DiskWriterEntry(*itr));
-    if((*itr)->isRequested()) {
-      entry->setDiskWriter(DefaultDiskWriterFactory().newDiskWriter());
-    } else {
-      SharedHandle<DiskWriter> dw(new DefaultDiskWriter());
-      entry->setDiskWriter(dw);
+
+  if(fileEntries.empty()) {
+    return;
+  }
+
+  DefaultDiskWriterFactory dwFactory;
+  if(pieceLength == 0) {
+    for(std::deque<SharedHandle<FileEntry> >::const_iterator itr =
+	  fileEntries.begin(); itr != fileEntries.end(); ++itr) {
+      if((*itr)->isRequested()) {
+	diskWriterEntries.push_back
+	  (createDiskWriterEntry(*itr, dwFactory, _directIOAllowed));
+      }
+    }
+  } else {
+    std::deque<SharedHandle<FileEntry> >::const_iterator done = fileEntries.begin();
+    for(std::deque<SharedHandle<FileEntry> >::const_iterator itr =
+	  fileEntries.begin(); itr != fileEntries.end();) {
+      if(!(*itr)->isRequested()) {
+	++itr;
+	continue;
+      }
+      off_t pieceStartOffset = ((*itr)->getOffset()/pieceLength)*pieceLength;
+      std::deque<SharedHandle<DiskWriterEntry> >::iterator insertionPoint =
+	diskWriterEntries.end();
+
+      if(itr != fileEntries.begin()) {
+	for(std::deque<SharedHandle<FileEntry> >::const_iterator i = itr-1;
+	    i != done; --i) {
+	  if((uint64_t)pieceStartOffset < (*i)->getOffset()+(*i)->getLength()) {
+	    insertionPoint = diskWriterEntries.insert
+	      (insertionPoint,
+	       createDiskWriterEntry(*i, dwFactory, _directIOAllowed));
+	  } else {
+	    break;
+	  }
+	}
+      }
+
+      diskWriterEntries.push_back
+	(createDiskWriterEntry(*itr, dwFactory, _directIOAllowed));
+
+      ++itr;
+
+      for(; itr != fileEntries.end(); ++itr) {
+	if((*itr)->getOffset() < pieceStartOffset+pieceLength) {
+	  diskWriterEntries.push_back
+	    (createDiskWriterEntry(*itr, dwFactory, _directIOAllowed));
+	} else {
+	  break;
+	}
+      }
+
+      done = itr-1;
     }
-    entry->getDiskWriter()->setDirectIOAllowed(_directIOAllowed);
-    diskWriterEntries.push_back(entry);
   }
 }
 
@@ -173,9 +230,9 @@ std::string MultiDiskAdaptor::getTopDirPath() const
 
 void MultiDiskAdaptor::mkdir(const std::string& topDirPath) const
 {
-  for(FileEntries::const_iterator itr = fileEntries.begin();
-      itr != fileEntries.end(); itr++) {
-    (*itr)->setupDir(topDirPath);
+  for(std::deque<SharedHandle<DiskWriterEntry> >::const_iterator i =
+	diskWriterEntries.begin(); i != diskWriterEntries.end(); ++i) {
+    (*i)->getFileEntry()->setupDir(topDirPath);
   }
 }
 
@@ -211,16 +268,16 @@ void MultiDiskAdaptor::openIfNot
 void MultiDiskAdaptor::openFile()
 {
   _cachedTopDirPath = getTopDirPath();
-  mkdir(_cachedTopDirPath);
   resetDiskWriterEntries();
+  mkdir(_cachedTopDirPath);
   // TODO we should call openIfNot here?
 }
 
 void MultiDiskAdaptor::initAndOpenFile()
 {
   _cachedTopDirPath = getTopDirPath();
-  mkdir(_cachedTopDirPath);
   resetDiskWriterEntries();
+  mkdir(_cachedTopDirPath);
   // Call DiskWriterEntry::initAndOpenFile to make files truncated.
   for(DiskWriterEntries::iterator itr = diskWriterEntries.begin();
       itr != diskWriterEntries.end(); ++itr) {
@@ -339,17 +396,15 @@ ssize_t MultiDiskAdaptor::readData(unsigned char* data, size_t len, off_t offset
 
 bool MultiDiskAdaptor::fileExists()
 {
-  if(diskWriterEntries.empty()) {
-    resetDiskWriterEntries();
-  }
   // Don't use _cachedTopDirPath because they are initialized after opening files.
   // This method could be called before opening files.
   std::string topDirPath = getTopDirPath();
-  for(DiskWriterEntries::iterator itr = diskWriterEntries.begin();
-      itr != diskWriterEntries.end(); ++itr) {
-    if((*itr)->fileExists(topDirPath)) {
+  for(std::deque<SharedHandle<FileEntry> >::iterator i =
+	fileEntries.begin(); i != fileEntries.end(); ++i) {
+    
+    if(File(topDirPath+"/"+(*i)->getPath()).isFile()) {
       return true;
-    }				   
+    }
   }
   return false;
 }

src/MultiFileAllocationIterator.cc

@@ -42,11 +42,9 @@ namespace aria2 {
 
 MultiFileAllocationIterator::MultiFileAllocationIterator(MultiDiskAdaptor* diskAdaptor):
   _diskAdaptor(diskAdaptor),
+  _entries(_diskAdaptor->diskWriterEntries),
   _offset(0)
-{
-  makeDiskWriterEntries(_entries, diskAdaptor->diskWriterEntries,
-			diskAdaptor->getPieceLength());
-}
+{}
 
 MultiFileAllocationIterator::~MultiFileAllocationIterator() {}
 
@@ -106,41 +104,4 @@ const DiskWriterEntries& MultiFileAllocationIterator::getDiskWriterEntries() con
   return _entries;
 }
 
-void MultiFileAllocationIterator::makeDiskWriterEntries
-(std::deque<SharedHandle<DiskWriterEntry> >& dwEntries,
- const DiskWriterEntries& srcEntries, size_t pieceLength) const
-{
-  if(pieceLength == 0) {
-    for(DiskWriterEntries::const_iterator itr = srcEntries.begin(); itr != srcEntries.end(); ++itr) {
-      if((*itr)->getFileEntry()->isRequested()) {
-	dwEntries.push_back(*itr);
-      }
-    }
-  } else {
-    DiskWriterEntries temp(srcEntries);
-    {
-      SharedHandle<FileEntry> f(new FileEntry());
-      SharedHandle<DiskWriterEntry> e(new DiskWriterEntry(f));
-      temp.push_front(e);
-    }
-    DiskWriterEntries::const_iterator done = temp.begin();
-    for(DiskWriterEntries::const_iterator itr = temp.begin()+1; itr != temp.end(); ++itr) {
-      const FileEntryHandle& fileEntry = (*itr)->getFileEntry();
-      if(!fileEntry->isRequested()) {
-	continue;
-      }
-      off_t pieceStartOffset = (fileEntry->getOffset()/pieceLength)*pieceLength;
-      for(DiskWriterEntries::const_iterator i = itr-1; i != done; --i) {
-	if((uint64_t)pieceStartOffset < (*i)->getFileEntry()->getOffset()+(*i)->getFileEntry()->getLength()) {
-	  dwEntries.push_back(*i);
-	} else {
-	  break;
-	}
-      }
-      dwEntries.push_back(*itr);
-      done = itr;
-    }
-  }
-}
-
 } // namespace aria2

src/MultiFileAllocationIterator.h

@@ -52,10 +52,6 @@ private:
   SharedHandle<SingleFileAllocationIterator> _fileAllocationIterator;
   off_t _offset;
 
-  void makeDiskWriterEntries
-  (std::deque<SharedHandle<DiskWriterEntry> >& dwEntries,
-   const std::deque<SharedHandle<DiskWriterEntry> >& srcEntries,
-   size_t pieceLength) const;
 public:
   MultiFileAllocationIterator(MultiDiskAdaptor* diskAdaptor);
 

test/MultiFileAllocationIteratorTest.cc

@@ -3,6 +3,7 @@
 #include "MultiDiskAdaptor.h"
 #include "FileEntry.h"
 #include "Exception.h"
+#include "array_fun.h"
 #include <algorithm>
 #include <iostream>
 #include <cppunit/extensions/HelperMacros.h>
@@ -31,27 +32,28 @@ void MultiFileAllocationIteratorTest::testMakeDiskWriterEntries()
 {
   SharedHandle<FileEntry> fs[] = {
     SharedHandle<FileEntry>(new FileEntry("file1", 1536, 0)),
-    SharedHandle<FileEntry>(new FileEntry("file2", 2048, 1536)),
+    SharedHandle<FileEntry>(new FileEntry("file2", 2048, 1536)),// req no
     SharedHandle<FileEntry>(new FileEntry("file3", 1024, 3584)),
-    SharedHandle<FileEntry>(new FileEntry("file4", 1024, 4608)),
-    SharedHandle<FileEntry>(new FileEntry("file5", 1024, 5632)),
-    SharedHandle<FileEntry>(new FileEntry("file6", 1024, 6656)),
-    SharedHandle<FileEntry>(new FileEntry("file7",  256, 7680)),
-    SharedHandle<FileEntry>(new FileEntry("file8",  768, 7936)),
-    SharedHandle<FileEntry>(new FileEntry("file9",  256, 8704)),
-    SharedHandle<FileEntry>(new FileEntry("fileA",  256, 8960)),
+    SharedHandle<FileEntry>(new FileEntry("file4", 1024, 4608)),// req no
+    SharedHandle<FileEntry>(new FileEntry("file5", 1024, 5632)),// req no
+    SharedHandle<FileEntry>(new FileEntry("file6", 1024, 6656)),// req no
+    SharedHandle<FileEntry>(new FileEntry("file7",  256, 7680)),// req no
+    SharedHandle<FileEntry>(new FileEntry("file8",  255, 7936)),
+    SharedHandle<FileEntry>(new FileEntry("file9", 1025, 8191)),// req no
+    SharedHandle<FileEntry>(new FileEntry("fileA", 1024, 9216)),// req no
+    SharedHandle<FileEntry>(new FileEntry("fileB", 1024, 10240)),
   };
-  fs[1]->setRequested(false);
-  fs[3]->setRequested(false);
-  fs[4]->setRequested(false);
-  fs[5]->setRequested(false);
-  fs[6]->setRequested(false);
-  fs[8]->setRequested(false);
-  fs[9]->setRequested(false);
+  fs[1]->setRequested(false); // file2
+  fs[3]->setRequested(false); // file4
+  fs[4]->setRequested(false); // file5
+  fs[5]->setRequested(false); // file6
+  fs[6]->setRequested(false); // file7
+  fs[8]->setRequested(false); // file9
+  fs[9]->setRequested(false); // fileA
   
   std::string storeDir = "/tmp/aria2_MultiFileAllocationIteratorTest_testMakeDiskWriterEntries";
   SharedHandle<MultiDiskAdaptor> diskAdaptor(new MultiDiskAdaptor());
-  diskAdaptor->setFileEntries(std::deque<SharedHandle<FileEntry> >(&fs[0], &fs[10]));
+  diskAdaptor->setFileEntries(std::deque<SharedHandle<FileEntry> >(&fs[0], &fs[arrayLength(fs)]));
   diskAdaptor->setPieceLength(1024);
   diskAdaptor->setStoreDir(storeDir);
   diskAdaptor->openFile();
@@ -63,7 +65,7 @@ void MultiFileAllocationIteratorTest::testMakeDiskWriterEntries()
 
   std::sort(entries.begin(), entries.end());
 
-  CPPUNIT_ASSERT_EQUAL((size_t)6, entries.size());
+  CPPUNIT_ASSERT_EQUAL((size_t)8, entries.size());
 
   CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file1"), entries[0]->getFilePath(storeDir));
   CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file2"), entries[1]->getFilePath(storeDir));
@@ -71,6 +73,8 @@ void MultiFileAllocationIteratorTest::testMakeDiskWriterEntries()
   CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file6"), entries[3]->getFilePath(storeDir));
   CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file7"), entries[4]->getFilePath(storeDir));
   CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file8"), entries[5]->getFilePath(storeDir));
+  CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file9"), entries[6]->getFilePath(storeDir));
+  CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/fileB"), entries[7]->getFilePath(storeDir));
 }
 
 void MultiFileAllocationIteratorTest::testAllocate()
@@ -145,9 +149,10 @@ void MultiFileAllocationIteratorTest::testAllocate()
     CPPUNIT_ASSERT_EQUAL((uint64_t)length1, File(dir+"/"+topDir+"/"+fname1).size());
     CPPUNIT_ASSERT_EQUAL((uint64_t)length2, File(dir+"/"+topDir+"/"+fname2).size());
     CPPUNIT_ASSERT_EQUAL((uint64_t)length3, File(dir+"/"+topDir+"/"+fname3).size());
-    CPPUNIT_ASSERT_EQUAL(0ULL, File(dir+"/"+topDir+"/"+fname4).size());
+    CPPUNIT_ASSERT(!File(dir+"/"+topDir+"/"+fname4).isFile());
+
     CPPUNIT_ASSERT_EQUAL((uint64_t)length5, File(dir+"/"+topDir+"/"+fname5).size());
-    CPPUNIT_ASSERT_EQUAL(0ULL, File(dir+"/"+topDir+"/"+fname6).size());
+    CPPUNIT_ASSERT(!File(dir+"/"+topDir+"/"+fname6).isFile());
 
   } catch(Exception& e) {
     CPPUNIT_FAIL(e.stackTrace());