2008-11-13 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

	Rewritten URI handling functions. They are now provided as a
	testable functions.
	* src/Makefile.am
	* src/RequestGroup.cc
	* src/RequestGroup.h
	* src/download_helper.cc
	* src/download_helper.h
	* src/main.cc
	* test/DownloadHelperTest.cc
	* test/Makefile.am
	* test/input_uris.txt

ChangeLog

@@ -1,3 +1,17 @@
+2008-11-13  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
+
+	Rewritten URI handling functions. They are now provided as a testable
+	functions.
+	* src/Makefile.am
+	* src/RequestGroup.cc
+	* src/RequestGroup.h
+	* src/download_helper.cc
+	* src/download_helper.h
+	* src/main.cc
+	* test/DownloadHelperTest.cc
+	* test/Makefile.am
+	* test/input_uris.txt
+
 2008-11-12  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
 
 	Quickly terminate commands when ctrl-c is pressed.

src/Makefile.am

@@ -1,7 +1,8 @@
 bin_PROGRAMS = aria2c
 aria2c_SOURCES = main.cc\
 	option_processing.cc\
-	version_usage.cc
+	version_usage.cc\
+	download_helper.cc download_helper.h
 SRCS =  Socket.h\
 	SocketCore.cc SocketCore.h\
 	BinaryStream.h\

src/Makefile.in

@@ -828,7 +828,7 @@ am__installdirs = "$(DESTDIR)$(bindir)"
 binPROGRAMS_INSTALL = $(INSTALL_PROGRAM)
 PROGRAMS = $(bin_PROGRAMS)
 am_aria2c_OBJECTS = main.$(OBJEXT) option_processing.$(OBJEXT) \
-	version_usage.$(OBJEXT)
+	version_usage.$(OBJEXT) download_helper.$(OBJEXT)
 aria2c_OBJECTS = $(am_aria2c_OBJECTS)
 aria2c_DEPENDENCIES = libaria2c.a @ALLOCA@
 DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
@@ -1025,7 +1025,8 @@ top_builddir = @top_builddir@
 top_srcdir = @top_srcdir@
 aria2c_SOURCES = main.cc\
 	option_processing.cc\
-	version_usage.cc
+	version_usage.cc\
+	download_helper.cc download_helper.h
 
 SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \
 	Command.h AbstractCommand.cc AbstractCommand.h \
@@ -1523,6 +1524,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/VersionMetalinkParserState.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/XML2SAXMetalinkProcessor.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asctime_r.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/download_helper.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gai_strerror.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/getaddrinfo.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gettimeofday.Po@am__quote@

src/RequestGroup.cc

@@ -113,7 +113,7 @@ RequestGroup::RequestGroup(const Option* option,
 			   const std::deque<std::string>& uris):
   _gid(++_gidCounter),
   _uris(uris),
-  _numConcurrentCommand(0),
+  _numConcurrentCommand(option->getAsInt(PREF_SPLIT)),
   _numStreamConnection(0),
   _numCommand(0),
   _segmentManFactory(new DefaultSegmentManFactory(option)),
@@ -501,6 +501,7 @@ void RequestGroup::createNextCommandWithAdj(std::deque<Command*>& commands,
     numCommand = 1+numAdj;
   } else {
     if(_numConcurrentCommand == 0) {
+      // TODO remove _uris.size() support
       numCommand = _uris.size();
     } else {
       numCommand = _numConcurrentCommand;
@@ -1074,4 +1075,9 @@ void RequestGroup::increaseAndValidateFileNotFoundCount()
   }
 }
 
+unsigned int RequestGroup::getNumConcurrentCommand() const
+{
+  return _numConcurrentCommand;
+}
+
 } // namespace aria2

src/RequestGroup.h

@@ -213,6 +213,8 @@ public:
     _numConcurrentCommand = num;
   }
 
+  unsigned int getNumConcurrentCommand() const;
+
   int32_t getGID() const
   {
     return _gid;

src/download_helper.cc

@@ -0,0 +1,308 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#include "download_helper.h"
+
+#include <iostream>
+#include <fstream>
+#include <algorithm>
+
+#include "RequestGroup.h"
+#include "Option.h"
+#include "prefs.h"
+#include "Metalink2RequestGroup.h"
+#include "ProtocolDetector.h"
+#include "ParameterizedStringParser.h"
+#include "PStringBuildVisitor.h"
+#include "UriListParser.h"
+#include "SingleFileDownloadContext.h"
+#include "RecoverableException.h"
+#include "FatalException.h"
+#include "message.h"
+#include "StringFormat.h"
+#include "DefaultBtContext.h"
+#include "FileEntry.h"
+#include "LogFactory.h"
+#include "File.h"
+
+namespace aria2 {
+
+static void unfoldURI
+(std::deque<std::string>& result, const std::deque<std::string>& args)
+{
+  ParameterizedStringParser p;
+  PStringBuildVisitor v;
+  for(std::deque<std::string>::const_iterator itr = args.begin();
+      itr != args.end(); ++itr) {
+    v.reset();
+    p.parse(*itr)->accept(&v);
+    result.insert(result.end(), v.getURIs().begin(), v.getURIs().end()); 
+  }
+}
+
+static void splitURI(std::deque<std::string>& result,
+		     std::deque<std::string>::const_iterator begin,
+		     std::deque<std::string>::const_iterator end,
+		     size_t numSplit)
+{
+  size_t numURIs = std::distance(begin, end);
+  if(numURIs >= numSplit) {
+    result.insert(result.end(), begin, end);
+  } else {
+    for(size_t i = 0; i < numSplit/numURIs; ++i) {
+      result.insert(result.end(), begin, end);
+    }
+    result.insert(result.end(), begin, begin+(numSplit%numURIs));
+  }
+}
+
+static SharedHandle<RequestGroup> createRequestGroup
+(const Option* op, const std::deque<std::string>& uris,
+ const Option& requestOption,
+ bool useOutOption = false)
+{
+  SharedHandle<RequestGroup> rg(new RequestGroup(op, uris));
+  SharedHandle<SingleFileDownloadContext> dctx
+    (new SingleFileDownloadContext(op->getAsInt(PREF_SEGMENT_SIZE),
+				   0,
+				   A2STR::NIL,
+				   useOutOption?
+				   requestOption.get(PREF_OUT):A2STR::NIL));
+  dctx->setDir(requestOption.get(PREF_DIR));
+  rg->setDownloadContext(dctx);
+  return rg;
+}
+
+#ifdef ENABLE_BITTORRENT
+
+static
+SharedHandle<RequestGroup>
+createBtRequestGroup(const std::string& torrentFilePath,
+		     Option* op,
+		     const std::deque<std::string>& auxUris,
+		     const Option& requestOption)
+{
+  SharedHandle<RequestGroup> rg(new RequestGroup(op, auxUris));
+  SharedHandle<DefaultBtContext> btContext(new DefaultBtContext());
+  btContext->load(torrentFilePath);// may throw exception
+  if(op->defined(PREF_PEER_ID_PREFIX)) {
+    btContext->setPeerIdPrefix(op->get(PREF_PEER_ID_PREFIX));
+  }
+  btContext->setDir(requestOption.get(PREF_DIR));
+  rg->setDownloadContext(btContext);
+  btContext->setOwnerRequestGroup(rg.get());
+  return rg;
+}
+
+void createRequestGroupForBitTorrent
+(std::deque<SharedHandle<RequestGroup> >& result, Option* op,
+ const std::deque<std::string>& uris)
+{
+  std::deque<std::string> nargs;
+  if(op->get(PREF_PARAMETERIZED_URI) == V_TRUE) {
+    unfoldURI(nargs, uris);
+  } else {
+    nargs = uris;
+  }
+  // we ignore -Z option here
+  size_t numSplit = op->getAsInt(PREF_SPLIT);
+  std::deque<std::string> auxUris;
+  splitURI(auxUris, nargs.begin(), nargs.end(), numSplit);
+  SharedHandle<RequestGroup> rg =
+    createBtRequestGroup(op->get(PREF_TORRENT_FILE), op, auxUris, *op);
+  rg->setNumConcurrentCommand(numSplit);
+  result.push_back(rg);
+}
+
+#endif // ENABLE_BITTORRENT
+
+#ifdef ENABLE_METALINK
+void createRequestGroupForMetalink
+(std::deque<SharedHandle<RequestGroup> >& result, Option* op)
+{
+  Metalink2RequestGroup(op).generate(result, op->get(PREF_METALINK_FILE), *op);
+  if(result.empty()) {
+    throw FatalException("No files to download.");
+  }
+}
+#endif // ENABLE_METALINK
+
+class AccRequestGroup {
+private:
+  std::deque<SharedHandle<RequestGroup> >& _requestGroups;
+  ProtocolDetector _detector;
+  Option* _op;
+  const Option& _requestOption;
+public:
+  AccRequestGroup(std::deque<SharedHandle<RequestGroup> >& requestGroups,
+		  Option* op,
+		  const Option& requestOption):
+    _requestGroups(requestGroups), _op(op), _requestOption(requestOption) {}
+
+  void
+  operator()(const std::string& uri)
+  {
+    if(_detector.isStreamProtocol(uri)) {
+      std::deque<std::string> streamURIs;
+      size_t numSplit = _op->getAsInt(PREF_SPLIT);
+      for(size_t i = 0; i < numSplit; ++i) {
+	streamURIs.push_back(uri);
+      }
+      SharedHandle<RequestGroup> rg =
+	createRequestGroup(_op, streamURIs, _requestOption);
+      rg->setNumConcurrentCommand(numSplit);
+      _requestGroups.push_back(rg);
+    }
+#ifdef ENABLE_BITTORRENT
+    else if(_detector.guessTorrentFile(uri)) {
+      try {
+	_requestGroups.push_back(createBtRequestGroup(uri, _op,
+						      std::deque<std::string>(),
+						      _requestOption));
+      } catch(RecoverableException& e) {
+	// error occurred while parsing torrent file.
+	// We simply ignore it.	
+	LogFactory::getInstance()->error(EX_EXCEPTION_CAUGHT, e);
+      }
+    }
+#endif // ENABLE_BITTORRENT
+#ifdef ENABLE_METALINK
+    else if(_detector.guessMetalinkFile(uri)) {
+      try {
+	Metalink2RequestGroup(_op).generate(_requestGroups, uri,
+					    _requestOption);
+      } catch(RecoverableException& e) {
+	// error occurred while parsing metalink file.
+	// We simply ignore it.
+	LogFactory::getInstance()->error(EX_EXCEPTION_CAUGHT, e);
+      }
+    }
+#endif // ENABLE_METALINK
+    else {
+      LogFactory::getInstance()->error(MSG_UNRECOGNIZED_URI, (uri).c_str());
+    }
+  }
+};
+
+class StreamProtocolFilter {
+private:
+  ProtocolDetector _detector;
+public:
+  bool operator()(const std::string& uri) {
+    return _detector.isStreamProtocol(uri);
+  }
+};
+
+static void copyIfndef(Option& dest, const Option& src, const std::string& name)
+{
+  if(!dest.defined(name)) {
+    dest.put(name, src.get(name));
+  }
+}
+
+static void createRequestGroupForUri
+(std::deque<SharedHandle<RequestGroup> >& result, Option* op,
+ const std::deque<std::string>& uris, const Option& requestOption)
+{
+  std::deque<std::string> nargs;
+  if(op->get(PREF_PARAMETERIZED_URI) == V_TRUE) {
+    unfoldURI(nargs, uris);
+  } else {
+    nargs = uris;
+  }
+  if(op->get(PREF_FORCE_SEQUENTIAL) == V_TRUE) {
+    std::for_each(nargs.begin(), nargs.end(),
+		  AccRequestGroup(result, op, requestOption));
+  } else {
+    std::deque<std::string>::iterator strmProtoEnd =
+      std::stable_partition(nargs.begin(), nargs.end(), StreamProtocolFilter());
+    // let's process http/ftp protocols first.
+    if(nargs.begin() != strmProtoEnd) {
+      size_t numSplit = op->getAsInt(PREF_SPLIT);
+      std::deque<std::string> streamURIs;
+      splitURI(streamURIs, nargs.begin(), strmProtoEnd,
+	       numSplit);
+      SharedHandle<RequestGroup> rg =
+	createRequestGroup(op, streamURIs, requestOption, true);
+      rg->setNumConcurrentCommand(numSplit);
+      result.push_back(rg);
+    }
+    // process remaining URIs(local metalink, BitTorrent files)
+    std::for_each(strmProtoEnd, nargs.end(),
+		  AccRequestGroup(result, op, requestOption));
+  }
+}
+
+void createRequestGroupForUri
+(std::deque<SharedHandle<RequestGroup> >& result, Option* op,
+ const std::deque<std::string>& uris)
+{
+  createRequestGroupForUri(result, op, uris, *op);
+}
+
+static void createRequestGroupForUriList
+(std::deque<SharedHandle<RequestGroup> >& result, Option* op, std::istream& in)
+{
+  UriListParser p(in);
+  while(p.hasNext()) {
+    std::deque<std::string> uris;
+    Option requestOption;
+    p.parseNext(uris, requestOption);
+    if(uris.empty()) {
+      continue;
+    }
+    copyIfndef(requestOption, *op, PREF_DIR);
+    copyIfndef(requestOption, *op, PREF_OUT);
+
+    createRequestGroupForUri(result, op, uris, requestOption);
+  }
+}
+
+void createRequestGroupForUriList
+(std::deque<SharedHandle<RequestGroup> >& result, Option* op)
+{
+  if(op->get(PREF_INPUT_FILE) == "-") {
+    createRequestGroupForUriList(result, op, std::cin);
+  } else {
+    if(!File(op->get(PREF_INPUT_FILE)).isFile()) {
+      throw FatalException
+	(StringFormat(EX_FILE_OPEN, op->get(PREF_INPUT_FILE).c_str(),
+		      "No such file").str());
+    }
+    std::ifstream f(op->get(PREF_INPUT_FILE).c_str());
+    createRequestGroupForUriList(result, op, f);
+  }
+}
+
+} // namespace aria2

src/download_helper.h

@@ -0,0 +1,76 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef _D_DOWNLOAD_HELPER_H_
+#define _D_DOWNLOAD_HELPER_H_
+
+#include "common.h"
+
+#include <string>
+#include <deque>
+
+#include "SharedHandle.h"
+
+namespace aria2 {
+
+class RequestGroup;
+class Option;
+
+// Create RequestGroup object using torrent file specified by torrent-file 
+// option. In this function, force-sequential is ignored.
+void createRequestGroupForBitTorrent
+(std::deque<SharedHandle<RequestGroup> >& result, Option* op,
+ const std::deque<std::string>& uris);
+
+// Create RequestGroup objects using Metalink file specified by metalink-file
+// option.
+void createRequestGroupForMetalink
+(std::deque<SharedHandle<RequestGroup> >& result, Option* op);
+
+// Create RequestGroup objects from reading file specified by input-file option.
+// If the value of input-file option is "-", stdin is used as a input source.
+// Each line is treated as if it is provided in command-line argument.
+// The additional out and dir options can be specified after each line of URIs.
+// This optional line must start with white space(s).
+void createRequestGroupForUriList
+(std::deque<SharedHandle<RequestGroup> >& result, Option* op);
+
+// Create RequestGroup object using provided uris.
+void createRequestGroupForUri
+(std::deque<SharedHandle<RequestGroup> >& result, Option* op,
+ const std::deque<std::string>& uris);
+
+} // namespace aria2
+
+#endif // _D_DOWNLOAD_HELPER_H_

src/main.cc

@@ -51,9 +51,7 @@
 #include "FeatureConfig.h"
 #include "MultiUrlRequestInfo.h"
 #include "SimpleRandomizer.h"
-#include "FatalException.h"
 #include "File.h"
-#include "UriListParser.h"
 #include "message.h"
 #include "prefs.h"
 #include "Option.h"
@@ -61,21 +59,14 @@
 #include "a2io.h"
 #include "a2time.h"
 #include "Platform.h"
-#include "ParameterizedStringParser.h"
-#include "PStringBuildVisitor.h"
-#include "SingleFileDownloadContext.h"
 #include "DefaultBtContext.h"
 #include "FileEntry.h"
 #include "RequestGroup.h"
-#include "ProtocolDetector.h"
 #include "ConsoleStatCalc.h"
 #include "NullStatCalc.h"
-#include "StringFormat.h"
-#include "A2STR.h"
-#include "RecoverableException.h"
+#include "download_helper.h"
 #ifdef ENABLE_METALINK
 # include "MetalinkHelper.h"
-# include "Metalink2RequestGroup.h"
 # include "MetalinkEntry.h"
 #endif // ENABLE_METALINK
 #ifdef ENABLE_MESSAGE_DIGEST
@@ -90,35 +81,6 @@ namespace aria2 {
 // output stream to /dev/null
 std::ofstream nullout(DEV_NULL);
 
-std::deque<std::string> unfoldURI(const std::deque<std::string>& args)
-{
-  std::deque<std::string> nargs;
-  ParameterizedStringParser p;
-  PStringBuildVisitor v;
-  for(std::deque<std::string>::const_iterator itr = args.begin(); itr != args.end();
-      ++itr) {
-    v.reset();
-    p.parse(*itr)->accept(&v);
-    nargs.insert(nargs.end(), v.getURIs().begin(), v.getURIs().end()); 
-  }
-  return nargs;
-}
-
-RequestGroupHandle createRequestGroup
-(const Option* op, const std::deque<std::string>& uris,
- const Option& requestOption)
-{
-  RequestGroupHandle rg(new RequestGroup(op, uris));
-  SingleFileDownloadContextHandle dctx
-    (new SingleFileDownloadContext(op->getAsInt(PREF_SEGMENT_SIZE),
-				   0,
-				   A2STR::NIL,
-				   requestOption.get(PREF_OUT)));
-  dctx->setDir(requestOption.get(PREF_DIR));
-  rg->setDownloadContext(dctx);
-  return rg;
-}
-
 SharedHandle<StatCalc> getStatCalc(const Option* op)
 {
   SharedHandle<StatCalc> statCalc;
@@ -141,227 +103,6 @@ std::ostream& getSummaryOut(const Option* op)
 
 extern Option* option_processing(int argc, char* const argv[]);
 
-#ifdef ENABLE_BITTORRENT
-
-SharedHandle<RequestGroup>
-createBtRequestGroup(const std::string& torrentFilePath,
-		     Option* op,
-		     const std::deque<std::string>& auxUris,
-		     const Option& requestOption)
-{
-  SharedHandle<RequestGroup> rg(new RequestGroup(op, auxUris));
-  SharedHandle<DefaultBtContext> btContext(new DefaultBtContext());
-  btContext->load(torrentFilePath);// may throw exception
-  if(op->defined(PREF_PEER_ID_PREFIX)) {
-    btContext->setPeerIdPrefix(op->get(PREF_PEER_ID_PREFIX));
-  }
-  btContext->setDir(requestOption.get(PREF_DIR));
-  rg->setDownloadContext(btContext);
-  btContext->setOwnerRequestGroup(rg.get());
-  return rg;
-}
-
-int32_t downloadBitTorrent(Option* op, const std::deque<std::string>& uris)
-{
-  std::deque<std::string> nargs;
-  if(op->get(PREF_PARAMETERIZED_URI) == V_TRUE) {
-    nargs = unfoldURI(uris);
-  } else {
-    nargs = uris;
-  }
-  RequestGroups groups;
-  size_t numSplit = op->getAsInt(PREF_SPLIT);
-  if(nargs.size() >= numSplit) {
-    RequestGroupHandle rg = createBtRequestGroup(op->get(PREF_TORRENT_FILE),
-						 op, nargs, *op);
-    rg->setNumConcurrentCommand(numSplit);
-    groups.push_back(rg);
-  } else {
-    std::deque<std::string> xargs;
-    if(!nargs.empty()) {
-      ncopy(nargs.begin(), nargs.end(), numSplit, std::back_inserter(xargs));
-      xargs.erase(xargs.begin()+numSplit, xargs.end());
-    }
-    RequestGroupHandle rg = createBtRequestGroup(op->get(PREF_TORRENT_FILE),
-						 op, xargs, *op);
-    rg->setNumConcurrentCommand(numSplit);
-    groups.push_back(rg);
-  }
-  return MultiUrlRequestInfo(groups, op, getStatCalc(op), getSummaryOut(op)).execute();
-}
-#endif // ENABLE_BITTORRENT
-
-#ifdef ENABLE_METALINK
-int32_t downloadMetalink(Option* op)
-{
-  RequestGroups groups;
-  Metalink2RequestGroup(op).generate(groups, op->get(PREF_METALINK_FILE), *op);
-  if(groups.empty()) {
-    throw FatalException("No files to download.");
-  }
-  return MultiUrlRequestInfo(groups, op, getStatCalc(op), getSummaryOut(op)).execute();
-}
-#endif // ENABLE_METALINK
-
-class AccRequestGroup {
-private:
-  std::deque<SharedHandle<RequestGroup> >& _requestGroups;
-  ProtocolDetector _detector;
-  Option* _op;
-  Option& _requestOption;
-public:
-  AccRequestGroup(std::deque<SharedHandle<RequestGroup> >& requestGroups,
-		  Option* op,
-		  Option& requestOption):
-    _requestGroups(requestGroups), _op(op), _requestOption(requestOption) {}
-
-  void
-  operator()(const std::string& uri)
-  {
-    if(_detector.isStreamProtocol(uri)) {
-      std::deque<std::string> xuris;
-      for(size_t count = _op->getAsInt(PREF_SPLIT); count; --count) {
-	xuris.push_back(uri);
-      }
-      RequestGroupHandle rg = createRequestGroup(_op, xuris, _requestOption);
-      _requestGroups.push_back(rg);
-    }
-#ifdef ENABLE_BITTORRENT
-    else if(_detector.guessTorrentFile(uri)) {
-      try {
-	_requestGroups.push_back(createBtRequestGroup(uri, _op,
-						      std::deque<std::string>(),
-						      _requestOption));
-      } catch(RecoverableException& e) {
-	// error occurred while parsing torrent file.
-	// We simply ignore it.	
-	LogFactory::getInstance()->error(EX_EXCEPTION_CAUGHT, e);
-      }
-    }
-#endif // ENABLE_BITTORRENT
-#ifdef ENABLE_METALINK
-    else if(_detector.guessMetalinkFile(uri)) {
-      try {
-	Metalink2RequestGroup(_op).generate(_requestGroups, uri,
-					    _requestOption);
-      } catch(RecoverableException& e) {
-	// error occurred while parsing metalink file.
-	// We simply ignore it.
-	LogFactory::getInstance()->error(EX_EXCEPTION_CAUGHT, e);
-      }
-    }
-#endif // ENABLE_METALINK
-    else {
-      LogFactory::getInstance()->error(MSG_UNRECOGNIZED_URI, (uri).c_str());
-    }
-  }
-};
-
-void copyIfndef(Option& dest, const Option& src, const std::string& name)
-{
-  if(!dest.defined(name)) {
-    dest.put(name, src.get(name));
-  }
-}
-
-int32_t downloadUriList(Option* op, std::istream& in)
-{
-  UriListParser p(in);
-  RequestGroups groups;
-  while(p.hasNext()) {
-    std::deque<std::string> uris;
-    Option requestOption;
-    p.parseNext(uris, requestOption);
-    copyIfndef(requestOption, *op, PREF_DIR);
-    copyIfndef(requestOption, *op, PREF_OUT);
-    if(uris.size() == 1 && op->get(PREF_PARAMETERIZED_URI) == V_TRUE) {
-      std::deque<std::string> unfoldedURIs = unfoldURI(uris);
-      std::for_each(unfoldedURIs.begin(), unfoldedURIs.end(),
-		    AccRequestGroup(groups, op, requestOption));
-    } else if(uris.size() == 1) {
-      std::for_each(uris.begin(), uris.end(),
-		    AccRequestGroup(groups, op, requestOption));
-    } else if(!uris.empty()) {
-      size_t numSplit = op->getAsInt(PREF_SPLIT);
-      if(uris.size() >= numSplit) {
-	SharedHandle<RequestGroup> rg =
-	  createRequestGroup(op, uris, requestOption);
-	rg->setNumConcurrentCommand(numSplit);
-	groups.push_back(rg);
-      } else {
-	std::deque<std::string> xuris;
-	ncopy(uris.begin(), uris.end(), numSplit, std::back_inserter(xuris));
-	xuris.erase(xuris.begin()+numSplit, xuris.end());
-	SharedHandle<RequestGroup> rg =
-	  createRequestGroup(op, xuris, requestOption);
-	rg->setNumConcurrentCommand(numSplit);
-	groups.push_back(rg);
-      }
-    }
-  }
-  return MultiUrlRequestInfo(groups, op, getStatCalc(op), getSummaryOut(op)).execute();
-}
-
-int32_t downloadUriList(Option* op)
-{
-  if(op->get(PREF_INPUT_FILE) == "-") {
-    return downloadUriList(op, std::cin);
-  } else {
-    if(!File(op->get(PREF_INPUT_FILE)).isFile()) {
-      throw FatalException
-	(StringFormat(EX_FILE_OPEN, op->get(PREF_INPUT_FILE).c_str(),
-		      "No such file").str());
-    }
-    std::ifstream f(op->get(PREF_INPUT_FILE).c_str());
-    return downloadUriList(op, f);
-  }
-}
-
-class StreamProtocolFilter {
-private:
-  ProtocolDetector _detector;
-public:
-  bool operator()(const std::string& uri) {
-    return _detector.isStreamProtocol(uri);
-  }
-};
-
-int32_t downloadUri(Option* op, const std::deque<std::string>& uris)
-{
-  std::deque<std::string> nargs;
-  if(op->get(PREF_PARAMETERIZED_URI) == V_TRUE) {
-    nargs = unfoldURI(uris);
-  } else {
-    nargs = uris;
-  }
-  RequestGroups groups;
-  if(op->get(PREF_FORCE_SEQUENTIAL) == V_TRUE) {
-    std::for_each(nargs.begin(), nargs.end(), AccRequestGroup(groups, op, *op));
-  } else {
-    std::deque<std::string>::iterator strmProtoEnd =
-      std::stable_partition(nargs.begin(), nargs.end(), StreamProtocolFilter());
-    // let's process http/ftp protocols first.
-    size_t numSplit = op->getAsInt(PREF_SPLIT);
-    size_t numURIs = std::distance(nargs.begin(), strmProtoEnd);
-    if(numURIs >= numSplit) {
-      std::deque<std::string> xargs(nargs.begin(), strmProtoEnd);
-      RequestGroupHandle rg = createRequestGroup(op, xargs, *op);
-      rg->setNumConcurrentCommand(numSplit);
-      groups.push_back(rg);
-    } else if(numURIs > 0) {
-      std::deque<std::string> xargs;
-      ncopy(nargs.begin(), strmProtoEnd, numSplit, std::back_inserter(xargs));
-      xargs.erase(xargs.begin()+numSplit, xargs.end());
-      RequestGroupHandle rg = createRequestGroup(op, xargs, *op);
-      rg->setNumConcurrentCommand(numSplit);
-      groups.push_back(rg);
-    }
-    // process remaining URIs(local metalink, BitTorrent files)
-    std::for_each(strmProtoEnd, nargs.end(), AccRequestGroup(groups, op, *op));
-  }
-  return MultiUrlRequestInfo(groups, op, getStatCalc(op), getSummaryOut(op)).execute();
-}
-
 int main(int argc, char* argv[])
 {
   Option* op = option_processing(argc, argv);
@@ -397,6 +138,7 @@ int main(int argc, char* argv[])
     Util::setGlobalSignalHandler(SIGPIPE, SIG_IGN, 0);
 #endif
     int32_t returnValue = 0;
+    std::deque<SharedHandle<RequestGroup> > requestGroups;
 #ifdef ENABLE_BITTORRENT
     if(op->defined(PREF_TORRENT_FILE)) {
       if(op->get(PREF_SHOW_FILES) == V_TRUE) {
@@ -404,7 +146,7 @@ int main(int argc, char* argv[])
 	btContext->load(op->get(PREF_TORRENT_FILE));
 	std::cout << btContext << std::endl;
       } else {
-	returnValue = downloadBitTorrent(op, args);
+	createRequestGroupForBitTorrent(requestGroups, op, args);
       }
     }
     else
@@ -419,16 +161,19 @@ int main(int argc, char* argv[])
 	  MetalinkEntry::toFileEntry(fileEntries, metalinkEntries);
 	  Util::toStream(std::cout, fileEntries);
 	} else {
-	  returnValue = downloadMetalink(op);
+	  createRequestGroupForMetalink(requestGroups, op);
 	}
       }
       else
 #endif // ENABLE_METALINK
 	if(op->defined(PREF_INPUT_FILE)) {
-	  returnValue = downloadUriList(op);
+	  createRequestGroupForUriList(requestGroups, op);
 	} else {
-	  returnValue = downloadUri(op, args);
+	  createRequestGroupForUri(requestGroups, op, args);
 	}
+
+    returnValue = MultiUrlRequestInfo(requestGroups, op, getStatCalc(op),
+				      getSummaryOut(op)).execute();
     if(returnValue == 1) {
       exitStatus = EXIT_FAILURE;
     }

test/DownloadHelperTest.cc

@@ -0,0 +1,380 @@
+#include "download_helper.h"
+
+#include <iostream>
+#include <string>
+#include <deque>
+#include <algorithm>
+
+#include <cppunit/extensions/HelperMacros.h>
+
+#include "RequestGroup.h"
+#include "DownloadContext.h"
+#include "Option.h"
+#include "array_fun.h"
+#include "prefs.h"
+#include "Exception.h"
+#include "Util.h"
+
+namespace aria2 {
+
+class DownloadHelperTest:public CppUnit::TestFixture {
+
+  CPPUNIT_TEST_SUITE(DownloadHelperTest);
+  CPPUNIT_TEST(testCreateRequestGroupForUri);
+  CPPUNIT_TEST(testCreateRequestGroupForUri_parameterized);
+  CPPUNIT_TEST(testCreateRequestGroupForUri_BitTorrent);
+  CPPUNIT_TEST(testCreateRequestGroupForUri_Metalink);
+  CPPUNIT_TEST(testCreateRequestGroupForUriList);
+  CPPUNIT_TEST(testCreateRequestGroupForBitTorrent);
+  CPPUNIT_TEST(testCreateRequestGroupForMetalink);
+  CPPUNIT_TEST_SUITE_END();
+public:
+  void setUp() {}
+
+  void tearDown() {}
+
+  void testCreateRequestGroupForUri();
+  void testCreateRequestGroupForUri_parameterized();
+  void testCreateRequestGroupForUri_BitTorrent();
+  void testCreateRequestGroupForUri_Metalink();
+  void testCreateRequestGroupForUriList();
+  void testCreateRequestGroupForBitTorrent();
+  void testCreateRequestGroupForMetalink();
+};
+
+
+CPPUNIT_TEST_SUITE_REGISTRATION(DownloadHelperTest);
+
+void DownloadHelperTest::testCreateRequestGroupForUri()
+{
+  std::string array[] = {
+    "http://alpha/file",
+    "http://bravo/file",
+    "http://charlie/file"
+  };
+  std::deque<std::string> uris(&array[0], &array[arrayLength(array)]);
+  Option op;
+  op.put(PREF_SPLIT, "3");
+  op.put(PREF_DIR, "/tmp");
+  op.put(PREF_OUT, "file.out");
+  {
+    std::deque<SharedHandle<RequestGroup> > result;
+    
+    createRequestGroupForUri(result, &op, uris);
+    
+    CPPUNIT_ASSERT_EQUAL((size_t)1, result.size());
+    SharedHandle<RequestGroup> group = result[0];
+    std::deque<std::string> uris;
+    group->getURIs(uris);
+    CPPUNIT_ASSERT_EQUAL((size_t)3, uris.size());
+    for(size_t i = 0; i < arrayLength(array); ++i) {
+      CPPUNIT_ASSERT_EQUAL(array[i], uris[i]);
+    }
+    CPPUNIT_ASSERT_EQUAL((unsigned int)3, group->getNumConcurrentCommand());
+    SharedHandle<DownloadContext> ctx = group->getDownloadContext();
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), ctx->getDir());
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp/file.out"),
+			 ctx->getActualBasePath());
+  }
+  op.put(PREF_SPLIT, "5");
+  {
+    std::deque<SharedHandle<RequestGroup> > result;
+    
+    createRequestGroupForUri(result, &op, uris);
+    
+    CPPUNIT_ASSERT_EQUAL((size_t)1, result.size());
+    SharedHandle<RequestGroup> group = result[0];
+    std::deque<std::string> uris;
+    group->getURIs(uris);
+    CPPUNIT_ASSERT_EQUAL((size_t)5, uris.size());
+    for(size_t i = 0; i < arrayLength(array); ++i) {
+      CPPUNIT_ASSERT_EQUAL(array[i], uris[i]);
+    }
+    for(size_t i = 0; i < 5-arrayLength(array); ++i) {
+      CPPUNIT_ASSERT_EQUAL(array[i], uris[i+arrayLength(array)]);
+    }
+    CPPUNIT_ASSERT_EQUAL((unsigned int)5, group->getNumConcurrentCommand());
+  }
+  op.put(PREF_SPLIT, "2");
+  {
+    std::deque<SharedHandle<RequestGroup> > result;
+    
+    createRequestGroupForUri(result, &op, uris);
+    
+    CPPUNIT_ASSERT_EQUAL((size_t)1, result.size());
+    SharedHandle<RequestGroup> group = result[0];
+    std::deque<std::string> uris;
+    group->getURIs(uris);
+    CPPUNIT_ASSERT_EQUAL((size_t)3, uris.size());
+    for(size_t i = 0; i < arrayLength(array); ++i) {
+      CPPUNIT_ASSERT_EQUAL(array[i], uris[i]);
+    }
+    CPPUNIT_ASSERT_EQUAL((unsigned int)2, group->getNumConcurrentCommand());
+  }
+  op.put(PREF_FORCE_SEQUENTIAL, V_TRUE);
+  {
+    std::deque<SharedHandle<RequestGroup> > result;
+    
+    createRequestGroupForUri(result, &op, uris);
+    
+    CPPUNIT_ASSERT_EQUAL((size_t)3, result.size());
+
+    // for alpha server
+    SharedHandle<RequestGroup> alphaGroup = result[0];
+    std::deque<std::string> alphaURIs;
+    alphaGroup->getURIs(alphaURIs);
+    CPPUNIT_ASSERT_EQUAL((size_t)2, alphaURIs.size());
+    for(size_t i = 0; i < 2; ++i) {
+      CPPUNIT_ASSERT_EQUAL(array[0], uris[0]);
+    }
+    CPPUNIT_ASSERT_EQUAL((unsigned int)2,
+			 alphaGroup->getNumConcurrentCommand());
+    SharedHandle<DownloadContext> alphaCtx = alphaGroup->getDownloadContext();
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), alphaCtx->getDir());
+    // See the value of PREF_OUT is not used as a file name.
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp/index.html"),
+			 alphaCtx->getActualBasePath());
+
+
+  }
+}
+
+void DownloadHelperTest::testCreateRequestGroupForUri_parameterized()
+{
+  std::string array[] = {
+    "http://{alpha, bravo}/file",
+    "http://charlie/file"
+  };
+  std::deque<std::string> uris(&array[0], &array[arrayLength(array)]);
+  Option op;
+  op.put(PREF_SPLIT, "3");
+  op.put(PREF_DIR, "/tmp");
+  op.put(PREF_OUT, "file.out");
+  op.put(PREF_PARAMETERIZED_URI, V_TRUE);
+  {
+    std::deque<SharedHandle<RequestGroup> > result;
+    
+    createRequestGroupForUri(result, &op, uris);
+    
+    CPPUNIT_ASSERT_EQUAL((size_t)1, result.size());
+    SharedHandle<RequestGroup> group = result[0];
+    std::deque<std::string> uris;
+    group->getURIs(uris);
+    CPPUNIT_ASSERT_EQUAL((size_t)3, uris.size());
+
+    CPPUNIT_ASSERT_EQUAL(std::string("http://alpha/file"), uris[0]);
+    CPPUNIT_ASSERT_EQUAL(std::string("http://bravo/file"), uris[1]);
+    CPPUNIT_ASSERT_EQUAL(std::string("http://charlie/file"), uris[2]);
+
+    CPPUNIT_ASSERT_EQUAL((unsigned int)3, group->getNumConcurrentCommand());
+    SharedHandle<DownloadContext> ctx = group->getDownloadContext();
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), ctx->getDir());
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp/file.out"),
+			 ctx->getActualBasePath());
+  }
+}
+
+void DownloadHelperTest::testCreateRequestGroupForUri_BitTorrent()
+{
+  std::string array[] = {
+    "http://alpha/file",
+    "test.torrent",
+    "http://bravo/file",
+    "http://charlie/file"
+  };
+  std::deque<std::string> uris(&array[0], &array[arrayLength(array)]);
+  Option op;
+  op.put(PREF_SPLIT, "3");
+  op.put(PREF_DIR, "/tmp");
+  op.put(PREF_OUT, "file.out");
+  {
+    std::deque<SharedHandle<RequestGroup> > result;
+    
+    createRequestGroupForUri(result, &op, uris);
+    
+    CPPUNIT_ASSERT_EQUAL((size_t)2, result.size());
+    SharedHandle<RequestGroup> group = result[0];
+    std::deque<std::string> uris;
+    group->getURIs(uris);
+    CPPUNIT_ASSERT_EQUAL((size_t)3, uris.size());
+
+    CPPUNIT_ASSERT_EQUAL(array[0], uris[0]);
+    CPPUNIT_ASSERT_EQUAL(array[2], uris[1]);
+    CPPUNIT_ASSERT_EQUAL(array[3], uris[2]);
+
+    CPPUNIT_ASSERT_EQUAL((unsigned int)3, group->getNumConcurrentCommand());
+    SharedHandle<DownloadContext> ctx = group->getDownloadContext();
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), ctx->getDir());
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp/file.out"),
+			 ctx->getActualBasePath());
+
+    SharedHandle<RequestGroup> torrentGroup = result[1];
+    std::deque<std::string> auxURIs;
+    torrentGroup->getURIs(auxURIs);
+    CPPUNIT_ASSERT(auxURIs.empty());
+    CPPUNIT_ASSERT_EQUAL((unsigned int)3,
+			 torrentGroup->getNumConcurrentCommand());
+    SharedHandle<DownloadContext> btctx = torrentGroup->getDownloadContext();
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), btctx->getDir());
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp/aria2-test"),
+			 btctx->getActualBasePath());    
+  }
+}
+
+void DownloadHelperTest::testCreateRequestGroupForUri_Metalink()
+{
+  std::string array[] = {
+    "http://alpha/file",
+    "http://bravo/file",
+    "http://charlie/file",
+    "test.xml"
+  };
+  std::deque<std::string> uris(&array[0], &array[arrayLength(array)]);
+  Option op;
+  op.put(PREF_SPLIT, "3");
+  op.put(PREF_METALINK_SERVERS, "2");
+  op.put(PREF_DIR, "/tmp");
+  op.put(PREF_OUT, "file.out");
+  {
+    std::deque<SharedHandle<RequestGroup> > result;
+    
+    createRequestGroupForUri(result, &op, uris);
+    
+    // group1: http://alpha/file, ...
+    // group2-7: 6 file entry in Metalink and 1 torrent file download
+    CPPUNIT_ASSERT_EQUAL((size_t)7, result.size());
+    SharedHandle<RequestGroup> group = result[0];
+    std::deque<std::string> uris;
+    group->getURIs(uris);
+    CPPUNIT_ASSERT_EQUAL((size_t)3, uris.size());
+    for(size_t i = 0; i < 3; ++i) {
+      CPPUNIT_ASSERT_EQUAL(array[i], uris[i]);
+    }
+    CPPUNIT_ASSERT_EQUAL((unsigned int)3, group->getNumConcurrentCommand());
+    SharedHandle<DownloadContext> ctx = group->getDownloadContext();
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), ctx->getDir());
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp/file.out"),
+			 ctx->getActualBasePath());
+
+    SharedHandle<RequestGroup> aria2052Group = result[1];
+    CPPUNIT_ASSERT_EQUAL((unsigned int)1, // because of maxconnections attribute
+			 aria2052Group->getNumConcurrentCommand());
+    SharedHandle<DownloadContext> aria2052Ctx =
+      aria2052Group->getDownloadContext();
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), aria2052Ctx->getDir());
+    CPPUNIT_ASSERT_EQUAL(std::string("/tmp/aria2-0.5.2.tar.bz2"),
+			 aria2052Ctx->getActualBasePath());
+    
+    SharedHandle<RequestGroup> aria2051Group = result[2];
+    CPPUNIT_ASSERT_EQUAL((unsigned int)2,
+			 aria2051Group->getNumConcurrentCommand());
+  }
+}
+
+void DownloadHelperTest::testCreateRequestGroupForUriList()
+{
+  Option op;
+  op.put(PREF_SPLIT, "3");
+  op.put(PREF_INPUT_FILE, "input_uris.txt");
+  op.put(PREF_DIR, "/tmp");
+  op.put(PREF_OUT, "file.out");
+
+  std::deque<SharedHandle<RequestGroup> > result;
+  
+  createRequestGroupForUriList(result, &op);
+
+  CPPUNIT_ASSERT_EQUAL((size_t)2, result.size());
+
+  SharedHandle<RequestGroup> fileGroup = result[0];
+  std::deque<std::string> fileURIs;
+  fileGroup->getURIs(fileURIs);
+  CPPUNIT_ASSERT_EQUAL(std::string("http://alpha/file"), fileURIs[0]);
+  CPPUNIT_ASSERT_EQUAL(std::string("http://bravo/file"), fileURIs[1]);
+  CPPUNIT_ASSERT_EQUAL(std::string("http://charlie/file"), fileURIs[2]);
+  CPPUNIT_ASSERT_EQUAL((unsigned int)3, fileGroup->getNumConcurrentCommand());
+  SharedHandle<DownloadContext> fileCtx = fileGroup->getDownloadContext();
+  CPPUNIT_ASSERT_EQUAL(std::string("/mydownloads"), fileCtx->getDir());
+  CPPUNIT_ASSERT_EQUAL(std::string("/mydownloads/myfile.out"),
+		       fileCtx->getActualBasePath());
+
+  SharedHandle<RequestGroup> fileISOGroup = result[1];
+  SharedHandle<DownloadContext> fileISOCtx = fileISOGroup->getDownloadContext();
+  CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), fileISOCtx->getDir());
+  CPPUNIT_ASSERT_EQUAL(std::string("/tmp/file.out"),
+		       fileISOCtx->getActualBasePath());
+}
+
+void DownloadHelperTest::testCreateRequestGroupForBitTorrent()
+{
+  std::string array[] = {
+    "http://alpha/file",
+    "http://bravo/file",
+    "http://charlie/file"
+  };
+
+  std::deque<std::string> auxURIs(&array[0], &array[arrayLength(array)]);
+  Option op;
+  op.put(PREF_SPLIT, "5");
+  op.put(PREF_TORRENT_FILE, "test.torrent");
+  op.put(PREF_DIR, "/tmp");
+  op.put(PREF_OUT, "file.out");
+  {
+    std::deque<SharedHandle<RequestGroup> > result;
+  
+    createRequestGroupForBitTorrent(result, &op, auxURIs);
+
+    CPPUNIT_ASSERT_EQUAL((size_t)1, result.size());
+
+    SharedHandle<RequestGroup> group = result[0];
+    std::deque<std::string> uris;
+    group->getURIs(uris);
+    CPPUNIT_ASSERT_EQUAL((size_t)5, uris.size());
+    for(size_t i = 0; i < arrayLength(array); ++i) {
+      CPPUNIT_ASSERT_EQUAL(array[i], uris[i]);
+    }
+    for(size_t i = 0; i < 5-arrayLength(array); ++i) {
+      CPPUNIT_ASSERT_EQUAL(array[i], uris[i+arrayLength(array)]);
+    }
+    CPPUNIT_ASSERT_EQUAL((unsigned int)5, group->getNumConcurrentCommand());
+  }
+  op.put(PREF_FORCE_SEQUENTIAL, V_TRUE);
+  {
+    std::deque<SharedHandle<RequestGroup> > result;
+  
+    createRequestGroupForBitTorrent(result, &op, auxURIs);
+
+    // See --force-requencial is ignored
+    CPPUNIT_ASSERT_EQUAL((size_t)1, result.size());
+  }
+}
+
+void DownloadHelperTest::testCreateRequestGroupForMetalink()
+{
+  Option op;
+  op.put(PREF_SPLIT, "3");
+  op.put(PREF_METALINK_FILE, "test.xml");
+  op.put(PREF_METALINK_SERVERS, "5");
+  op.put(PREF_DIR, "/tmp");
+  op.put(PREF_OUT, "file.out");
+  {
+    std::deque<SharedHandle<RequestGroup> > result;
+  
+    createRequestGroupForMetalink(result, &op);
+
+    CPPUNIT_ASSERT_EQUAL((size_t)6, result.size());
+
+    SharedHandle<RequestGroup> group = result[0];
+    std::deque<std::string> uris;
+    group->getURIs(uris);
+    std::sort(uris.begin(), uris.end());
+    CPPUNIT_ASSERT_EQUAL((size_t)2, uris.size());
+    CPPUNIT_ASSERT_EQUAL(std::string("ftp://ftphost/aria2-0.5.2.tar.bz2"),
+			 uris[0]);
+    CPPUNIT_ASSERT_EQUAL(std::string("http://httphost/aria2-0.5.2.tar.bz2"),
+			 uris[1]);
+    // See numConcurrentCommand is 1 because of maxconnections attribute.
+    CPPUNIT_ASSERT_EQUAL((unsigned int)1, group->getNumConcurrentCommand());
+  }
+}
+
+} // namespace aria2

test/Makefile.am

@@ -62,7 +62,8 @@ aria2c_SOURCES = AllTest.cc\
 	CopyDiskAdaptorTest.cc\
 	FtpConnectionTest.cc\
 	OptionParserTest.cc\
-	SimpleDNSCacheTest.cc
+	SimpleDNSCacheTest.cc\
+	DownloadHelperTest.cc
 
 if HAVE_LIBZ
 aria2c_SOURCES += GZipDecoderTest.cc
@@ -185,7 +186,7 @@ endif # ENABLE_METALINK
 #aria2c_CXXFLAGS = ${CPPUNIT_CFLAGS} -I../src -I../lib -Wall -D_FILE_OFFSET_BITS=64
 #aria2c_LDFLAGS = ${CPPUNIT_LIBS}
 
-aria2c_LDADD = ../src/libaria2c.a\
+aria2c_LDADD = ../src/libaria2c.a ../src/download_helper.o\
     @LIBINTL@ @LIBGNUTLS_LIBS@\
 	@LIBGCRYPT_LIBS@ @OPENSSL_LIBS@ @XML_LIBS@ @LIBARES_LIBS@\
 	@LIBCARES_LIBS@ @WINSOCK_LIBS@ @LIBEXPAT_LIBS@ @LIBZ_LIBS@\

test/Makefile.in

@@ -196,7 +196,8 @@ am__aria2c_SOURCES_DIST = AllTest.cc TestUtil.cc TestUtil.h \
 	ServerStatTest.cc NsCookieParserTest.cc \
 	DirectDiskAdaptorTest.cc CookieTest.cc CookieStorageTest.cc \
 	TimeTest.cc CopyDiskAdaptorTest.cc FtpConnectionTest.cc \
-	OptionParserTest.cc SimpleDNSCacheTest.cc GZipDecoderTest.cc \
+	OptionParserTest.cc SimpleDNSCacheTest.cc \
+	DownloadHelperTest.cc GZipDecoderTest.cc \
 	Sqlite3MozCookieParserTest.cc MessageDigestHelperTest.cc \
 	IteratableChunkChecksumValidatorTest.cc \
 	IteratableChecksumValidatorTest.cc BtAllowedFastMessageTest.cc \
@@ -369,11 +370,13 @@ am_aria2c_OBJECTS = AllTest.$(OBJEXT) TestUtil.$(OBJEXT) \
 	CookieTest.$(OBJEXT) CookieStorageTest.$(OBJEXT) \
 	TimeTest.$(OBJEXT) CopyDiskAdaptorTest.$(OBJEXT) \
 	FtpConnectionTest.$(OBJEXT) OptionParserTest.$(OBJEXT) \
-	SimpleDNSCacheTest.$(OBJEXT) $(am__objects_1) $(am__objects_2) \
-	$(am__objects_3) $(am__objects_4) $(am__objects_5)
+	SimpleDNSCacheTest.$(OBJEXT) DownloadHelperTest.$(OBJEXT) \
+	$(am__objects_1) $(am__objects_2) $(am__objects_3) \
+	$(am__objects_4) $(am__objects_5)
 aria2c_OBJECTS = $(am_aria2c_OBJECTS)
 am__DEPENDENCIES_1 =
-aria2c_DEPENDENCIES = ../src/libaria2c.a $(am__DEPENDENCIES_1)
+aria2c_DEPENDENCIES = ../src/libaria2c.a ../src/download_helper.o \
+	$(am__DEPENDENCIES_1)
 DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
 depcomp = $(SHELL) $(top_srcdir)/depcomp
 am__depfiles_maybe = depfiles
@@ -592,13 +595,13 @@ aria2c_SOURCES = AllTest.cc TestUtil.cc TestUtil.h SocketCoreTest.cc \
 	ServerStatTest.cc NsCookieParserTest.cc \
 	DirectDiskAdaptorTest.cc CookieTest.cc CookieStorageTest.cc \
 	TimeTest.cc CopyDiskAdaptorTest.cc FtpConnectionTest.cc \
-	OptionParserTest.cc SimpleDNSCacheTest.cc $(am__append_1) \
-	$(am__append_2) $(am__append_3) $(am__append_4) \
-	$(am__append_5)
+	OptionParserTest.cc SimpleDNSCacheTest.cc \
+	DownloadHelperTest.cc $(am__append_1) $(am__append_2) \
+	$(am__append_3) $(am__append_4) $(am__append_5)
 
 #aria2c_CXXFLAGS = ${CPPUNIT_CFLAGS} -I../src -I../lib -Wall -D_FILE_OFFSET_BITS=64
 #aria2c_LDFLAGS = ${CPPUNIT_LIBS}
-aria2c_LDADD = ../src/libaria2c.a\
+aria2c_LDADD = ../src/libaria2c.a ../src/download_helper.o\
     @LIBINTL@ @LIBGNUTLS_LIBS@\
 	@LIBGCRYPT_LIBS@ @OPENSSL_LIBS@ @XML_LIBS@ @LIBARES_LIBS@\
 	@LIBCARES_LIBS@ @WINSOCK_LIBS@ @LIBEXPAT_LIBS@ @LIBZ_LIBS@\
@@ -758,6 +761,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DictionaryTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DirectDiskAdaptorTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadHandlerFactoryTest.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadHelperTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ExceptionTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfigTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileEntryTest.Po@am__quote@

test/input_uris.txt

@@ -0,0 +1,5 @@
+http://alpha/file	http://bravo/file	http://charlie/file
+  out=myfile.out
+  dir=/mydownloads
+http://delta/file.iso
+