Added Metalink/HTTP Link and Digest header field parser.

src/DownloadContext.cc

@@ -53,7 +53,8 @@ DownloadContext::DownloadContext():
   knowsTotalLength_(true),
   ownerRequestGroup_(0),
   downloadStartTime_(0),
-  downloadStopTime_(downloadStartTime_) {}
+  downloadStopTime_(downloadStartTime_),
+  metalinkServerContacted_(false) {}
 
 DownloadContext::DownloadContext(size_t pieceLength,
                                  uint64_t totalLength,
@@ -63,7 +64,8 @@ DownloadContext::DownloadContext(size_t pieceLength,
   knowsTotalLength_(true),
   ownerRequestGroup_(0),
   downloadStartTime_(0),
-  downloadStopTime_(0)
+  downloadStopTime_(0),
+  metalinkServerContacted_(false)
 {
   SharedHandle<FileEntry> fileEntry(new FileEntry(path, totalLength, 0));
   fileEntries_.push_back(fileEntry);

src/DownloadContext.h

@@ -84,6 +84,9 @@ private:
   Timer downloadStopTime_;
 
   SharedHandle<Signature> signature_;
+  // This member variable is required to avoid to parse Metalink/HTTP
+  // Link header fields multiple times.
+  bool metalinkServerContacted_;
 public:
   DownloadContext();
 
@@ -224,6 +227,15 @@ public:
   SharedHandle<FileEntry> findFileEntryByOffset(off_t offset) const;
 
   void releaseRuntimeResource();
+
+  void setMetalinkServerContacted(bool f)
+  {
+    metalinkServerContacted_ = f;
+  }
+  bool getMetalinkServerContacted() const
+  {
+    return metalinkServerContacted_;
+  }
 };
 
 } // namespace aria2

src/HttpHeader.cc

@@ -66,6 +66,10 @@ const std::string HttpHeader::LAST_MODIFIED("Last-Modified");
 
 const std::string HttpHeader::ACCEPT_ENCODING("Accept-Encoding");
 
+const std::string HttpHeader::LINK("Link");
+
+const std::string HttpHeader::DIGEST("Digest");
+
 const char HttpHeader::HTTP_1_1[] = "HTTP/1.1";
 const char HttpHeader::CLOSE[] = "close";
 const char HttpHeader::CHUNKED[] = "chunked";
@@ -110,6 +114,14 @@ std::vector<std::string> HttpHeader::get(const std::string& name) const
   return v;
 }
 
+std::pair<std::multimap<std::string, std::string>::const_iterator,
+          std::multimap<std::string, std::string>::const_iterator>
+HttpHeader::getIterator(const std::string& name) const
+{
+  std::string n(util::toLower(name));
+  return table_.equal_range(n);
+}
+
 unsigned int HttpHeader::getFirstAsUInt(const std::string& name) const {
   return getFirstAsULLInt(name);
 }

src/HttpHeader.h

@@ -71,6 +71,9 @@ public:
   bool defined(const std::string& name) const;
   const std::string& getFirst(const std::string& name) const;
   std::vector<std::string> get(const std::string& name) const;
+  std::pair<std::multimap<std::string, std::string>::const_iterator,
+            std::multimap<std::string, std::string>::const_iterator>
+  getIterator(const std::string& name) const;
   unsigned int getFirstAsUInt(const std::string& name) const;
   uint64_t getFirstAsULLInt(const std::string& name) const;
 
@@ -136,6 +139,10 @@ public:
 
   static const std::string ACCEPT_ENCODING;
 
+  static const std::string LINK;
+
+  static const std::string DIGEST;
+
   static const char HTTP_1_1[];
 
   static const char CLOSE[];

src/HttpResponse.cc

@@ -51,6 +51,15 @@
 #include "AuthConfig.h"
 #include "ChunkedDecodingStreamFilter.h"
 #include "error_code.h"
+#include "prefs.h"
+#include "Option.h"
+#include "Checksum.h"
+#include "uri.h"
+#include "MetalinkHttpEntry.h"
+#include "Base64.h"
+#ifdef ENABLE_MESSAGE_DIGEST
+#include "MessageDigest.h"
+#endif // ENABLE_MESSAGE_DIGEST
 #ifdef HAVE_ZLIB
 # include "GZipDecodingStreamFilter.h"
 #endif // HAVE_ZLIB
@@ -286,4 +295,134 @@ bool HttpResponse::supportsPersistentConnection() const
      != std::string::npos);
 }
 
+namespace {
+bool parseMetalinkHttpLink(MetalinkHttpEntry& result, const std::string& s)
+{
+  std::string::const_iterator first = std::find(s.begin(), s.end(), '<');
+  if(first == s.end()) {
+    return false;
+  }
+  std::string::const_iterator last = std::find(first, s.end(), '>');
+  if(last == s.end()) {
+    return false;
+  }
+  std::string uri(first+1, last);
+  uri::UriStruct us;
+  if(uri::parse(us, uri)) {
+    result.uri = uri;
+  } else {
+    return false;
+  }
+  last = std::find(last, s.end(), ';');
+  if(last != s.end()) {
+    ++last;
+  }
+  bool ok = false;
+  while(1) {
+    std::string name, value;
+    std::pair<std::string::const_iterator, bool> r =
+      util::nextParam(name, value, last, s.end(), ';');
+    last = r.first;
+    if(!r.second) {
+      break;
+    }
+    if(value.empty()) {
+      if(name == "pref") {
+        result.pref = true;
+      }
+    } else {
+      if(name == "rel") {
+        if(value == "duplicate") {
+          ok = true;
+        } else {
+          ok = false;
+        }
+      } else if(name == "pri") {
+        int32_t priValue;
+        if(util::parseIntNoThrow(priValue, value)) {
+          if(1 <= priValue && priValue <= 999999) {
+            result.pri = priValue;
+          }
+        }
+      } else if(name == "geo") {
+        util::lowercase(value);
+        result.geo = value;
+      }
+    }
+  }
+  return ok;
+}
+} // namespace
+
+// Metalink/HTTP is defined by http://tools.ietf.org/html/rfc6249.
+// Link header field is defined by http://tools.ietf.org/html/rfc5988.
+void HttpResponse::getMetalinKHttpEntries
+(std::vector<MetalinkHttpEntry>& result,
+ const SharedHandle<Option>& option) const
+{
+  std::pair<std::multimap<std::string, std::string>::const_iterator,
+            std::multimap<std::string, std::string>::const_iterator> p =
+    httpHeader_->getIterator(HttpHeader::LINK);
+  for(; p.first != p.second; ++p.first) {
+    MetalinkHttpEntry e;
+    if(parseMetalinkHttpLink(e, (*p.first).second)) {
+      result.push_back(e);
+    }
+  }
+  if(!result.empty()) {
+    std::vector<std::string> locs;
+    if(option->defined(PREF_METALINK_LOCATION)) {
+      util::split(util::toLower(option->get(PREF_METALINK_LOCATION)),
+                  std::back_inserter(locs), ",", true);
+    }
+    for(std::vector<MetalinkHttpEntry>::iterator i = result.begin(),
+          eoi = result.end(); i != eoi; ++i) {
+      if(std::find(locs.begin(), locs.end(), (*i).geo) != locs.end()) {
+        (*i).pri -= 999999;
+      }
+    }
+  }
+  std::sort(result.begin(), result.end());
+}
+
+#ifdef ENABLE_MESSAGE_DIGEST
+// Digest header field is defined by
+// http://tools.ietf.org/html/rfc3230.
+SharedHandle<Checksum> HttpResponse::getDigest() const
+{
+  SharedHandle<Checksum> res;
+  std::pair<std::multimap<std::string, std::string>::const_iterator,
+            std::multimap<std::string, std::string>::const_iterator> p =
+    httpHeader_->getIterator(HttpHeader::DIGEST);
+  for(; p.first != p.second; ++p.first) {
+    const std::string& s = (*p.first).second;
+    std::string::const_iterator itr = s.begin();
+    while(1) {
+      std::string hashType, digest;
+      std::pair<std::string::const_iterator, bool> r =
+        util::nextParam(hashType, digest, itr, s.end(), ',');
+      itr = r.first;
+      if(!r.second) {
+        break;
+      }
+      util::lowercase(hashType);
+      if(!MessageDigest::supports(hashType)) {
+        continue;
+      }
+      std::string hexDigest = util::toHex(Base64::decode(digest));
+      if(!MessageDigest::isValidHash(hashType, hexDigest)) {
+        continue;
+      }
+      if(!res) {
+        res.reset(new Checksum(hashType, hexDigest));
+      } else if(MessageDigest::isStronger(hashType, res->getAlgo())) {
+        res->setAlgo(hashType);
+        res->setMessageDigest(hexDigest);
+      }
+    }
+  }
+  return res;
+}
+#endif // ENABLE_MESSAGE_DIGEST
+
 } // namespace aria2

src/HttpResponse.h

@@ -38,6 +38,7 @@
 #include "common.h"
 
 #include <stdint.h>
+#include <vector>
 
 #include "SharedHandle.h"
 #include "TimeA2.h"
@@ -48,6 +49,9 @@ namespace aria2 {
 class HttpRequest;
 class HttpHeader;
 class StreamFilter;
+class MetalinkHttpEntry;
+class Option;
+class Checksum;
 
 class HttpResponse {
 private:
@@ -127,6 +131,17 @@ public:
   Time getLastModifiedTime() const;
 
   bool supportsPersistentConnection() const;
+
+  void getMetalinKHttpEntries
+  (std::vector<MetalinkHttpEntry>& result,
+   const SharedHandle<Option>& option) const;
+#ifdef ENABLE_MESSAGE_DIGEST
+  // Returns digest specified in Digest header field.  If multiple
+  // digest algorithm is available, use strongest one defined in
+  // MessageDigest. If several same digest algorithms are available,
+  // but they have different value, they are all ignored.
+  SharedHandle<Checksum> getDigest() const;
+#endif // ENABLE_MESSAGE_DIGEST
 };
 
 } // namespace aria2

src/util.h

@@ -438,6 +438,60 @@ std::string makeString(const char* str);
 // strerror returns NULL, this function returns empty string.
 std::string safeStrerror(int errNum);
 
+// Parses sequence [first, last) and find name=value pair delimited by
+// delim character. If name(and optionally value) is found, returns
+// pair of iterator which can use as first parameter of next call of
+// this function, and true. If no name is found, returns the pair of
+// last and false.
+template<typename Iterator>
+std::pair<Iterator, bool>
+nextParam
+(std::string& name,
+ std::string& value,
+ Iterator first,
+ Iterator last,
+ char delim)
+{
+  Iterator end = last;
+  while(first != end) {
+    last = first;
+    Iterator parmnameFirst = first;
+    Iterator parmnameLast = first;
+    bool eqFound = false;
+    for(; last != end; ++last) {
+      if(*last == delim) {
+        break;
+      } else if(!eqFound && *last == '=') {
+        eqFound = true;
+        parmnameFirst = first;
+        parmnameLast = last;
+      }
+    }
+    std::string tname, tvalue;
+    if(parmnameFirst == parmnameLast) {
+      if(!eqFound) {
+        parmnameFirst = first;
+        parmnameLast = last;
+        tname = util::stripIter(parmnameFirst, parmnameLast);
+      }
+    } else {
+      first = parmnameLast+1;
+      tname = util::stripIter(parmnameFirst, parmnameLast);
+      tvalue = util::stripIter(first, last);
+    }
+    if(last != end) {
+      ++last;
+    }
+    if(!tname.empty()) {
+      name.swap(tname);
+      value.swap(tvalue);
+      return std::make_pair(last, true);
+    }
+    first = last;
+  }
+  return std::make_pair(end, false);
+}
+
 } // namespace util
 
 } // namespace aria2

test/HttpResponseTest.cc

@@ -17,6 +17,9 @@
 #include "AuthConfigFactory.h"
 #include "AuthConfig.h"
 #include "StreamFilter.h"
+#include "MetalinkHttpEntry.h"
+#include "Option.h"
+#include "Checksum.h"
 
 namespace aria2 {
 
@@ -49,6 +52,10 @@ class HttpResponseTest : public CppUnit::TestFixture {
   CPPUNIT_TEST(testProcessRedirect);
   CPPUNIT_TEST(testRetrieveCookie);
   CPPUNIT_TEST(testSupportsPersistentConnection);
+  CPPUNIT_TEST(testGetMetalinKHttpEntries);
+#ifdef ENABLE_MESSAGE_DIGEST
+  CPPUNIT_TEST(testGetDigest);
+#endif // ENABLE_MESSAGE_DIGEST
   CPPUNIT_TEST_SUITE_END();
 private:
 
@@ -81,6 +88,10 @@ public:
   void testProcessRedirect();
   void testRetrieveCookie();
   void testSupportsPersistentConnection();
+  void testGetMetalinKHttpEntries();
+#ifdef ENABLE_MESSAGE_DIGEST
+  void testGetDigest();
+#endif // ENABLE_MESSAGE_DIGEST
 };
 
 
@@ -590,4 +601,73 @@ void HttpResponseTest::testSupportsPersistentConnection()
   httpHeader->clearField();
 }
 
+void HttpResponseTest::testGetMetalinKHttpEntries()
+{
+  HttpResponse httpResponse;
+  SharedHandle<HttpHeader> httpHeader(new HttpHeader());
+  httpResponse.setHttpHeader(httpHeader);
+  SharedHandle<Option> option(new Option());
+
+  httpHeader->put("Link", "<http://uri1/>; rel=duplicate; pri=1; pref; geo=JP");
+  httpHeader->put("Link", "<http://uri2/>; rel=duplicate");
+  httpHeader->put("Link", "<http://uri3/>;;;;;;;;rel=duplicate;;;;;pri=2;;;;;");
+  httpHeader->put("Link", "<http://uri4/>;rel=duplicate;=pri=1;pref");
+  httpHeader->put("Link", "<http://describedby>; rel=describedby");
+  httpHeader->put("Link", "<baduri>; rel=duplicate");
+  httpHeader->put("Link", "<http://norel/>");
+  httpHeader->put("Link", "<http://badpri/>; rel=duplicate; pri=-1;");
+  std::vector<MetalinkHttpEntry> result;
+  httpResponse.getMetalinKHttpEntries(result, option);
+  CPPUNIT_ASSERT_EQUAL((size_t)5, result.size());
+
+  MetalinkHttpEntry e = result[0];
+  CPPUNIT_ASSERT_EQUAL(std::string("http://uri1/"), e.uri);
+  CPPUNIT_ASSERT_EQUAL(1, e.pri);
+  CPPUNIT_ASSERT(e.pref);
+  CPPUNIT_ASSERT_EQUAL(std::string("jp"), e.geo);
+
+  e = result[1];
+  CPPUNIT_ASSERT_EQUAL(std::string("http://uri4/"), e.uri);
+  CPPUNIT_ASSERT_EQUAL(999999, e.pri);
+  CPPUNIT_ASSERT(e.pref);
+  CPPUNIT_ASSERT(e.geo.empty());
+
+  e = result[2];
+  CPPUNIT_ASSERT_EQUAL(std::string("http://uri3/"), e.uri);
+  CPPUNIT_ASSERT_EQUAL(2, e.pri);
+  CPPUNIT_ASSERT(!e.pref);
+  CPPUNIT_ASSERT(e.geo.empty());
+
+  e = result[3];
+  CPPUNIT_ASSERT_EQUAL(std::string("http://uri2/"), e.uri);
+  CPPUNIT_ASSERT_EQUAL(999999, e.pri);
+  CPPUNIT_ASSERT(!e.pref);
+  CPPUNIT_ASSERT(e.geo.empty());
+
+  e = result[4];
+  CPPUNIT_ASSERT_EQUAL(std::string("http://badpri/"), e.uri);
+  CPPUNIT_ASSERT_EQUAL(999999, e.pri);
+  CPPUNIT_ASSERT(!e.pref);
+  CPPUNIT_ASSERT(e.geo.empty());
+}
+
+#ifdef ENABLE_MESSAGE_DIGEST
+void HttpResponseTest::testGetDigest()
+{
+  HttpResponse httpResponse;
+  SharedHandle<HttpHeader> httpHeader(new HttpHeader());
+  httpResponse.setHttpHeader(httpHeader);
+  SharedHandle<Option> option(new Option());
+
+  httpHeader->put("Digest", "SHA-1=82AD8itGL/oYQ5BTPFANiYnp9oE=");
+  httpHeader->put("Digest", "NOT_SUPPORTED");
+  httpHeader->put("Digest",
+                  "SHA-256=+D8nGudz3G/kpkVKQeDrI3xD57v0UeQmzGCZOk03nsU=,"
+                  "MD5=LJDK2+9ClF8Nz/K5WZd/+A==");
+  SharedHandle<Checksum> c = httpResponse.getDigest();
+  CPPUNIT_ASSERT(c);
+  CPPUNIT_ASSERT_EQUAL(std::string("sha-256"), c->getAlgo());
+}
+#endif // ENABLE_MESSAGE_DIGEST
+
 } // namespace aria2

test/UtilTest.cc

@@ -71,6 +71,7 @@ class UtilTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testGetCidrPrefix);
   CPPUNIT_TEST(testInSameCidrBlock);
   CPPUNIT_TEST(testIsUtf8String);
+  CPPUNIT_TEST(testNextParam);
   CPPUNIT_TEST_SUITE_END();
 private:
 
@@ -130,6 +131,7 @@ public:
   void testGetCidrPrefix();
   void testInSameCidrBlock();
   void testIsUtf8String();
+  void testNextParam();
 };
 
 
@@ -1247,4 +1249,42 @@ void UtilTest::testIsUtf8String()
   CPPUNIT_ASSERT(!util::isUtf8(util::fromHex("00")));
 }
 
+void UtilTest::testNextParam()
+{
+  std::string s1 = "    :a  :  b=c :d=b::::g::";
+  std::pair<std::string::iterator, bool> r;
+  std::string name, value;
+  r = util::nextParam(name, value, s1.begin(), s1.end(), ':');
+  CPPUNIT_ASSERT(r.second);
+  CPPUNIT_ASSERT_EQUAL(std::string("a"), name);
+  CPPUNIT_ASSERT_EQUAL(std::string(), value);
+
+  r = util::nextParam(name, value, r.first, s1.end(), ':');
+  CPPUNIT_ASSERT(r.second);
+  CPPUNIT_ASSERT_EQUAL(std::string("b"), name);
+  CPPUNIT_ASSERT_EQUAL(std::string("c"), value);
+
+  r = util::nextParam(name, value, r.first, s1.end(), ':');
+  CPPUNIT_ASSERT(r.second);
+  CPPUNIT_ASSERT_EQUAL(std::string("d"), name);
+  CPPUNIT_ASSERT_EQUAL(std::string("b"), value);
+
+  r = util::nextParam(name, value, r.first, s1.end(), ':');
+  CPPUNIT_ASSERT(r.second);
+  CPPUNIT_ASSERT_EQUAL(std::string("g"), name);
+  CPPUNIT_ASSERT_EQUAL(std::string(), value);
+
+  std::string s2 = "";
+  r = util::nextParam(name, value, s2.begin(), s2.end(), ':');
+  CPPUNIT_ASSERT(!r.second);
+
+  std::string s3 = "   ";
+  r = util::nextParam(name, value, s3.begin(), s3.end(), ':');
+  CPPUNIT_ASSERT(!r.second);
+
+  std::string s4 = ":::";
+  r = util::nextParam(name, value, s4.begin(), s4.end(), ':');
+  CPPUNIT_ASSERT(!r.second);
+}
+
 } // namespace aria2