Parallel A and AAAA record lookups with c-ares

src/AbstractCommand.cc

@@ -69,6 +69,7 @@
 #include "SocketRecvBuffer.h"
 #ifdef ENABLE_ASYNC_DNS
 #include "AsyncNameResolver.h"
+#include "AsyncNameResolverMan.h"
 #endif // ENABLE_ASYNC_DNS
 #ifdef ENABLE_MESSAGE_DIGEST
 # include "ChecksumCheckIntegrityEntry.h"
@@ -90,8 +91,10 @@ AbstractCommand::AbstractCommand
     requestGroup_(requestGroup),
     req_(req), fileEntry_(fileEntry), e_(e), socket_(s),
     socketRecvBuffer_(socketRecvBuffer),
+#ifdef ENABLE_ASYNC_DNS
+    asyncNameResolverMan_(new AsyncNameResolverMan()),
+#endif // ENABLE_ASYNC_DNS
     checkSocketIsReadable_(false), checkSocketIsWritable_(false),
-    nameResolverCheck_(false),
     incNumConnection_(incNumConnection),
     serverStatTimer_(global::wallclock())
 {
@@ -103,13 +106,18 @@ AbstractCommand::AbstractCommand
   }
   requestGroup_->increaseStreamCommand();
   requestGroup_->increaseNumCommand();
+#ifdef ENABLE_ASYNC_DNS
+  if(e_->getOption()->getAsBool(PREF_DISABLE_IPV6)) {
+    asyncNameResolverMan_->setIPv6(false);
+  }
+#endif // ENABLE_ASYNC_DNS
 }
 
 AbstractCommand::~AbstractCommand() {
   disableReadCheckSocket();
   disableWriteCheckSocket();
 #ifdef ENABLE_ASYNC_DNS
-  disableNameResolverCheck(asyncNameResolver_);
+  asyncNameResolverMan_->disableNameResolverCheck(e_, this);
 #endif // ENABLE_ASYNC_DNS
   requestGroup_->decreaseNumCommand();
   requestGroup_->decreaseStreamCommand();
@@ -206,10 +214,14 @@ bool AbstractCommand::execute() {
        (checkSocketIsWritable_ && writeEventEnabled()) ||
        hupEventEnabled() ||
 #ifdef ENABLE_ASYNC_DNS
-       (nameResolverCheck_ && nameResolveFinished()) ||
+       (asyncNameResolverMan_->resolverChecked() &&
+        asyncNameResolverMan_->getStatus() != 0) ||
 #endif // ENABLE_ASYNC_DNS
-       (!checkSocketIsReadable_ && !checkSocketIsWritable_ &&
-        !nameResolverCheck_)) {
+       (!checkSocketIsReadable_ && !checkSocketIsWritable_
+#ifdef ENABLE_ASYNC_DNS
+        && !asyncNameResolverMan_->resolverChecked()
+#endif // ENABLE_ASYNC_DNS
+        )) {
       checkPoint_ = global::wallclock();
       if(getPieceStorage()) {
         if(!req_ || req_->getMaxPipelinedRequest() == 1 ||
@@ -688,86 +700,6 @@ SharedHandle<Request> AbstractCommand::createProxyRequest() const
   return proxyRequest;
 }
 
-#ifdef ENABLE_ASYNC_DNS
-
-bool AbstractCommand::isAsyncNameResolverInitialized() const
-{
-  return asyncNameResolver_;
-}
-
-void AbstractCommand::initAsyncNameResolver(const std::string& hostname)
-{
-  int family;
-  if(getOption()->getAsBool(PREF_ENABLE_ASYNC_DNS6)) {
-    family = AF_UNSPEC;
-  } else {
-    family = AF_INET;
-  }
-  asyncNameResolver_.reset
-    (new AsyncNameResolver(family
-#ifdef HAVE_ARES_ADDR_NODE
-                           ,
-                           e_->getAsyncDNSServers()
-#endif // HAVE_ARES_ADDR_NODE
-                           ));
-  A2_LOG_INFO(fmt(MSG_RESOLVING_HOSTNAME,
-                  getCuid(),
-                  hostname.c_str()));
-  asyncNameResolver_->resolve(hostname);
-  setNameResolverCheck(asyncNameResolver_);
-}
-
-bool AbstractCommand::asyncResolveHostname()
-{
-  switch(asyncNameResolver_->getStatus()) {
-  case AsyncNameResolver::STATUS_SUCCESS:
-    disableNameResolverCheck(asyncNameResolver_);
-    return true;
-  case AsyncNameResolver::STATUS_ERROR:
-    disableNameResolverCheck(asyncNameResolver_);
-    if(!isProxyRequest(req_->getProtocol(), getOption())) {
-      e_->getRequestGroupMan()->getOrCreateServerStat
-        (req_->getHost(), req_->getProtocol())->setError();
-    }
-    throw DL_ABORT_EX2
-      (fmt(MSG_NAME_RESOLUTION_FAILED,
-           getCuid(),
-           asyncNameResolver_->getHostname().c_str(),
-           asyncNameResolver_->getError().c_str()),
-       error_code::NAME_RESOLVE_ERROR);
-  default:
-    return false;
-  }
-}
-
-const std::vector<std::string>& AbstractCommand::getResolvedAddresses()
-{
-  return asyncNameResolver_->getResolvedAddresses();
-}
-
-void AbstractCommand::setNameResolverCheck
-(const SharedHandle<AsyncNameResolver>& resolver) {
-  if(resolver) {
-    nameResolverCheck_ = true;
-    e_->addNameResolverCheck(resolver, this);
-  }
-}
-
-void AbstractCommand::disableNameResolverCheck
-(const SharedHandle<AsyncNameResolver>& resolver) {
-  if(resolver) {
-    nameResolverCheck_ = false;
-    e_->deleteNameResolverCheck(resolver, this);
-  }
-}
-
-bool AbstractCommand::nameResolveFinished() const {
-  return
-    asyncNameResolver_->getStatus() ==  AsyncNameResolver::STATUS_SUCCESS ||
-    asyncNameResolver_->getStatus() == AsyncNameResolver::STATUS_ERROR;
-}
-#endif // ENABLE_ASYNC_DNS
-
 std::string AbstractCommand::resolveHostname
 (std::vector<std::string>& addrs, const std::string& hostname, uint16_t port)
 {
@@ -780,13 +712,24 @@ std::string AbstractCommand::resolveHostname
   if(addrs.empty()) {
 #ifdef ENABLE_ASYNC_DNS
     if(getOption()->getAsBool(PREF_ASYNC_DNS)) {
-      if(!isAsyncNameResolverInitialized()) {
-        initAsyncNameResolver(hostname);
+      if(!asyncNameResolverMan_->started()) {
+        asyncNameResolverMan_->startAsync(hostname, e_, this);
       }
-      if(asyncResolveHostname()) {
-        addrs = getResolvedAddresses();
-      } else {
+      switch(asyncNameResolverMan_->getStatus()) {
+      case -1:
+        if(!isProxyRequest(req_->getProtocol(), getOption())) {
+          e_->getRequestGroupMan()->getOrCreateServerStat
+            (req_->getHost(), req_->getProtocol())->setError();
+        }
+        throw DL_ABORT_EX2
+          (fmt(MSG_NAME_RESOLUTION_FAILED, getCuid(), hostname.c_str(),
+               asyncNameResolverMan_->getLastError().c_str()),
+           error_code::NAME_RESOLVE_ERROR);
+      case 0:
         return A2STR::NIL;
+      case 1:
+        asyncNameResolverMan_->getResolvedAddress(addrs);
+        break;
       }
     } else
 #endif // ENABLE_ASYNC_DNS

src/AbstractCommand.h

@@ -59,6 +59,7 @@ class Option;
 class SocketRecvBuffer;
 #ifdef ENABLE_ASYNC_DNS
 class AsyncNameResolver;
+class AsyncNameResolverMan;
 #endif // ENABLE_ASYNC_DNS
 
 class AbstractCommand : public Command {
@@ -75,28 +76,19 @@ private:
   std::vector<SharedHandle<Segment> > segments_;
 
 #ifdef ENABLE_ASYNC_DNS
-  SharedHandle<AsyncNameResolver> asyncNameResolver_;
+  SharedHandle<AsyncNameResolverMan> asyncNameResolverMan_;
 #endif // ENABLE_ASYNC_DNS
 
   bool checkSocketIsReadable_;
   bool checkSocketIsWritable_;
   SharedHandle<SocketCore> readCheckTarget_;
   SharedHandle<SocketCore> writeCheckTarget_;
-  bool nameResolverCheck_;
 
   bool incNumConnection_;
   Timer serverStatTimer_;
 
   int32_t calculateMinSplitSize() const;
   void useFasterRequest(const SharedHandle<Request>& fasterRequest);
-#ifdef ENABLE_ASYNC_DNS
-  void setNameResolverCheck(const SharedHandle<AsyncNameResolver>& resolver);
-
-  void disableNameResolverCheck
-  (const SharedHandle<AsyncNameResolver>& resolver);
-
-  bool nameResolveFinished() const;
-#endif // ENABLE_ASYNC_DNS
 protected:
   RequestGroup* getRequestGroup() const
   {
@@ -145,16 +137,6 @@ protected:
     return segments_;
   }
 
-#ifdef ENABLE_ASYNC_DNS
-  bool isAsyncNameResolverInitialized() const;
-
-  void initAsyncNameResolver(const std::string& hostname);
-
-  bool asyncResolveHostname();
-
-  const std::vector<std::string>& getResolvedAddresses();
-#endif // ENABLE_ASYNC_DNS
-
   // Resolves hostname.  The resolved addresses are stored in addrs
   // and first element is returned.  If resolve is not finished,
   // return empty string. In this case, call this function with same

src/AsyncNameResolver.cc

@@ -58,7 +58,7 @@ void callback(void* arg, int status, int timeouts, struct hostent* host)
     }
   }
   if(resolverPtr->resolvedAddresses_.empty()) {
-    resolverPtr->error_ = "address conversion failed";
+    resolverPtr->error_ = "no address returned or address conversion failed";
     resolverPtr->status_ = AsyncNameResolver::STATUS_ERROR;
   } else {
     resolverPtr->status_ = AsyncNameResolver::STATUS_SUCCESS;

src/AsyncNameResolverMan.cc

@@ -0,0 +1,180 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2013 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#include "AsyncNameResolverMan.h"
+#include "AsyncNameResolver.h"
+#include "DownloadEngine.h"
+#include "Command.h"
+#include "message.h"
+#include "fmt.h"
+#include "LogFactory.h"
+
+namespace aria2 {
+
+AsyncNameResolverMan::AsyncNameResolverMan()
+  : numResolver_(0),
+    resolverCheck_(0),
+    ipv4_(true),
+    ipv6_(true)
+{}
+
+AsyncNameResolverMan::~AsyncNameResolverMan()
+{
+  assert(!resolverCheck_);
+}
+
+bool AsyncNameResolverMan::started() const
+{
+  for(size_t i = 0; i < numResolver_; ++i) {
+    if(asyncNameResolver_[i]) {
+      return true;
+    }
+  }
+  return false;
+}
+
+void AsyncNameResolverMan::startAsync(const std::string& hostname,
+                                      DownloadEngine* e,
+                                      Command* command)
+{
+  numResolver_ = 0;
+  if(ipv4_) {
+    startAsyncFamily(hostname, AF_INET, e, command);
+    ++numResolver_;
+  }
+  if(ipv6_) {
+    startAsyncFamily(hostname, AF_INET6, e, command);
+    ++numResolver_;
+  }
+  A2_LOG_INFO(fmt(MSG_RESOLVING_HOSTNAME, command->getCuid(),
+                  hostname.c_str()));
+}
+
+void AsyncNameResolverMan::startAsyncFamily(const std::string& hostname,
+                                            int family,
+                                            DownloadEngine* e,
+                                            Command* command)
+{
+  asyncNameResolver_[numResolver_].reset
+    (new AsyncNameResolver(family
+#ifdef HAVE_ARES_ADDR_NODE
+                           ,
+                           e->getAsyncDNSServers()
+#endif // HAVE_ARES_ADDR_NODE
+                           ));
+  asyncNameResolver_[numResolver_]->resolve(hostname);
+  setNameResolverCheck(numResolver_, e, command);
+}
+
+void AsyncNameResolverMan::getResolvedAddress(std::vector<std::string>& res)
+const
+{
+  for(size_t i = 0; i < numResolver_; ++i) {
+    if(asyncNameResolver_[i]->getStatus() ==
+       AsyncNameResolver::STATUS_SUCCESS) {
+      const std::vector<std::string>& addrs =
+        asyncNameResolver_[i]->getResolvedAddresses();
+      res.insert(res.end(), addrs.begin(), addrs.end());
+    }
+  }
+  return;
+}
+
+void AsyncNameResolverMan::setNameResolverCheck(DownloadEngine* e,
+                                                Command* command)
+{
+  for(size_t i = 0; i < numResolver_; ++i) {
+    setNameResolverCheck(i, e, command);
+  }
+}
+
+void AsyncNameResolverMan::setNameResolverCheck(size_t index,
+                                                DownloadEngine* e,
+                                                Command* command)
+{
+  if(asyncNameResolver_[index]) {
+    assert((resolverCheck_ & (1 << index)) == 0);
+    resolverCheck_ |= 1 << index;
+    e->addNameResolverCheck(asyncNameResolver_[index], command);
+  }
+}
+
+void AsyncNameResolverMan::disableNameResolverCheck(DownloadEngine* e,
+                                                    Command* command)
+{
+  for(size_t i = 0; i < numResolver_; ++i) {
+    disableNameResolverCheck(i, e, command);
+  }
+}
+
+void AsyncNameResolverMan::disableNameResolverCheck(size_t index,
+                                                    DownloadEngine* e,
+                                                    Command* command)
+{
+  if(asyncNameResolver_[index] && (resolverCheck_ & (1 << index))) {
+    resolverCheck_ &= ~(1 << index);
+    e->deleteNameResolverCheck(asyncNameResolver_[index], command);
+  }
+}
+
+int AsyncNameResolverMan::getStatus() const
+{
+  size_t error = 0;
+  for(size_t i = 0; i < numResolver_; ++i) {
+    switch(asyncNameResolver_[i]->getStatus()) {
+    case AsyncNameResolver::STATUS_SUCCESS:
+      return 1;
+    case AsyncNameResolver::STATUS_ERROR:
+      ++error;
+      break;
+    default:
+      break;
+    }
+  }
+  return error == numResolver_ ? -1 : 0;
+}
+
+const std::string& AsyncNameResolverMan::getLastError() const
+{
+  for(size_t i = 0; i < numResolver_; ++i) {
+    if(asyncNameResolver_[i]->getStatus() == AsyncNameResolver::STATUS_ERROR) {
+      // TODO This is not last error chronologically.
+      return asyncNameResolver_[i]->getError();
+    }
+  }
+  return A2STR::NIL;
+}
+
+} // namespace aria2
+

src/AsyncNameResolverMan.h

@@ -0,0 +1,106 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2013 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef D_ASYNC_NAME_RESOLVER_MAN_H
+#define D_ASYNC_NAME_RESOLVER_MAN_H
+
+#include "common.h"
+
+#include <vector>
+#include <string>
+
+#include "SharedHandle.h"
+
+namespace aria2 {
+
+class AsyncNameResolver;
+class DownloadEngine;
+class Command;
+
+class AsyncNameResolverMan {
+public:
+  AsyncNameResolverMan();
+  // Destructor does not call disableNameResolverCheck(). Application
+  // must call it before the destruction of this object.
+  ~AsyncNameResolverMan();
+  // Enable IPv4 address lookup. default: true
+  void setIPv4(bool ipv4)
+  {
+    ipv4_ = ipv4;
+  }
+  // Enable IPv6 address lookup. default: true
+  void setIPv6(bool ipv6)
+  {
+    ipv6_ = ipv6;
+  }
+  // Returns true if asynchronous name resolution has been started.
+  bool started() const;
+  // Starts asynchronous name resolution.
+  void startAsync(const std::string& hostname, DownloadEngine* e,
+                  Command* command);
+  // Appends resolved addresses to |res|.
+  void getResolvedAddress(std::vector<std::string>& res) const;
+  // Adds resolvers to DownloadEngine to check event notification.
+  void setNameResolverCheck(DownloadEngine* e, Command* command);
+  // Removes resolvers from DownloadEngine.
+  void disableNameResolverCheck(DownloadEngine* e, Command* command);
+  // Returns true if any of resolvers are added in DownloadEngine.
+  bool resolverChecked() const
+  {
+    return resolverCheck_;
+  }
+  // Returns status value: 0 for inprogress, 1 for success and -1 for
+  // failure.
+  int getStatus() const;
+  // Returns last error string
+  const std::string& getLastError() const;
+private:
+  void startAsyncFamily(const std::string& hostname,
+                        int family,
+                        DownloadEngine* e, Command* command);
+  void setNameResolverCheck(size_t resolverIndex, DownloadEngine* e,
+                            Command* command);
+  void disableNameResolverCheck(size_t index, DownloadEngine* e,
+                                Command* command);
+
+  SharedHandle<AsyncNameResolver> asyncNameResolver_[2];
+  size_t numResolver_;
+  int resolverCheck_;
+  bool ipv4_;
+  bool ipv6_;
+};
+
+} // namespace aria2
+
+#endif // D_ASYNC_NAME_RESOLVER_MAN_H

src/Makefile.am

@@ -335,7 +335,8 @@ SRCS += Sqlite3CookieParser.cc Sqlite3CookieParser.h\
 endif # HAVE_SQLITE3
 
 if ENABLE_ASYNC_DNS
-SRCS += AsyncNameResolver.cc AsyncNameResolver.h
+SRCS += AsyncNameResolver.cc AsyncNameResolver.h\
+	AsyncNameResolverMan.cc AsyncNameResolverMan.h
 endif # ENABLE_ASYNC_DNS
 
 if ENABLE_MESSAGE_DIGEST