aria2/src/FileEntry.cc

/* <!-- copyright */
/*
 * aria2 - The high speed download utility
 *
 * Copyright (C) 2006 Tatsuhiro Tsujikawa
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * In addition, as a special exception, the copyright holders give
 * permission to link the code of portions of this program with the
 * OpenSSL library under certain conditions as described in each
 * individual source file, and distribute linked combinations
 * including the two.
 * You must obey the GNU General Public License in all respects
 * for all of the code used other than OpenSSL.  If you modify
 * file(s) with this exception, you may extend this exception to your
 * version of the file(s), but you are not obligated to do so.  If you
 * do not wish to do so, delete this exception statement from your
 * version.  If you delete this exception statement from all source
 * files in the program, then also delete it here.
 */
/* copyright --> */
#include "FileEntry.h"

#include <cassert>
#include <algorithm>

#include "util.h"
#include "URISelector.h"
#include "LogFactory.h"
#include "wallclock.h"

namespace aria2 {

FileEntry::FileEntry(const std::string& path,
                     uint64_t length,
                     off_t offset,
                     const std::vector<std::string>& uris):
  path(path), _uris(uris.begin(), uris.end()), length(length), offset(offset),
  requested(true),
  _singleHostMultiConnection(true),
  _logger(LogFactory::getInstance()) {}

FileEntry::FileEntry():
  length(0), offset(0), requested(false),
  _singleHostMultiConnection(true),
  _logger(LogFactory::getInstance()) {}

FileEntry::~FileEntry() {}

void FileEntry::setupDir()
{
  util::mkdirs(File(path).getDirname());
}

FileEntry& FileEntry::operator=(const FileEntry& entry)
{
  if(this != &entry) {
    path = entry.path;
    length = entry.length;
    offset = entry.offset;
    requested = entry.requested;
  }
  return *this;
}

bool FileEntry::operator<(const FileEntry& fileEntry) const
{
  return offset < fileEntry.offset;
}

bool FileEntry::exists() const
{
  return File(getPath()).exists();
}

off_t FileEntry::gtoloff(off_t goff) const
{
  assert(offset <= goff);
  return goff-offset;
}

void FileEntry::getUris(std::vector<std::string>& uris) const
{
  uris.insert(uris.end(), _spentUris.begin(), _spentUris.end());
  uris.insert(uris.end(), _uris.begin(), _uris.end());
}

std::string FileEntry::selectUri(const SharedHandle<URISelector>& uriSelector)
{
  return uriSelector->select(this);
}

template<typename InputIterator>
static bool inFlightHost(InputIterator first, InputIterator last,
                         const std::string& hostname)
{
  // TODO redirection should be considered here. We need to parse
  // original URI to get hostname.
  for(; first != last; ++first) {
    if((*first)->getHost() == hostname) {
      return true;
    }
  }
  return false;
}

SharedHandle<Request>
FileEntry::getRequest
(const SharedHandle<URISelector>& selector,
 const std::string& referer,
 const std::string& method)
{
  SharedHandle<Request> req;
  if(_requestPool.empty()) {
    std::vector<std::string> pending;
    while(1) {
      std::string uri = selector->select(this);
      if(uri.empty()) {
        return req;
      }
      req.reset(new Request());
      if(req->setUri(uri)) {
        if(!_singleHostMultiConnection) {
          if(inFlightHost(_inFlightRequests.begin(), _inFlightRequests.end(),
                          req->getHost())) {
            pending.push_back(uri);
            req.reset();
            continue;
          }
        }
        req->setReferer(referer);
        req->setMethod(method);
        _spentUris.push_back(uri);
        _inFlightRequests.push_back(req);
        break;
      } else {
        req.reset();
      }
    }
    _uris.insert(_uris.begin(), pending.begin(), pending.end());
  } else {
    req = _requestPool.front();
    _requestPool.pop_front();
    _inFlightRequests.push_back(req);
  }
  return req;
}

SharedHandle<Request>
FileEntry::findFasterRequest(const SharedHandle<Request>& base)
{
  if(_requestPool.empty()) {
    return SharedHandle<Request>();
  }
  const SharedHandle<PeerStat>& fastest = _requestPool.front()->getPeerStat();
  if(fastest.isNull()) {
    return SharedHandle<Request>();
  }
  const SharedHandle<PeerStat>& basestat = base->getPeerStat();
  // TODO hard coded value. See PREF_STARTUP_IDLE_TIME
  const int startupIdleTime = 10;
  if(basestat.isNull() ||
     (basestat->getDownloadStartTime().
      difference(global::wallclock) >= startupIdleTime &&
      fastest->getAvgDownloadSpeed()*0.8 > basestat->calculateDownloadSpeed())){
    // TODO we should consider that "fastest" is very slow.
    SharedHandle<Request> fastestRequest = _requestPool.front();
    _requestPool.pop_front();
    _inFlightRequests.push_back(fastestRequest);
    return fastestRequest;
  }
  return SharedHandle<Request>();
}

class RequestFaster {
public:
  bool operator()(const SharedHandle<Request>& lhs,
                  const SharedHandle<Request>& rhs) const
  {
    if(lhs->getPeerStat().isNull()) {
      return false;
    }
    if(rhs->getPeerStat().isNull()) {
      return true;
    }
    return
      lhs->getPeerStat()->getAvgDownloadSpeed() > rhs->getPeerStat()->getAvgDownloadSpeed();
  }
};

void FileEntry::storePool(const SharedHandle<Request>& request)
{
  const SharedHandle<PeerStat>& peerStat = request->getPeerStat();
  if(!peerStat.isNull()) {
    // We need to calculate average download speed here in order to
    // store Request in the right position in the pool.
    peerStat->calculateAvgDownloadSpeed();
  }
  std::deque<SharedHandle<Request> >::iterator i =
    std::lower_bound(_requestPool.begin(), _requestPool.end(), request,
                     RequestFaster());
  _requestPool.insert(i, request);
}

void FileEntry::poolRequest(const SharedHandle<Request>& request)
{
  removeRequest(request);
  if(!request->removalRequested()) {
    storePool(request);
  }
}

bool FileEntry::removeRequest(const SharedHandle<Request>& request)
{
  for(std::deque<SharedHandle<Request> >::iterator i =
        _inFlightRequests.begin(), eoi = _inFlightRequests.end();
      i != eoi; ++i) {
    if((*i).get() == request.get()) {
      _inFlightRequests.erase(i);
      return true;
    }
  }
  return false;
}

void FileEntry::removeURIWhoseHostnameIs(const std::string& hostname)
{
  std::deque<std::string> newURIs;
  Request req;
  for(std::deque<std::string>::const_iterator itr = _uris.begin(),
        eoi = _uris.end(); itr != eoi; ++itr) {
    if(((*itr).find(hostname) == std::string::npos) ||
       (req.setUri(*itr) && (req.getHost() != hostname))) {
      newURIs.push_back(*itr);
    }
  }
  if(_logger->debug()) {
    _logger->debug("Removed %d duplicate hostname URIs for path=%s",
                   _uris.size()-newURIs.size(), getPath().c_str());
  }
  _uris = newURIs;
}

void FileEntry::removeIdenticalURI(const std::string& uri)
{
  _uris.erase(std::remove(_uris.begin(), _uris.end(), uri), _uris.end());
}

void FileEntry::addURIResult(std::string uri, downloadresultcode::RESULT result)
{
  _uriResults.push_back(URIResult(uri, result));
}

class FindURIResultByResult {
private:
  downloadresultcode::RESULT _r;
public:
  FindURIResultByResult(downloadresultcode::RESULT r):_r(r) {}

  bool operator()(const URIResult& uriResult) const
  {
    return uriResult.getResult() == _r;
  }
};

void FileEntry::extractURIResult
(std::deque<URIResult>& res, downloadresultcode::RESULT r)
{
  std::deque<URIResult>::iterator i =
    std::stable_partition(_uriResults.begin(), _uriResults.end(),
                          FindURIResultByResult(r));
  std::copy(_uriResults.begin(), i, std::back_inserter(res));
  _uriResults.erase(_uriResults.begin(), i);
}

void FileEntry::reuseUri(size_t num)
{
  std::deque<std::string> uris = _spentUris;
  std::sort(uris.begin(), uris.end());
  uris.erase(std::unique(uris.begin(), uris.end()), uris.end());

  std::vector<std::string> errorUris(_uriResults.size());
  std::transform(_uriResults.begin(), _uriResults.end(),
                 errorUris.begin(), std::mem_fun_ref(&URIResult::getURI));
  std::sort(errorUris.begin(), errorUris.end());
  errorUris.erase(std::unique(errorUris.begin(), errorUris.end()),
                  errorUris.end());
     
  std::vector<std::string> reusableURIs;
  std::set_difference(uris.begin(), uris.end(),
                      errorUris.begin(), errorUris.end(),
                      std::back_inserter(reusableURIs));
  size_t ininum = reusableURIs.size();
  if(_logger->debug()) {
    _logger->debug("Found %u reusable URIs", static_cast<unsigned int>(ininum));
    for(std::vector<std::string>::const_iterator i = reusableURIs.begin(),
          eoi = reusableURIs.end(); i != eoi; ++i) {
      _logger->debug("URI=%s", (*i).c_str());
    }
  }
  // Reuse at least num URIs here to avoid to
  // run this process repeatedly.
  if(ininum > 0) {
    for(size_t i = 0; i < num/ininum; ++i) {
      _uris.insert(_uris.end(), reusableURIs.begin(), reusableURIs.end());
    }
    _uris.insert(_uris.end(), reusableURIs.begin(),
                 reusableURIs.begin()+(num%ininum));
    if(_logger->debug()) {
      _logger->debug("Duplication complete: now %u URIs for reuse",
                     static_cast<unsigned int>(_uris.size()));
    }
  }
}

void FileEntry::releaseRuntimeResource()
{
  _requestPool.clear();
  _inFlightRequests.clear();
}

template<typename InputIterator, typename T>
static InputIterator findRequestByUri
(InputIterator first, InputIterator last, const T& uri)
{
  for(; first != last; ++first) {
    if(!(*first)->removalRequested() && (*first)->getUri() == uri) {
      return first;
    }
  }
  return last;
}

bool FileEntry::removeUri(const std::string& uri)
{
  std::deque<std::string>::iterator itr =
    std::find(_spentUris.begin(), _spentUris.end(), uri);
  if(itr == _spentUris.end()) {
    itr = std::find(_uris.begin(), _uris.end(), uri);
    if(itr == _uris.end()) {
      return false;
    } else {
      _uris.erase(itr);
      return true;
    }
  } else {
    _spentUris.erase(itr);
    SharedHandle<Request> req;
    std::deque<SharedHandle<Request> >::iterator riter =
      findRequestByUri(_inFlightRequests.begin(), _inFlightRequests.end(), uri);
    if(riter == _inFlightRequests.end()) {
      riter = findRequestByUri(_requestPool.begin(), _requestPool.end(), uri);
      if(riter == _requestPool.end()) {
        return true;
      } else {
        req = *riter;
        _requestPool.erase(riter);
      }
    } else {
      req = *riter;
    }
    req->requestRemoval();
    return true;
  }
}

} // namespace aria2