aria2/src/HttpResponseCommand.cc

/* <!-- copyright */
/*
 * aria2 - The high speed download utility
 *
 * Copyright (C) 2006 Tatsuhiro Tsujikawa
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * In addition, as a special exception, the copyright holders give
 * permission to link the code of portions of this program with the
 * OpenSSL library under certain conditions as described in each
 * individual source file, and distribute linked combinations
 * including the two.
 * You must obey the GNU General Public License in all respects
 * for all of the code used other than OpenSSL.  If you modify
 * file(s) with this exception, you may extend this exception to your
 * version of the file(s), but you are not obligated to do so.  If you
 * do not wish to do so, delete this exception statement from your
 * version.  If you delete this exception statement from all source
 * files in the program, then also delete it here.
 */
/* copyright --> */
#include "HttpResponseCommand.h"
#include "DownloadEngine.h"
#include "DownloadContext.h"
#include "FileEntry.h"
#include "RequestGroup.h"
#include "RequestGroupMan.h"
#include "Request.h"
#include "HttpRequest.h"
#include "HttpResponse.h"
#include "HttpConnection.h"
#include "SegmentMan.h"
#include "Segment.h"
#include "HttpDownloadCommand.h"
#include "DiskAdaptor.h"
#include "PieceStorage.h"
#include "DefaultBtProgressInfoFile.h"
#include "DownloadFailureException.h"
#include "DlAbortEx.h"
#include "util.h"
#include "File.h"
#include "Option.h"
#include "Logger.h"
#include "SocketCore.h"
#include "message.h"
#include "prefs.h"
#include "fmt.h"
#include "HttpSkipResponseCommand.h"
#include "HttpHeader.h"
#include "LogFactory.h"
#include "CookieStorage.h"
#include "AuthConfigFactory.h"
#include "AuthConfig.h"
#include "a2functional.h"
#include "URISelector.h"
#include "CheckIntegrityEntry.h"
#include "StreamFilter.h"
#include "SinkStreamFilter.h"
#include "ChunkedDecodingStreamFilter.h"
#include "uri.h"
#include "SocketRecvBuffer.h"
#include "MetalinkHttpEntry.h"
#ifdef ENABLE_MESSAGE_DIGEST
# include "Checksum.h"
# include "ChecksumCheckIntegrityEntry.h"
#endif // ENABLE_MESSAGE_DIGEST
#ifdef HAVE_ZLIB
# include "GZipDecodingStreamFilter.h"
#endif // HAVE_ZLIB

namespace aria2 {

namespace {
std::unique_ptr<StreamFilter> getTransferEncodingStreamFilter
(HttpResponse* httpResponse,
 std::unique_ptr<StreamFilter> delegate = std::unique_ptr<StreamFilter>{})
{
  if(httpResponse->isTransferEncodingSpecified()) {
    auto filter = httpResponse->getTransferEncodingStreamFilter();
    if(!filter) {
      throw DL_ABORT_EX
        (fmt(EX_TRANSFER_ENCODING_NOT_SUPPORTED,
             httpResponse->getTransferEncoding().c_str()));
    }
    filter->init();
    filter->installDelegate(std::move(delegate));
    return filter;
  }
  return delegate;
}
} // namespace

namespace {
std::unique_ptr<StreamFilter> getContentEncodingStreamFilter
(HttpResponse* httpResponse,
 std::unique_ptr<StreamFilter> delegate = std::unique_ptr<StreamFilter>{})
{
  if(httpResponse->isContentEncodingSpecified()) {
    auto filter = httpResponse->getContentEncodingStreamFilter();
    if(!filter) {
      A2_LOG_INFO
        (fmt("Content-Encoding %s is specified, but the current implementation"
             "doesn't support it. The decoding process is skipped and the"
             "downloaded content will be still encoded.",
             httpResponse->getContentEncoding().c_str()));
    }
    filter->init();
    filter->installDelegate(std::move(delegate));
    return filter;
  }
  return delegate;
}
} // namespace

HttpResponseCommand::HttpResponseCommand
(cuid_t cuid,
 const std::shared_ptr<Request>& req,
 const std::shared_ptr<FileEntry>& fileEntry,
 RequestGroup* requestGroup,
 const std::shared_ptr<HttpConnection>& httpConnection,
 DownloadEngine* e,
 const std::shared_ptr<SocketCore>& s)
  : AbstractCommand(cuid, req, fileEntry, requestGroup, e, s,
                    httpConnection->getSocketRecvBuffer()),
    httpConnection_(httpConnection)
{
  checkSocketRecvBuffer();
}

HttpResponseCommand::~HttpResponseCommand() {}

bool HttpResponseCommand::executeInternal()
{
  auto httpResponse = httpConnection_->receiveResponse();
  if(!httpResponse) {
    // The server has not responded to our request yet.
    // For socket->wantRead() == true, setReadCheckSocket(socket) is already
    // done in the constructor.
    setWriteCheckSocketIf(getSocket(), getSocket()->wantWrite());
    addCommandSelf();
    return false;
  }
  // check HTTP status number
  httpResponse->validateResponse();
  httpResponse->retrieveCookie();

  const auto& httpHeader = httpResponse->getHttpHeader();
  // Disable persistent connection if:
  //   Connection: close is received or the remote server is not HTTP/1.1.
  // We don't care whether non-HTTP/1.1 server returns Connection: keep-alive.
  getRequest()->supportsPersistentConnection
    (httpResponse->supportsPersistentConnection());
  if(getRequest()->isPipeliningEnabled()) {
    getRequest()->setMaxPipelinedRequest
      (getOption()->getAsInt(PREF_MAX_HTTP_PIPELINING));
  } else {
    getRequest()->setMaxPipelinedRequest(1);
  }

  int statusCode = httpResponse->getStatusCode();

  if(statusCode == 304) {
    int64_t totalLength = httpResponse->getEntityLength();
    getFileEntry()->setLength(totalLength);
    getRequestGroup()->initPieceStorage();
    getPieceStorage()->markAllPiecesDone();
    // Just set checksum verification done.
    getDownloadContext()->setChecksumVerified(true);
    A2_LOG_NOTICE
      (fmt(MSG_DOWNLOAD_ALREADY_COMPLETED,
           GroupId::toHex(getRequestGroup()->getGID()).c_str(),
           getRequestGroup()->getFirstFilePath().c_str()));
    poolConnection();
    getFileEntry()->poolRequest(getRequest());
    return true;
  }
  if(!getPieceStorage()) {
    // Metalink/HTTP
    if(getDownloadContext()->getAcceptMetalink()) {
      if(httpHeader->defined(HttpHeader::LINK)) {
        getDownloadContext()->setAcceptMetalink(false);
        std::vector<MetalinkHttpEntry> entries;
        httpResponse->getMetalinKHttpEntries(entries, getOption());
        for(std::vector<MetalinkHttpEntry>::iterator i = entries.begin(),
              eoi = entries.end(); i != eoi; ++i) {
          getFileEntry()->addUri((*i).uri);
          A2_LOG_DEBUG(fmt("Adding URI=%s", (*i).uri.c_str()));
        }
      }
    }
#ifdef ENABLE_MESSAGE_DIGEST
    if(httpHeader->defined(HttpHeader::DIGEST)) {
      std::vector<Checksum> checksums;
      httpResponse->getDigest(checksums);
      for(std::vector<Checksum>::iterator i = checksums.begin(),
            eoi = checksums.end(); i != eoi; ++i) {
        if(getDownloadContext()->getHashType().empty()) {
          A2_LOG_DEBUG(fmt("Setting digest: type=%s, digest=%s",
                           (*i).getHashType().c_str(),
                           (*i).getDigest().c_str()));
          getDownloadContext()->setDigest((*i).getHashType(), (*i).getDigest());
          break;
        } else {
          if(checkChecksum(getDownloadContext(), *i)) {
            break;
          }
        }
      }
    }
#endif // ENABLE_MESSAGE_DIGEST
  }
  if(statusCode >= 300) {
    if(statusCode == 404) {
      getRequestGroup()->increaseAndValidateFileNotFoundCount();
    }
    return skipResponseBody(std::move(httpResponse));
  }
  if(getFileEntry()->isUniqueProtocol()) {
    // Redirection should be considered here. We need to parse
    // original URI to get hostname.
    const std::string& uri = getRequest()->getUri();
    uri_split_result us;
    if(uri_split(&us, uri.c_str()) == 0) {
      std::string host = uri::getFieldString(us, USR_HOST, uri.c_str());
      getFileEntry()->removeURIWhoseHostnameIs(host);
    }
  }
  if(!getPieceStorage()) {
    getDownloadContext()->setAcceptMetalink(false);
    int64_t totalLength = httpResponse->getEntityLength();
    getFileEntry()->setLength(totalLength);
    if(getFileEntry()->getPath().empty()) {
      getFileEntry()->setPath
        (util::createSafePath
         (getOption()->get(PREF_DIR), httpResponse->determinFilename()));
    }
    getFileEntry()->setContentType(httpResponse->getContentType());
    getRequestGroup()->preDownloadProcessing();
    if(getDownloadEngine()->getRequestGroupMan()->
       isSameFileBeingDownloaded(getRequestGroup())) {
      throw DOWNLOAD_FAILURE_EXCEPTION2
        (fmt(EX_DUPLICATE_FILE_DOWNLOAD,
             getRequestGroup()->getFirstFilePath().c_str()),
         error_code::DUPLICATE_DOWNLOAD);
    }
    // update last modified time
    updateLastModifiedTime(httpResponse->getLastModifiedTime());

    // If both transfer-encoding and total length is specified, we
    // assume we can do segmented downloading
    if(totalLength == 0 || shouldInflateContentEncoding(httpResponse.get())) {
      // we ignore content-length when inflate is required
      getFileEntry()->setLength(0);
      if(getRequest()->getMethod() == Request::METHOD_GET &&
         (totalLength != 0 ||
          !httpResponse->getHttpHeader()->defined(HttpHeader::CONTENT_LENGTH))){
        // DownloadContext::knowsTotalLength() == true only when
        // server says the size of file is 0 explicitly.
        getDownloadContext()->markTotalLengthIsUnknown();
      }
      return handleOtherEncoding(std::move(httpResponse));
    } else {
      return handleDefaultEncoding(std::move(httpResponse));
    }
  } else {
#ifdef ENABLE_MESSAGE_DIGEST
    if(!getDownloadContext()->getHashType().empty() &&
       httpHeader->defined(HttpHeader::DIGEST)) {
      std::vector<Checksum> checksums;
      httpResponse->getDigest(checksums);
      for(std::vector<Checksum>::iterator i = checksums.begin(),
            eoi = checksums.end(); i != eoi; ++i) {
        if(checkChecksum(getDownloadContext(), *i)) {
          break;
        }
      }
    }
#endif // ENABLE_MESSAGE_DIGEST
    // validate totalsize
    getRequestGroup()->validateTotalLength(getFileEntry()->getLength(),
                                       httpResponse->getEntityLength());
    // update last modified time
    updateLastModifiedTime(httpResponse->getLastModifiedTime());
    if(getRequestGroup()->getTotalLength() == 0) {
      // Since total length is unknown, the file size in previously
      // failed download could be larger than the size this time.
      // Also we can't resume in this case too.  So truncate the file
      // anyway.
      getPieceStorage()->getDiskAdaptor()->truncate(0);
      auto teFilter = getTransferEncodingStreamFilter
        (httpResponse.get(),
         getContentEncodingStreamFilter(httpResponse.get()));
      getDownloadEngine()->addCommand
        (createHttpDownloadCommand(std::move(httpResponse),
                                   std::move(teFilter)));
    } else {
      auto teFilter = getTransferEncodingStreamFilter(httpResponse.get());
      getDownloadEngine()->addCommand
        (createHttpDownloadCommand(std::move(httpResponse),
                                   std::move(teFilter)));
    }
    return true;
  }
}

void HttpResponseCommand::updateLastModifiedTime(const Time& lastModified)
{
  if(getOption()->getAsBool(PREF_REMOTE_TIME)) {
    getRequestGroup()->updateLastModifiedTime(lastModified);
  }
}

bool HttpResponseCommand::shouldInflateContentEncoding
(HttpResponse* httpResponse)
{
  // Basically, on the fly inflation cannot be made with segment
  // download, because in each segment we don't know where the date
  // should be written.  So turn off segmented downloading.
  // Meanwhile, Some server returns content-encoding: gzip for .tgz
  // files.  I think those files should not be inflated by clients,
  // because it is the original format of those files. Current
  // implementation just inflates these files nonetheless.
  const std::string& ce = httpResponse->getContentEncoding();
  return httpResponse->getHttpRequest()->acceptGZip() &&
    (ce == "gzip" || ce == "deflate");
}

bool HttpResponseCommand::handleDefaultEncoding
(std::unique_ptr<HttpResponse> httpResponse)
{
  auto progressInfoFile = std::make_shared<DefaultBtProgressInfoFile>
    (getDownloadContext(), std::shared_ptr<PieceStorage>{}, getOption().get());
  getRequestGroup()->adjustFilename(progressInfoFile);
  getRequestGroup()->initPieceStorage();

  if(getOption()->getAsBool(PREF_DRY_RUN)) {
    onDryRunFileFound();
    return true;
  }

  auto checkEntry = getRequestGroup()->createCheckIntegrityEntry();
  if(!checkEntry) {
    return true;
  }
  File file(getRequestGroup()->getFirstFilePath());
  // We have to make sure that command that has Request object must
  // have segment after PieceStorage is initialized. See
  // AbstractCommand::execute()
  auto segment = getSegmentMan()->getSegmentWithIndex(getCuid(), 0);
  // pipelining requires implicit range specified. But the request for
  // this response most likely dones't contains range header. This means
  // we can't continue to use this socket because server sends all entity
  // body instead of a segment.
  // Therefore, we shutdown the socket here if pipelining is enabled.
  if(getRequest()->getMethod() == Request::METHOD_GET &&
     segment && segment->getPositionToWrite() == 0 &&
     !getRequest()->isPipeliningEnabled()) {
    auto teFilter = getTransferEncodingStreamFilter(httpResponse.get());
    checkEntry->pushNextCommand
      (createHttpDownloadCommand(std::move(httpResponse),
                                 std::move(teFilter)));
  } else {
    getSegmentMan()->cancelSegment(getCuid());
    getFileEntry()->poolRequest(getRequest());
  }

  prepareForNextAction(std::move(checkEntry));

  if(getRequest()->getMethod() == Request::METHOD_HEAD) {
    poolConnection();
    getRequest()->setMethod(Request::METHOD_GET);
  }
  return true;
}

bool HttpResponseCommand::handleOtherEncoding
(std::unique_ptr<HttpResponse> httpResponse) {
  // We assume that RequestGroup::getTotalLength() == 0 here
  if(getOption()->getAsBool(PREF_DRY_RUN)) {
    getRequestGroup()->initPieceStorage();
    onDryRunFileFound();
    return true;
  }

  if(getRequest()->getMethod() == Request::METHOD_HEAD) {
    poolConnection();
    getRequest()->setMethod(Request::METHOD_GET);
    return prepareForRetry(0);
  }

  // In this context, knowsTotalLength() is true only when the file is
  // really zero-length.

  auto streamFilter = getTransferEncodingStreamFilter
    (httpResponse.get(), getContentEncodingStreamFilter(httpResponse.get()));
  // If chunked transfer-encoding is specified, we have to read end of
  // chunk markers(0\r\n\r\n, for example).
  bool chunkedUsed = streamFilter &&
    streamFilter->getName() == ChunkedDecodingStreamFilter::NAME;

  // For zero-length file, check existing file comparing its size
  if(!chunkedUsed && getDownloadContext()->knowsTotalLength() &&
     getRequestGroup()->downloadFinishedByFileLength()) {
    getRequestGroup()->initPieceStorage();
#ifdef ENABLE_MESSAGE_DIGEST
    // TODO Known issue: if .aria2 file exists, it will not be deleted
    // on successful verification, because .aria2 file is not loaded.
    // See also FtpNegotiationCommand::onFileSizeDetermined()
    if(getDownloadContext()->isChecksumVerificationNeeded()) {
      A2_LOG_DEBUG("Zero length file exists. Verify checksum.");
      auto  entry = make_unique<ChecksumCheckIntegrityEntry>
        (getRequestGroup());
      entry->initValidator();
      getPieceStorage()->getDiskAdaptor()->openExistingFile();
      getDownloadEngine()->getCheckIntegrityMan()->pushEntry(std::move(entry));
    } else
#endif // ENABLE_MESSAGE_DIGEST
      {
        getPieceStorage()->markAllPiecesDone();
        getDownloadContext()->setChecksumVerified(true);
        A2_LOG_NOTICE
          (fmt(MSG_DOWNLOAD_ALREADY_COMPLETED,
               GroupId::toHex(getRequestGroup()->getGID()).c_str(),
               getRequestGroup()->getFirstFilePath().c_str()));
      }
    poolConnection();
    return true;
  }

  getRequestGroup()->shouldCancelDownloadForSafety();
  getRequestGroup()->initPieceStorage();
  getPieceStorage()->getDiskAdaptor()->initAndOpenFile();

  // Local file size becomes zero when DiskAdaptor::initAndOpenFile()
  // is called. So zero-length file is complete if chunked encoding is
  // not used.
  if(!chunkedUsed && getDownloadContext()->knowsTotalLength()) {
    A2_LOG_DEBUG("File length becomes zero and it means download completed.");
    // TODO Known issue: if .aria2 file exists, it will not be deleted
    // on successful verification, because .aria2 file is not loaded.
    // See also FtpNegotiationCommand::onFileSizeDetermined()
#ifdef ENABLE_MESSAGE_DIGEST
    if(getDownloadContext()->isChecksumVerificationNeeded()) {
      A2_LOG_DEBUG("Verify checksum for zero-length file");
      auto entry = make_unique<ChecksumCheckIntegrityEntry>
        (getRequestGroup());
      entry->initValidator();
      getDownloadEngine()->getCheckIntegrityMan()->pushEntry(std::move(entry));
    } else
#endif // ENABLE_MESSAGE_DIGEST
      {
        getRequestGroup()->getPieceStorage()->markAllPiecesDone();
      }
    poolConnection();
    return true;
  }
  // We have to make sure that command that has Request object must
  // have segment after PieceStorage is initialized. See
  // AbstractCommand::execute()
  getSegmentMan()->getSegmentWithIndex(getCuid(), 0);

  getDownloadEngine()->addCommand
    (createHttpDownloadCommand(std::move(httpResponse),
                               std::move(streamFilter)));
  return true;
}

bool HttpResponseCommand::skipResponseBody
(std::unique_ptr<HttpResponse> httpResponse)
{
  auto filter = getTransferEncodingStreamFilter(httpResponse.get());
  // We don't use Content-Encoding here because this response body is just
  // thrown away.
  auto httpResponsePtr = httpResponse.get();
  auto command = make_unique<HttpSkipResponseCommand>
    (getCuid(), getRequest(), getFileEntry(), getRequestGroup(),
     httpConnection_, std::move(httpResponse),
     getDownloadEngine(), getSocket());
  command->installStreamFilter(std::move(filter));

  // If request method is HEAD or the response body is zero-length,
  // set command's status to real time so that avoid read check blocking
  if(getRequest()->getMethod() == Request::METHOD_HEAD ||
     (httpResponsePtr->getEntityLength() == 0 &&
      !httpResponsePtr->isTransferEncodingSpecified())) {
    command->setStatusRealtime();
    // If entity length == 0, then socket read/write check must be disabled.
    command->disableSocketCheck();
    getDownloadEngine()->setNoWait(true);
  }

  getDownloadEngine()->addCommand(std::move(command));
  return true;
}

namespace {
bool decideFileAllocation(StreamFilter* filter)
{
#ifdef HAVE_ZLIB
  for(StreamFilter* f = filter; f; f = f->getDelegate().get()){
    // Since the compressed file's length are returned in the response header
    // and the decompressed file size is unknown at this point, disable file
    // allocation here.
    if(f->getName() == GZipDecodingStreamFilter::NAME) {
      return false;
    }
  }
#endif // HAVE_ZLIB
  return true;
}
} // namespace

std::unique_ptr<HttpDownloadCommand>
HttpResponseCommand::createHttpDownloadCommand
(std::unique_ptr<HttpResponse> httpResponse,
 std::unique_ptr<StreamFilter> filter)
{

  auto command = make_unique<HttpDownloadCommand>
    (getCuid(), getRequest(), getFileEntry(),
     getRequestGroup(),
     std::move(httpResponse), httpConnection_,
     getDownloadEngine(), getSocket());
  command->setStartupIdleTime(getOption()->getAsInt(PREF_STARTUP_IDLE_TIME));
  command->setLowestDownloadSpeedLimit
    (getOption()->getAsInt(PREF_LOWEST_SPEED_LIMIT));
  if(getRequestGroup()->isFileAllocationEnabled() &&
     !decideFileAllocation(filter.get())) {
    getRequestGroup()->setFileAllocationEnabled(false);
  }
  command->installStreamFilter(std::move(filter));
  getRequestGroup()->getURISelector()->tuneDownloadCommand
    (getFileEntry()->getRemainingUris(), command.get());

  return std::move(command);
}

void HttpResponseCommand::poolConnection()
{
  if(getRequest()->supportsPersistentConnection()) {
    getDownloadEngine()->poolSocket(getRequest(), createProxyRequest(),
                                    getSocket());
  }
}

void HttpResponseCommand::onDryRunFileFound()
{
  getPieceStorage()->markAllPiecesDone();
  getDownloadContext()->setChecksumVerified(true);
  poolConnection();
}

#ifdef ENABLE_MESSAGE_DIGEST
bool HttpResponseCommand::checkChecksum
(const std::shared_ptr<DownloadContext>& dctx,
 const Checksum& checksum)
{
  if(dctx->getHashType() == checksum.getHashType()) {
    if(dctx->getDigest() == checksum.getDigest()) {
      A2_LOG_INFO("Valid hash found in Digest header field.");
      return true;
    } else {
      throw DL_ABORT_EX("Invalid hash found in Digest header field.");
    }
  }
  return false;
}
#endif // ENABLE_MESSAGE_DIGEST

} // namespace aria2