/* */ #include "HttpSkipResponseCommand.h" #include "HttpConnection.h" #include "HttpResponse.h" #include "message.h" #include "SocketCore.h" #include "DlRetryEx.h" #include "Request.h" #include "DownloadEngine.h" #include "Logger.h" #include "LogFactory.h" #include "HttpRequest.h" #include "Segment.h" #include "util.h" #include "fmt.h" #include "DlAbortEx.h" #include "HttpHeader.h" #include "prefs.h" #include "Option.h" #include "CookieStorage.h" #include "AuthConfigFactory.h" #include "AuthConfig.h" #include "DownloadContext.h" #include "StreamFilter.h" #include "BinaryStream.h" #include "NullSinkStreamFilter.h" #include "SinkStreamFilter.h" #include "error_code.h" #include "SocketRecvBuffer.h" namespace aria2 { HttpSkipResponseCommand::HttpSkipResponseCommand( cuid_t cuid, const std::shared_ptr& req, const std::shared_ptr& fileEntry, RequestGroup* requestGroup, const std::shared_ptr& httpConnection, std::unique_ptr httpResponse, DownloadEngine* e, const std::shared_ptr& s) : AbstractCommand(cuid, req, fileEntry, requestGroup, e, s, httpConnection->getSocketRecvBuffer()), sinkFilterOnly_(true), totalLength_(httpResponse->getEntityLength()), receivedBytes_(0), httpConnection_(httpConnection), httpResponse_(std::move(httpResponse)), streamFilter_(make_unique()) { checkSocketRecvBuffer(); } HttpSkipResponseCommand::~HttpSkipResponseCommand() {} void HttpSkipResponseCommand::installStreamFilter( std::unique_ptr streamFilter) { if (!streamFilter) { return; } streamFilter->installDelegate(std::move(streamFilter_)); streamFilter_ = std::move(streamFilter); const std::string& name = streamFilter_->getName(); sinkFilterOnly_ = util::endsWith(name, SinkStreamFilter::NAME); } bool HttpSkipResponseCommand::executeInternal() { if (getRequest()->getMethod() == Request::METHOD_HEAD || (totalLength_ == 0 && sinkFilterOnly_)) { // If request method is HEAD or content-length header is present and // it's value is 0, then pool socket for reuse. // If content-length header is not present, then EOF is expected in the end. // In this case, the content is thrown away and socket cannot be pooled. if (getRequest()->getMethod() == Request::METHOD_HEAD || httpResponse_->getHttpHeader()->defined(HttpHeader::CONTENT_LENGTH)) { poolConnection(); } return processResponse(); } bool eof = false; try { size_t bufSize; if (getSocketRecvBuffer()->bufferEmpty()) { eof = getSocketRecvBuffer()->recv() == 0 && !getSocket()->wantRead() && !getSocket()->wantWrite(); } if (!eof) { if (sinkFilterOnly_) { if (totalLength_ > 0) { bufSize = std::min( totalLength_ - receivedBytes_, static_cast(getSocketRecvBuffer()->getBufferLength())); } else { bufSize = getSocketRecvBuffer()->getBufferLength(); } receivedBytes_ += bufSize; } else { // receivedBytes_ is not updated if transferEncoding is set. // The return value is safely ignored here. streamFilter_->transform(std::shared_ptr(), std::shared_ptr(), getSocketRecvBuffer()->getBuffer(), getSocketRecvBuffer()->getBufferLength()); bufSize = streamFilter_->getBytesProcessed(); } getSocketRecvBuffer()->drain(bufSize); } if (totalLength_ != 0 && eof) { throw DL_RETRY_EX(EX_GOT_EOF); } } catch (RecoverableException& e) { A2_LOG_DEBUG_EX(EX_EXCEPTION_CAUGHT, e) return processResponse(); } if (eof) { // we may get EOF before non-sink streamFilter reports its // completion. There are some broken servers to prevent // streamFilter from completion. Since we just discard the // response body anyway, so we assume that the response is // completed. return processResponse(); } bool finished = false; if (sinkFilterOnly_) { finished = (totalLength_ == receivedBytes_); } else { finished = streamFilter_->finished(); } if (finished) { if (getSegments().size() <= 1) { // Don't pool connection if the command has multiple // segments. This means it did HTTP pipelined request. If this // response is for the first request, then successive response // may arrived to the socket. poolConnection(); } return processResponse(); } else { setWriteCheckSocketIf(getSocket(), getSocket()->wantWrite()); addCommandSelf(); return false; } } void HttpSkipResponseCommand::poolConnection() const { if (getRequest()->supportsPersistentConnection()) { getDownloadEngine()->poolSocket(getRequest(), createProxyRequest(), getSocket()); } } bool HttpSkipResponseCommand::processResponse() { if (httpResponse_->isRedirect()) { int rnum = httpResponse_->getHttpRequest()->getRequest()->getRedirectCount(); if (rnum >= Request::MAX_REDIRECT) { throw DL_ABORT_EX2(fmt("Too many redirects: count=%u", rnum), error_code::HTTP_TOO_MANY_REDIRECTS); } httpResponse_->processRedirect(); return prepareForRetry(0); } auto statusCode = httpResponse_->getStatusCode(); if (statusCode >= 400) { switch (statusCode) { case 401: if (getOption()->getAsBool(PREF_HTTP_AUTH_CHALLENGE) && !httpResponse_->getHttpRequest()->authenticationUsed() && getDownloadEngine()->getAuthConfigFactory()->activateBasicCred( getRequest()->getHost(), getRequest()->getPort(), getRequest()->getDir(), getOption().get())) { return prepareForRetry(0); } throw DL_ABORT_EX2(EX_AUTH_FAILED, error_code::HTTP_AUTH_FAILED); case 404: if (getOption()->getAsInt(PREF_MAX_FILE_NOT_FOUND) == 0) { throw DL_ABORT_EX2(MSG_RESOURCE_NOT_FOUND, error_code::RESOURCE_NOT_FOUND); } throw DL_RETRY_EX2(MSG_RESOURCE_NOT_FOUND, error_code::RESOURCE_NOT_FOUND); case 503: // Only retry if pretry-wait > 0. Hammering 'busy' server is not // a good idea. if (getOption()->getAsInt(PREF_RETRY_WAIT) > 0) { throw DL_RETRY_EX2(fmt(EX_BAD_STATUS, statusCode), error_code::HTTP_SERVICE_UNAVAILABLE); } throw DL_ABORT_EX2(fmt(EX_BAD_STATUS, statusCode), error_code::HTTP_SERVICE_UNAVAILABLE); case 504: // This is Gateway Timeout, so try again throw DL_RETRY_EX2(fmt(EX_BAD_STATUS, statusCode), error_code::HTTP_SERVICE_UNAVAILABLE); }; throw DL_ABORT_EX2(fmt(EX_BAD_STATUS, statusCode), error_code::HTTP_PROTOCOL_ERROR); } return prepareForRetry(0); } void HttpSkipResponseCommand::disableSocketCheck() { disableReadCheckSocket(); disableWriteCheckSocket(); } } // namespace aria2