HttpRequest.cc 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461
  1. /* <!-- copyright */
  2. /*
  3. * aria2 - The high speed download utility
  4. *
  5. * Copyright (C) 2006 Tatsuhiro Tsujikawa
  6. *
  7. * This program is free software; you can redistribute it and/or modify
  8. * it under the terms of the GNU General Public License as published by
  9. * the Free Software Foundation; either version 2 of the License, or
  10. * (at your option) any later version.
  11. *
  12. * This program is distributed in the hope that it will be useful,
  13. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. * GNU General Public License for more details.
  16. *
  17. * You should have received a copy of the GNU General Public License
  18. * along with this program; if not, write to the Free Software
  19. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20. *
  21. * In addition, as a special exception, the copyright holders give
  22. * permission to link the code of portions of this program with the
  23. * OpenSSL library under certain conditions as described in each
  24. * individual source file, and distribute linked combinations
  25. * including the two.
  26. * You must obey the GNU General Public License in all respects
  27. * for all of the code used other than OpenSSL. If you modify
  28. * file(s) with this exception, you may extend this exception to your
  29. * version of the file(s), but you are not obligated to do so. If you
  30. * do not wish to do so, delete this exception statement from your
  31. * version. If you delete this exception statement from all source
  32. * files in the program, then also delete it here.
  33. */
  34. /* copyright --> */
  35. #include "HttpRequest.h"
  36. #include <cassert>
  37. #include <numeric>
  38. #include <vector>
  39. #include "Segment.h"
  40. #include "Range.h"
  41. #include "CookieStorage.h"
  42. #include "Option.h"
  43. #include "util.h"
  44. #include "base64.h"
  45. #include "prefs.h"
  46. #include "AuthConfigFactory.h"
  47. #include "AuthConfig.h"
  48. #include "a2functional.h"
  49. #include "TimeA2.h"
  50. #include "array_fun.h"
  51. #include "Request.h"
  52. namespace aria2 {
  53. const std::string HttpRequest::USER_AGENT("aria2");
  54. HttpRequest::HttpRequest():contentEncodingEnabled_(true),
  55. userAgent_(USER_AGENT),
  56. noCache_(true),
  57. acceptGzip_(false),
  58. endOffsetOverride_(0)
  59. {}
  60. HttpRequest::~HttpRequest() {}
  61. void HttpRequest::setSegment(const SharedHandle<Segment>& segment)
  62. {
  63. segment_ = segment;
  64. }
  65. void HttpRequest::setRequest(const SharedHandle<Request>& request)
  66. {
  67. request_ = request;
  68. }
  69. off_t HttpRequest::getStartByte() const
  70. {
  71. if(!segment_) {
  72. return 0;
  73. } else {
  74. return fileEntry_->gtoloff(segment_->getPositionToWrite());
  75. }
  76. }
  77. off_t HttpRequest::getEndByte() const
  78. {
  79. if(!segment_ || !request_) {
  80. return 0;
  81. } else {
  82. if(request_->isPipeliningEnabled()) {
  83. off_t endByte =
  84. fileEntry_->gtoloff(segment_->getPosition()+segment_->getLength()-1);
  85. return std::min(endByte, static_cast<off_t>(fileEntry_->getLength()-1));
  86. } else {
  87. return 0;
  88. }
  89. }
  90. }
  91. RangeHandle HttpRequest::getRange() const
  92. {
  93. // content-length is always 0
  94. if(!segment_) {
  95. return SharedHandle<Range>(new Range());
  96. } else {
  97. return SharedHandle<Range>(new Range(getStartByte(), getEndByte(),
  98. fileEntry_->getLength()));
  99. }
  100. }
  101. bool HttpRequest::isRangeSatisfied(const RangeHandle& range) const
  102. {
  103. if(!segment_) {
  104. return true;
  105. }
  106. if((getStartByte() == range->getStartByte()) &&
  107. ((getEndByte() == 0) ||
  108. (getEndByte() == range->getEndByte())) &&
  109. ((fileEntry_->getLength() == 0) ||
  110. (fileEntry_->getLength() == range->getEntityLength()))) {
  111. return true;
  112. } else {
  113. return false;
  114. }
  115. }
  116. namespace {
  117. std::string getHostText(const std::string& host, uint16_t port)
  118. {
  119. std::string hosttext = host;
  120. if(!(port == 80 || port == 443)) {
  121. strappend(hosttext, ":", util::uitos(port));
  122. }
  123. return hosttext;
  124. }
  125. } // namespace
  126. std::string HttpRequest::createRequest()
  127. {
  128. authConfig_ = authConfigFactory_->createAuthConfig(request_, option_);
  129. std::string requestLine = request_->getMethod();
  130. requestLine += " ";
  131. if(proxyRequest_) {
  132. if(getProtocol() == Request::PROTO_FTP &&
  133. request_->getUsername().empty() && authConfig_) {
  134. // Insert user into URI, like ftp://USER@host/
  135. std::string uri = getCurrentURI();
  136. assert(uri.size() >= 6);
  137. uri.insert(6, util::percentEncode(authConfig_->getUser())+"@");
  138. requestLine += uri;
  139. } else {
  140. requestLine += getCurrentURI();
  141. }
  142. } else {
  143. if(getDir() == A2STR::SLASH_C) {
  144. requestLine += getDir();
  145. } else {
  146. requestLine += getDir();
  147. requestLine += A2STR::SLASH_C;
  148. }
  149. requestLine += getFile();
  150. requestLine += getQuery();
  151. }
  152. requestLine += " HTTP/1.1\r\n";
  153. std::vector<std::pair<std::string, std::string> > builtinHds;
  154. builtinHds.reserve(20);
  155. builtinHds.push_back(std::make_pair("User-Agent:", userAgent_));
  156. std::string acceptTypes = "*/*";
  157. for(std::vector<std::string>::const_iterator i = acceptTypes_.begin(),
  158. eoi = acceptTypes_.end(); i != eoi; ++i) {
  159. strappend(acceptTypes, ",", (*i));
  160. }
  161. builtinHds.push_back(std::make_pair("Accept:", acceptTypes));
  162. if(contentEncodingEnabled_) {
  163. std::string acceptableEncodings;
  164. #ifdef HAVE_ZLIB
  165. if(acceptGzip_) {
  166. acceptableEncodings += "deflate, gzip";
  167. }
  168. #endif // HAVE_ZLIB
  169. if(!acceptableEncodings.empty()) {
  170. builtinHds.push_back
  171. (std::make_pair("Accept-Encoding:", acceptableEncodings));
  172. }
  173. }
  174. builtinHds.push_back
  175. (std::make_pair("Host:", getHostText(getURIHost(), getPort())));
  176. if(noCache_) {
  177. builtinHds.push_back(std::make_pair("Pragma:", "no-cache"));
  178. builtinHds.push_back(std::make_pair("Cache-Control:", "no-cache"));
  179. }
  180. if(!request_->isKeepAliveEnabled() && !request_->isPipeliningEnabled()) {
  181. builtinHds.push_back(std::make_pair("Connection:", "close"));
  182. }
  183. if(segment_ && segment_->getLength() > 0 &&
  184. (request_->isPipeliningEnabled() || getStartByte() > 0)) {
  185. std::string rangeHeader = "bytes=";
  186. rangeHeader += util::itos(getStartByte());
  187. rangeHeader += "-";
  188. if(request_->isPipeliningEnabled()) {
  189. rangeHeader += util::itos(getEndByte());
  190. } else if(getProtocol() != Request::PROTO_FTP && endOffsetOverride_ > 0) {
  191. // FTP via http proxy does not support endbytes
  192. rangeHeader += util::itos(endOffsetOverride_-1);
  193. }
  194. builtinHds.push_back(std::make_pair("Range:", rangeHeader));
  195. }
  196. if(proxyRequest_) {
  197. if(request_->isKeepAliveEnabled() || request_->isPipeliningEnabled()) {
  198. builtinHds.push_back(std::make_pair("Proxy-Connection:", "Keep-Alive"));
  199. } else {
  200. builtinHds.push_back(std::make_pair("Proxy-Connection:", "close"));
  201. }
  202. }
  203. if(proxyRequest_ && !proxyRequest_->getUsername().empty()) {
  204. builtinHds.push_back(getProxyAuthString());
  205. }
  206. if(authConfig_) {
  207. std::string authText = authConfig_->getAuthText();
  208. std::string val = "Basic ";
  209. val += base64::encode(authText.begin(), authText.end());
  210. builtinHds.push_back(std::make_pair("Authorization:", val));
  211. }
  212. if(getPreviousURI().size()) {
  213. builtinHds.push_back(std::make_pair("Referer:", getPreviousURI()));
  214. }
  215. if(cookieStorage_) {
  216. std::string cookiesValue;
  217. std::string path = getDir();
  218. if(getDir() == "/") {
  219. path += getFile();
  220. } else {
  221. path += "/";
  222. path += getFile();
  223. }
  224. std::vector<Cookie> cookies =
  225. cookieStorage_->criteriaFind(getHost(), path,
  226. Time().getTime(),
  227. getProtocol() == Request::PROTO_HTTPS);
  228. for(std::vector<Cookie>::const_iterator itr = cookies.begin(),
  229. eoi = cookies.end(); itr != eoi; ++itr) {
  230. strappend(cookiesValue, (*itr).toString(), ";");
  231. }
  232. if(!cookiesValue.empty()) {
  233. builtinHds.push_back(std::make_pair("Cookie:", cookiesValue));
  234. }
  235. }
  236. if(!ifModSinceHeader_.empty()) {
  237. builtinHds.push_back
  238. (std::make_pair("If-Modified-Since:", ifModSinceHeader_));
  239. }
  240. for(std::vector<std::pair<std::string, std::string> >::const_iterator i =
  241. builtinHds.begin(), eoi = builtinHds.end(); i != eoi; ++i) {
  242. std::vector<std::string>::const_iterator j = headers_.begin();
  243. std::vector<std::string>::const_iterator jend = headers_.end();
  244. for(; j != jend; ++j) {
  245. if(util::startsWith((*j).begin(), (*j).end(),
  246. (*i).first.begin(), (*i).first.end())) {
  247. break;
  248. }
  249. }
  250. if(j == jend) {
  251. strappend(requestLine, (*i).first, " ", (*i).second, A2STR::CRLF);
  252. }
  253. }
  254. // append additional headers given by user.
  255. for(std::vector<std::string>::const_iterator i = headers_.begin(),
  256. eoi = headers_.end(); i != eoi; ++i) {
  257. strappend(requestLine, (*i), A2STR::CRLF);
  258. }
  259. requestLine += A2STR::CRLF;
  260. return requestLine;
  261. }
  262. std::string HttpRequest::createProxyRequest() const
  263. {
  264. assert(proxyRequest_);
  265. std::string hostport = getURIHost();
  266. strappend(hostport, ":", util::uitos(getPort()));
  267. std::string requestLine = "CONNECT ";
  268. strappend(requestLine, hostport, " HTTP/1.1\r\n");
  269. strappend(requestLine, "User-Agent: ", userAgent_, "\r\n");
  270. strappend(requestLine, "Host: ", hostport, "\r\n");
  271. // TODO Is "Proxy-Connection" needed here?
  272. // if(request->isKeepAliveEnabled() || request->isPipeliningEnabled()) {
  273. // requestLine += "Proxy-Connection: Keep-Alive\r\n";
  274. // }else {
  275. // requestLine += "Proxy-Connection: close\r\n";
  276. // }
  277. if(!proxyRequest_->getUsername().empty()) {
  278. std::pair<std::string, std::string> auth = getProxyAuthString();
  279. strappend(requestLine, auth.first, " ", auth.second, A2STR::CRLF);
  280. }
  281. requestLine += A2STR::CRLF;
  282. return requestLine;
  283. }
  284. std::pair<std::string, std::string> HttpRequest::getProxyAuthString() const
  285. {
  286. std::string authText = proxyRequest_->getUsername();
  287. authText += ':';
  288. authText += proxyRequest_->getPassword();
  289. std::string val = "Basic ";
  290. val += base64::encode(authText.begin(), authText.end());
  291. return std::make_pair("Proxy-Authorization:", val);
  292. }
  293. void HttpRequest::enableContentEncoding()
  294. {
  295. contentEncodingEnabled_ = true;
  296. }
  297. void HttpRequest::disableContentEncoding()
  298. {
  299. contentEncodingEnabled_ = false;
  300. }
  301. void HttpRequest::addHeader(const std::string& headersString)
  302. {
  303. util::split(headersString.begin(), headersString.end(),
  304. std::back_inserter(headers_), '\n', true);
  305. }
  306. void HttpRequest::clearHeader()
  307. {
  308. headers_.clear();
  309. }
  310. void HttpRequest::addAcceptType(const std::string& type)
  311. {
  312. acceptTypes_.push_back(type);
  313. }
  314. void HttpRequest::setCookieStorage
  315. (const SharedHandle<CookieStorage>& cookieStorage)
  316. {
  317. cookieStorage_ = cookieStorage;
  318. }
  319. void HttpRequest::setAuthConfigFactory
  320. (const SharedHandle<AuthConfigFactory>& factory, const Option* option)
  321. {
  322. authConfigFactory_ = factory;
  323. option_ = option;
  324. }
  325. void HttpRequest::setProxyRequest(const SharedHandle<Request>& proxyRequest)
  326. {
  327. proxyRequest_ = proxyRequest;
  328. }
  329. bool HttpRequest::isProxyRequestSet() const
  330. {
  331. return proxyRequest_;
  332. }
  333. bool HttpRequest::authenticationUsed() const
  334. {
  335. return authConfig_;
  336. }
  337. const SharedHandle<AuthConfig>& HttpRequest::getAuthConfig() const
  338. {
  339. return authConfig_;
  340. }
  341. uint64_t HttpRequest::getEntityLength() const
  342. {
  343. assert(fileEntry_);
  344. return fileEntry_->getLength();
  345. }
  346. const std::string& HttpRequest::getHost() const
  347. {
  348. return request_->getHost();
  349. }
  350. uint16_t HttpRequest::getPort() const
  351. {
  352. return request_->getPort();
  353. }
  354. const std::string& HttpRequest::getMethod() const
  355. {
  356. return request_->getMethod();
  357. }
  358. const std::string& HttpRequest::getProtocol() const
  359. {
  360. return request_->getProtocol();
  361. }
  362. const std::string& HttpRequest::getCurrentURI() const
  363. {
  364. return request_->getCurrentUri();
  365. }
  366. const std::string& HttpRequest::getDir() const
  367. {
  368. return request_->getDir();
  369. }
  370. const std::string& HttpRequest::getFile() const
  371. {
  372. return request_->getFile();
  373. }
  374. const std::string& HttpRequest::getQuery() const
  375. {
  376. return request_->getQuery();
  377. }
  378. const std::string& HttpRequest::getPreviousURI() const
  379. {
  380. return request_->getPreviousUri();
  381. }
  382. std::string HttpRequest::getURIHost() const
  383. {
  384. return request_->getURIHost();
  385. }
  386. void HttpRequest::setUserAgent(const std::string& userAgent)
  387. {
  388. userAgent_ = userAgent;
  389. }
  390. void HttpRequest::setFileEntry(const SharedHandle<FileEntry>& fileEntry)
  391. {
  392. fileEntry_ = fileEntry;
  393. }
  394. void HttpRequest::setIfModifiedSinceHeader(const std::string& hd)
  395. {
  396. ifModSinceHeader_ = hd;
  397. }
  398. bool HttpRequest::conditionalRequest() const
  399. {
  400. if(!ifModSinceHeader_.empty()) {
  401. return true;
  402. }
  403. static const char A2_IF_MOD_SINCE[] = "if-modified-since";
  404. static const char A2_IF_NONE_MATCH[] = "if-none-match";
  405. for(std::vector<std::string>::const_iterator i = headers_.begin(),
  406. eoi = headers_.end(); i != eoi; ++i) {
  407. if(util::istartsWith((*i).begin(), (*i).end(),
  408. A2_IF_MOD_SINCE, vend(A2_IF_MOD_SINCE)-1) ||
  409. util::istartsWith((*i).begin(), (*i).end(),
  410. A2_IF_NONE_MATCH, vend(A2_IF_NONE_MATCH)-1)) {
  411. return true;
  412. }
  413. }
  414. return false;
  415. }
  416. } // namespace aria2