download_helper.cc 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553
  1. /* <!-- copyright */
  2. /*
  3. * aria2 - The high speed download utility
  4. *
  5. * Copyright (C) 2006 Tatsuhiro Tsujikawa
  6. *
  7. * This program is free software; you can redistribute it and/or modify
  8. * it under the terms of the GNU General Public License as published by
  9. * the Free Software Foundation; either version 2 of the License, or
  10. * (at your option) any later version.
  11. *
  12. * This program is distributed in the hope that it will be useful,
  13. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. * GNU General Public License for more details.
  16. *
  17. * You should have received a copy of the GNU General Public License
  18. * along with this program; if not, write to the Free Software
  19. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20. *
  21. * In addition, as a special exception, the copyright holders give
  22. * permission to link the code of portions of this program with the
  23. * OpenSSL library under certain conditions as described in each
  24. * individual source file, and distribute linked combinations
  25. * including the two.
  26. * You must obey the GNU General Public License in all respects
  27. * for all of the code used other than OpenSSL. If you modify
  28. * file(s) with this exception, you may extend this exception to your
  29. * version of the file(s), but you are not obligated to do so. If you
  30. * do not wish to do so, delete this exception statement from your
  31. * version. If you delete this exception statement from all source
  32. * files in the program, then also delete it here.
  33. */
  34. /* copyright --> */
  35. #include "download_helper.h"
  36. #include <algorithm>
  37. #include <sstream>
  38. #include "RequestGroup.h"
  39. #include "Option.h"
  40. #include "prefs.h"
  41. #include "Metalink2RequestGroup.h"
  42. #include "ProtocolDetector.h"
  43. #include "paramed_string.h"
  44. #include "UriListParser.h"
  45. #include "DownloadContext.h"
  46. #include "RecoverableException.h"
  47. #include "DlAbortEx.h"
  48. #include "message.h"
  49. #include "fmt.h"
  50. #include "FileEntry.h"
  51. #include "LogFactory.h"
  52. #include "File.h"
  53. #include "util.h"
  54. #include "array_fun.h"
  55. #include "OptionHandler.h"
  56. #include "ByteArrayDiskWriter.h"
  57. #include "a2functional.h"
  58. #include "ByteArrayDiskWriterFactory.h"
  59. #include "MetadataInfo.h"
  60. #include "OptionParser.h"
  61. #include "SegList.h"
  62. #ifdef ENABLE_BITTORRENT
  63. # include "bittorrent_helper.h"
  64. # include "BtConstants.h"
  65. # include "UTMetadataPostDownloadHandler.h"
  66. # include "ValueBaseBencodeParser.h"
  67. #endif // ENABLE_BITTORRENT
  68. namespace aria2 {
  69. namespace {
  70. void unfoldURI
  71. (std::vector<std::string>& result, const std::vector<std::string>& args)
  72. {
  73. for(const auto& i : args) {
  74. paramed_string::expand(std::begin(i), std::end(i),
  75. std::back_inserter(result));
  76. }
  77. }
  78. } // namespace
  79. namespace {
  80. template<typename InputIterator>
  81. void splitURI(std::vector<std::string>& result,
  82. InputIterator begin,
  83. InputIterator end,
  84. size_t numSplit,
  85. size_t maxIter)
  86. {
  87. size_t numURIs = std::distance(begin, end);
  88. if(numURIs >= numSplit) {
  89. result.insert(std::end(result), begin, end);
  90. } else if(numURIs > 0) {
  91. size_t num = std::min(numSplit/numURIs, maxIter);
  92. for(size_t i = 0; i < num; ++i) {
  93. result.insert(std::end(result), begin, end);
  94. }
  95. if(num < maxIter) {
  96. result.insert(std::end(result), begin, begin+(numSplit%numURIs));
  97. }
  98. }
  99. }
  100. } // namespace
  101. namespace {
  102. std::shared_ptr<GroupId> getGID(const std::shared_ptr<Option>& option)
  103. {
  104. std::shared_ptr<GroupId> gid;
  105. if(option->defined(PREF_GID)) {
  106. a2_gid_t n;
  107. if(GroupId::toNumericId(n, option->get(PREF_GID).c_str()) != 0) {
  108. throw DL_ABORT_EX(fmt("%s is invalid for GID.",
  109. option->get(PREF_GID).c_str()));
  110. }
  111. gid = GroupId::import(n);
  112. if(!gid) {
  113. throw DL_ABORT_EX(fmt("GID %s is not unique.",
  114. option->get(PREF_GID).c_str()));
  115. }
  116. } else {
  117. gid = GroupId::create();
  118. }
  119. return gid;
  120. }
  121. } // namespace
  122. namespace {
  123. std::shared_ptr<RequestGroup> createRequestGroup
  124. (const std::shared_ptr<Option>& optionTemplate,
  125. const std::vector<std::string>& uris,
  126. bool useOutOption = false)
  127. {
  128. auto option = util::copy(optionTemplate);
  129. auto rg = std::make_shared<RequestGroup>(getGID(option), option);
  130. auto dctx = std::make_shared<DownloadContext>
  131. (option->getAsInt(PREF_PIECE_LENGTH),
  132. 0,
  133. useOutOption&&!option->blank(PREF_OUT)?
  134. util::applyDir(option->get(PREF_DIR),
  135. option->get(PREF_OUT)):A2STR::NIL);
  136. dctx->getFirstFileEntry()->setUris(uris);
  137. dctx->getFirstFileEntry()->setMaxConnectionPerServer
  138. (option->getAsInt(PREF_MAX_CONNECTION_PER_SERVER));
  139. #ifdef ENABLE_MESSAGE_DIGEST
  140. const std::string& checksum = option->get(PREF_CHECKSUM);
  141. if(!checksum.empty()) {
  142. auto p = util::divide(std::begin(checksum), std::end(checksum), '=');
  143. std::string hashType(p.first.first, p.first.second);
  144. std::string hexDigest(p.second.first, p.second.second);
  145. util::lowercase(hashType);
  146. dctx->setDigest(hashType,
  147. util::fromHex(std::begin(hexDigest), std::end(hexDigest)));
  148. }
  149. #endif // ENABLE_MESSAGE_DIGEST
  150. rg->setDownloadContext(dctx);
  151. rg->setPauseRequested(option->getAsBool(PREF_PAUSE));
  152. removeOneshotOption(option);
  153. return rg;
  154. }
  155. } // namespace
  156. #if defined ENABLE_BITTORRENT || ENABLE_METALINK
  157. namespace {
  158. std::shared_ptr<MetadataInfo> createMetadataInfo
  159. (const std::shared_ptr<GroupId>& gid, const std::string& uri)
  160. {
  161. return std::make_shared<MetadataInfo>(gid, uri);
  162. }
  163. } // namespace
  164. namespace {
  165. std::shared_ptr<MetadataInfo> createMetadataInfoDataOnly()
  166. {
  167. return std::make_shared<MetadataInfo>();
  168. }
  169. } // namespace
  170. #endif // ENABLE_BITTORRENT || ENABLE_METALINK
  171. #ifdef ENABLE_BITTORRENT
  172. namespace {
  173. std::shared_ptr<RequestGroup>
  174. createBtRequestGroup(const std::string& metaInfoUri,
  175. const std::shared_ptr<Option>& optionTemplate,
  176. const std::vector<std::string>& auxUris,
  177. const ValueBase* torrent,
  178. bool adjustAnnounceUri = true)
  179. {
  180. auto option = util::copy(optionTemplate);
  181. auto gid = getGID(option);
  182. auto rg = std::make_shared<RequestGroup>(gid, option);
  183. auto dctx = std::make_shared<DownloadContext>();
  184. // may throw exception
  185. bittorrent::loadFromMemory(torrent, dctx, option, auxUris,
  186. metaInfoUri.empty() ? "default" : metaInfoUri);
  187. if(metaInfoUri.empty()) {
  188. rg->setMetadataInfo(createMetadataInfoDataOnly());
  189. } else {
  190. rg->setMetadataInfo(createMetadataInfo(gid, metaInfoUri));
  191. }
  192. if(adjustAnnounceUri) {
  193. bittorrent::adjustAnnounceUri(bittorrent::getTorrentAttrs(dctx), option);
  194. }
  195. SegList<int> sgl;
  196. util::parseIntSegments(sgl, option->get(PREF_SELECT_FILE));
  197. sgl.normalize();
  198. dctx->setFileFilter(sgl);
  199. std::istringstream indexOutIn(option->get(PREF_INDEX_OUT));
  200. auto indexPaths = util::createIndexPaths(indexOutIn);
  201. for(const auto& i : indexPaths) {
  202. dctx->setFilePathWithIndex
  203. (i.first, util::applyDir(option->get(PREF_DIR), i.second));
  204. }
  205. rg->setDownloadContext(dctx);
  206. rg->setPauseRequested(option->getAsBool(PREF_PAUSE));
  207. // Remove "metalink" from Accept Type list to avoid server from
  208. // responding Metalink file for web-seeding URIs.
  209. dctx->setAcceptMetalink(false);
  210. removeOneshotOption(option);
  211. return rg;
  212. }
  213. } // namespace
  214. namespace {
  215. std::shared_ptr<RequestGroup>
  216. createBtMagnetRequestGroup
  217. (const std::string& magnetLink,
  218. const std::shared_ptr<Option>& optionTemplate)
  219. {
  220. auto option = util::copy(optionTemplate);
  221. auto gid = getGID(option);
  222. auto rg = std::make_shared<RequestGroup>(gid, option);
  223. auto dctx = std::make_shared<DownloadContext>(METADATA_PIECE_SIZE, 0,
  224. A2STR::NIL);
  225. // We only know info hash. Total Length is unknown at this moment.
  226. dctx->markTotalLengthIsUnknown();
  227. rg->setFileAllocationEnabled(false);
  228. rg->setPreLocalFileCheckEnabled(false);
  229. bittorrent::loadMagnet(magnetLink, dctx);
  230. auto torrentAttrs = bittorrent::getTorrentAttrs(dctx);
  231. bittorrent::adjustAnnounceUri(torrentAttrs, option);
  232. dctx->getFirstFileEntry()->setPath(torrentAttrs->name);
  233. rg->setDownloadContext(dctx);
  234. rg->clearPostDownloadHandler();
  235. auto utMetadataPostHandler =
  236. std::make_shared<UTMetadataPostDownloadHandler>();
  237. rg->addPostDownloadHandler(utMetadataPostHandler);
  238. rg->setDiskWriterFactory(std::make_shared<ByteArrayDiskWriterFactory>());
  239. rg->setMetadataInfo(createMetadataInfo(gid, magnetLink));
  240. rg->markInMemoryDownload();
  241. rg->setPauseRequested(option->getAsBool(PREF_PAUSE));
  242. removeOneshotOption(option);
  243. return rg;
  244. }
  245. } // namespace
  246. void createRequestGroupForBitTorrent
  247. (std::vector<std::shared_ptr<RequestGroup>>& result,
  248. const std::shared_ptr<Option>& option,
  249. const std::vector<std::string>& uris,
  250. const std::string& metaInfoUri,
  251. const std::string& torrentData,
  252. bool adjustAnnounceUri)
  253. {
  254. std::unique_ptr<ValueBase> torrent;
  255. bittorrent::ValueBaseBencodeParser parser;
  256. if(torrentData.empty()) {
  257. torrent = parseFile(parser, metaInfoUri);
  258. } else {
  259. ssize_t error;
  260. torrent = parser.parseFinal(torrentData.c_str(), torrentData.size(),
  261. error);
  262. }
  263. if(!torrent) {
  264. throw DL_ABORT_EX2("Bencode decoding failed",
  265. error_code::BENCODE_PARSE_ERROR);
  266. }
  267. createRequestGroupForBitTorrent(result, option, uris, metaInfoUri,
  268. torrent.get());
  269. }
  270. void createRequestGroupForBitTorrent
  271. (std::vector<std::shared_ptr<RequestGroup>>& result,
  272. const std::shared_ptr<Option>& option,
  273. const std::vector<std::string>& uris,
  274. const std::string& metaInfoUri,
  275. const ValueBase* torrent,
  276. bool adjustAnnounceUri)
  277. {
  278. std::vector<std::string> nargs;
  279. if(option->get(PREF_PARAMETERIZED_URI) == A2_V_TRUE) {
  280. unfoldURI(nargs, uris);
  281. } else {
  282. nargs = uris;
  283. }
  284. // we ignore -Z option here
  285. size_t numSplit = option->getAsInt(PREF_SPLIT);
  286. auto rg = createBtRequestGroup(metaInfoUri, option, nargs,
  287. torrent, adjustAnnounceUri);
  288. rg->setNumConcurrentCommand(numSplit);
  289. result.push_back(rg);
  290. }
  291. #endif // ENABLE_BITTORRENT
  292. #ifdef ENABLE_METALINK
  293. void createRequestGroupForMetalink
  294. (std::vector<std::shared_ptr<RequestGroup>>& result,
  295. const std::shared_ptr<Option>& option,
  296. const std::string& metalinkData)
  297. {
  298. if(metalinkData.empty()) {
  299. Metalink2RequestGroup().generate(result,
  300. option->get(PREF_METALINK_FILE),
  301. option,
  302. option->get(PREF_METALINK_BASE_URI));
  303. } else {
  304. auto dw = std::make_shared<ByteArrayDiskWriter>();
  305. dw->setString(metalinkData);
  306. Metalink2RequestGroup().generate(result, dw, option,
  307. option->get(PREF_METALINK_BASE_URI));
  308. }
  309. }
  310. #endif // ENABLE_METALINK
  311. namespace {
  312. class AccRequestGroup {
  313. private:
  314. std::vector<std::shared_ptr<RequestGroup>>& requestGroups_;
  315. ProtocolDetector detector_;
  316. std::shared_ptr<Option> option_;
  317. bool ignoreLocalPath_;
  318. bool throwOnError_;
  319. public:
  320. AccRequestGroup(std::vector<std::shared_ptr<RequestGroup>>& requestGroups,
  321. const std::shared_ptr<Option>& option,
  322. bool ignoreLocalPath = false,
  323. bool throwOnError = false):
  324. requestGroups_(requestGroups), option_(option),
  325. ignoreLocalPath_(ignoreLocalPath),
  326. throwOnError_(throwOnError)
  327. {}
  328. void
  329. operator()(const std::string& uri)
  330. {
  331. if(detector_.isStreamProtocol(uri)) {
  332. std::vector<std::string> streamURIs;
  333. size_t numIter = option_->getAsInt(PREF_MAX_CONNECTION_PER_SERVER);
  334. size_t numSplit = option_->getAsInt(PREF_SPLIT);
  335. size_t num = std::min(numIter, numSplit);
  336. for(size_t i = 0; i < num; ++i) {
  337. streamURIs.push_back(uri);
  338. }
  339. auto rg = createRequestGroup(option_, streamURIs);
  340. rg->setNumConcurrentCommand(numSplit);
  341. requestGroups_.push_back(rg);
  342. }
  343. #ifdef ENABLE_BITTORRENT
  344. else if(detector_.guessTorrentMagnet(uri)) {
  345. requestGroups_.push_back(createBtMagnetRequestGroup(uri, option_));
  346. } else if(!ignoreLocalPath_ && detector_.guessTorrentFile(uri)) {
  347. try {
  348. bittorrent::ValueBaseBencodeParser parser;
  349. auto torrent = parseFile(parser, uri);
  350. if(!torrent) {
  351. throw DL_ABORT_EX2("Bencode decoding failed",
  352. error_code::BENCODE_PARSE_ERROR);
  353. }
  354. requestGroups_.push_back
  355. (createBtRequestGroup(uri, option_, {}, torrent.get()));
  356. } catch(RecoverableException& e) {
  357. if(throwOnError_) {
  358. throw;
  359. } else {
  360. // error occurred while parsing torrent file.
  361. // We simply ignore it.
  362. A2_LOG_ERROR_EX(EX_EXCEPTION_CAUGHT, e);
  363. }
  364. }
  365. }
  366. #endif // ENABLE_BITTORRENT
  367. #ifdef ENABLE_METALINK
  368. else if(!ignoreLocalPath_ && detector_.guessMetalinkFile(uri)) {
  369. try {
  370. Metalink2RequestGroup().generate(requestGroups_, uri, option_,
  371. option_->get(PREF_METALINK_BASE_URI));
  372. } catch(RecoverableException& e) {
  373. if(throwOnError_) {
  374. throw;
  375. } else {
  376. // error occurred while parsing metalink file.
  377. // We simply ignore it.
  378. A2_LOG_ERROR_EX(EX_EXCEPTION_CAUGHT, e);
  379. }
  380. }
  381. }
  382. #endif // ENABLE_METALINK
  383. else {
  384. if(throwOnError_) {
  385. throw DL_ABORT_EX(fmt(MSG_UNRECOGNIZED_URI, uri.c_str()));
  386. } else {
  387. A2_LOG_ERROR(fmt(MSG_UNRECOGNIZED_URI, uri.c_str()));
  388. }
  389. }
  390. }
  391. };
  392. } // namespace
  393. namespace {
  394. class StreamProtocolFilter {
  395. private:
  396. ProtocolDetector detector_;
  397. public:
  398. bool operator()(const std::string& uri) {
  399. return detector_.isStreamProtocol(uri);
  400. }
  401. };
  402. } // namespace
  403. void createRequestGroupForUri
  404. (std::vector<std::shared_ptr<RequestGroup>>& result,
  405. const std::shared_ptr<Option>& option,
  406. const std::vector<std::string>& uris,
  407. bool ignoreForceSequential,
  408. bool ignoreLocalPath,
  409. bool throwOnError)
  410. {
  411. std::vector<std::string> nargs;
  412. if(option->get(PREF_PARAMETERIZED_URI) == A2_V_TRUE) {
  413. unfoldURI(nargs, uris);
  414. } else {
  415. nargs = uris;
  416. }
  417. if(!ignoreForceSequential &&
  418. option->get(PREF_FORCE_SEQUENTIAL) == A2_V_TRUE) {
  419. std::for_each(std::begin(nargs), std::end(nargs),
  420. AccRequestGroup(result, option, ignoreLocalPath,
  421. throwOnError));
  422. } else {
  423. auto strmProtoEnd =
  424. std::stable_partition(std::begin(nargs), std::end(nargs),
  425. StreamProtocolFilter());
  426. // let's process http/ftp protocols first.
  427. if(std::begin(nargs) != strmProtoEnd) {
  428. size_t numIter = option->getAsInt(PREF_MAX_CONNECTION_PER_SERVER);
  429. size_t numSplit = option->getAsInt(PREF_SPLIT);
  430. std::vector<std::string> streamURIs;
  431. splitURI(streamURIs, std::begin(nargs), strmProtoEnd, numSplit, numIter);
  432. try {
  433. auto rg = createRequestGroup(option, streamURIs, true);
  434. rg->setNumConcurrentCommand(numSplit);
  435. result.push_back(rg);
  436. } catch(RecoverableException& e) {
  437. if(throwOnError) {
  438. throw;
  439. } else {
  440. A2_LOG_ERROR_EX(EX_EXCEPTION_CAUGHT, e);
  441. }
  442. }
  443. }
  444. // process remaining URIs(local metalink, BitTorrent files)
  445. std::for_each(strmProtoEnd, std::end(nargs),
  446. AccRequestGroup(result, option, ignoreLocalPath,
  447. throwOnError));
  448. }
  449. }
  450. bool createRequestGroupFromUriListParser
  451. (std::vector<std::shared_ptr<RequestGroup>>& result,
  452. const Option* option,
  453. UriListParser* uriListParser)
  454. {
  455. // Since result already contains some entries, we cache the size of
  456. // it. Later, we use this value to determine RequestGroup is
  457. // actually created.
  458. size_t num = result.size();
  459. while(uriListParser->hasNext()) {
  460. std::vector<std::string> uris;
  461. Option tempOption;
  462. uriListParser->parseNext(uris, tempOption);
  463. if(uris.empty()) {
  464. continue;
  465. }
  466. auto requestOption = std::make_shared<Option>(*option);
  467. requestOption->remove(PREF_OUT);
  468. const auto& oparser = OptionParser::getInstance();
  469. for(size_t i = 1, len = option::countOption(); i < len; ++i) {
  470. auto pref = option::i2p(i);
  471. auto h = oparser->find(pref);
  472. if(h && h->getInitialOption() && tempOption.defined(pref)) {
  473. requestOption->put(pref, tempOption.get(pref));
  474. }
  475. }
  476. // This does not throw exception because throwOnError = false.
  477. createRequestGroupForUri(result, requestOption, uris);
  478. if(num < result.size()) {
  479. return true;
  480. }
  481. }
  482. return false;
  483. }
  484. std::shared_ptr<UriListParser> openUriListParser(const std::string& filename)
  485. {
  486. std::string listPath;
  487. if(filename == "-") {
  488. listPath = DEV_STDIN;
  489. } else {
  490. if(!File(filename).isFile()) {
  491. throw DL_ABORT_EX
  492. (fmt(EX_FILE_OPEN, filename.c_str(), "No such file"));
  493. }
  494. listPath = filename;
  495. }
  496. return std::make_shared<UriListParser>(listPath);
  497. }
  498. void createRequestGroupForUriList
  499. (std::vector<std::shared_ptr<RequestGroup>>& result,
  500. const std::shared_ptr<Option>& option)
  501. {
  502. auto uriListParser = openUriListParser(option->get(PREF_INPUT_FILE));
  503. while(createRequestGroupFromUriListParser(result, option.get(),
  504. uriListParser.get()));
  505. }
  506. std::shared_ptr<MetadataInfo>
  507. createMetadataInfoFromFirstFileEntry
  508. (const std::shared_ptr<GroupId>& gid,
  509. const std::shared_ptr<DownloadContext>& dctx)
  510. {
  511. if(dctx->getFileEntries().empty()) {
  512. return nullptr;
  513. } else {
  514. std::vector<std::string> uris;
  515. dctx->getFileEntries()[0]->getUris(uris);
  516. if(uris.empty()) {
  517. return nullptr;
  518. }
  519. return std::make_shared<MetadataInfo>(gid, uris[0]);
  520. }
  521. }
  522. void removeOneshotOption(const std::shared_ptr<Option>& option)
  523. {
  524. option->remove(PREF_PAUSE);
  525. option->remove(PREF_GID);
  526. }
  527. } // namespace aria2