Added Referer support

TODO

@@ -1,6 +1,5 @@
 * Add HTTP POST support
 * Add expires handling for Cookie
-* Add Referer support
 * Fix no wait retry in HttpInitiateConnectionCommand, HttpRequestCommand, HttpResponseCommand, except for redirection.
 * Add FTP support
 * Add SSL server cert verification

src/HttpConnection.cc

@@ -56,7 +56,7 @@ string HttpConnection::createRequest(const Request* req, const Segment& segment)
     req->getCurrentUrl()+
     //(req->getDir() == "/" ? "/" : req->getDir()+"/")+req->getFile()+
     string(" HTTP/1.1\r\n")+
-    "Referer: \r\n"+
+    "Referer: "+req->getPreviousUrl()+"\r\n"+
     "User-Agent: aria2\r\n"+
     "Connection: close\r\n"+
     "Accept: */*\r\n"+

src/Request.cc

@@ -45,10 +45,12 @@ bool Request::setUrl(string url) {
 }
 
 bool Request::resetUrl() {
+  previousUrl = referer;
   return setUrl(url);
 }
 
 bool Request::redirectUrl(string url) {
+  previousUrl = currentUrl;
   return parseUrl(url);
 }
 

src/Request.h

@@ -41,6 +41,14 @@ class Request {
 private:
   string url;
   string currentUrl;
+  /**
+   * URL previously requested to the server. This is used as Referer
+   */
+  string previousUrl;
+  /**
+   * URL used as Referer in the initial request
+   */
+  string referer;
   string protocol;
   string host;
   int port;
@@ -70,6 +78,10 @@ public:
 
   string getUrl() const { return url; }
   string getCurrentUrl() const { return currentUrl; }
+  string getPreviousUrl() const { return previousUrl; }
+  void setPreviousUrl(string url) { previousUrl = url; }
+  string getReferer() const { return referer; }
+  void setReferer(string url) { referer = previousUrl = url; }
   string getProtocol() const { return protocol; }
   string getHost() const { return host; }
   int getPort() const { return port; }

src/main.cc

@@ -60,8 +60,9 @@ void handler(int signal) {
   exit(0);
 }
 
-void addCommand(int cuid, const char* url, vector<Request*> requests) {
+void addCommand(int cuid, const char* url, string referer, vector<Request*> requests) {
   Request* req = new Request();
+  req->setReferer(referer);
   if(req->setUrl(url)) {
     e->commands.push(InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, req, e));
     requests.push_back(req);
@@ -112,7 +113,10 @@ void showUsage() {
   cout << " --http-proxy-user=USER     Set HTTP proxy user. This affects to all URLs" << endl;
   cout << " --http-proxy-passwd=PASSWD Set HTTP proxy password. This affects to all URLs." << endl;
   cout << " --http-auth-scheme=SCHEME  Set HTTP authentication scheme. Currently, BASIC" << endl;
-  cout << "                            is the only supported scheme." << endl;
+  cout << "                            is the only supported scheme. You MUST specify" << endl;
+  cout << "                            this option in order to use HTTP authentication" << endl;
+  cout << "                            as well as --http-proxy option." << endl;
+  cout << " --referer                  Set Referer. This affects to all URLs." << endl;
   cout << " -v, --version              Print the version number and exit." << endl;
   cout << " -h, --help                 Print this message and exit." << endl;
   cout << "URL:" << endl;
@@ -135,6 +139,7 @@ int main(int argc, char* argv[]) {
   string ufilename;
   int split = 0;
   bool daemonMode = false;
+  string referer;
 
   int c;
   Option* op = new Option();
@@ -154,6 +159,7 @@ int main(int argc, char* argv[]) {
       { "http-proxy-user", required_argument, &lopt, 4 },
       { "http-proxy-passwd", required_argument, &lopt, 5 },
       { "http-auth-scheme", required_argument, &lopt, 6 },
+      { "referer", required_argument, &lopt, 7 },
       { "version", no_argument, NULL, 'v' },
       { "help", no_argument, NULL, 'h' },
       { 0, 0, 0, 0 }
@@ -200,6 +206,9 @@ int main(int argc, char* argv[]) {
 	  cerr << "Currently, supported authentication scheme is BASIC." << endl;
 	}
 	break;
+      case 7:
+	referer = optarg;
+	break;
       }
       break;
     }
@@ -274,11 +283,11 @@ int main(int argc, char* argv[]) {
   vector<Request*> requests;
   if(split > 0) {
     for(int i = 1; i <= split; i++) {
-      addCommand(i, argv[optind], requests);
+      addCommand(i, argv[optind], referer, requests);
     }
   } else {
     for(int i = 1; optind < argc; i++) {
-      addCommand(i, argv[optind++], requests); 
+      addCommand(i, argv[optind++], referer, requests); 
     }
   }
 

test/RequestTest.cc

@@ -16,6 +16,7 @@ class RequestTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testSetUrl9);
   CPPUNIT_TEST(testSetUrl10);
   CPPUNIT_TEST(testRedirectUrl);
+  CPPUNIT_TEST(testRedirectUrl2);
   CPPUNIT_TEST(testResetUrl);
   CPPUNIT_TEST(testSafeChar);
   CPPUNIT_TEST_SUITE_END();
@@ -32,6 +33,7 @@ public:
   void testSetUrl9();
   void testSetUrl10();
   void testRedirectUrl();
+  void testRedirectUrl2();
   void testResetUrl();
   void testSafeChar();
 };
@@ -46,6 +48,7 @@ void RequestTest::testSetUrl1() {
   CPPUNIT_ASSERT(v);
   CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com/"), req.getUrl());
   CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com/"), req.getCurrentUrl());
+  CPPUNIT_ASSERT_EQUAL(string(""), req.getPreviousUrl());
   CPPUNIT_ASSERT_EQUAL(string("http"), req.getProtocol());
   CPPUNIT_ASSERT_EQUAL(80, req.getPort());
   CPPUNIT_ASSERT_EQUAL(string("aria.rednoah.com"), req.getHost());
@@ -56,8 +59,14 @@ void RequestTest::testSetUrl1() {
 void RequestTest::testSetUrl2() {
   Request req;
   bool v = req.setUrl("http://aria.rednoah.com:8080/index.html");
+  req.setReferer("http://aria.rednoah.com:8080");
 
   CPPUNIT_ASSERT(v);
+
+  // referer is unchaged
+  CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com:8080"), req.getReferer());
+  // previousUrl must equal to referer;
+  CPPUNIT_ASSERT_EQUAL(req.getReferer(), req.getPreviousUrl());
   CPPUNIT_ASSERT_EQUAL(string("http"), req.getProtocol());
   CPPUNIT_ASSERT_EQUAL(8080, req.getPort());
   CPPUNIT_ASSERT_EQUAL(string("aria.rednoah.com"), req.getHost());
@@ -152,6 +161,8 @@ void RequestTest::testRedirectUrl() {
 		       req.getUrl());
   // currentUrl must be updated
   CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.co.jp/"), req.getCurrentUrl());
+  // previousUrl must be updated
+  CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com:8080/aria2/index.html"), req.getPreviousUrl());
   CPPUNIT_ASSERT_EQUAL(string("http"), req.getProtocol());
   CPPUNIT_ASSERT_EQUAL(string("aria.rednoah.co.jp"), req.getHost());
   CPPUNIT_ASSERT_EQUAL(80, req.getPort());
@@ -159,10 +170,30 @@ void RequestTest::testRedirectUrl() {
   CPPUNIT_ASSERT_EQUAL(string(""), req.getFile());
 }
 
+void RequestTest::testRedirectUrl2() {
+  Request req;
+  bool v = req.setUrl("http://aria.rednoah.com/download.html");
+  CPPUNIT_ASSERT_EQUAL(string(""), req.getPreviousUrl());
+  req.setReferer("http://aria.rednoah.com/");
+  // previousUrl is updated when referer is specified
+  CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com/"), req.getPreviousUrl());
+  bool v2 = req.redirectUrl("http://aria.rednoah.com/403.html");
+
+  // previousUrl is updated
+  CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com/download.html"), req.getPreviousUrl());
+  // referer is unchagned
+  CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com/"), req.getReferer());
+
+  bool v3 = req.redirectUrl("http://aria.rednoah.com/error.html");
+
+  // previousUrl is update
+  CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com/403.html"), req.getPreviousUrl());
+}
+  
 void RequestTest::testResetUrl() {
   Request req;
   bool v = req.setUrl("http://aria.rednoah.com:8080/aria2/index.html");
-  
+  req.setReferer("http://aria.rednoah.com:8080/");
   bool v2 = req.redirectUrl("ftp://aria.rednoah.co.jp/");
 
   bool v3 = req.resetUrl();
@@ -170,6 +201,10 @@ void RequestTest::testResetUrl() {
   // currentUrl must equal to url
   CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com:8080/aria2/index.html"), req.getUrl());
   CPPUNIT_ASSERT_EQUAL(req.getUrl(), req.getCurrentUrl());
+  // previousUrl must equal to referer
+  CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com:8080/"), req.getPreviousUrl());
+  // referer is unchanged
+  CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com:8080/"), req.getReferer());
   CPPUNIT_ASSERT_EQUAL(string("http"), req.getProtocol());
   CPPUNIT_ASSERT_EQUAL(8080, req.getPort());
   CPPUNIT_ASSERT_EQUAL(string("aria.rednoah.com"), req.getHost());