2008-09-08 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

	Implemented the ability to get timestamp from remote FTP server 
using
	MDTM command described in RFC3659.
	* src/FtpConnection.cc
	* src/FtpConnection.h
	* src/FtpNegotiationCommand.cc
	* src/FtpNegotiationCommand.h
	* test/FtpConnectionTest.cc
	* test/Makefile.am

ChangeLog

@@ -1,3 +1,14 @@
+2008-09-08  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Implemented the ability to get timestamp from remote FTP server using
+	MDTM command described in RFC3659.
+	* src/FtpConnection.cc
+	* src/FtpConnection.h
+	* src/FtpNegotiationCommand.cc
+	* src/FtpNegotiationCommand.h
+	* test/FtpConnectionTest.cc
+	* test/Makefile.am
+
 2008-09-07  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	Implemented the ability to get timestamp from remote HTTP server and

src/FtpConnection.cc

@@ -47,6 +47,7 @@
 #include "DlAbortEx.h"
 #include "Socket.h"
 #include "A2STR.h"
+#include <cstring>
 
 namespace aria2 {
 
@@ -95,6 +96,13 @@ void FtpConnection::sendCwd() const
   socket->writeData(request);
 }
 
+void FtpConnection::sendMdtm() const
+{
+  std::string request = "MDTM "+Util::urlencode(req->getFile())+"\r\n";
+  logger->info(MSG_SENDING_REQUEST, cuid, request.c_str());
+  socket->writeData(request);
+}
+
 void FtpConnection::sendSize() const
 {
   std::string request = "SIZE "+Util::urldecode(req->getFile())+"\r\n";
@@ -257,6 +265,26 @@ unsigned int FtpConnection::receiveSizeResponse(uint64_t& size)
   }
 }
 
+unsigned int FtpConnection::receiveMdtmResponse(Time& time)
+{
+  // MDTM command, specified in RFC3659.
+  std::pair<unsigned int, std::string> response;
+  if(bulkReceiveResponse(response)) {
+    if(response.first == 213) {
+      char buf[15]; // YYYYMMDDhhmmss+\0, milli second part is dropped.
+      sscanf(response.second.c_str(), "%*u %14s", buf);
+      if(strlen(buf) == 14) {
+	time = Time::parse(buf, "%Y%m%d%H%M%S");
+      } else {
+	time.setTimeInSec(-1);
+      }
+    }
+    return response.first;
+  } else {
+    return 0;
+  }
+}
+
 unsigned int FtpConnection::receivePasvResponse(std::pair<std::string, uint16_t>& dest)
 {
   std::pair<unsigned int, std::string> response;

src/FtpConnection.h

@@ -37,6 +37,7 @@
 
 #include "common.h"
 #include "SharedHandle.h"
+#include "TimeA2.h"
 #include <utility>
 #include <string>
 
@@ -74,6 +75,7 @@ public:
   void sendPass() const;
   void sendType() const;
   void sendCwd() const;
+  void sendMdtm() const;
   void sendSize() const;
   void sendPasv() const;
   SharedHandle<SocketCore> sendPort() const;
@@ -82,6 +84,13 @@ public:
 
   unsigned int receiveResponse();
   unsigned int receiveSizeResponse(uint64_t& size);
+  // Returns status code of MDTM reply. If the status code is 213, parses
+  // time-val and store it in time.
+  // If a code other than 213 is returned, time is not touched.
+  // Expect MDTM reply is YYYYMMDDhhmmss in GMT. If status is 213 but returned
+  // date cannot be parsed, then executes time.setTimeInSec(-1).
+  // If reply is not received yet, returns 0.
+  unsigned int receiveMdtmResponse(Time& time);
   unsigned int receivePasvResponse(std::pair<std::string, uint16_t>& dest);
 };
 

src/FtpNegotiationCommand.cc

@@ -202,10 +202,49 @@ bool FtpNegotiationCommand::recvCwd() {
     poolConnection();
     throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str());
   }
-  sequence = SEQ_SEND_SIZE;
+  if(e->option->getAsBool(PREF_REMOTE_TIME)) {
+    sequence = SEQ_SEND_MDTM;
+  } else {
+    sequence = SEQ_SEND_SIZE;
+  }
   return true;
 }
 
+bool FtpNegotiationCommand::sendMdtm()
+{
+  ftp->sendMdtm();
+  sequence = SEQ_RECV_MDTM;
+  return false;
+}
+
+bool FtpNegotiationCommand::recvMdtm()
+{
+  Time lastModifiedTime(-1);
+  unsigned int status = ftp->receiveMdtmResponse(lastModifiedTime);
+  if(status == 0) {
+    return false;
+  }
+  if(status == 213) {
+    if(lastModifiedTime.good()) {
+      _requestGroup->updateLastModifiedTime(lastModifiedTime);
+      time_t t = lastModifiedTime.getTime();
+      struct tm* tms = gmtime(&t); // returned struct is statically allocated.
+      if(tms) {
+	logger->debug("MDTM result was parsed as: %s GMT", asctime(tms));
+      } else {
+	logger->debug("gmtime() failed for MDTM result.");
+      }
+    } else {
+      logger->debug("MDTM response was returned, but it seems not to be a time"
+		    " value as in specified in RFC3659.");
+    }
+  } else {
+    logger->info("CUID#%d - MDTM command failed.", cuid);
+  }
+  sequence = SEQ_SEND_SIZE;
+  return true;  
+}
+
 bool FtpNegotiationCommand::sendSize() {
   ftp->sendSize();
   sequence = SEQ_RECV_SIZE;
@@ -445,6 +484,10 @@ bool FtpNegotiationCommand::processSequence(const SegmentHandle& segment) {
     return sendCwd();
   case SEQ_RECV_CWD:
     return recvCwd();
+  case SEQ_SEND_MDTM:
+    return sendMdtm();
+  case SEQ_RECV_MDTM:
+    return recvMdtm();
   case SEQ_SEND_SIZE:
     return sendSize();
   case SEQ_RECV_SIZE:

src/FtpNegotiationCommand.h

@@ -54,6 +54,8 @@ public:
     SEQ_RECV_TYPE,
     SEQ_SEND_CWD,
     SEQ_RECV_CWD,
+    SEQ_SEND_MDTM,
+    SEQ_RECV_MDTM,
     SEQ_SEND_SIZE,
     SEQ_RECV_SIZE,
     SEQ_SEND_PORT,
@@ -82,6 +84,8 @@ private:
   bool recvType();
   bool sendCwd();
   bool recvCwd();
+  bool sendMdtm();
+  bool recvMdtm();
   bool sendSize();
   bool recvSize();
   bool sendPort();

test/FtpConnectionTest.cc

@@ -0,0 +1,105 @@
+#include "FtpConnection.h"
+#include "Exception.h"
+#include "Util.h"
+#include "SocketCore.h"
+#include "Request.h"
+#include "Option.h"
+#include <iostream>
+#include <cppunit/extensions/HelperMacros.h>
+
+namespace aria2 {
+
+class FtpConnectionTest:public CppUnit::TestFixture {
+
+  CPPUNIT_TEST_SUITE(FtpConnectionTest);
+  CPPUNIT_TEST(testSendMdtm);
+  CPPUNIT_TEST(testReceiveMdtmResponse);
+  CPPUNIT_TEST_SUITE_END();
+private:
+  SharedHandle<SocketCore> _serverSocket;
+  uint16_t _listenPort;
+  SharedHandle<FtpConnection> _ftp;
+  Option _option;
+public:
+  void setUp()
+  {
+    //_ftpServerSocket.reset(new SocketCore());
+    SharedHandle<SocketCore> listenSocket(new SocketCore());
+    listenSocket->bind(0);
+    listenSocket->beginListen();
+    std::pair<std::string, uint16_t> addrinfo;
+    listenSocket->getAddrInfo(addrinfo);
+    _listenPort = addrinfo.second;
+
+    SharedHandle<Request> req(new Request());
+    req->setUrl("ftp://localhost/dir/file.img");
+
+    SharedHandle<SocketCore> clientSocket(new SocketCore());
+    clientSocket->establishConnection("127.0.0.1", _listenPort);
+
+    while(!clientSocket->isWritable(0));
+    clientSocket->setBlockingMode();
+
+    _serverSocket.reset(listenSocket->acceptConnection());
+    _ftp.reset(new FtpConnection(1, clientSocket, req, &_option));
+  }
+
+  void tearDown() {}
+
+  void testSendMdtm();
+  void testReceiveMdtmResponse();
+};
+
+
+CPPUNIT_TEST_SUITE_REGISTRATION(FtpConnectionTest);
+
+void FtpConnectionTest::testSendMdtm()
+{
+  _ftp->sendMdtm();
+  char data[32];
+  size_t len = sizeof(data);
+  _serverSocket->readData(data, len);
+  CPPUNIT_ASSERT_EQUAL((size_t)15, len);
+  data[len] = '\0';
+  CPPUNIT_ASSERT_EQUAL(std::string("MDTM file.img\r\n"), std::string(data));
+}
+
+void FtpConnectionTest::testReceiveMdtmResponse()
+{
+  {
+    Time t;
+    _serverSocket->writeData("213 20080908124312");
+    CPPUNIT_ASSERT_EQUAL((unsigned int)0, _ftp->receiveMdtmResponse(t));
+    _serverSocket->writeData("\r\n");
+    CPPUNIT_ASSERT_EQUAL((unsigned int)213, _ftp->receiveMdtmResponse(t));
+    CPPUNIT_ASSERT_EQUAL((time_t)1220877792, t.getTime());
+  }
+  {
+    // see milli second part is ignored
+    Time t;
+    _serverSocket->writeData("213 20080908124312.014\r\n");
+    CPPUNIT_ASSERT_EQUAL((unsigned int)213, _ftp->receiveMdtmResponse(t));
+    CPPUNIT_ASSERT_EQUAL((time_t)1220877792, t.getTime());
+  }
+  {
+    // hhmmss part is missing
+    Time t;
+    _serverSocket->writeData("213 20080908\r\n");
+    CPPUNIT_ASSERT_EQUAL((unsigned int)213, _ftp->receiveMdtmResponse(t));
+    CPPUNIT_ASSERT_EQUAL((time_t)-1, t.getTime());
+  }
+  {
+    // invalid month: 19
+    Time t;
+    _serverSocket->writeData("213 20081908124312\r\n");
+    CPPUNIT_ASSERT_EQUAL((unsigned int)213, _ftp->receiveMdtmResponse(t));
+    CPPUNIT_ASSERT_EQUAL((time_t)-1, t.getTime());
+  }
+  {
+    Time t;
+    _serverSocket->writeData("550 File Not Found\r\n");
+    CPPUNIT_ASSERT_EQUAL((unsigned int)550, _ftp->receiveMdtmResponse(t));
+  }
+}
+
+} // namespace aria2

test/Makefile.am

@@ -62,7 +62,8 @@ aria2c_SOURCES = AllTest.cc\
 	CookieTest.cc\
 	CookieStorageTest.cc\
 	TimeTest.cc\
-	CopyDiskAdaptorTest.cc
+	CopyDiskAdaptorTest.cc\
+	FtpConnectionTest.cc
 
 if HAVE_LIBZ
 aria2c_SOURCES += GZipDecoderTest.cc

test/Makefile.in

@@ -194,8 +194,9 @@ am__aria2c_SOURCES_DIST = AllTest.cc TestUtil.cc TestUtil.h \
 	ServerStatURISelectorTest.cc InOrderURISelectorTest.cc \
 	ServerStatTest.cc NsCookieParserTest.cc \
 	DirectDiskAdaptorTest.cc CookieTest.cc CookieStorageTest.cc \
-	TimeTest.cc CopyDiskAdaptorTest.cc GZipDecoderTest.cc \
-	Sqlite3MozCookieParserTest.cc MessageDigestHelperTest.cc \
+	TimeTest.cc CopyDiskAdaptorTest.cc FtpConnectionTest.cc \
+	GZipDecoderTest.cc Sqlite3MozCookieParserTest.cc \
+	MessageDigestHelperTest.cc \
 	IteratableChunkChecksumValidatorTest.cc \
 	IteratableChecksumValidatorTest.cc BtAllowedFastMessageTest.cc \
 	BtBitfieldMessageTest.cc BtCancelMessageTest.cc \
@@ -367,8 +368,8 @@ am_aria2c_OBJECTS = AllTest.$(OBJEXT) TestUtil.$(OBJEXT) \
 	NsCookieParserTest.$(OBJEXT) DirectDiskAdaptorTest.$(OBJEXT) \
 	CookieTest.$(OBJEXT) CookieStorageTest.$(OBJEXT) \
 	TimeTest.$(OBJEXT) CopyDiskAdaptorTest.$(OBJEXT) \
-	$(am__objects_1) $(am__objects_2) $(am__objects_3) \
-	$(am__objects_4) $(am__objects_5)
+	FtpConnectionTest.$(OBJEXT) $(am__objects_1) $(am__objects_2) \
+	$(am__objects_3) $(am__objects_4) $(am__objects_5)
 aria2c_OBJECTS = $(am_aria2c_OBJECTS)
 am__DEPENDENCIES_1 =
 aria2c_DEPENDENCIES = ../src/libaria2c.a $(am__DEPENDENCIES_1)
@@ -590,9 +591,9 @@ aria2c_SOURCES = AllTest.cc TestUtil.cc TestUtil.h SocketCoreTest.cc \
 	ServerStatURISelectorTest.cc InOrderURISelectorTest.cc \
 	ServerStatTest.cc NsCookieParserTest.cc \
 	DirectDiskAdaptorTest.cc CookieTest.cc CookieStorageTest.cc \
-	TimeTest.cc CopyDiskAdaptorTest.cc $(am__append_1) \
-	$(am__append_2) $(am__append_3) $(am__append_4) \
-	$(am__append_5)
+	TimeTest.cc CopyDiskAdaptorTest.cc FtpConnectionTest.cc \
+	$(am__append_1) $(am__append_2) $(am__append_3) \
+	$(am__append_4) $(am__append_5)
 
 #aria2c_CXXFLAGS = ${CPPUNIT_CFLAGS} -I../src -I../lib -Wall -D_FILE_OFFSET_BITS=64
 #aria2c_LDFLAGS = ${CPPUNIT_LIBS}
@@ -759,6 +760,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfigTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileEntryTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileTest.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpConnectionTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GZipDecoderTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GrowSegmentTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HandshakeExtensionMessageTest.Po@am__quote@