Support for gzipped -i and --save-session

Saved sessions may very large, as in hundreds and even thousands of
megabyte when dealing with large queues.
Add support to save and reload sessions to gzipped files, when libz is
available.

The session serializer will output gzipped contents when the file ends
with .gz, while the input file reader (UriListParser) will accept
whatever is thrown at it.

configure.ac

@@ -100,6 +100,7 @@ if test "x$with_libz" = "xyes"; then
     AC_DEFINE([HAVE_ZLIB], [1], [Define to 1 if you have zlib.])
     LIBS="$ZLIB_LIBS $LIBS"
     CPPFLAGS="$ZLIB_CFLAGS $CPPFLAGS"
+    AC_CHECK_FUNCS([gzbuffer gzsetparams])
   else
     AC_MSG_WARN([$ZLIB_PKG_ERRORS])
     if test "x$with_libz_requested" = "xyes"; then

src/BufferedFile.cc

@@ -59,7 +59,7 @@ BufferedFile::BufferedFile(const char* filename, const char* mode)
 {}
 
 BufferedFile::BufferedFile(FILE* fp)
-  : fp_(fp), open_(true), supportsColor_(fp_ ? isatty(fileno(fp_)) : false)
+  : fp_(fp), open_(fp_), supportsColor_(fp_ ? isatty(fileno(fp_)) : false)
 {}
 
 BufferedFile::~BufferedFile()
@@ -69,7 +69,8 @@ BufferedFile::~BufferedFile()
 
 BufferedFile::operator unspecified_bool_type() const
 {
-  return (!open_ || ferror(fp_)) ? 0 : &BufferedFile::good_state;
+  bool ok = isOpen() && !isError();
+  return ok ? &BufferedFile::good_state : 0;
 }
 
 size_t BufferedFile::read(void* ptr, size_t count)
@@ -94,7 +95,7 @@ char* BufferedFile::gets(char* s, int size)
 
 char* BufferedFile::getsn(char* s, int size)
 {
-  char* ptr = fgets(s, size, fp_);
+  char* ptr = gets(s, size);
   if(ptr) {
     int len = strlen(ptr);
     if(ptr[len-1] == '\n') {
@@ -128,17 +129,16 @@ std::string BufferedFile::getLine()
 
 int BufferedFile::close()
 {
-  if(open_) {
+  if (open_) {
     open_ = false;
     return fclose(fp_);
-  } else {
-    return 0;
   }
+  return 0;
 }
 
 bool BufferedFile::eof()
 {
-  return open_ && feof(fp_);
+  return !isOpen()|| isEOF();
 }
 
 size_t BufferedFile::transfer(std::ostream& out)

src/BufferedFile.h

@@ -56,18 +56,18 @@ public:
   // returns false.
   operator unspecified_bool_type() const;
   // wrapper for fread. Using 1 for 2nd argument of fread.
-  size_t read(void* ptr, size_t count);
+  virtual size_t read(void* ptr, size_t count);
   // wrapper for fwrite. Using 1 for 2nd argument of fwrite.
-  size_t write(const void* ptr, size_t count);
+  virtual size_t write(const void* ptr, size_t count);
   virtual size_t write(const char* str);
   // wrapper for fgets
-  char* gets(char* s, int size);
+  virtual char* gets(char* s, int size);
   // wrapper for fgets, but trailing '\n' is replaced with '\0'.
   char* getsn(char* s, int size);
   // Reads one line and returns it. The last '\n' is removed.
   std::string getLine();
   // wrapper for fclose
-  int close();
+  virtual int close();
   // Return true if open_ && feof(fp_) != 0. Otherwise returns false.
   bool eof();
   // Convenient method. Read data to end of file and write them into
@@ -83,15 +83,21 @@ public:
   static const char WRITE[];
   // Mode for append
   static const char APPEND[];
+
 private:
   // Don't allow copying
   BufferedFile(const BufferedFile&);
   BufferedFile& operator=(const BufferedFile&);
 
   FILE* fp_;
-  // true when file has been opened.
   bool open_;
   bool supportsColor_;
+
+protected:
+  virtual bool isError() const { return ferror(fp_); }
+  virtual bool isEOF() const { return feof(fp_); }
+  virtual bool isOpen() const { return open_; }
+
 };
 
 } // namespace aria2

src/GZipFile.cc

@@ -0,0 +1,161 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2013 Nils Maier
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+
+#include "GZipFile.h"
+
+#include <algorithm>
+#include <limits>
+
+#include "a2io.h"
+#include "util.h"
+
+namespace aria2 {
+
+GZipFile::GZipFile(const char* filename, const char* mode)
+  : BufferedFile(0), fp_(0), open_(false)
+{
+  FILE* fp =
+#ifdef __MINGW32__
+  a2fopen(utf8ToWChar(filename).c_str(), utf8ToWChar(mode).c_str());
+#else // !__MINGW32__
+  a2fopen(filename, mode);
+#endif // !__MINGW32__
+
+  open_  = fp;
+  if (open_) {
+    int fd = dup(fileno(fp));
+    if ((open_ = fd) >= 0) {
+      open_ = (fp_ = gzdopen(fd, mode));
+      if (!open_) {
+        ::close(fd);
+      }
+    }
+    if (open_) {
+#if HAVE_GZBUFFER
+      gzbuffer(fp_, 1<<17);
+#endif
+#if HAVE_GZSETPARAMS
+      gzsetparams(fp_, 2, Z_DEFAULT_STRATEGY);
+#endif
+    }
+    fclose(fp);
+  }
+}
+
+int GZipFile::close()
+{
+  if (open_) {
+    open_ = false;
+    return gzclose(fp_);
+  }
+  return 0;
+}
+
+bool GZipFile::isError() const
+{
+  int rv = 0;
+  const char *e = gzerror(fp_, &rv);
+  return (e != 0 && *e != 0) || rv != 0;
+}
+
+size_t GZipFile::read(void* ptr, size_t count)
+{
+  char *data = reinterpret_cast<char*>(ptr);
+  size_t read = 0;
+  while (count) {
+    size_t len = std::min(count, (size_t)std::numeric_limits<unsigned>::max());
+    int rv = gzread(fp_, data, len);
+    if (rv <= 0) {
+      break;
+    }
+    count -= rv;
+    read += rv;
+    data += rv;
+  }
+  return read;
+}
+
+size_t GZipFile::write(const void* ptr, size_t count)
+{
+  const char *data = reinterpret_cast<const char*>(ptr);
+  size_t written = 0;
+  while (count) {
+    size_t len = std::min(count, (size_t)std::numeric_limits<unsigned>::max());
+    int rv = gzwrite(fp_, data, len);
+    if (rv <= 0) {
+      break;
+    }
+    count -= rv;
+    written += rv;
+    data += rv;
+  }
+  return written;
+}
+
+char* GZipFile::gets(char* s, int size)
+{
+  return gzgets(fp_, s, size);
+}
+
+int GZipFile::flush()
+{
+  return gzflush(fp_, 0);
+}
+
+int GZipFile::vprintf(const char* format, va_list va)
+{
+  char *buf = 0;
+  size_t len;
+
+  int rv = ::vasprintf(&buf, format, va);
+  if (rv <= 0) {
+    // buf is undefined at this point
+    // do not attempt to free it
+    return rv;
+  }
+
+  len = strlen(buf);
+  if (len) {
+    rv = gzwrite(fp_, buf, len);
+  }
+
+  if (buf) {
+    free(buf);
+  }
+  return rv;
+}
+
+
+} // namespace aria2

src/GZipFile.h

@@ -0,0 +1,71 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2013 Nils Maier
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+
+#ifndef D_GZIP_FILE_H
+#define D_GZIP_FILE_H
+
+#include <zlib.h>
+
+#include "BufferedFile.h"
+
+namespace aria2 {
+
+class GZipFile: public BufferedFile {
+public:
+  GZipFile(const char* filename, const char* mode);
+  virtual ~GZipFile() {}
+  virtual int close();
+  virtual size_t read(void* ptr, size_t count);
+  virtual size_t write(const void* ptr, size_t count);
+  virtual char* gets(char* s, int size);
+  virtual int vprintf(const char* format, va_list va);
+  virtual int flush();
+private:
+  // Don't allow copying
+  GZipFile(const GZipFile&);
+  GZipFile& operator=(const GZipFile&);
+
+  gzFile fp_;
+  bool open_;
+
+protected:
+  virtual bool isError() const;
+  virtual bool isEOF() const { return gzeof(fp_); }
+  virtual bool isOpen() const { return open_; }
+};
+
+} // namespace aria2
+
+#endif // D_GZIP_FILE_H

src/Makefile.am

@@ -325,7 +325,8 @@ endif # HAVE_OPENSSL
 
 if HAVE_ZLIB
 SRCS += GZipEncoder.cc GZipEncoder.h\
-	GZipDecodingStreamFilter.cc GZipDecodingStreamFilter.h
+	GZipDecodingStreamFilter.cc GZipDecodingStreamFilter.h\
+	GZipFile.cc GzipFile.h
 endif # HAVE_ZLIB
 
 if HAVE_SQLITE3

src/SessionSerializer.cc

@@ -52,6 +52,10 @@
 #include "OptionParser.h"
 #include "OptionHandler.h"
 
+#if HAVE_ZLIB
+#include "GZipFile.h"
+#endif
+
 namespace aria2 {
 
 SessionSerializer::SessionSerializer
@@ -66,11 +70,20 @@ bool SessionSerializer::save(const std::string& filename) const
   std::string tempFilename = filename;
   tempFilename += "__temp";
   {
-    BufferedFile fp(tempFilename.c_str(), BufferedFile::WRITE);
-    if(!fp) {
+    SharedHandle<BufferedFile> fp;
+#if HAVE_ZLIB
+    if (filename.compare(filename.length() - 3, 3, ".gz") == 0) {
+      fp.reset(new GZipFile(tempFilename.c_str(), BufferedFile::WRITE));
+    }
+    else
+#endif
+    {
+     fp.reset(new BufferedFile(tempFilename.c_str(), BufferedFile::WRITE));
+    }
+    if(!fp || !*fp) {
       return false;
     }
-    if(!save(fp) || fp.close() == EOF) {
+    if(!save(*fp) || fp->close() == EOF) {
       return false;
     }
   }
@@ -92,12 +105,36 @@ bool writeOption(BufferedFile& fp, const SharedHandle<Option>& op)
                     false, false);
         for(std::vector<std::string>::const_iterator j = v.begin(),
               eoj = v.end(); j != eoj; ++j) {
-          if(fp.printf(" %s=%s\n", pref->k, (*j).c_str()) < 0) {
+          if (fp.write(" ", 1) != 1) {
+            return false;
+          }
+          if (fp.write(pref->k) == 0) {
+            return false;
+          }
+          if (fp.write("=", 1) != 1) {
+            return false;
+          }
+          if (!j->empty() && fp.write(j->c_str()) == 0) {
+            return false;
+          }
+          if (fp.write("\n", 1) != 1) {
             return false;
           }
         }
       } else {
-        if(fp.printf(" %s=%s\n", pref->k, op->get(pref).c_str()) < 0) {
+        if (fp.write(" ", 1) != 1) {
+          return false;
+        }
+        if (fp.write(pref->k) == 0) {
+          return false;
+        }
+        if (fp.write("=", 1) != 1) {
+          return false;
+        }
+        if (fp.write(op->get(pref).c_str()) == 0) {
+          return false;
+        }
+        if (fp.write("\n", 1) != 1) {
           return false;
         }
       }
@@ -152,14 +189,17 @@ bool writeDownloadResult
     }
     for(std::vector<std::string>::const_iterator i = uris.begin(),
           eoi = uris.end(); i != eoi; ++i) {
-      if(fp.printf("%s\t", (*i).c_str()) < 0) {
+      if (fp.write(i->c_str()) == 0 || fp.write("\t", 1) != 1) {
         return false;
       }
     }
-    if(fp.write("\n", 1) != 1) {
+    if(fp.write("\n gid=", 6) != 6) {
       return false;
     }
-    if(fp.printf(" gid=%s\n", dr->gid->toHex().c_str()) < 0) {
+    if(fp.write(dr->gid->toHex().c_str()) == 0) {
+       return false;
+    }
+    if(fp.write("\n", 1) != 1) {
       return false;
     }
   } else {
@@ -167,12 +207,12 @@ bool writeDownloadResult
       return true;
     } else {
       metainfoCache.insert(mi->getGID());
-      if(fp.printf("%s\n", mi->getUri().c_str()) < 0) {
+      if (fp.write(mi->getUri().c_str()) == 0 || fp.write("\n", 1) != 1) {
         return false;
       }
       // For downloads generated by metadata (e.g., BitTorrent,
       // Metalink), save gid of Metadata download.
-      if(fp.printf(" gid=%s\n", GroupId::toHex(mi->getGID()).c_str()) < 0) {
+      if (fp.write(" gid=", 5) != 5 || fp.write(GroupId::toHex(mi->getGID()).c_str()) == 0 || fp.write("\n", 1) != 1) {
         return false;
       }
     }
@@ -241,7 +281,7 @@ bool SessionSerializer::save(BufferedFile& fp) const
       // PREF_PAUSE was removed from option, so save it here looking
       // property separately.
       if(rg->isPauseRequested()) {
-        if(fp.printf(" %s=true\n", PREF_PAUSE->k) < 0) {
+        if (fp.write(" ", 1) != 1 || fp.write(PREF_PAUSE->k) == 0 || fp.write("\n", 1) != 1) {
           return false;
         }
       }

src/UriListParser.cc

@@ -45,10 +45,18 @@
 #include "BufferedFile.h"
 #include "OptionParser.h"
 
+#if HAVE_ZLIB
+#include "GZipFile.h"
+#endif
+
 namespace aria2 {
 
 UriListParser::UriListParser(const std::string& filename)
-  : fp_(filename.c_str(), BufferedFile::READ)
+#if HAVE_ZLIB
+  : fp_(new GZipFile(filename.c_str(), BufferedFile::READ))
+#else
+  : fp_(new BufferedFile(filename.c_str(), BufferedFile::READ))
+#endif
 {}
 
 UriListParser::~UriListParser() {}
@@ -63,9 +71,9 @@ void UriListParser::parseNext(std::vector<std::string>& uris, Option& op)
       // Read options
       std::stringstream ss;
       while(1) {
-        line_ = fp_.getLine();
+        line_ = fp_->getLine();
         if(line_.empty()) {
-          if(fp_.eof()) {
+          if(fp_->eof()) {
             break;
           } else if(!fp_) {
             throw DL_ABORT_EX("UriListParser:I/O error.");
@@ -84,9 +92,9 @@ void UriListParser::parseNext(std::vector<std::string>& uris, Option& op)
       optparser->parse(op, ss);
       return;
     }
-    line_ = fp_.getLine();
+    line_ = fp_->getLine();
     if(line_.empty()) {
-      if(fp_.eof()) {
+      if(fp_->eof()) {
         return;
       } else if(!fp_) {
         throw DL_ABORT_EX("UriListParser:I/O error.");
@@ -97,7 +105,11 @@ void UriListParser::parseNext(std::vector<std::string>& uris, Option& op)
 
 bool UriListParser::hasNext()
 {
-  return !line_.empty() || (fp_ && !fp_.eof());
+  bool rv = !line_.empty() || (fp_ && *fp_ && !fp_->eof());
+  if (!rv) {
+    fp_->close();
+  }
+  return rv;
 }
 
 } // namespace aria2

src/UriListParser.h

@@ -43,12 +43,13 @@
 
 #include "Option.h"
 #include "BufferedFile.h"
+#include "SharedHandle.h"
 
 namespace aria2 {
 
 class UriListParser {
 private:
-  BufferedFile fp_;
+  SharedHandle<BufferedFile> fp_;
 
   std::string line_;
 public:

test/Makefile.am

@@ -3,7 +3,6 @@ TESTS = aria2c
 check_PROGRAMS = $(TESTS)
 aria2c_SOURCES = AllTest.cc\
 	TestUtil.cc TestUtil.h\
-	GZipDecoder.cc GZipDecoder.h\
 	SocketCoreTest.cc\
 	array_funTest.cc\
 	Base64Test.cc\
@@ -100,8 +99,9 @@ aria2c_SOURCES += FallocFileAllocationIteratorTest.cc
 endif  # HAVE_SOME_FALLOCATE
 
 if HAVE_ZLIB
-aria2c_SOURCES += GZipDecoderTest.cc\
-	GZipEncoderTest.cc\
+aria2c_SOURCES += \
+	GZipDecoder.cc GZipDecoder.h\
+	GZipDecoderTest.cc GZipEncoderTest.cc\
 	GZipDecodingStreamFilterTest.cc
 endif # HAVE_ZLIB
 
@@ -281,4 +281,4 @@ EXTRA_DIST = 4096chunk.txt\
 	base_uri.xml
 
 clean-local:
-	-rm -rf ${a2_test_outdir}
+	-rm -rf ${a2_test_outdir}