IMDb: Split downloading from parsing into SQLite database

[cinaest] / src / imdb / ftp-downloader.vala

diff --git a/src/imdb/ftp-downloader.vala b/src/imdb/ftp-downloader.vala
new file mode 100644 (file)
index 0000000..2e873c9
--- /dev/null
+++ b/src/imdb/ftp-downloader.vala
@@ -0,0 +1,139 @@
+class FtpDownloader {
+       private Curl.EasyHandle curl;
+       private Cancellable cancellable;
+       private FileStream file;
+       private string dirname;
+       private HashTable<string,int> file_size;
+
+       public FtpDownloader (Cancellable? _cancellable) {
+               cancellable = _cancellable;
+               curl = new Curl.EasyHandle ();
+       }
+
+       [CCode (instance_pos = -1)]
+       size_t write_callback (void *buffer, size_t size, size_t nmemb) {
+               if (cancellable != null && cancellable.is_cancelled ())
+                       return 0;
+
+               unowned uint8[] buf = (uint8[]) buffer;
+               buf.length = (int) (size * nmemb);
+
+               file.write (buf);
+
+               return buf.length;
+       }
+
+       private int last_dlnow;
+
+       int progress_callback (double dltotal, double dlnow, double ultotal, double ulnow) {
+               if (cancellable != null && cancellable.is_cancelled ())
+                       return 1;
+               if (last_dlnow != (int) dlnow) {
+                       last_dlnow = (int) dlnow;
+                       progress ((int) dltotal, last_dlnow);
+               }
+               return 0;
+       }
+
+       public void download (string url, string filename) throws IOError {
+               print ("download (\"%s\", \"%s\")\n", url, filename);
+               download_dir (Path.get_dirname (url) + "/");
+               string basename = Path.get_basename (url);
+               int size = file_size.lookup (basename);
+               if (size > 0) {
+                       Posix.Stat st;
+                       Posix.stat (filename, out st);
+                       if (size == st.st_size) {
+                               return;
+                       }
+               }
+
+               curl.setopt (Curl.Option.URL, url);
+               curl.setopt (Curl.Option.WRITEFUNCTION, write_callback);
+               curl.setopt (Curl.Option.WRITEDATA, this);
+               curl.setopt (Curl.Option.NOPROGRESS, 0L);
+               curl.setopt (Curl.Option.PROGRESSFUNCTION, progress_callback);
+               curl.setopt (Curl.Option.PROGRESSDATA, this);
+
+               last_dlnow = -1;
+               file = FileStream.open (filename, "w");
+
+               var res = curl.perform ();
+               if (Curl.Code.ABORTED_BY_CALLBACK == res) {
+                               throw new IOError.CANCELLED ("Download cancelled.");
+               } else if (res != 0) {
+                       stderr.printf ("cURL performed: %d\n", res);
+               }
+
+               file = null;
+       }
+
+       void parse_dir_entry (string line) {
+               try {
+                       Regex re_dir_entry = new Regex ("^.* ([0-9]*) [A-Z][a-z]* *[0-9]* [0-9]* [0-9]*:[0-9]* ([^ ]*)$");
+                       MatchInfo match_info;
+                       if (re_dir_entry.match (line, 0, out match_info)) {
+                               string name = match_info.fetch (2);
+                               int size = match_info.fetch (1).to_int ();
+                               file_size.insert (name, size);
+                       }
+               } catch (RegexError e) {
+               }
+       }
+
+       string last_line = null;
+       [CCode (instance_pos = -1)]
+       size_t dir_callback (void *buffer, size_t size, size_t nmemb) {
+               if (cancellable != null && cancellable.is_cancelled ())
+                       return 0;
+
+               unowned char[] buf = (char[]) buffer;
+               buf.length = (int) (size * nmemb);
+
+               char *p = buf;
+               int i;
+               int j;
+               for (i = 0, j = 0; i < buf.length; i++, j++) {
+                       if (buf[i] == '\n') {
+                               buf[i] = 0;
+                               if (last_line != null) {
+                                       parse_dir_entry (last_line + (string) p);
+                                       last_line = null;
+                               } else {
+                                       parse_dir_entry ((string) p);
+                               }
+                               p += j + 1;
+                               j = -1;
+                       }
+               }
+               if (j > 0)
+                       last_line = ((string) p).ndup (j);
+
+               return buf.length;
+       }
+
+       public void download_dir (string url) throws IOError {
+               if (dirname != null && dirname == url)
+                       return;
+               print ("download_dir (\"%s\")\n", url);
+
+               curl.setopt (Curl.Option.URL, url);
+               curl.setopt (Curl.Option.WRITEFUNCTION, dir_callback);
+               curl.setopt (Curl.Option.WRITEDATA, this);
+               curl.setopt (Curl.Option.NOPROGRESS, 1L);
+               curl.setopt (Curl.Option.PROGRESSFUNCTION, null);
+               curl.setopt (Curl.Option.PROGRESSDATA, null);
+
+               file_size = new HashTable<string, int> (str_hash, int_equal);
+
+               var res = curl.perform ();
+               if (Curl.Code.ABORTED_BY_CALLBACK == res) {
+                               throw new IOError.CANCELLED ("Dir listing cancelled.");
+               } else if (res != 0) {
+                       stderr.printf ("cURL performed: %d\n", res);
+               }
+               dirname = url;
+       }
+
+       public signal void progress (int dltotal, int dlnow);
+}