From 98b81ad1b2a281dd1a6ceb7b8bdae2552268bd42 Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Tue, 24 Nov 2009 17:37:42 +0100 Subject: [PATCH] IMDb plaintext downloader: switch to cURL instead of GIO for the FTP downloads This should fix the GIO related download hangups that people have experienced on the N900. --- Makefile.am | 12 +-- configure.ac | 4 + src/imdb/imdb-ftp-downloader.vala | 98 +++++++++++++++++++++++ src/imdb/imdb-plaintext-downloader.vala | 133 ++++++++----------------------- 4 files changed, 143 insertions(+), 104 deletions(-) create mode 100644 src/imdb/imdb-ftp-downloader.vala diff --git a/Makefile.am b/Makefile.am index 81c66d9..42b9d74 100644 --- a/Makefile.am +++ b/Makefile.am @@ -170,22 +170,22 @@ src/plugins/imdb-plugin.c: ${libimdb_plugin_la_VALASOURCES} imdb_plaintext_downloader_SOURCES = \ src/imdb/imdb-plaintext-downloader.c \ - src/imdb/gzip-input-stream.c \ + src/imdb/imdb-ftp-downloader.c \ src/imdb/imdb-line-parser.c \ src/imdb/imdb-sqlite.c \ src/imdb/plaintext-downloader-interface.c imdb_plaintext_downloader_VALASOURCES = \ src/imdb/imdb-plaintext-downloader.vala \ - src/imdb/gzip-input-stream.vala \ + src/imdb/imdb-ftp-downloader.vala \ src/imdb/imdb-line-parser.vala \ src/imdb/imdb-sqlite.vala \ src/imdb/plaintext-downloader-interface.vala -imdb_plaintext_downloader_VALAFLAGS = --vapidir ./vapi --pkg cinaest \ - --pkg dbus-glib-1 --pkg gio-2.0 --pkg sqlite3 --pkg zlib -X -lz -imdb_plaintext_downloader_CFLAGS = ${CINAEST_CFLAGS} ${DBUS_CFLAGS} ${GIO_CFLAGS} ${SQLITE3_CFLAGS} -imdb_plaintext_downloader_LDADD = ${CINAEST_LIBS} ${DBUS_LIBS} ${GIO_LIBS} ${SQLITE3_LIBS} -lz +imdb_plaintext_downloader_VALAFLAGS = --thread --vapidir ./vapi --pkg cinaest \ + --pkg dbus-glib-1 --pkg gio-2.0 --pkg libcurl --pkg sqlite3 --pkg zlib -X -lz +imdb_plaintext_downloader_CFLAGS = ${CINAEST_CFLAGS} ${CURL_CFLAGS} ${DBUS_CFLAGS} ${GIO_CFLAGS} ${SQLITE3_CFLAGS} +imdb_plaintext_downloader_LDADD = ${CINAEST_LIBS} ${CURL_LIBS} ${DBUS_LIBS} ${GIO_LIBS} ${SQLITE3_LIBS} -lz src/imdb/imdb-plaintext-downloader.c: ${imdb_plaintext_downloader_VALASOURCES} ${VALAC} -C ${imdb_plaintext_downloader_VALASOURCES} ${imdb_plaintext_downloader_VALAFLAGS} diff --git a/configure.ac b/configure.ac index 191fbf4..6bf8fe1 100644 --- a/configure.ac +++ b/configure.ac @@ -27,6 +27,10 @@ PKG_CHECK_MODULES(CINAEST, cinaest) AC_SUBST(CINAEST_LIBS) AC_SUBST(CINAEST_CFLAGS) +PKG_CHECK_MODULES(CURL, libcurl) +AC_SUBST(CURL_LIBS) +AC_SUBST(CURL_CFLAGS) + PKG_CHECK_MODULES(GMODULE, gmodule-2.0 >= 2.20.3) AC_SUBST(GMODULE_LIBS) AC_SUBST(GMODULE_CFLAGS) diff --git a/src/imdb/imdb-ftp-downloader.vala b/src/imdb/imdb-ftp-downloader.vala new file mode 100644 index 0000000..4c78aac --- /dev/null +++ b/src/imdb/imdb-ftp-downloader.vala @@ -0,0 +1,98 @@ +class IMDbFtpDownloader { + Curl.EasyHandle curl; + private ZLib.InflateStream strm; + private int percent; + private char[] buf_out; + private uint have; + private LineParser parser; + private Cancellable cancellable; + + [CCode (instance_pos = -1)] + size_t write_callback (void *buffer, size_t size, size_t nmemb) { + if (cancellable != null && cancellable.is_cancelled ()) + return 0; + strm.next_in = buffer; + strm.avail_in = (uint) (size * nmemb); + if (strm.avail_in == 0) + return 0; + + do { + strm.next_out = (char*) buf_out + have; + strm.avail_out = buf_out.length - have; + + char* p = (char*) buf_out; + + var ret = strm.inflate (ZLib.Flush.NO_FLUSH); + assert (ret != ZLib.Status.STREAM_ERROR); + if (ret == ZLib.Status.NEED_DICT) + ret = ZLib.Status.DATA_ERROR; + switch (ret) { + case ZLib.Status.DATA_ERROR: + case ZLib.Status.MEM_ERROR: + return ret; + } + + have = buf_out.length - strm.avail_out; + + char* l = p; + int j = 0; + for (int i = 0; i < have; i++, j++) { + if (p[i] == '\n') { + p[i] = 0; + if (parser != null) + parser.parse_line ((string) l); + j = -1; + l = p + i + 1; + } + } + if (j > 0) { + Memory.copy (p, l, j); + have = j; + } else { + have = 0; + } + } while (strm.avail_out == 0); + + return size * nmemb; + } + + int progress_callback (double dltotal, double dlnow, double ultotal, double ulnow) { + if (cancellable != null && cancellable.is_cancelled ()) + return 1; + if (dltotal > 0) { + int p = (int) (100 * dlnow / dltotal); + if (p > percent) { + percent = p; + progress_changed (p); + } + } + return 0; + } + + public IMDbFtpDownloader (Cancellable? _cancellable) { + cancellable = _cancellable; + curl = new Curl.EasyHandle (); + curl.setopt (Curl.Option.WRITEFUNCTION, write_callback); + curl.setopt (Curl.Option.WRITEDATA, this); + curl.setopt (Curl.Option.NOPROGRESS, 0L); + curl.setopt (Curl.Option.PROGRESSFUNCTION, progress_callback); + curl.setopt (Curl.Option.PROGRESSDATA, this); + buf_out = new char[16384]; + } + + public void download (string url, LineParser? _parser) throws IOError { + curl.setopt (Curl.Option.URL, url); + percent = 0; + parser = _parser; + have = 0; + + strm = ZLib.InflateStream.full (15 | 32); + + var res = curl.perform (); + if (Curl.Code.ABORTED_BY_CALLBACK == res) { + throw new IOError.CANCELLED ("Download cancelled."); + } + } + + public signal void progress_changed (int percent); +} diff --git a/src/imdb/imdb-plaintext-downloader.vala b/src/imdb/imdb-plaintext-downloader.vala index 74efd39..e3df558 100644 --- a/src/imdb/imdb-plaintext-downloader.vala +++ b/src/imdb/imdb-plaintext-downloader.vala @@ -3,8 +3,6 @@ using GLib; class IMDbDownloadServer : Object, IMDbDownloader { MainLoop loop; Cancellable cancellable; - int64 sofar; - int64 total; bool running; uint source_id; unowned IMDbSqlite sqlite; @@ -13,6 +11,9 @@ class IMDbDownloadServer : Object, IMDbDownloader { "ftp.funet.fi/pub/mirrors/ftp.imdb.com/pub/", "ftp.sunet.se/pub/tv+movies/imdb/" }; + string url; + int flags; + int percent_finished; delegate void ParseLineFunction (string line); @@ -23,9 +24,9 @@ class IMDbDownloadServer : Object, IMDbDownloader { // IMDbDownloader implementation - public void download (string mirror, int flags) throws DBus.Error { + public void download (string mirror, int _flags) throws DBus.Error { if (running) { - message ("Download in progress. Abort.\n"); + stdout.printf ("Download in progress. Abort.\n"); return; } running = true; @@ -33,10 +34,16 @@ class IMDbDownloadServer : Object, IMDbDownloader { Source.remove (source_id); } - message ("Download started (%x).", flags); + stdout.printf ("Download started (%x).", flags); progress (0); - download_imdb_async.begin ("ftp://anonymous@" + mirror, flags, Priority.DEFAULT_IDLE); - message ("Download finished."); + url = "ftp://anonymous@" + mirror; + flags = _flags; + try { + Thread.create(download_thread, false); + } catch (ThreadError e) { + critical ("Failed to create download thread\n"); + return; + } } public void cancel () throws DBus.Error { @@ -49,56 +56,10 @@ class IMDbDownloadServer : Object, IMDbDownloader { // Private methods - async void download_imdb_async (string mirror, int flags, int io_priority) { - Mount m; - File movies = File.new_for_uri (mirror + "/movies.list.gz"); - File genres = File.new_for_uri (mirror + "/genres.list.gz"); - File ratings = File.new_for_uri (mirror + "/ratings.list.gz"); - + private void* download_thread () { description_changed ("Connecting to FTP ..."); progress (0); - - try { - m = yield movies.find_enclosing_mount_async(io_priority, cancellable); - } catch (Error e0) { - try { - bool mounted = yield movies.mount_enclosing_volume (MountMountFlags.NONE, null, cancellable); - if (mounted) { - m = yield movies.find_enclosing_mount_async(io_priority, cancellable); - } else { - running = false; - timeout_quit (); - return; - } - } catch (Error e1) { - critical ("Failed to mount: %s\n", e1.message); - running = false; - timeout_quit (); - return; - } - } - stdout.printf ("Mounted: %s\n", m.get_name ()); - - description_changed ("Querying file sizes ..."); - try { - FileInfo info; - - if (MOVIES in flags) { - info = yield movies.query_info_async ("", FileQueryInfoFlags.NONE, io_priority, cancellable); - total += info.get_size (); - } - if (GENRES in flags) { - info = yield genres.query_info_async ("", FileQueryInfoFlags.NONE, io_priority, cancellable); - total += info.get_size (); - } - if (RATINGS in flags) { - info = yield ratings.query_info_async ("", FileQueryInfoFlags.NONE, io_priority, cancellable); - total += info.get_size (); - } - } catch (Error e3) { - warning ("Failed to get size: %s\n", e3.message); - total = 0; - } + percent_finished = 0; var cache_dir = Path.build_filename (Environment.get_user_cache_dir (), "cinaest"); DirUtils.create_with_parents (cache_dir, 0770); @@ -111,38 +72,33 @@ class IMDbDownloadServer : Object, IMDbDownloader { var movie_parser = new MovieLineParser (sqlite); var genre_parser = new GenreLineParser (sqlite); var rating_parser = new RatingLineParser (sqlite); - sofar = 0; + + var downloader = new IMDbFtpDownloader (cancellable); + downloader.progress_changed.connect (on_progress_changed); if (MOVIES in flags) { description_changed ("Downloading movie list ..."); - yield download_async(movies, movie_parser, io_priority); + downloader.download (url + "movies.list.gz", movie_parser); description_changed ("Creating title index ..."); sqlite.create_title_index (); } + percent_finished = 33; if (GENRES in flags) { description_changed ("Downloading genre data ..."); - yield download_async(genres, genre_parser, io_priority); + downloader.download (url + "genres.list.gz", genre_parser); } + percent_finished = 66; if (RATINGS in flags) { description_changed ("Downloading rating data ..."); - yield download_async(ratings, rating_parser, io_priority); + downloader.download (url + "ratings.list.gz", rating_parser); } } catch (Error e2) { if (e2 is IOError.CANCELLED) - message ("Download cancelled.\n"); + stdout.printf ("Download cancelled.\n"); else warning ("Failed to open/read stream: %s\n", e2.message); } - try { - bool unmounted = yield m.unmount(MountUnmountFlags.NONE, null); - if (!unmounted) { - warning ("Failed to unmount.\n"); - } - } catch (Error e4) { - warning ("Failed to unmount: %s\n", e4.message); - } - description_changed ("Creating indices ..."); sqlite.create_votes_index (); @@ -155,6 +111,12 @@ class IMDbDownloadServer : Object, IMDbDownloader { running = false; timeout_quit (); + + return null; + } + + private void on_progress_changed (int percent) { + progress (percent_finished + percent / 3); } private void timeout_quit () { @@ -168,40 +130,13 @@ class IMDbDownloadServer : Object, IMDbDownloader { return false; } - async void download_async (File f, LineParser parser, int io_priority) throws Error { - int percent = 0; - unowned string line = null; - - var stream = new GzipInputStream (yield f.read_async (io_priority, cancellable)); - var data = new DataInputStream(stream); - - do { - size_t l; - - line = yield data.read_line_async (io_priority, cancellable, out l); - if (line != null) - parser.parse_line (line); - - if (total == 0) - continue; - int p = (int) (100 * (sofar + stream.total_in ()) / total); - if (p > percent) { - percent = p; - if (p < 100) - progress (p); - } - } while (line != null); - - sofar += stream.total_in (); - - yield stream.close_async (io_priority, cancellable); - } - public void run () { loop.run (); } public static void main () { + Curl.global_init (Curl.GLOBAL_DEFAULT); + try { var conn = DBus.Bus.get (DBus.BusType.SESSION); dynamic DBus.Object bus = conn.get_object ("org.freedesktop.DBus", @@ -223,5 +158,7 @@ class IMDbDownloadServer : Object, IMDbDownloader { } catch (Error e) { critical ("Oops: %s\n", e.message); } + + Curl.global_cleanup (); } } -- 1.7.9.5