imdb_plaintext_downloader_SOURCES = \
src/imdb/imdb-plaintext-downloader.c \
- src/imdb/gzip-input-stream.c \
+ src/imdb/imdb-ftp-downloader.c \
src/imdb/imdb-line-parser.c \
src/imdb/imdb-sqlite.c \
src/imdb/plaintext-downloader-interface.c
imdb_plaintext_downloader_VALASOURCES = \
src/imdb/imdb-plaintext-downloader.vala \
- src/imdb/gzip-input-stream.vala \
+ src/imdb/imdb-ftp-downloader.vala \
src/imdb/imdb-line-parser.vala \
src/imdb/imdb-sqlite.vala \
src/imdb/plaintext-downloader-interface.vala
-imdb_plaintext_downloader_VALAFLAGS = --vapidir ./vapi --pkg cinaest \
- --pkg dbus-glib-1 --pkg gio-2.0 --pkg sqlite3 --pkg zlib -X -lz
-imdb_plaintext_downloader_CFLAGS = ${CINAEST_CFLAGS} ${DBUS_CFLAGS} ${GIO_CFLAGS} ${SQLITE3_CFLAGS}
-imdb_plaintext_downloader_LDADD = ${CINAEST_LIBS} ${DBUS_LIBS} ${GIO_LIBS} ${SQLITE3_LIBS} -lz
+imdb_plaintext_downloader_VALAFLAGS = --thread --vapidir ./vapi --pkg cinaest \
+ --pkg dbus-glib-1 --pkg gio-2.0 --pkg libcurl --pkg sqlite3 --pkg zlib -X -lz
+imdb_plaintext_downloader_CFLAGS = ${CINAEST_CFLAGS} ${CURL_CFLAGS} ${DBUS_CFLAGS} ${GIO_CFLAGS} ${SQLITE3_CFLAGS}
+imdb_plaintext_downloader_LDADD = ${CINAEST_LIBS} ${CURL_LIBS} ${DBUS_LIBS} ${GIO_LIBS} ${SQLITE3_LIBS} -lz
src/imdb/imdb-plaintext-downloader.c: ${imdb_plaintext_downloader_VALASOURCES}
${VALAC} -C ${imdb_plaintext_downloader_VALASOURCES} ${imdb_plaintext_downloader_VALAFLAGS}
--- /dev/null
+class IMDbFtpDownloader {
+ Curl.EasyHandle curl;
+ private ZLib.InflateStream strm;
+ private int percent;
+ private char[] buf_out;
+ private uint have;
+ private LineParser parser;
+ private Cancellable cancellable;
+
+ [CCode (instance_pos = -1)]
+ size_t write_callback (void *buffer, size_t size, size_t nmemb) {
+ if (cancellable != null && cancellable.is_cancelled ())
+ return 0;
+ strm.next_in = buffer;
+ strm.avail_in = (uint) (size * nmemb);
+ if (strm.avail_in == 0)
+ return 0;
+
+ do {
+ strm.next_out = (char*) buf_out + have;
+ strm.avail_out = buf_out.length - have;
+
+ char* p = (char*) buf_out;
+
+ var ret = strm.inflate (ZLib.Flush.NO_FLUSH);
+ assert (ret != ZLib.Status.STREAM_ERROR);
+ if (ret == ZLib.Status.NEED_DICT)
+ ret = ZLib.Status.DATA_ERROR;
+ switch (ret) {
+ case ZLib.Status.DATA_ERROR:
+ case ZLib.Status.MEM_ERROR:
+ return ret;
+ }
+
+ have = buf_out.length - strm.avail_out;
+
+ char* l = p;
+ int j = 0;
+ for (int i = 0; i < have; i++, j++) {
+ if (p[i] == '\n') {
+ p[i] = 0;
+ if (parser != null)
+ parser.parse_line ((string) l);
+ j = -1;
+ l = p + i + 1;
+ }
+ }
+ if (j > 0) {
+ Memory.copy (p, l, j);
+ have = j;
+ } else {
+ have = 0;
+ }
+ } while (strm.avail_out == 0);
+
+ return size * nmemb;
+ }
+
+ int progress_callback (double dltotal, double dlnow, double ultotal, double ulnow) {
+ if (cancellable != null && cancellable.is_cancelled ())
+ return 1;
+ if (dltotal > 0) {
+ int p = (int) (100 * dlnow / dltotal);
+ if (p > percent) {
+ percent = p;
+ progress_changed (p);
+ }
+ }
+ return 0;
+ }
+
+ public IMDbFtpDownloader (Cancellable? _cancellable) {
+ cancellable = _cancellable;
+ curl = new Curl.EasyHandle ();
+ curl.setopt (Curl.Option.WRITEFUNCTION, write_callback);
+ curl.setopt (Curl.Option.WRITEDATA, this);
+ curl.setopt (Curl.Option.NOPROGRESS, 0L);
+ curl.setopt (Curl.Option.PROGRESSFUNCTION, progress_callback);
+ curl.setopt (Curl.Option.PROGRESSDATA, this);
+ buf_out = new char[16384];
+ }
+
+ public void download (string url, LineParser? _parser) throws IOError {
+ curl.setopt (Curl.Option.URL, url);
+ percent = 0;
+ parser = _parser;
+ have = 0;
+
+ strm = ZLib.InflateStream.full (15 | 32);
+
+ var res = curl.perform ();
+ if (Curl.Code.ABORTED_BY_CALLBACK == res) {
+ throw new IOError.CANCELLED ("Download cancelled.");
+ }
+ }
+
+ public signal void progress_changed (int percent);
+}
class IMDbDownloadServer : Object, IMDbDownloader {
MainLoop loop;
Cancellable cancellable;
- int64 sofar;
- int64 total;
bool running;
uint source_id;
unowned IMDbSqlite sqlite;
"ftp.funet.fi/pub/mirrors/ftp.imdb.com/pub/",
"ftp.sunet.se/pub/tv+movies/imdb/"
};
+ string url;
+ int flags;
+ int percent_finished;
delegate void ParseLineFunction (string line);
// IMDbDownloader implementation
- public void download (string mirror, int flags) throws DBus.Error {
+ public void download (string mirror, int _flags) throws DBus.Error {
if (running) {
- message ("Download in progress. Abort.\n");
+ stdout.printf ("Download in progress. Abort.\n");
return;
}
running = true;
Source.remove (source_id);
}
- message ("Download started (%x).", flags);
+ stdout.printf ("Download started (%x).", flags);
progress (0);
- download_imdb_async.begin ("ftp://anonymous@" + mirror, flags, Priority.DEFAULT_IDLE);
- message ("Download finished.");
+ url = "ftp://anonymous@" + mirror;
+ flags = _flags;
+ try {
+ Thread.create(download_thread, false);
+ } catch (ThreadError e) {
+ critical ("Failed to create download thread\n");
+ return;
+ }
}
public void cancel () throws DBus.Error {
// Private methods
- async void download_imdb_async (string mirror, int flags, int io_priority) {
- Mount m;
- File movies = File.new_for_uri (mirror + "/movies.list.gz");
- File genres = File.new_for_uri (mirror + "/genres.list.gz");
- File ratings = File.new_for_uri (mirror + "/ratings.list.gz");
-
+ private void* download_thread () {
description_changed ("Connecting to FTP ...");
progress (0);
-
- try {
- m = yield movies.find_enclosing_mount_async(io_priority, cancellable);
- } catch (Error e0) {
- try {
- bool mounted = yield movies.mount_enclosing_volume (MountMountFlags.NONE, null, cancellable);
- if (mounted) {
- m = yield movies.find_enclosing_mount_async(io_priority, cancellable);
- } else {
- running = false;
- timeout_quit ();
- return;
- }
- } catch (Error e1) {
- critical ("Failed to mount: %s\n", e1.message);
- running = false;
- timeout_quit ();
- return;
- }
- }
- stdout.printf ("Mounted: %s\n", m.get_name ());
-
- description_changed ("Querying file sizes ...");
- try {
- FileInfo info;
-
- if (MOVIES in flags) {
- info = yield movies.query_info_async ("", FileQueryInfoFlags.NONE, io_priority, cancellable);
- total += info.get_size ();
- }
- if (GENRES in flags) {
- info = yield genres.query_info_async ("", FileQueryInfoFlags.NONE, io_priority, cancellable);
- total += info.get_size ();
- }
- if (RATINGS in flags) {
- info = yield ratings.query_info_async ("", FileQueryInfoFlags.NONE, io_priority, cancellable);
- total += info.get_size ();
- }
- } catch (Error e3) {
- warning ("Failed to get size: %s\n", e3.message);
- total = 0;
- }
+ percent_finished = 0;
var cache_dir = Path.build_filename (Environment.get_user_cache_dir (), "cinaest");
DirUtils.create_with_parents (cache_dir, 0770);
var movie_parser = new MovieLineParser (sqlite);
var genre_parser = new GenreLineParser (sqlite);
var rating_parser = new RatingLineParser (sqlite);
- sofar = 0;
+
+ var downloader = new IMDbFtpDownloader (cancellable);
+ downloader.progress_changed.connect (on_progress_changed);
if (MOVIES in flags) {
description_changed ("Downloading movie list ...");
- yield download_async(movies, movie_parser, io_priority);
+ downloader.download (url + "movies.list.gz", movie_parser);
description_changed ("Creating title index ...");
sqlite.create_title_index ();
}
+ percent_finished = 33;
if (GENRES in flags) {
description_changed ("Downloading genre data ...");
- yield download_async(genres, genre_parser, io_priority);
+ downloader.download (url + "genres.list.gz", genre_parser);
}
+ percent_finished = 66;
if (RATINGS in flags) {
description_changed ("Downloading rating data ...");
- yield download_async(ratings, rating_parser, io_priority);
+ downloader.download (url + "ratings.list.gz", rating_parser);
}
} catch (Error e2) {
if (e2 is IOError.CANCELLED)
- message ("Download cancelled.\n");
+ stdout.printf ("Download cancelled.\n");
else
warning ("Failed to open/read stream: %s\n", e2.message);
}
- try {
- bool unmounted = yield m.unmount(MountUnmountFlags.NONE, null);
- if (!unmounted) {
- warning ("Failed to unmount.\n");
- }
- } catch (Error e4) {
- warning ("Failed to unmount: %s\n", e4.message);
- }
-
description_changed ("Creating indices ...");
sqlite.create_votes_index ();
running = false;
timeout_quit ();
+
+ return null;
+ }
+
+ private void on_progress_changed (int percent) {
+ progress (percent_finished + percent / 3);
}
private void timeout_quit () {
return false;
}
- async void download_async (File f, LineParser parser, int io_priority) throws Error {
- int percent = 0;
- unowned string line = null;
-
- var stream = new GzipInputStream (yield f.read_async (io_priority, cancellable));
- var data = new DataInputStream(stream);
-
- do {
- size_t l;
-
- line = yield data.read_line_async (io_priority, cancellable, out l);
- if (line != null)
- parser.parse_line (line);
-
- if (total == 0)
- continue;
- int p = (int) (100 * (sofar + stream.total_in ()) / total);
- if (p > percent) {
- percent = p;
- if (p < 100)
- progress (p);
- }
- } while (line != null);
-
- sofar += stream.total_in ();
-
- yield stream.close_async (io_priority, cancellable);
- }
-
public void run () {
loop.run ();
}
public static void main () {
+ Curl.global_init (Curl.GLOBAL_DEFAULT);
+
try {
var conn = DBus.Bus.get (DBus.BusType.SESSION);
dynamic DBus.Object bus = conn.get_object ("org.freedesktop.DBus",
} catch (Error e) {
critical ("Oops: %s\n", e.message);
}
+
+ Curl.global_cleanup ();
}
}
projects / cinaest / commitdiff