Don't reprocess downloaded articles that are already up to date.

[feedingit] / src / rss_sqlite.py

diff --git a/src/rss_sqlite.py b/src/rss_sqlite.py
index 5735de2..077fce0 100644 (file)
--- a/src/rss_sqlite.py
+++ b/src/rss_sqlite.py
@@ -73,6 +73,27 @@ def downloader(progress_handler=None, proxy=None):
 
     return urllib2.build_opener(*openers)
 
+def transfer_stats(sent, received, **kwargs):
+    """
+    This function takes two arguments: sent is the number of bytes
+    sent so far, received is the number of bytes received.  The
+    function returns a continuation that you can call later.
+
+    The continuation takes the same two arguments.  It returns a tuple
+    of the number of bytes sent, the number of bytes received and the
+    time since the original function was invoked.
+    """
+    start_time = time.time()
+    start_sent = sent
+    start_received = received
+
+    def e(sent, received, **kwargs):
+        return (sent - start_sent,
+                received - start_received,
+                time.time() - start_time)
+
+    return e
+
 # If not None, a subprocess.Popen object corresponding to a
 # update_feeds.py process.
 update_feed_process = None
@@ -297,10 +318,12 @@ class Feed(BaseObject):
                 time.sleep(1)
 
     def _updateFeed(self, configdir, url, etag, modified, expiryTime=24, proxy=None, imageCache=False, postFeedUpdateFunc=None, *postFeedUpdateFuncArgs):
+        logger.debug("Updating %s" % url)
+
         success = False
         have_serial_execution_lock = False
         try:
-            download_start = time.time ()
+            update_start = time.time ()
 
             progress_handler = HTTPProgressHandler(download_callback)
 
@@ -309,9 +332,11 @@ class Feed(BaseObject):
                 openers.append (proxy)
             kwargs = {'handlers':openers}
             
+            feed_transfer_stats = transfer_stats(0, 0)
+
             tmp=feedparser.parse(url, etag=etag, modified=modified, **kwargs)
-            download_duration = time.time () - download_start
-    
+            download_duration = time.time () - update_start
+
             opener = downloader(progress_handler, proxy)
 
             if JobManager().do_quit:
@@ -323,7 +348,10 @@ class Feed(BaseObject):
             expiry = float(expiryTime) * 3600.
     
             currentTime = 0
-    
+            
+            updated_objects = 0
+            new_objects = 0
+
             def wc_success():
                 try:
                     wc().stream_register (self.key, "", 6 * 60 * 60)
@@ -335,10 +363,11 @@ class Feed(BaseObject):
                                    |woodchuck.Indicator.StreamWide),
                         transferred_down=progress_handler.stats['received'],
                         transferred_up=progress_handler.stats['sent'],
-                        transfer_time=download_start,
+                        transfer_time=update_start,
                         transfer_duration=download_duration,
-                        new_objects=len (tmp.entries),
-                        objects_inline=len (tmp.entries))
+                        new_objects=new_objects,
+                        updated_objects=updated_objects,
+                        objects_inline=new_objects + updated_objects)
                 except KeyError:
                     logger.warn(
                         "Failed to register update of %s with woodchuck!"
@@ -428,11 +457,12 @@ class Feed(BaseObject):
                    # responsive.
                    time.sleep(0)
     
+                   entry_transfer_stats = transfer_stats(
+                       *feed_transfer_stats(**progress_handler.stats)[0:2])
+
                    if JobManager().do_quit:
                        raise KeyboardInterrupt
 
-                   received_base = progress_handler.stats['received']
-                   sent_base = progress_handler.stats['sent']
                    object_size = 0
 
                    date = self.extractDate(entry)
@@ -452,11 +482,30 @@ class Feed(BaseObject):
                        entry["id"] = None
                    content = self.extractContent(entry)
                    object_size = len (content)
-                   received_base -= len (content)
                    tmpEntry = {"title":entry["title"], "content":content,
                                 "date":date, "link":entry["link"], "author":entry["author"], "id":entry["id"]}
                    id = self.generateUniqueId(tmpEntry)
                    
+                   current_version \
+                       = self.db.execute('select date from feed where id=?',
+                                         (id,)).fetchone()
+                   if (current_version is not None
+                       and current_version[0] == date):
+                       logger.debug("ALREADY DOWNLOADED %s (%s)"
+                                    % (entry["title"], entry["link"]))
+                       continue                       
+
+                   if current_version is not None:
+                       # The version was updated.  Mark it as unread.
+                       logger.debug("UPDATED: %s (%s)"
+                                    % (entry["title"], entry["link"]))
+                       self.setEntryUnread(id)
+                       updated_objects += 1
+                   else:
+                       logger.debug("NEW: %s (%s)"
+                                    % (entry["title"], entry["link"]))
+                       new_objects += 1
+
                    #articleTime = time.mktime(self.entries[id]["dateTuple"])
                    soup = BeautifulSoup(self.getArticle(tmpEntry)) #tmpEntry["content"])
                    images = soup('img')
@@ -544,9 +593,14 @@ class Feed(BaseObject):
                        else:
                            publication_time = None
 
-                       sent = progress_handler.stats['sent'] - sent_base
-                       received = (progress_handler.stats['received']
-                                   - received_base)
+                       sent, received, _ \
+                           = entry_transfer_stats(**progress_handler.stats)
+                       # sent and received are for objects (in
+                       # particular, images) associated with this
+                       # item.  We also want to attribute the data
+                       # transferred for the item's content.  This is
+                       # a good first approximation.
+                       received += len(content)
 
                        mainthread.execute(
                            register_object_transferred(
@@ -558,12 +612,11 @@ class Feed(BaseObject):
                            async=True)
                self.db.commit()
 
+               sent, received, _ \
+                   = feed_transfer_stats(**progress_handler.stats)
                logger.debug (
                    "%s: Update successful: transferred: %d/%d; objects: %d)"
-                   % (self.key,
-                      progress_handler.stats['sent'],
-                      progress_handler.stats['received'],
-                      len (tmp.entries)))
+                   % (url, sent, received, len (tmp.entries)))
                mainthread.execute (wc_success, async=True)
                success = True