vcs.maemo.org Git - feedingit/blob - src/rss_sqlite.py

   1 #!/usr/bin/env python2.5
   2
   3 #
   4 # Copyright (c) 2007-2008 INdT.
   5 # Copyright (c) 2011 Neal H. Walfield
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU Lesser General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 #  This program is distributed in the hope that it will be useful,
  12 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 #  GNU Lesser General Public License for more details.
  15 #
  16 #  You should have received a copy of the GNU Lesser General Public License
  17 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18 #
  19
  20 # ============================================================================
  21 # Name        : FeedingIt.py
  22 # Author      : Yves Marcoz
  23 # Version     : 0.5.4
  24 # Description : Simple RSS Reader
  25 # ============================================================================
  26
  27 from __future__ import with_statement
  28
  29 import sqlite3
  30 from os.path import isfile, isdir
  31 from shutil import rmtree
  32 from os import mkdir, remove, utime
  33 import os
  34 import md5
  35 import feedparser
  36 import time
  37 import urllib2
  38 from BeautifulSoup import BeautifulSoup
  39 from urlparse import urljoin
  40 from calendar import timegm
  41 import threading
  42 import traceback
  43 from wc import wc, wc_init, woodchuck
  44 import subprocess
  45 import dbus
  46 from updatedbus import update_server_object
  47
  48 from jobmanager import JobManager
  49 import mainthread
  50 from httpprogresshandler import HTTPProgressHandler
  51 import random
  52 import sys
  53 import logging
  54 logger = logging.getLogger(__name__)
  55
  56 def getId(string):
  57     if issubclass(string.__class__, unicode):
  58         string = string.encode('utf8', 'replace')
  59
  60     return md5.new(string).hexdigest()
  61
  62 def download_callback(connection):
  63     if JobManager().do_quit:
  64         raise KeyboardInterrupt
  65
  66 def downloader(progress_handler=None, proxy=None):
  67     openers = []
  68
  69     if progress_handler is not None:
  70         openers.append(progress_handler)
  71     else:
  72         openers.append(HTTPProgressHandler(download_callback))
  73
  74     if proxy:
  75         openers.append(proxy)
  76
  77     return urllib2.build_opener(*openers)
  78
  79 def transfer_stats(sent, received, **kwargs):
  80     """
  81     This function takes two arguments: sent is the number of bytes
  82     sent so far, received is the number of bytes received.  The
  83     function returns a continuation that you can call later.
  84
  85     The continuation takes the same two arguments.  It returns a tuple
  86     of the number of bytes sent, the number of bytes received and the
  87     time since the original function was invoked.
  88     """
  89     start_time = time.time()
  90     start_sent = sent
  91     start_received = received
  92
  93     def e(sent, received, **kwargs):
  94         return (sent - start_sent,
  95                 received - start_received,
  96                 time.time() - start_time)
  97
  98     return e
  99
 100 # If not None, a subprocess.Popen object corresponding to a
 101 # update_feeds.py process.
 102 update_feed_process = None
 103
 104 update_feeds_iface = None
 105
 106 jobs_at_start = 0
 107
 108 class BaseObject(object):
 109     # Columns to cache.  Classes that inherit from this and use the
 110     # cache mechanism should set this to a list of tuples, each of
 111     # which contains two entries: the table and the column.  Note that
 112     # both are case sensitive.
 113     cached_columns = ()
 114
 115     def cache_invalidate(self, table=None):
 116         """
 117         Invalidate the cache.
 118
 119         If table is not None, invalidate only the specified table.
 120         Otherwise, drop the whole cache.
 121         """
 122         if not hasattr(self, 'cache'):
 123             return
 124
 125         if table is None:
 126             del self.cache
 127         else:
 128             if table in self.cache:
 129                 del self.cache[table]
 130
 131     def lookup(self, table, column, id=None):
 132         """
 133         Look up a column or value.  Uses a cache for columns in
 134         cached_columns.  Note: the column is returned unsorted.
 135         """
 136         if not hasattr(self, 'cache'):
 137             self.cache = {}
 138
 139         # Cache data for at most 60 seconds.
 140         now = time.time()
 141         try:
 142             cache = self.cache[table]
 143
 144             if time.time() - cache[None] > 60:
 145                 # logger.debug("%s: Cache too old: clearing" % (table,))
 146                 del self.cache[table]
 147                 cache = None
 148         except KeyError:
 149             cache = None
 150
 151         if (cache is None
 152             or (table, column) not in self.cached_columns):
 153             # The cache is empty or the caller wants a column that we
 154             # don't cache.
 155             if (table, column) in self.cached_columns:
 156                 # logger.debug("%s: Rebuilding cache" % (table,))
 157
 158                 do_cache = True
 159
 160                 self.cache[table] = cache = {}
 161                 columns = []
 162                 for t, c in self.cached_columns:
 163                     if table == t:
 164                         cache[c] = {}
 165                         columns.append(c)
 166
 167                 columns.append('id')
 168                 where = ""
 169             else:
 170                 do_cache = False
 171
 172                 columns = (colums,)
 173                 if id is not None:
 174                     where = "where id = '%s'" % id
 175                 else:
 176                     where = ""
 177
 178             results = self.db.execute(
 179                 "SELECT %s FROM %s %s" % (','.join(columns), table, where))
 180
 181             if do_cache:
 182                 for r in results:
 183                     values = list(r)
 184                     i = values.pop()
 185                     for index, value in enumerate(values):
 186                         cache[columns[index]][i] = value
 187
 188                 cache[None] = now
 189             else:
 190                 results = []
 191                 for r in results:
 192                     if id is not None:
 193                         return values[0]
 194
 195                     results.append(values[0])
 196
 197                 return results
 198         else:
 199             cache = self.cache[table]
 200
 201         try:
 202             if id is not None:
 203                 value = cache[column][id]
 204                 # logger.debug("%s.%s:%s -> %s" % (table, column, id, value))
 205                 return value
 206             else:
 207                 return cache[column].values()
 208         except KeyError:
 209             # logger.debug("%s.%s:%s -> Not found" % (table, column, id))
 210             return None
 211
 212 class Feed(BaseObject):
 213     # Columns to cache.
 214     cached_columns = (('feed', 'read'),
 215                       ('feed', 'title'))
 216
 217     serial_execution_lock = threading.Lock()
 218
 219     def _getdb(self):
 220         try:
 221             db = self.tls.db
 222         except AttributeError:
 223             db = sqlite3.connect("%s/%s.db" % (self.dir, self.key), timeout=120)
 224             self.tls.db = db
 225         return db
 226     db = property(_getdb)
 227
 228     def __init__(self, configdir, key):
 229         self.key = key
 230         self.configdir = configdir
 231         self.dir = "%s/%s.d" %(self.configdir, self.key)
 232         self.tls = threading.local()
 233
 234         if not isdir(self.dir):
 235             mkdir(self.dir)
 236         filename = "%s/%s.db" % (self.dir, self.key)
 237         if not isfile(filename):
 238             self.db.execute("CREATE TABLE feed (id text, title text, contentLink text, contentHash text, date float, updated float, link text, read int);")
 239             self.db.execute("CREATE TABLE images (id text, imagePath text);")
 240             self.db.commit()
 241         else:
 242             try:
 243                 self.db.execute("ALTER TABLE feed ADD COLUMN contentHash text")
 244                 self.db.commit()
 245             except sqlite3.OperationalError, e:
 246                 if 'duplicate column name' in str(e):
 247                     pass
 248                 else:
 249                     logger.exception("Add column contentHash to %s", filename)
 250
 251     def addImage(self, configdir, key, baseurl, url, proxy=None, opener=None):
 252         filename = configdir+key+".d/"+getId(url)
 253         if not isfile(filename):
 254             try:
 255                 if not opener:
 256                     opener = downloader(proxy=proxy)
 257
 258                 abs_url = urljoin(baseurl,url)
 259                 f = opener.open(abs_url)
 260                 try:
 261                     with open(filename, "w") as outf:
 262                         for data in f:
 263                             outf.write(data)
 264                 finally:
 265                     f.close()
 266             except (urllib2.HTTPError, urllib2.URLError, IOError), exception:
 267                 logger.info("Could not download image %s: %s"
 268                             % (abs_url, str (exception)))
 269                 return None
 270             except:
 271                 exception = sys.exc_info()[0]
 272
 273                 logger.info("Downloading image %s: %s" %
 274                             (abs_url, traceback.format_exc()))
 275                 try:
 276                     remove(filename)
 277                 except OSError:
 278                     pass
 279
 280                 return None
 281         else:
 282             #open(filename,"a").close()  # "Touch" the file
 283             file = open(filename,"a")
 284             utime(filename, None)
 285             file.close()
 286         return filename
 287
 288     def updateFeed(self, configdir, url, etag, modified, expiryTime=24, proxy=None, imageCache=False, priority=0, postFeedUpdateFunc=None, *postFeedUpdateFuncArgs):
 289         if (os.path.basename(sys.argv[0]) == 'update_feeds.py'):
 290             def doit():
 291                 def it():
 292                     self._updateFeed(configdir, url, etag, modified, expiryTime, proxy, imageCache, postFeedUpdateFunc, *postFeedUpdateFuncArgs)
 293                 return it
 294             JobManager().execute(doit(), self.key, priority=priority)
 295         else:
 296             def send_update_request():
 297                 global update_feeds_iface
 298                 if update_feeds_iface is None:
 299                     bus=dbus.SessionBus()
 300                     remote_object = bus.get_object(
 301                         "org.marcoz.feedingit", # Connection name
 302                         "/org/marcoz/feedingit/update" # Object's path
 303                         )
 304                     update_feeds_iface = dbus.Interface(
 305                         remote_object, 'org.marcoz.feedingit')
 306
 307                 try:
 308                     update_feeds_iface.Update(self.key)
 309                 except Exception, e:
 310                     logger.error("Invoking org.marcoz.feedingit.Update: %s"
 311                                  % str(e))
 312                     update_feeds_iface = None
 313                 else:
 314                     return True
 315
 316             if send_update_request():
 317                 # Success!  It seems we were able to start the update
 318                 # daemon via dbus (or, it was already running).
 319                 return
 320
 321             global update_feed_process
 322             if (update_feed_process is None
 323                 or update_feed_process.poll() is not None):
 324                 # The update_feeds process is not running.  Start it.
 325                 update_feeds = os.path.join(os.path.dirname(__file__),
 326                                             'update_feeds.py')
 327                 argv = ['/usr/bin/env', 'python', update_feeds, '--daemon' ]
 328                 logger.debug("Starting update_feeds: running %s"
 329                              % (str(argv),))
 330                 update_feed_process = subprocess.Popen(argv)
 331                 # Make sure the dbus calls go to the right process:
 332                 # rebind.
 333                 update_feeds_iface = None
 334
 335             for _ in xrange(5):
 336                 if send_update_request():
 337                     break
 338                 time.sleep(1)
 339
 340     def _updateFeed(self, configdir, url, etag, modified, expiryTime=24, proxy=None, imageCache=False, postFeedUpdateFunc=None, *postFeedUpdateFuncArgs):
 341         logger.debug("Updating %s" % url)
 342
 343         success = False
 344         have_serial_execution_lock = False
 345         try:
 346             update_start = time.time ()
 347
 348             progress_handler = HTTPProgressHandler(download_callback)
 349
 350             openers = [progress_handler]
 351             if proxy:
 352                 openers.append (proxy)
 353             kwargs = {'handlers':openers}
 354
 355             feed_transfer_stats = transfer_stats(0, 0)
 356
 357             tmp=feedparser.parse(url, etag=etag, modified=modified, **kwargs)
 358             download_duration = time.time () - update_start
 359
 360             opener = downloader(progress_handler, proxy)
 361
 362             if JobManager().do_quit:
 363                 raise KeyboardInterrupt
 364
 365             process_start = time.time()
 366
 367             # Expiry time is in hours
 368             expiry = float(expiryTime) * 3600.
 369
 370             currentTime = 0
 371
 372             updated_objects = 0
 373             new_objects = 0
 374
 375             def wc_success():
 376                 try:
 377                     wc().stream_register (self.key, "", 6 * 60 * 60)
 378                 except woodchuck.ObjectExistsError:
 379                     pass
 380                 try:
 381                     wc()[self.key].updated (
 382                         indicator=(woodchuck.Indicator.ApplicationVisual
 383                                    |woodchuck.Indicator.StreamWide),
 384                         transferred_down=progress_handler.stats['received'],
 385                         transferred_up=progress_handler.stats['sent'],
 386                         transfer_time=update_start,
 387                         transfer_duration=download_duration,
 388                         new_objects=new_objects,
 389                         updated_objects=updated_objects,
 390                         objects_inline=new_objects + updated_objects)
 391                 except KeyError:
 392                     logger.warn(
 393                         "Failed to register update of %s with woodchuck!"
 394                         % (self.key))
 395
 396             http_status = tmp.get ('status', 200)
 397
 398             # Check if the parse was succesful.  If the http status code
 399             # is 304, then the download was successful, but there is
 400             # nothing new.  Indeed, no content is returned.  This make a
 401             # 304 look like an error because there are no entries and the
 402             # parse fails.  But really, everything went great!  Check for
 403             # this first.
 404             if http_status == 304:
 405                 logger.debug("%s: No changes to feed." % (self.key,))
 406                 mainthread.execute(wc_success, async=True)
 407                 success = True
 408             elif len(tmp["entries"])==0 and not tmp.get('version', None):
 409                 # An error occured fetching or parsing the feed.  (Version
 410                 # will be either None if e.g. the connection timed our or
 411                 # '' if the data is not a proper feed)
 412                 logger.error(
 413                     "Error fetching %s: version is: %s: error: %s"
 414                     % (url, str (tmp.get('version', 'unset')),
 415                        str (tmp.get ('bozo_exception', 'Unknown error'))))
 416                 logger.debug(tmp)
 417                 def register_stream_update_failed(http_status):
 418                     def doit():
 419                         logger.debug("%s: stream update failed!" % self.key)
 420
 421                         try:
 422                             # It's not easy to get the feed's title from here.
 423                             # At the latest, the next time the application is
 424                             # started, we'll fix up the human readable name.
 425                             wc().stream_register (self.key, "", 6 * 60 * 60)
 426                         except woodchuck.ObjectExistsError:
 427                             pass
 428                         ec = woodchuck.TransferStatus.TransientOther
 429                         if 300 <= http_status and http_status < 400:
 430                             ec = woodchuck.TransferStatus.TransientNetwork
 431                         if 400 <= http_status and http_status < 500:
 432                             ec = woodchuck.TransferStatus.FailureGone
 433                         if 500 <= http_status and http_status < 600:
 434                             ec = woodchuck.TransferStatus.TransientNetwork
 435                         wc()[self.key].update_failed(ec)
 436                     return doit
 437                 if wc().available:
 438                     mainthread.execute(
 439                         register_stream_update_failed(
 440                             http_status=http_status),
 441                         async=True)
 442             else:
 443                currentTime = time.time()
 444                # The etag and modified value should only be updated if the content was not null
 445                try:
 446                    etag = tmp["etag"]
 447                except KeyError:
 448                    etag = None
 449                try:
 450                    modified = tmp["modified"]
 451                except KeyError:
 452                    modified = None
 453                try:
 454                    abs_url = urljoin(tmp["feed"]["link"],"/favicon.ico")
 455                    f = opener.open(abs_url)
 456                    data = f.read()
 457                    f.close()
 458                    outf = open(self.dir+"/favicon.ico", "w")
 459                    outf.write(data)
 460                    outf.close()
 461                    del data
 462                except (urllib2.HTTPError, urllib2.URLError), exception:
 463                    logger.debug("Could not download favicon %s: %s"
 464                                 % (abs_url, str (exception)))
 465
 466                self.serial_execution_lock.acquire ()
 467                have_serial_execution_lock = True
 468
 469                #reversedEntries = self.getEntries()
 470                #reversedEntries.reverse()
 471
 472                tmp["entries"].reverse()
 473                for entry in tmp["entries"]:
 474                    # Yield so as to make the main thread a bit more
 475                    # responsive.
 476                    time.sleep(0)
 477
 478                    entry_transfer_stats = transfer_stats(
 479                        *feed_transfer_stats(**progress_handler.stats)[0:2])
 480
 481                    if JobManager().do_quit:
 482                        raise KeyboardInterrupt
 483
 484                    object_size = 0
 485
 486                    date = self.extractDate(entry)
 487                    try:
 488                        entry["title"]
 489                    except KeyError:
 490                        entry["title"] = "No Title"
 491                    try :
 492                        entry["link"]
 493                    except KeyError:
 494                        entry["link"] = ""
 495                    try:
 496                        entry["author"]
 497                    except KeyError:
 498                        entry["author"] = None
 499                    if(not(entry.has_key("id"))):
 500                        entry["id"] = None
 501                    content = self.extractContent(entry)
 502                    contentHash = getId(content)
 503                    object_size = len (content)
 504                    tmpEntry = {"title":entry["title"], "content":content,
 505                                 "date":date, "link":entry["link"], "author":entry["author"], "id":entry["id"]}
 506                    id = self.generateUniqueId(tmpEntry)
 507
 508                    current_version = self.db.execute(
 509                        'select date, ROWID, contentHash from feed where id=?',
 510                        (id,)).fetchone()
 511                    if (current_version is not None
 512                        # To detect updates, don't compare by date:
 513                        # compare by content.
 514                        #
 515                        # - If an article update is just a date change
 516                        #   and the content remains the same, we don't
 517                        #   want to register an update.
 518                        #
 519                        # - If an article's content changes but not the
 520                        #   date, we want to recognize an update.
 521                        and current_version[2] == contentHash):
 522                        logger.debug("ALREADY DOWNLOADED %s (%s)"
 523                                     % (entry["title"], entry["link"]))
 524                        ## This article is already present in the feed listing. Update the "updated" time, so it doesn't expire
 525                        self.db.execute("UPDATE feed SET updated=? WHERE id=?;",(currentTime,id))
 526                        try:
 527                            logger.debug("Updating already downloaded files for %s" %(id))
 528                            filename = configdir+self.key+".d/"+id+".html"
 529                            file = open(filename,"a")
 530                            utime(filename, None)
 531                            file.close()
 532                            images = self.db.execute("SELECT imagePath FROM images where id=?;", (id, )).fetchall()
 533                            for image in images:
 534                                 file = open(image[0],"a")
 535                                 utime(image[0], None)
 536                                 file.close()
 537                        except:
 538                            logger.debug("Error in refreshing images for %s" % (id))
 539                        self.db.commit()
 540                        continue
 541
 542                    if current_version is not None:
 543                        # The version was updated.  Mark it as unread.
 544                        logger.debug("UPDATED: %s (%s)"
 545                                     % (entry["title"], entry["link"]))
 546                        updated_objects += 1
 547                    else:
 548                        logger.debug("NEW: %s (%s)"
 549                                     % (entry["title"], entry["link"]))
 550                        new_objects += 1
 551
 552                    #articleTime = time.mktime(self.entries[id]["dateTuple"])
 553                    soup = BeautifulSoup(self.getArticle(tmpEntry)) #tmpEntry["content"])
 554                    images = soup('img')
 555                    baseurl = tmpEntry["link"]
 556                    if imageCache and len(images) > 0:
 557                        self.serial_execution_lock.release ()
 558                        have_serial_execution_lock = False
 559                        for img in images:
 560                            if not img.has_key('src'):
 561                                continue
 562
 563                            filename = self.addImage(
 564                                configdir, self.key, baseurl, img['src'],
 565                                opener=opener)
 566                            if filename:
 567                                 img['src']="file://%s" %filename
 568                                 count = self.db.execute("SELECT count(1) FROM images where id=? and imagePath=?;", (id, filename )).fetchone()[0]
 569                                 if count == 0:
 570                                     self.db.execute("INSERT INTO images (id, imagePath) VALUES (?, ?);", (id, filename) )
 571                                     self.db.commit()
 572
 573                                 try:
 574                                     object_size += os.path.getsize (filename)
 575                                 except os.error, exception:
 576                                     logger.error ("Error getting size of %s: %s"
 577                                                   % (filename, exception))
 578                        self.serial_execution_lock.acquire ()
 579                        have_serial_execution_lock = True
 580
 581                    tmpEntry["contentLink"] = configdir+self.key+".d/"+id+".html"
 582                    file = open(tmpEntry["contentLink"], "w")
 583                    file.write(soup.prettify())
 584                    file.close()
 585
 586                    values = {'id': id,
 587                              'title': tmpEntry["title"],
 588                              'contentLink': tmpEntry["contentLink"],
 589                              'contentHash': contentHash,
 590                              'date': tmpEntry["date"],
 591                              'updated': currentTime,
 592                              'link': tmpEntry["link"],
 593                              'read': 0}
 594
 595                    if current_version is not None:
 596                        # This is an update.  Ensure that the existing
 597                        # entry is replaced.
 598                        values['ROWID'] = current_version[1]
 599
 600                    cols, values = zip(*values.items())
 601                    self.db.execute(
 602                        "INSERT OR REPLACE INTO feed (%s) VALUES (%s);"
 603                        % (','.join(cols), ','.join(('?',) * len(values))),
 604                        values)
 605                    self.db.commit()
 606
 607                    # Register the object with Woodchuck and mark it as
 608                    # downloaded.
 609                    def register_object_transferred(
 610                            id, title, publication_time,
 611                            sent, received, object_size):
 612                        def doit():
 613                            logger.debug("Registering transfer of object %s"
 614                                         % title)
 615                            try:
 616                                obj = wc()[self.key].object_register(
 617                                    object_identifier=id,
 618                                    human_readable_name=title)
 619                            except woodchuck.ObjectExistsError:
 620                                obj = wc()[self.key][id]
 621                            else:
 622                                obj.publication_time = publication_time
 623                                obj.transferred(
 624                                    indicator=(
 625                                        woodchuck.Indicator.ApplicationVisual
 626                                        |woodchuck.Indicator.StreamWide),
 627                                    transferred_down=received,
 628                                    transferred_up=sent,
 629                                    object_size=object_size)
 630                        return doit
 631                    if wc().available:
 632                        # If the entry does not contain a publication
 633                        # time, the attribute won't exist.
 634                        pubtime = entry.get('date_parsed', None)
 635                        if pubtime:
 636                            publication_time = time.mktime (pubtime)
 637                        else:
 638                            publication_time = None
 639
 640                        sent, received, _ \
 641                            = entry_transfer_stats(**progress_handler.stats)
 642                        # sent and received are for objects (in
 643                        # particular, images) associated with this
 644                        # item.  We also want to attribute the data
 645                        # transferred for the item's content.  This is
 646                        # a good first approximation.
 647                        received += len(content)
 648
 649                        mainthread.execute(
 650                            register_object_transferred(
 651                                id=id,
 652                                title=tmpEntry["title"],
 653                                publication_time=publication_time,
 654                                sent=sent, received=received,
 655                                object_size=object_size),
 656                            async=True)
 657                self.db.commit()
 658
 659                sent, received, _ \
 660                    = feed_transfer_stats(**progress_handler.stats)
 661                logger.debug (
 662                    "%s: Update successful: transferred: %d/%d; objects: %d)"
 663                    % (url, sent, received, len (tmp.entries)))
 664                mainthread.execute (wc_success, async=True)
 665                success = True
 666
 667             rows = self.db.execute("SELECT id FROM feed WHERE (read=0 AND updated<?) OR (read=1 AND updated<?);", (currentTime-2*expiry, currentTime-expiry))
 668             for row in rows:
 669                self.removeEntry(row[0])
 670
 671             from glob import glob
 672             from os import stat
 673             for file in glob(configdir+self.key+".d/*"):
 674                 #
 675                 stats = stat(file)
 676                 #
 677                 # put the two dates into matching format
 678                 #
 679                 lastmodDate = stats[8]
 680                 #
 681                 expDate = time.time()-expiry*3
 682                 # check if image-last-modified-date is outdated
 683                 #
 684                 if expDate > lastmodDate:
 685                     #
 686                     try:
 687                         #
 688                         #print 'Removing', file
 689                         #
 690                         # XXX: Tell woodchuck.
 691                         remove(file) # commented out for testing
 692                         #
 693                     except OSError, exception:
 694                         #
 695                         logger.error('Could not remove %s: %s'
 696                                      % (file, str (exception)))
 697             logger.debug("updated %s: %fs in download, %fs in processing"
 698                          % (self.key, download_duration,
 699                             time.time () - process_start))
 700         except:
 701             logger.error("Updating %s: %s" % (self.key, traceback.format_exc()))
 702         finally:
 703             self.db.commit ()
 704
 705             if have_serial_execution_lock:
 706                 self.serial_execution_lock.release ()
 707
 708             updateTime = 0
 709             try:
 710                 rows = self.db.execute("SELECT MAX(date) FROM feed;")
 711                 for row in rows:
 712                     updateTime=row[0]
 713             except Exception, e:
 714                 logger.error("Fetching update time: %s: %s"
 715                              % (str(e), traceback.format_exc()))
 716             finally:
 717                 if not success:
 718                     etag = None
 719                     modified = None
 720                 title = None
 721                 try:
 722                     title = tmp.feed.title
 723                 except (AttributeError, UnboundLocalError), exception:
 724                     pass
 725                 if postFeedUpdateFunc is not None:
 726                     postFeedUpdateFunc (self.key, updateTime, etag, modified,
 727                                         title, *postFeedUpdateFuncArgs)
 728
 729         self.cache_invalidate()
 730
 731     def setEntryRead(self, id):
 732         self.db.execute("UPDATE feed SET read=1 WHERE id=?;", (id,) )
 733         self.db.commit()
 734
 735         def doit():
 736             try:
 737                 wc()[self.key][id].used()
 738             except KeyError:
 739                 pass
 740         if wc().available():
 741             mainthread.execute(doit, async=True)
 742         self.cache_invalidate('feed')
 743
 744     def setEntryUnread(self, id):
 745         self.db.execute("UPDATE feed SET read=0 WHERE id=?;", (id,) )
 746         self.db.commit()
 747         self.cache_invalidate('feed')
 748
 749     def markAllAsRead(self):
 750         self.db.execute("UPDATE feed SET read=1 WHERE read=0;")
 751         self.db.commit()
 752         self.cache_invalidate('feed')
 753
 754     def isEntryRead(self, id):
 755         return self.lookup('feed', 'read', id) == 1
 756
 757     def getTitle(self, id):
 758         return self.lookup('feed', 'title', id)
 759
 760     def getContentLink(self, id):
 761         return self.db.execute("SELECT contentLink FROM feed WHERE id=?;", (id,) ).fetchone()[0]
 762
 763     def getContentHash(self, id):
 764         return self.db.execute("SELECT contentHash FROM feed WHERE id=?;", (id,) ).fetchone()[0]
 765
 766     def getExternalLink(self, id):
 767         return self.db.execute("SELECT link FROM feed WHERE id=?;", (id,) ).fetchone()[0]
 768
 769     def getDate(self, id):
 770         dateStamp = self.db.execute("SELECT date FROM feed WHERE id=?;", (id,) ).fetchone()[0]
 771         return time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime(dateStamp))
 772
 773     def getDateTuple(self, id):
 774         dateStamp = self.db.execute("SELECT date FROM feed WHERE id=?;", (id,) ).fetchone()[0]
 775         return time.localtime(dateStamp)
 776
 777     def getDateStamp(self, id):
 778         return self.db.execute("SELECT date FROM feed WHERE id=?;", (id,) ).fetchone()[0]
 779
 780     def generateUniqueId(self, entry):
 781         """
 782         Generate a stable identifier for the article.  For the same
 783         entry, this should result in the same identifier.  If
 784         possible, the identifier should remain the same even if the
 785         article is updated.
 786         """
 787         # Prefer the entry's id, which is supposed to be globally
 788         # unique.
 789         key = entry.get('id', None)
 790         if not key:
 791             # Next, try the link to the content.
 792             key = entry.get('link', None)
 793         if not key:
 794             # Ok, the title and the date concatenated are likely to be
 795             # relatively stable.
 796             key = entry.get('title', None) + entry.get('date', None)
 797         if not key:
 798             # Hmm, the article's content will at least guarantee no
 799             # false negatives (i.e., missing articles)
 800             key = entry.get('content', None)
 801         if not key:
 802             # If all else fails, just use a random number.
 803             key = str (random.random ())
 804         return getId (key)
 805
 806     def getIds(self, onlyUnread=False):
 807         if onlyUnread:
 808             rows = self.db.execute("SELECT id FROM feed where read=0 ORDER BY date DESC;").fetchall()
 809         else:
 810             rows = self.db.execute("SELECT id FROM feed ORDER BY date DESC;").fetchall()
 811         ids = []
 812         for row in rows:
 813             ids.append(row[0])
 814         #ids.reverse()
 815         return ids
 816
 817     def getNextId(self, id, forward=True):
 818         if forward:
 819             delta = 1
 820         else:
 821             delta = -1
 822         ids = self.getIds()
 823         index = ids.index(id)
 824         return ids[(index + delta) % len(ids)]
 825
 826     def getPreviousId(self, id):
 827         return self.getNextId(id, forward=False)
 828
 829     def getNumberOfUnreadItems(self):
 830         return self.db.execute("SELECT count(*) FROM feed WHERE read=0;").fetchone()[0]
 831
 832     def getNumberOfEntries(self):
 833         return self.db.execute("SELECT count(*) FROM feed;").fetchone()[0]
 834
 835     def getArticle(self, entry):
 836         #self.setEntryRead(id)
 837         #entry = self.entries[id]
 838         title = entry['title']
 839         #content = entry.get('content', entry.get('summary_detail', {}))
 840         content = entry["content"]
 841
 842         link = entry['link']
 843         author = entry['author']
 844         date = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime(entry["date"]) )
 845
 846         #text = '''<div style="color: black; background-color: white;">'''
 847         text = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">'
 848         text += "<html><head><title>" + title + "</title>"
 849         text += '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>\n'
 850         #text += '<style> body {-webkit-user-select: none;} </style>'
 851         text += '</head><body bgcolor=\"#ffffff\"><div><a href=\"' + link + '\">' + title + "</a>"
 852         if author != None:
 853             text += "<BR /><small><i>Author: " + author + "</i></small>"
 854         text += "<BR /><small><i>Date: " + date + "</i></small></div>"
 855         text += "<BR /><BR />"
 856         text += content
 857         text += "</body></html>"
 858         return text
 859
 860     def getContent(self, id):
 861         """
 862         Return the content of the article with the specified ID.  If
 863         the content is not available, returns None.
 864         """
 865         contentLink = self.getContentLink(id)
 866         try:
 867             with open(contentLink, 'r') as file:
 868                 content = file.read()
 869         except Exception:
 870             logger.exception("Failed get content for %s: reading %s failed",
 871                              id, contentLink)
 872             content = None
 873         return content
 874
 875     def extractDate(self, entry):
 876         if entry.has_key("updated_parsed"):
 877             try:
 878                 return timegm(entry.updated_parsed)
 879             except (TypeError, IndexError):
 880                 # entry.updated_parsed is garbage.
 881                 pass
 882         if entry.has_key("published_parsed"):
 883             try:
 884                 return timegm(entry.published_parsed)
 885             except (TypeError, IndexError):
 886                 # entry.published_parsed is garbage.
 887                 pass
 888
 889         return time.time()
 890
 891     def extractContent(self, entry):
 892         content = ""
 893         if entry.has_key('summary'):
 894             content = entry.get('summary', '')
 895         if entry.has_key('content'):
 896             if len(entry.content[0].value) > len(content):
 897                 content = entry.content[0].value
 898         if content == "":
 899             content = entry.get('description', '')
 900         return content
 901
 902     def removeEntry(self, id):
 903         contentLink = self.db.execute("SELECT contentLink FROM feed WHERE id=?;", (id,)).fetchone()[0]
 904         if contentLink:
 905             try:
 906                 remove(contentLink)
 907             except OSError, exception:
 908                 logger.error("Deleting %s: %s" % (contentLink, str (exception)))
 909         self.db.execute("DELETE FROM feed WHERE id=?;", (id,) )
 910         self.db.execute("DELETE FROM images WHERE id=?;", (id,) )
 911         self.db.commit()
 912
 913         def doit():
 914             try:
 915                 wc()[self.key][id].files_deleted (
 916                     woodchuck.DeletionResponse.Deleted)
 917                 del wc()[self.key][id]
 918             except KeyError:
 919                 pass
 920         if wc().available():
 921             mainthread.execute (doit, async=True)
 922
 923 class ArchivedArticles(Feed):
 924     def addArchivedArticle(self, title, link, date, configdir):
 925         id = self.generateUniqueId({"date":date, "title":title})
 926         values = (id, title, link, date, 0, link, 0)
 927         self.db.execute("INSERT INTO feed (id, title, contentLink, date, updated, link, read) VALUES (?, ?, ?, ?, ?, ?, ?);", values)
 928         self.db.commit()
 929
 930     # Feed.UpdateFeed calls this function.
 931     def _updateFeed(self, configdir, url, etag, modified, expiryTime=24, proxy=None, imageCache=False, priority=0, postFeedUpdateFunc=None, *postFeedUpdateFuncArgs):
 932         currentTime = 0
 933         rows = self.db.execute("SELECT id, link FROM feed WHERE updated=0;")
 934         for row in rows:
 935             try:
 936                 currentTime = time.time()
 937                 id = row[0]
 938                 link = row[1]
 939                 f = urllib2.urlopen(link)
 940                 #entry["content"] = f.read()
 941                 html = f.read()
 942                 f.close()
 943                 soup = BeautifulSoup(html)
 944                 images = soup('img')
 945                 baseurl = link
 946                 for img in images:
 947                     filename = self.addImage(configdir, self.key, baseurl, img['src'], proxy=proxy)
 948                     img['src']=filename
 949                     self.db.execute("INSERT INTO images (id, imagePath) VALUES (?, ?);", (id, filename) )
 950                     self.db.commit()
 951                 contentLink = configdir+self.key+".d/"+id+".html"
 952                 file = open(contentLink, "w")
 953                 file.write(soup.prettify())
 954                 file.close()
 955
 956                 self.db.execute("UPDATE feed SET read=0, contentLink=?, updated=? WHERE id=?;", (contentLink, time.time(), id) )
 957                 self.db.commit()
 958             except:
 959                 logger.error("Error updating Archived Article: %s %s"
 960                              % (link,traceback.format_exc(),))
 961
 962         if postFeedUpdateFunc is not None:
 963             postFeedUpdateFunc (self.key, currentTime, None, None, None,
 964                                 *postFeedUpdateFuncArgs)
 965
 966     def purgeReadArticles(self):
 967         rows = self.db.execute("SELECT id FROM feed WHERE read=1;")
 968         #ids = self.getIds()
 969         for row in rows:
 970             self.removeArticle(row[0])
 971
 972     def removeArticle(self, id):
 973         rows = self.db.execute("SELECT imagePath FROM images WHERE id=?;", (id,) )
 974         for row in rows:
 975             try:
 976                 count = self.db.execute("SELECT count(*) FROM images WHERE id!=? and imagePath=?;", (id,row[0]) ).fetchone()[0]
 977                 if count == 0:
 978                     os.remove(row[0])
 979             except:
 980                 pass
 981         self.removeEntry(id)
 982
 983 class Listing(BaseObject):
 984     # Columns to cache.
 985     cached_columns = (('feeds', 'updateTime'),
 986                       ('feeds', 'unread'),
 987                       ('feeds', 'title'),
 988                       ('categories', 'title'))
 989
 990     def _getdb(self):
 991         try:
 992             db = self.tls.db
 993         except AttributeError:
 994             db = sqlite3.connect("%s/feeds.db" % self.configdir, timeout=120)
 995             self.tls.db = db
 996         return db
 997     db = property(_getdb)
 998
 999     # Lists all the feeds in a dictionary, and expose the data
1000     def __init__(self, config, configdir):
1001         self.config = config
1002         self.configdir = configdir
1003
1004         self.tls = threading.local ()
1005
1006         try:
1007             table = self.db.execute("SELECT sql FROM sqlite_master").fetchone()
1008             if table == None:
1009                 self.db.execute("CREATE TABLE feeds(id text, url text, title text, unread int, updateTime float, rank int, etag text, modified text, widget int, category int);")
1010                 self.db.execute("CREATE TABLE categories(id text, title text, unread int, rank int);")
1011                 self.addCategory("Default Category")
1012                 if isfile(self.configdir+"feeds.pickle"):
1013                     self.importOldFormatFeeds()
1014                 else:
1015                     self.addFeed("Maemo News", "http://maemo.org/news/items.xml")
1016             else:
1017                 from string import find, upper
1018                 if find(upper(table[0]), "WIDGET")<0:
1019                     self.db.execute("ALTER TABLE feeds ADD COLUMN widget int;")
1020                     self.db.execute("UPDATE feeds SET widget=1;")
1021                     self.db.commit()
1022                 if find(upper(table[0]), "CATEGORY")<0:
1023                     self.db.execute("CREATE TABLE categories(id text, title text, unread int, rank int);")
1024                     self.addCategory("Default Category")
1025                     self.db.execute("ALTER TABLE feeds ADD COLUMN category int;")
1026                     self.db.execute("UPDATE feeds SET category=1;")
1027             self.db.commit()
1028         except:
1029             pass
1030
1031         # Check that Woodchuck's state is up to date with respect our
1032         # state.
1033         try:
1034             updater = os.path.basename(sys.argv[0]) == 'update_feeds.py'
1035             wc_init(config, self, True if updater else False)
1036             if wc().available() and updater:
1037                 # The list of known streams.
1038                 streams = wc().streams_list ()
1039                 stream_ids = [s.identifier for s in streams]
1040
1041                 # Register any unknown streams.  Remove known streams from
1042                 # STREAMS_IDS.
1043                 for key in self.getListOfFeeds():
1044                     title = self.getFeedTitle(key)
1045                     # XXX: We should also check whether the list of
1046                     # articles/objects in each feed/stream is up to date.
1047                     if key not in stream_ids:
1048                         logger.debug(
1049                             "Registering previously unknown channel: %s (%s)"
1050                             % (key, title,))
1051                         wc().stream_register(
1052                             key, title,
1053                             self.config.getUpdateInterval() * 60 * 60)
1054                     else:
1055                         # Make sure the human readable name is up to date.
1056                         if wc()[key].human_readable_name != title:
1057                             wc()[key].human_readable_name = title
1058                         stream_ids.remove (key)
1059                         wc()[key].freshness \
1060                             = self.config.getUpdateInterval() * 60 * 60
1061
1062
1063                 # Unregister any streams that are no longer subscribed to.
1064                 for id in stream_ids:
1065                     logger.debug("Unregistering %s" % (id,))
1066                     wc().stream_unregister (id)
1067         except Exception:
1068             logger.exception("Registering streams with Woodchuck")
1069
1070     def importOldFormatFeeds(self):
1071         """This function loads feeds that are saved in an outdated format, and converts them to sqlite"""
1072         import rss
1073         listing = rss.Listing(self.configdir)
1074         rank = 0
1075         for id in listing.getListOfFeeds():
1076             try:
1077                 rank += 1
1078                 values = (id, listing.getFeedTitle(id) , listing.getFeedUrl(id), 0, time.time(), rank, None, "None", 1)
1079                 self.db.execute("INSERT INTO feeds (id, title, url, unread, updateTime, rank, etag, modified, widget, category) VALUES (?, ?, ? ,? ,? ,?, ?, ?, ?, 1);", values)
1080                 self.db.commit()
1081
1082                 feed = listing.getFeed(id)
1083                 new_feed = self.getFeed(id)
1084
1085                 items = feed.getIds()[:]
1086                 items.reverse()
1087                 for item in items:
1088                         if feed.isEntryRead(item):
1089                             read_status = 1
1090                         else:
1091                             read_status = 0
1092                         date = timegm(feed.getDateTuple(item))
1093                         title = feed.getTitle(item)
1094                         newId = new_feed.generateUniqueId({"date":date, "title":title})
1095                         values = (newId, title , feed.getContentLink(item), date, tuple(time.time()), feed.getExternalLink(item), read_status)
1096                         new_feed.db.execute("INSERT INTO feed (id, title, contentLink, date, updated, link, read) VALUES (?, ?, ?, ?, ?, ?, ?);", values)
1097                         new_feed.db.commit()
1098                         try:
1099                             images = feed.getImages(item)
1100                             for image in images:
1101                                 new_feed.db.execute("INSERT INTO images (id, imagePath) VALUES (?, ?);", (item, image) )
1102                                 new_feed.db.commit()
1103                         except:
1104                             pass
1105                 self.updateUnread(id)
1106             except:
1107                 logger.error("importOldFormatFeeds: %s"
1108                              % (traceback.format_exc(),))
1109         remove(self.configdir+"feeds.pickle")
1110
1111
1112     def addArchivedArticle(self, key, index):
1113         feed = self.getFeed(key)
1114         title = feed.getTitle(index)
1115         link = feed.getExternalLink(index)
1116         date = feed.getDate(index)
1117         count = self.db.execute("SELECT count(*) FROM feeds where id=?;", ("ArchivedArticles",) ).fetchone()[0]
1118         if count == 0:
1119             self.addFeed("Archived Articles", "", id="ArchivedArticles")
1120
1121         archFeed = self.getFeed("ArchivedArticles")
1122         archFeed.addArchivedArticle(title, link, date, self.configdir)
1123         self.updateUnread("ArchivedArticles")
1124
1125     def updateFeed(self, key, expiryTime=None, proxy=None, imageCache=None,
1126                    priority=0):
1127         if expiryTime is None:
1128             expiryTime = self.config.getExpiry()
1129         if not expiryTime:
1130             # Default to 24 hours
1131             expriyTime = 24
1132         if proxy is None:
1133             (use_proxy, proxy) = self.config.getProxy()
1134             if not use_proxy:
1135                 proxy = None
1136         if imageCache is None:
1137             imageCache = self.config.getImageCache()
1138
1139         feed = self.getFeed(key)
1140         (url, etag, modified) = self.db.execute("SELECT url, etag, modified FROM feeds WHERE id=?;", (key,) ).fetchone()
1141         try:
1142             modified = time.struct_time(eval(modified))
1143         except:
1144             modified = None
1145         feed.updateFeed(
1146             self.configdir, url, etag, modified, expiryTime, proxy, imageCache,
1147             priority, postFeedUpdateFunc=self._queuePostFeedUpdate)
1148
1149     def _queuePostFeedUpdate(self, *args, **kwargs):
1150         mainthread.execute (self._postFeedUpdate, async=True, *args, **kwargs)
1151
1152     def _postFeedUpdate(self, key, updateTime, etag, modified, title):
1153         if modified==None:
1154             modified="None"
1155         else:
1156             modified=str(tuple(modified))
1157         if updateTime > 0:
1158             self.db.execute("UPDATE feeds SET updateTime=?, etag=?, modified=? WHERE id=?;", (updateTime, etag, modified, key) )
1159         else:
1160             self.db.execute("UPDATE feeds SET etag=?, modified=? WHERE id=?;", (etag, modified, key) )
1161
1162         if title is not None:
1163             self.db.execute("UPDATE feeds SET title=(case WHEN title=='' THEN ? ELSE title END) where id=?;",
1164                             (title, key))
1165         self.db.commit()
1166         self.cache_invalidate('feeds')
1167         self.updateUnread(key)
1168
1169         update_server_object().ArticleCountUpdated()
1170
1171         stats = JobManager().stats()
1172         global jobs_at_start
1173         completed = stats['jobs-completed'] - jobs_at_start
1174         in_progress = stats['jobs-in-progress']
1175         queued = stats['jobs-queued']
1176
1177         try:
1178             percent = (100 * ((completed + in_progress / 2.))
1179                        / (completed + in_progress + queued))
1180         except ZeroDivisionError:
1181             percent = 100
1182
1183         update_server_object().UpdateProgress(
1184             percent, completed, in_progress, queued, 0, 0, 0, key)
1185
1186         if in_progress == 0 and queued == 0:
1187             jobs_at_start = stats['jobs-completed']
1188
1189     def getFeed(self, key):
1190         if key == "ArchivedArticles":
1191             return ArchivedArticles(self.configdir, key)
1192         return Feed(self.configdir, key)
1193
1194     def editFeed(self, key, title, url, category=None):
1195         if category:
1196             self.db.execute("UPDATE feeds SET title=?, url=?, category=? WHERE id=?;", (title, url, category, key))
1197         else:
1198             self.db.execute("UPDATE feeds SET title=?, url=? WHERE id=?;", (title, url, key))
1199         self.db.commit()
1200         self.cache_invalidate('feeds')
1201
1202         if wc().available():
1203             try:
1204                 wc()[key].human_readable_name = title
1205             except KeyError:
1206                 logger.debug("Feed %s (%s) unknown." % (key, title))
1207
1208     def getFeedUpdateTime(self, key):
1209         update_time = self.lookup('feeds', 'updateTime', key)
1210
1211         if not update_time:
1212             return "Never"
1213
1214         delta = time.time() - update_time
1215
1216         delta_hours = delta / (60. * 60.)
1217         if delta_hours < .1:
1218             return "A few minutes ago"
1219         if delta_hours < .75:
1220             return "Less than an hour ago"
1221         if delta_hours < 1.5:
1222             return "About an hour ago"
1223         if delta_hours < 18:
1224             return "About %d hours ago" % (int(delta_hours + 0.5),)
1225
1226         delta_days = delta_hours / 24.
1227         if delta_days < 1.5:
1228             return "About a day ago"
1229         if delta_days < 18:
1230             return "%d days ago" % (int(delta_days + 0.5),)
1231
1232         delta_weeks = delta_days / 7.
1233         if delta_weeks <= 8:
1234             return "%d weeks ago" % int(delta_weeks + 0.5)
1235
1236         delta_months = delta_days / 30.
1237         if delta_months <= 30:
1238             return "%d months ago" % int(delta_months + 0.5)
1239
1240         return time.strftime("%x", time.gmtime(update_time))
1241
1242     def getFeedNumberOfUnreadItems(self, key):
1243         return self.lookup('feeds', 'unread', key)
1244
1245     def getFeedTitle(self, key):
1246         title = self.lookup('feeds', 'title', key)
1247         if title:
1248             return title
1249
1250         return self.getFeedUrl(key)
1251
1252     def getFeedUrl(self, key):
1253         return self.db.execute("SELECT url FROM feeds WHERE id=?;", (key,)).fetchone()[0]
1254
1255     def getFeedCategory(self, key):
1256         return self.db.execute("SELECT category FROM feeds WHERE id=?;", (key,)).fetchone()[0]
1257
1258     def getListOfFeeds(self, category=None):
1259         if category:
1260             rows = self.db.execute("SELECT id FROM feeds WHERE category=? ORDER BY rank;", (category, ) )
1261         else:
1262             rows = self.db.execute("SELECT id FROM feeds ORDER BY rank;" )
1263         keys = []
1264         for row in rows:
1265             if row[0]:
1266                 keys.append(row[0])
1267         return keys
1268
1269     def getListOfCategories(self):
1270         return list(row[0] for row in self.db.execute(
1271                 "SELECT id FROM categories ORDER BY rank;"))
1272
1273     def getCategoryTitle(self, id):
1274         return self.lookup('categories', 'title', id)
1275
1276     def getCategoryUnread(self, id):
1277         count = 0
1278         for key in self.getListOfFeeds(category=id):
1279             try:
1280                 count = count + self.getFeedNumberOfUnreadItems(key)
1281             except:
1282                 pass
1283         return count
1284
1285     def getSortedListOfKeys(self, order, onlyUnread=False, category=1):
1286         if   order == "Most unread":
1287             tmp = "ORDER BY unread DESC"
1288             #keyorder = sorted(feedInfo, key = lambda k: feedInfo[k][1], reverse=True)
1289         elif order == "Least unread":
1290             tmp = "ORDER BY unread"
1291             #keyorder = sorted(feedInfo, key = lambda k: feedInfo[k][1])
1292         elif order == "Most recent":
1293             tmp = "ORDER BY updateTime DESC"
1294             #keyorder = sorted(feedInfo, key = lambda k: feedInfo[k][2], reverse=True)
1295         elif order == "Least recent":
1296             tmp = "ORDER BY updateTime"
1297             #keyorder = sorted(feedInfo, key = lambda k: feedInfo[k][2])
1298         else: # order == "Manual" or invalid value...
1299             tmp = "ORDER BY rank"
1300             #keyorder = sorted(feedInfo, key = lambda k: feedInfo[k][0])
1301         if onlyUnread:
1302             sql = "SELECT id FROM feeds WHERE unread>0 AND category=%s " %category + tmp
1303         else:
1304             sql = "SELECT id FROM feeds WHERE category=%s " %category + tmp
1305         rows = self.db.execute(sql)
1306         keys = []
1307         for row in rows:
1308             if row[0]:
1309                 keys.append(row[0])
1310         return keys
1311
1312     def getFavicon(self, key):
1313         filename = "%s%s.d/favicon.ico" % (self.configdir, key)
1314         if isfile(filename):
1315             return filename
1316         else:
1317             return False
1318
1319     def updateUnread(self, key):
1320         feed = self.getFeed(key)
1321         self.db.execute("UPDATE feeds SET unread=? WHERE id=?;", (feed.getNumberOfUnreadItems(), key))
1322         self.db.commit()
1323         self.cache_invalidate('feeds')
1324
1325     def addFeed(self, title, url, id=None, category=1):
1326         if not id:
1327             id = getId(url)
1328         count = self.db.execute("SELECT count(*) FROM feeds WHERE id=?;", (id,) ).fetchone()[0]
1329         if count == 0:
1330             max_rank = self.db.execute("SELECT MAX(rank) FROM feeds;").fetchone()[0]
1331             if max_rank == None:
1332                 max_rank = 0
1333             values = (id, title, url, 0, 0, max_rank+1, None, "None", 1, category)
1334             self.db.execute("INSERT INTO feeds (id, title, url, unread, updateTime, rank, etag, modified, widget, category) VALUES (?, ?, ? ,? ,? ,?, ?, ?, ?,?);", values)
1335             self.db.commit()
1336             # Ask for the feed object, it will create the necessary tables
1337             self.getFeed(id)
1338
1339             if wc().available():
1340                 # Register the stream with Woodchuck.  Update approximately
1341                 # every 6 hours.
1342                 wc().stream_register(stream_identifier=id,
1343                                      human_readable_name=title,
1344                                      freshness=6*60*60)
1345
1346             self.cache_invalidate('feeds')
1347             return True
1348         else:
1349             return False
1350
1351     def addCategory(self, title):
1352         rank = self.db.execute("SELECT MAX(rank)+1 FROM categories;").fetchone()[0]
1353         if rank==None:
1354             rank=1
1355         id = self.db.execute("SELECT MAX(id)+1 FROM categories;").fetchone()[0]
1356         if id==None:
1357             id=1
1358         self.db.execute("INSERT INTO categories (id, title, unread, rank) VALUES (?, ?, 0, ?)", (id, title, rank))
1359         self.db.commit()
1360         self.cache_invalidate('categories')
1361
1362     def removeFeed(self, key):
1363         if wc().available ():
1364             try:
1365                 del wc()[key]
1366             except KeyError, woodchuck.Error:
1367                 logger.debug("Removing unregistered feed %s failed" % (key,))
1368
1369         rank = self.db.execute("SELECT rank FROM feeds WHERE id=?;", (key,) ).fetchone()[0]
1370         self.db.execute("DELETE FROM feeds WHERE id=?;", (key, ))
1371         self.db.execute("UPDATE feeds SET rank=rank-1 WHERE rank>?;", (rank,) )
1372         self.db.commit()
1373
1374         if isdir(self.configdir+key+".d/"):
1375            rmtree(self.configdir+key+".d/")
1376         self.cache_invalidate('feeds')
1377
1378     def removeCategory(self, key):
1379         if self.db.execute("SELECT count(*) FROM categories;").fetchone()[0] > 1:
1380             rank = self.db.execute("SELECT rank FROM categories WHERE id=?;", (key,) ).fetchone()[0]
1381             self.db.execute("DELETE FROM categories WHERE id=?;", (key, ))
1382             self.db.execute("UPDATE categories SET rank=rank-1 WHERE rank>?;", (rank,) )
1383             self.db.execute("UPDATE feeds SET category=1 WHERE category=?;", (key,) )
1384             self.db.commit()
1385             self.cache_invalidate('categories')
1386
1387     #def saveConfig(self):
1388     #    self.listOfFeeds["feedingit-order"] = self.sortedKeys
1389     #    file = open(self.configdir+"feeds.pickle", "w")
1390     #    pickle.dump(self.listOfFeeds, file)
1391     #    file.close()
1392
1393     def moveUp(self, key):
1394         rank = self.db.execute("SELECT rank FROM feeds WHERE id=?;", (key,)).fetchone()[0]
1395         if rank>0:
1396             self.db.execute("UPDATE feeds SET rank=? WHERE rank=?;", (rank, rank-1) )
1397             self.db.execute("UPDATE feeds SET rank=? WHERE id=?;", (rank-1, key) )
1398             self.db.commit()
1399
1400     def moveCategoryUp(self, key):
1401         rank = self.db.execute("SELECT rank FROM categories WHERE id=?;", (key,)).fetchone()[0]
1402         if rank>0:
1403             self.db.execute("UPDATE categories SET rank=? WHERE rank=?;", (rank, rank-1) )
1404             self.db.execute("UPDATE categories SET rank=? WHERE id=?;", (rank-1, key) )
1405             self.db.commit()
1406
1407     def moveDown(self, key):
1408         rank = self.db.execute("SELECT rank FROM feeds WHERE id=?;", (key,)).fetchone()[0]
1409         max_rank = self.db.execute("SELECT MAX(rank) FROM feeds;").fetchone()[0]
1410         if rank<max_rank:
1411             self.db.execute("UPDATE feeds SET rank=? WHERE rank=?;", (rank, rank+1) )
1412             self.db.execute("UPDATE feeds SET rank=? WHERE id=?;", (rank+1, key) )
1413             self.db.commit()
1414
1415     def moveCategoryDown(self, key):
1416         rank = self.db.execute("SELECT rank FROM categories WHERE id=?;", (key,)).fetchone()[0]
1417         max_rank = self.db.execute("SELECT MAX(rank) FROM categories;").fetchone()[0]
1418         if rank<max_rank:
1419             self.db.execute("UPDATE categories SET rank=? WHERE rank=?;", (rank, rank+1) )
1420             self.db.execute("UPDATE categories SET rank=? WHERE id=?;", (rank+1, key) )
1421             self.db.commit()
1422
1423