vcs.maemo.org Git - feedingit/blob - src/rss_sqlite.py

   1 #!/usr/bin/env python2.5
   2
   3 #
   4 # Copyright (c) 2007-2008 INdT.
   5 # Copyright (c) 2011 Neal H. Walfield
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU Lesser General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 #  This program is distributed in the hope that it will be useful,
  12 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 #  GNU Lesser General Public License for more details.
  15 #
  16 #  You should have received a copy of the GNU Lesser General Public License
  17 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18 #
  19
  20 # ============================================================================
  21 # Name        : FeedingIt.py
  22 # Author      : Yves Marcoz
  23 # Version     : 0.5.4
  24 # Description : Simple RSS Reader
  25 # ============================================================================
  26
  27 from __future__ import with_statement
  28
  29 import sqlite3
  30 from os.path import isfile, isdir
  31 from shutil import rmtree
  32 from os import mkdir, remove, utime
  33 import os
  34 import md5
  35 import feedparser
  36 import time
  37 import urllib2
  38 from BeautifulSoup import BeautifulSoup
  39 from urlparse import urljoin
  40 from calendar import timegm
  41 import threading
  42 import traceback
  43 from wc import wc, wc_init, woodchuck
  44 import subprocess
  45 import dbus
  46 from updatedbus import update_server_object
  47
  48 from jobmanager import JobManager
  49 import mainthread
  50 from httpprogresshandler import HTTPProgressHandler
  51 import random
  52 import sys
  53 import logging
  54 logger = logging.getLogger(__name__)
  55
  56 def getId(string):
  57     return md5.new(string).hexdigest()
  58
  59 def download_callback(connection):
  60     if JobManager().do_quit:
  61         raise KeyboardInterrupt
  62
  63 def downloader(progress_handler=None, proxy=None):
  64     openers = []
  65
  66     if progress_handler is not None:
  67         openers.append(progress_handler)
  68     else:
  69         openers.append(HTTPProgressHandler(download_callback))
  70
  71     if proxy:
  72         openers.append(proxy)
  73
  74     return urllib2.build_opener(*openers)
  75
  76 def transfer_stats(sent, received, **kwargs):
  77     """
  78     This function takes two arguments: sent is the number of bytes
  79     sent so far, received is the number of bytes received.  The
  80     function returns a continuation that you can call later.
  81
  82     The continuation takes the same two arguments.  It returns a tuple
  83     of the number of bytes sent, the number of bytes received and the
  84     time since the original function was invoked.
  85     """
  86     start_time = time.time()
  87     start_sent = sent
  88     start_received = received
  89
  90     def e(sent, received, **kwargs):
  91         return (sent - start_sent,
  92                 received - start_received,
  93                 time.time() - start_time)
  94
  95     return e
  96
  97 # If not None, a subprocess.Popen object corresponding to a
  98 # update_feeds.py process.
  99 update_feed_process = None
 100
 101 update_feeds_iface = None
 102
 103 jobs_at_start = 0
 104
 105 class BaseObject(object):
 106     # Columns to cache.  Classes that inherit from this and use the
 107     # cache mechanism should set this to a list of tuples, each of
 108     # which contains two entries: the table and the column.  Note that
 109     # both are case sensitive.
 110     cached_columns = ()
 111
 112     def cache_invalidate(self, table=None):
 113         """
 114         Invalidate the cache.
 115
 116         If table is not None, invalidate only the specified table.
 117         Otherwise, drop the whole cache.
 118         """
 119         if not hasattr(self, 'cache'):
 120             return
 121
 122         if table is None:
 123             del self.cache
 124         else:
 125             if table in self.cache:
 126                 del self.cache[table]
 127
 128     def lookup(self, table, column, id=None):
 129         """
 130         Look up a column or value.  Uses a cache for columns in
 131         cached_columns.  Note: the column is returned unsorted.
 132         """
 133         if not hasattr(self, 'cache'):
 134             self.cache = {}
 135
 136         # Cache data for at most 60 seconds.
 137         now = time.time()
 138         try:
 139             cache = self.cache[table]
 140
 141             if time.time() - cache[None] > 60:
 142                 # logger.debug("%s: Cache too old: clearing" % (table,))
 143                 del self.cache[table]
 144                 cache = None
 145         except KeyError:
 146             cache = None
 147
 148         if (cache is None
 149             or (table, column) not in self.cached_columns):
 150             # The cache is empty or the caller wants a column that we
 151             # don't cache.
 152             if (table, column) in self.cached_columns:
 153                 # logger.debug("%s: Rebuilding cache" % (table,))
 154
 155                 do_cache = True
 156
 157                 self.cache[table] = cache = {}
 158                 columns = []
 159                 for t, c in self.cached_columns:
 160                     if table == t:
 161                         cache[c] = {}
 162                         columns.append(c)
 163
 164                 columns.append('id')
 165                 where = ""
 166             else:
 167                 do_cache = False
 168
 169                 columns = (colums,)
 170                 if id is not None:
 171                     where = "where id = '%s'" % id
 172                 else:
 173                     where = ""
 174
 175             results = self.db.execute(
 176                 "SELECT %s FROM %s %s" % (','.join(columns), table, where))
 177
 178             if do_cache:
 179                 for r in results:
 180                     values = list(r)
 181                     i = values.pop()
 182                     for index, value in enumerate(values):
 183                         cache[columns[index]][i] = value
 184
 185                 cache[None] = now
 186             else:
 187                 results = []
 188                 for r in results:
 189                     if id is not None:
 190                         return values[0]
 191
 192                     results.append(values[0])
 193
 194                 return results
 195         else:
 196             cache = self.cache[table]
 197
 198         try:
 199             if id is not None:
 200                 value = cache[column][id]
 201                 # logger.debug("%s.%s:%s -> %s" % (table, column, id, value))
 202                 return value
 203             else:
 204                 return cache[column].values()
 205         except KeyError:
 206             # logger.debug("%s.%s:%s -> Not found" % (table, column, id))
 207             return None
 208
 209 class Feed(BaseObject):
 210     # Columns to cache.
 211     cached_columns = (('feed', 'read'),
 212                       ('feed', 'title'))
 213
 214     serial_execution_lock = threading.Lock()
 215
 216     def _getdb(self):
 217         try:
 218             db = self.tls.db
 219         except AttributeError:
 220             db = sqlite3.connect("%s/%s.db" % (self.dir, self.key), timeout=120)
 221             self.tls.db = db
 222         return db
 223     db = property(_getdb)
 224
 225     def __init__(self, configdir, key):
 226         self.key = key
 227         self.configdir = configdir
 228         self.dir = "%s/%s.d" %(self.configdir, self.key)
 229         self.tls = threading.local ()
 230
 231         if not isdir(self.dir):
 232             mkdir(self.dir)
 233         if not isfile("%s/%s.db" %(self.dir, self.key)):
 234             self.db.execute("CREATE TABLE feed (id text, title text, contentLink text, date float, updated float, link text, read int);")
 235             self.db.execute("CREATE TABLE images (id text, imagePath text);")
 236             self.db.commit()
 237
 238     def addImage(self, configdir, key, baseurl, url, proxy=None, opener=None):
 239         filename = configdir+key+".d/"+getId(url)
 240         if not isfile(filename):
 241             try:
 242                 if not opener:
 243                     opener = downloader(proxy=proxy)
 244
 245                 abs_url = urljoin(baseurl,url)
 246                 f = opener.open(abs_url)
 247                 try:
 248                     with open(filename, "w") as outf:
 249                         for data in f:
 250                             outf.write(data)
 251                 finally:
 252                     f.close()
 253             except (urllib2.HTTPError, urllib2.URLError, IOError), exception:
 254                 logger.info("Could not download image %s: %s"
 255                             % (abs_url, str (exception)))
 256                 return None
 257             except:
 258                 exception = sys.exc_info()[0]
 259
 260                 logger.info("Downloading image %s: %s" %
 261                             (abs_url, traceback.format_exc()))
 262                 try:
 263                     remove(filename)
 264                 except OSError:
 265                     pass
 266
 267                 return None
 268         else:
 269             #open(filename,"a").close()  # "Touch" the file
 270             file = open(filename,"a")
 271             utime(filename, None)
 272             file.close()
 273         return filename
 274
 275     def updateFeed(self, configdir, url, etag, modified, expiryTime=24, proxy=None, imageCache=False, priority=0, postFeedUpdateFunc=None, *postFeedUpdateFuncArgs):
 276         if (os.path.basename(sys.argv[0]) == 'update_feeds.py'):
 277             def doit():
 278                 def it():
 279                     self._updateFeed(configdir, url, etag, modified, expiryTime, proxy, imageCache, postFeedUpdateFunc, *postFeedUpdateFuncArgs)
 280                 return it
 281             JobManager().execute(doit(), self.key, priority=priority)
 282         else:
 283             def send_update_request():
 284                 global update_feeds_iface
 285                 if update_feeds_iface is None:
 286                     bus=dbus.SessionBus()
 287                     remote_object = bus.get_object(
 288                         "org.marcoz.feedingit", # Connection name
 289                         "/org/marcoz/feedingit/update" # Object's path
 290                         )
 291                     update_feeds_iface = dbus.Interface(
 292                         remote_object, 'org.marcoz.feedingit')
 293
 294                 try:
 295                     update_feeds_iface.Update(self.key)
 296                 except Exception, e:
 297                     logger.error("Invoking org.marcoz.feedingit.Update: %s"
 298                                  % str(e))
 299                     update_feeds_iface = None
 300                 else:
 301                     return True
 302
 303             if send_update_request():
 304                 # Success!  It seems we were able to start the update
 305                 # daemon via dbus (or, it was already running).
 306                 return
 307
 308             global update_feed_process
 309             if (update_feed_process is None
 310                 or update_feed_process.poll() is not None):
 311                 # The update_feeds process is not running.  Start it.
 312                 update_feeds = os.path.join(os.path.dirname(__file__),
 313                                             'update_feeds.py')
 314                 argv = ['/usr/bin/env', 'python', update_feeds, '--daemon' ]
 315                 logger.debug("Starting update_feeds: running %s"
 316                              % (str(argv),))
 317                 update_feed_process = subprocess.Popen(argv)
 318                 # Make sure the dbus calls go to the right process:
 319                 # rebind.
 320                 update_feeds_iface = None
 321
 322             for _ in xrange(5):
 323                 if send_update_request():
 324                     break
 325                 time.sleep(1)
 326
 327     def _updateFeed(self, configdir, url, etag, modified, expiryTime=24, proxy=None, imageCache=False, postFeedUpdateFunc=None, *postFeedUpdateFuncArgs):
 328         logger.debug("Updating %s" % url)
 329
 330         success = False
 331         have_serial_execution_lock = False
 332         try:
 333             update_start = time.time ()
 334
 335             progress_handler = HTTPProgressHandler(download_callback)
 336
 337             openers = [progress_handler]
 338             if proxy:
 339                 openers.append (proxy)
 340             kwargs = {'handlers':openers}
 341
 342             feed_transfer_stats = transfer_stats(0, 0)
 343
 344             tmp=feedparser.parse(url, etag=etag, modified=modified, **kwargs)
 345             download_duration = time.time () - update_start
 346
 347             opener = downloader(progress_handler, proxy)
 348
 349             if JobManager().do_quit:
 350                 raise KeyboardInterrupt
 351
 352             process_start = time.time()
 353
 354             # Expiry time is in hours
 355             expiry = float(expiryTime) * 3600.
 356
 357             currentTime = 0
 358
 359             updated_objects = 0
 360             new_objects = 0
 361
 362             def wc_success():
 363                 try:
 364                     wc().stream_register (self.key, "", 6 * 60 * 60)
 365                 except woodchuck.ObjectExistsError:
 366                     pass
 367                 try:
 368                     wc()[self.key].updated (
 369                         indicator=(woodchuck.Indicator.ApplicationVisual
 370                                    |woodchuck.Indicator.StreamWide),
 371                         transferred_down=progress_handler.stats['received'],
 372                         transferred_up=progress_handler.stats['sent'],
 373                         transfer_time=update_start,
 374                         transfer_duration=download_duration,
 375                         new_objects=new_objects,
 376                         updated_objects=updated_objects,
 377                         objects_inline=new_objects + updated_objects)
 378                 except KeyError:
 379                     logger.warn(
 380                         "Failed to register update of %s with woodchuck!"
 381                         % (self.key))
 382
 383             http_status = tmp.get ('status', 200)
 384
 385             # Check if the parse was succesful.  If the http status code
 386             # is 304, then the download was successful, but there is
 387             # nothing new.  Indeed, no content is returned.  This make a
 388             # 304 look like an error because there are no entries and the
 389             # parse fails.  But really, everything went great!  Check for
 390             # this first.
 391             if http_status == 304:
 392                 logger.debug("%s: No changes to feed." % (self.key,))
 393                 mainthread.execute(wc_success, async=True)
 394                 success = True
 395             elif len(tmp["entries"])==0 and not tmp.version:
 396                 # An error occured fetching or parsing the feed.  (Version
 397                 # will be either None if e.g. the connection timed our or
 398                 # '' if the data is not a proper feed)
 399                 logger.error(
 400                     "Error fetching %s: version is: %s: error: %s"
 401                     % (url, str (tmp.version),
 402                        str (tmp.get ('bozo_exception', 'Unknown error'))))
 403                 logger.debug(tmp)
 404                 def register_stream_update_failed(http_status):
 405                     def doit():
 406                         logger.debug("%s: stream update failed!" % self.key)
 407
 408                         try:
 409                             # It's not easy to get the feed's title from here.
 410                             # At the latest, the next time the application is
 411                             # started, we'll fix up the human readable name.
 412                             wc().stream_register (self.key, "", 6 * 60 * 60)
 413                         except woodchuck.ObjectExistsError:
 414                             pass
 415                         ec = woodchuck.TransferStatus.TransientOther
 416                         if 300 <= http_status and http_status < 400:
 417                             ec = woodchuck.TransferStatus.TransientNetwork
 418                         if 400 <= http_status and http_status < 500:
 419                             ec = woodchuck.TransferStatus.FailureGone
 420                         if 500 <= http_status and http_status < 600:
 421                             ec = woodchuck.TransferStatus.TransientNetwork
 422                         wc()[self.key].update_failed(ec)
 423                     return doit
 424                 if wc().available:
 425                     mainthread.execute(
 426                         register_stream_update_failed(
 427                             http_status=http_status),
 428                         async=True)
 429             else:
 430                currentTime = time.time()
 431                # The etag and modified value should only be updated if the content was not null
 432                try:
 433                    etag = tmp["etag"]
 434                except KeyError:
 435                    etag = None
 436                try:
 437                    modified = tmp["modified"]
 438                except KeyError:
 439                    modified = None
 440                try:
 441                    abs_url = urljoin(tmp["feed"]["link"],"/favicon.ico")
 442                    f = opener.open(abs_url)
 443                    data = f.read()
 444                    f.close()
 445                    outf = open(self.dir+"/favicon.ico", "w")
 446                    outf.write(data)
 447                    outf.close()
 448                    del data
 449                except (urllib2.HTTPError, urllib2.URLError), exception:
 450                    logger.debug("Could not download favicon %s: %s"
 451                                 % (abs_url, str (exception)))
 452
 453                self.serial_execution_lock.acquire ()
 454                have_serial_execution_lock = True
 455
 456                #reversedEntries = self.getEntries()
 457                #reversedEntries.reverse()
 458
 459                tmp["entries"].reverse()
 460                for entry in tmp["entries"]:
 461                    # Yield so as to make the main thread a bit more
 462                    # responsive.
 463                    time.sleep(0)
 464
 465                    entry_transfer_stats = transfer_stats(
 466                        *feed_transfer_stats(**progress_handler.stats)[0:2])
 467
 468                    if JobManager().do_quit:
 469                        raise KeyboardInterrupt
 470
 471                    object_size = 0
 472
 473                    date = self.extractDate(entry)
 474                    try:
 475                        entry["title"]
 476                    except KeyError:
 477                        entry["title"] = "No Title"
 478                    try :
 479                        entry["link"]
 480                    except KeyError:
 481                        entry["link"] = ""
 482                    try:
 483                        entry["author"]
 484                    except KeyError:
 485                        entry["author"] = None
 486                    if(not(entry.has_key("id"))):
 487                        entry["id"] = None
 488                    content = self.extractContent(entry)
 489                    object_size = len (content)
 490                    tmpEntry = {"title":entry["title"], "content":content,
 491                                 "date":date, "link":entry["link"], "author":entry["author"], "id":entry["id"]}
 492                    id = self.generateUniqueId(tmpEntry)
 493
 494                    current_version = self.db.execute(
 495                        'select date, ROWID from feed where id=?',
 496                        (id,)).fetchone()
 497                    if (current_version is not None
 498                        and current_version[0] == date):
 499                        logger.debug("ALREADY DOWNLOADED %s (%s)"
 500                                     % (entry["title"], entry["link"]))
 501                        continue
 502
 503                    if current_version is not None:
 504                        # The version was updated.  Mark it as unread.
 505                        logger.debug("UPDATED: %s (%s)"
 506                                     % (entry["title"], entry["link"]))
 507                        updated_objects += 1
 508                    else:
 509                        logger.debug("NEW: %s (%s)"
 510                                     % (entry["title"], entry["link"]))
 511                        new_objects += 1
 512
 513                    #articleTime = time.mktime(self.entries[id]["dateTuple"])
 514                    soup = BeautifulSoup(self.getArticle(tmpEntry)) #tmpEntry["content"])
 515                    images = soup('img')
 516                    baseurl = tmpEntry["link"]
 517                    if imageCache and len(images) > 0:
 518                        self.serial_execution_lock.release ()
 519                        have_serial_execution_lock = False
 520                        for img in images:
 521                            if not 'src' in img:
 522                                continue
 523
 524                            filename = self.addImage(
 525                                configdir, self.key, baseurl, img['src'],
 526                                opener=opener)
 527                            if filename:
 528                                 img['src']="file://%s" %filename
 529                                 count = self.db.execute("SELECT count(1) FROM images where id=? and imagePath=?;", (id, filename )).fetchone()[0]
 530                                 if count == 0:
 531                                     self.db.execute("INSERT INTO images (id, imagePath) VALUES (?, ?);", (id, filename) )
 532                                     self.db.commit()
 533
 534                                 try:
 535                                     object_size += os.path.getsize (filename)
 536                                 except os.error, exception:
 537                                     logger.error ("Error getting size of %s: %s"
 538                                                   % (filename, exception))
 539                        self.serial_execution_lock.acquire ()
 540                        have_serial_execution_lock = True
 541
 542                    tmpEntry["contentLink"] = configdir+self.key+".d/"+id+".html"
 543                    file = open(tmpEntry["contentLink"], "w")
 544                    file.write(soup.prettify())
 545                    file.close()
 546
 547                    values = {'id': id,
 548                              'title': tmpEntry["title"],
 549                              'contentLink': tmpEntry["contentLink"],
 550                              'date': tmpEntry["date"],
 551                              'updated': currentTime,
 552                              'link': tmpEntry["link"],
 553                              'read': 0}
 554
 555                    if current_version is not None:
 556                        # This is an update.  Ensure that the existing
 557                        # entry is replaced.
 558                        values['ROWID'] = current_version[1]
 559
 560                    cols, values = zip(*values.items())
 561                    self.db.execute(
 562                        "INSERT OR REPLACE INTO feed (%s) VALUES (%s);"
 563                        % (','.join(cols), ','.join(('?',) * len(values))),
 564                        values)
 565                    self.db.commit()
 566
 567                    # Register the object with Woodchuck and mark it as
 568                    # downloaded.
 569                    def register_object_transferred(
 570                            id, title, publication_time,
 571                            sent, received, object_size):
 572                        def doit():
 573                            logger.debug("Registering transfer of object %s"
 574                                         % title)
 575                            try:
 576                                obj = wc()[self.key].object_register(
 577                                    object_identifier=id,
 578                                    human_readable_name=title)
 579                            except woodchuck.ObjectExistsError:
 580                                obj = wc()[self.key][id]
 581                            else:
 582                                obj.publication_time = publication_time
 583                                obj.transferred(
 584                                    indicator=(
 585                                        woodchuck.Indicator.ApplicationVisual
 586                                        |woodchuck.Indicator.StreamWide),
 587                                    transferred_down=received,
 588                                    transferred_up=sent,
 589                                    object_size=object_size)
 590                        return doit
 591                    if wc().available:
 592                        # If the entry does not contain a publication
 593                        # time, the attribute won't exist.
 594                        pubtime = entry.get('date_parsed', None)
 595                        if pubtime:
 596                            publication_time = time.mktime (pubtime)
 597                        else:
 598                            publication_time = None
 599
 600                        sent, received, _ \
 601                            = entry_transfer_stats(**progress_handler.stats)
 602                        # sent and received are for objects (in
 603                        # particular, images) associated with this
 604                        # item.  We also want to attribute the data
 605                        # transferred for the item's content.  This is
 606                        # a good first approximation.
 607                        received += len(content)
 608
 609                        mainthread.execute(
 610                            register_object_transferred(
 611                                id=id,
 612                                title=tmpEntry["title"],
 613                                publication_time=publication_time,
 614                                sent=sent, received=received,
 615                                object_size=object_size),
 616                            async=True)
 617                self.db.commit()
 618
 619                sent, received, _ \
 620                    = feed_transfer_stats(**progress_handler.stats)
 621                logger.debug (
 622                    "%s: Update successful: transferred: %d/%d; objects: %d)"
 623                    % (url, sent, received, len (tmp.entries)))
 624                mainthread.execute (wc_success, async=True)
 625                success = True
 626
 627             rows = self.db.execute("SELECT id FROM feed WHERE (read=0 AND updated<?) OR (read=1 AND updated<?);", (currentTime-2*expiry, currentTime-expiry))
 628             for row in rows:
 629                self.removeEntry(row[0])
 630
 631             from glob import glob
 632             from os import stat
 633             for file in glob(configdir+self.key+".d/*"):
 634                 #
 635                 stats = stat(file)
 636                 #
 637                 # put the two dates into matching format
 638                 #
 639                 lastmodDate = stats[8]
 640                 #
 641                 expDate = time.time()-expiry*3
 642                 # check if image-last-modified-date is outdated
 643                 #
 644                 if expDate > lastmodDate:
 645                     #
 646                     try:
 647                         #
 648                         #print 'Removing', file
 649                         #
 650                         # XXX: Tell woodchuck.
 651                         remove(file) # commented out for testing
 652                         #
 653                     except OSError, exception:
 654                         #
 655                         logger.error('Could not remove %s: %s'
 656                                      % (file, str (exception)))
 657             logger.debug("updated %s: %fs in download, %fs in processing"
 658                          % (self.key, download_duration,
 659                             time.time () - process_start))
 660         except:
 661             logger.error("Updating %s: %s" % (self.key, traceback.format_exc()))
 662         finally:
 663             self.db.commit ()
 664
 665             if have_serial_execution_lock:
 666                 self.serial_execution_lock.release ()
 667
 668             updateTime = 0
 669             try:
 670                 rows = self.db.execute("SELECT MAX(date) FROM feed;")
 671                 for row in rows:
 672                     updateTime=row[0]
 673             except Exception, e:
 674                 logger.error("Fetching update time: %s: %s"
 675                              % (str(e), traceback.format_exc()))
 676             finally:
 677                 if not success:
 678                     etag = None
 679                     modified = None
 680                 title = None
 681                 try:
 682                     title = tmp.feed.title
 683                 except (AttributeError, UnboundLocalError), exception:
 684                     pass
 685                 if postFeedUpdateFunc is not None:
 686                     postFeedUpdateFunc (self.key, updateTime, etag, modified,
 687                                         title, *postFeedUpdateFuncArgs)
 688
 689         self.cache_invalidate()
 690
 691     def setEntryRead(self, id):
 692         self.db.execute("UPDATE feed SET read=1 WHERE id=?;", (id,) )
 693         self.db.commit()
 694
 695         def doit():
 696             try:
 697                 wc()[self.key][id].used()
 698             except KeyError:
 699                 pass
 700         if wc().available():
 701             mainthread.execute(doit, async=True)
 702         self.cache_invalidate('feed')
 703
 704     def setEntryUnread(self, id):
 705         self.db.execute("UPDATE feed SET read=0 WHERE id=?;", (id,) )
 706         self.db.commit()
 707         self.cache_invalidate('feed')
 708
 709     def markAllAsRead(self):
 710         self.db.execute("UPDATE feed SET read=1 WHERE read=0;")
 711         self.db.commit()
 712         self.cache_invalidate('feed')
 713
 714     def isEntryRead(self, id):
 715         return self.lookup('feed', 'read', id) == 1
 716
 717     def getTitle(self, id):
 718         return self.lookup('feed', 'title', id)
 719
 720     def getContentLink(self, id):
 721         return self.db.execute("SELECT contentLink FROM feed WHERE id=?;", (id,) ).fetchone()[0]
 722
 723     def getExternalLink(self, id):
 724         return self.db.execute("SELECT link FROM feed WHERE id=?;", (id,) ).fetchone()[0]
 725
 726     def getDate(self, id):
 727         dateStamp = self.db.execute("SELECT date FROM feed WHERE id=?;", (id,) ).fetchone()[0]
 728         return time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime(dateStamp))
 729
 730     def getDateTuple(self, id):
 731         dateStamp = self.db.execute("SELECT date FROM feed WHERE id=?;", (id,) ).fetchone()[0]
 732         return time.localtime(dateStamp)
 733
 734     def getDateStamp(self, id):
 735         return self.db.execute("SELECT date FROM feed WHERE id=?;", (id,) ).fetchone()[0]
 736
 737     def generateUniqueId(self, entry):
 738         """
 739         Generate a stable identifier for the article.  For the same
 740         entry, this should result in the same identifier.  If
 741         possible, the identifier should remain the same even if the
 742         article is updated.
 743         """
 744         # Prefer the entry's id, which is supposed to be globally
 745         # unique.
 746         key = entry.get('id', None)
 747         if not key:
 748             # Next, try the link to the content.
 749             key = entry.get('link', None)
 750         if not key:
 751             # Ok, the title and the date concatenated are likely to be
 752             # relatively stable.
 753             key = entry.get('title', None) + entry.get('date', None)
 754         if not key:
 755             # Hmm, the article's content will at least guarantee no
 756             # false negatives (i.e., missing articles)
 757             key = entry.get('content', None)
 758         if not key:
 759             # If all else fails, just use a random number.
 760             key = str (random.random ())
 761         return getId (key)
 762
 763     def getIds(self, onlyUnread=False):
 764         if onlyUnread:
 765             rows = self.db.execute("SELECT id FROM feed where read=0 ORDER BY date DESC;").fetchall()
 766         else:
 767             rows = self.db.execute("SELECT id FROM feed ORDER BY date DESC;").fetchall()
 768         ids = []
 769         for row in rows:
 770             ids.append(row[0])
 771         #ids.reverse()
 772         return ids
 773
 774     def getNextId(self, id, forward=True):
 775         if forward:
 776             delta = 1
 777         else:
 778             delta = -1
 779         ids = self.getIds()
 780         index = ids.index(id)
 781         return ids[(index + delta) % len(ids)]
 782
 783     def getPreviousId(self, id):
 784         return self.getNextId(id, forward=False)
 785
 786     def getNumberOfUnreadItems(self):
 787         return self.db.execute("SELECT count(*) FROM feed WHERE read=0;").fetchone()[0]
 788
 789     def getNumberOfEntries(self):
 790         return self.db.execute("SELECT count(*) FROM feed;").fetchone()[0]
 791
 792     def getArticle(self, entry):
 793         #self.setEntryRead(id)
 794         #entry = self.entries[id]
 795         title = entry['title']
 796         #content = entry.get('content', entry.get('summary_detail', {}))
 797         content = entry["content"]
 798
 799         link = entry['link']
 800         author = entry['author']
 801         date = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime(entry["date"]) )
 802
 803         #text = '''<div style="color: black; background-color: white;">'''
 804         text = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">'
 805         text += "<html><head><title>" + title + "</title>"
 806         text += '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>\n'
 807         #text += '<style> body {-webkit-user-select: none;} </style>'
 808         text += '</head><body bgcolor=\"#ffffff\"><div><a href=\"' + link + '\">' + title + "</a>"
 809         if author != None:
 810             text += "<BR /><small><i>Author: " + author + "</i></small>"
 811         text += "<BR /><small><i>Date: " + date + "</i></small></div>"
 812         text += "<BR /><BR />"
 813         text += content
 814         text += "</body></html>"
 815         return text
 816
 817     def getContent(self, id):
 818         contentLink = self.db.execute("SELECT contentLink FROM feed WHERE id=?;", (id,)).fetchone()[0]
 819         try:
 820             file = open(self.entries[id]["contentLink"])
 821             content = file.read()
 822             file.close()
 823         except:
 824             content = "Content unavailable"
 825         return content
 826
 827     def extractDate(self, entry):
 828         if entry.has_key("updated_parsed"):
 829             return timegm(entry["updated_parsed"])
 830         elif entry.has_key("published_parsed"):
 831             return timegm(entry["published_parsed"])
 832         else:
 833             return time.time()
 834
 835     def extractContent(self, entry):
 836         content = ""
 837         if entry.has_key('summary'):
 838             content = entry.get('summary', '')
 839         if entry.has_key('content'):
 840             if len(entry.content[0].value) > len(content):
 841                 content = entry.content[0].value
 842         if content == "":
 843             content = entry.get('description', '')
 844         return content
 845
 846     def removeEntry(self, id):
 847         contentLink = self.db.execute("SELECT contentLink FROM feed WHERE id=?;", (id,)).fetchone()[0]
 848         if contentLink:
 849             try:
 850                 remove(contentLink)
 851             except OSError, exception:
 852                 logger.error("Deleting %s: %s" % (contentLink, str (exception)))
 853         self.db.execute("DELETE FROM feed WHERE id=?;", (id,) )
 854         self.db.execute("DELETE FROM images WHERE id=?;", (id,) )
 855         self.db.commit()
 856
 857         def doit():
 858             try:
 859                 wc()[self.key][id].files_deleted (
 860                     woodchuck.DeletionResponse.Deleted)
 861                 del wc()[self.key][id]
 862             except KeyError:
 863                 pass
 864         if wc().available():
 865             mainthread.execute (doit, async=True)
 866
 867 class ArchivedArticles(Feed):
 868     def addArchivedArticle(self, title, link, date, configdir):
 869         id = self.generateUniqueId({"date":date, "title":title})
 870         values = (id, title, link, date, 0, link, 0)
 871         self.db.execute("INSERT INTO feed (id, title, contentLink, date, updated, link, read) VALUES (?, ?, ?, ?, ?, ?, ?);", values)
 872         self.db.commit()
 873
 874     def updateFeed(self, configdir, url, etag, modified, expiryTime=24, proxy=None, imageCache=False):
 875         currentTime = 0
 876         rows = self.db.execute("SELECT id, link FROM feed WHERE updated=0;")
 877         for row in rows:
 878             currentTime = time.time()
 879             id = row[0]
 880             link = row[1]
 881             f = urllib2.urlopen(link)
 882             #entry["content"] = f.read()
 883             html = f.read()
 884             f.close()
 885             soup = BeautifulSoup(html)
 886             images = soup('img')
 887             baseurl = link
 888             for img in images:
 889                 filename = self.addImage(configdir, self.key, baseurl, img['src'], proxy=proxy)
 890                 img['src']=filename
 891                 self.db.execute("INSERT INTO images (id, imagePath) VALUES (?, ?);", (id, filename) )
 892                 self.db.commit()
 893             contentLink = configdir+self.key+".d/"+id+".html"
 894             file = open(contentLink, "w")
 895             file.write(soup.prettify())
 896             file.close()
 897
 898             self.db.execute("UPDATE feed SET read=0, contentLink=?, updated=? WHERE id=?;", (contentLink, time.time(), id) )
 899             self.db.commit()
 900         return (currentTime, None, None)
 901
 902     def purgeReadArticles(self):
 903         rows = self.db.execute("SELECT id FROM feed WHERE read=1;")
 904         #ids = self.getIds()
 905         for row in rows:
 906             self.removeArticle(row[0])
 907
 908     def removeArticle(self, id):
 909         rows = self.db.execute("SELECT imagePath FROM images WHERE id=?;", (id,) )
 910         for row in rows:
 911             try:
 912                 count = self.db.execute("SELECT count(*) FROM images WHERE id!=? and imagePath=?;", (id,row[0]) ).fetchone()[0]
 913                 if count == 0:
 914                     os.remove(row[0])
 915             except:
 916                 pass
 917         self.removeEntry(id)
 918
 919 class Listing(BaseObject):
 920     # Columns to cache.
 921     cached_columns = (('feeds', 'updateTime'),
 922                       ('feeds', 'unread'),
 923                       ('feeds', 'title'),
 924                       ('categories', 'title'))
 925
 926     def _getdb(self):
 927         try:
 928             db = self.tls.db
 929         except AttributeError:
 930             db = sqlite3.connect("%s/feeds.db" % self.configdir, timeout=120)
 931             self.tls.db = db
 932         return db
 933     db = property(_getdb)
 934
 935     # Lists all the feeds in a dictionary, and expose the data
 936     def __init__(self, config, configdir):
 937         self.config = config
 938         self.configdir = configdir
 939
 940         self.tls = threading.local ()
 941
 942         try:
 943             table = self.db.execute("SELECT sql FROM sqlite_master").fetchone()
 944             if table == None:
 945                 self.db.execute("CREATE TABLE feeds(id text, url text, title text, unread int, updateTime float, rank int, etag text, modified text, widget int, category int);")
 946                 self.db.execute("CREATE TABLE categories(id text, title text, unread int, rank int);")
 947                 self.addCategory("Default Category")
 948                 if isfile(self.configdir+"feeds.pickle"):
 949                     self.importOldFormatFeeds()
 950                 else:
 951                     self.addFeed("Maemo News", "http://maemo.org/news/items.xml")
 952             else:
 953                 from string import find, upper
 954                 if find(upper(table[0]), "WIDGET")<0:
 955                     self.db.execute("ALTER TABLE feeds ADD COLUMN widget int;")
 956                     self.db.execute("UPDATE feeds SET widget=1;")
 957                     self.db.commit()
 958                 if find(upper(table[0]), "CATEGORY")<0:
 959                     self.db.execute("CREATE TABLE categories(id text, title text, unread int, rank int);")
 960                     self.addCategory("Default Category")
 961                     self.db.execute("ALTER TABLE feeds ADD COLUMN category int;")
 962                     self.db.execute("UPDATE feeds SET category=1;")
 963             self.db.commit()
 964         except:
 965             pass
 966
 967         # Check that Woodchuck's state is up to date with respect our
 968         # state.
 969         try:
 970             updater = os.path.basename(sys.argv[0]) == 'update_feeds.py'
 971             wc_init (self, True if updater else False)
 972             if wc().available() and updater:
 973                 # The list of known streams.
 974                 streams = wc().streams_list ()
 975                 stream_ids = [s.identifier for s in streams]
 976
 977                 # Register any unknown streams.  Remove known streams from
 978                 # STREAMS_IDS.
 979                 for key in self.getListOfFeeds():
 980                     title = self.getFeedTitle(key)
 981                     # XXX: We should also check whether the list of
 982                     # articles/objects in each feed/stream is up to date.
 983                     if key not in stream_ids:
 984                         logger.debug(
 985                             "Registering previously unknown channel: %s (%s)"
 986                             % (key, title,))
 987                         # Use a default refresh interval of 6 hours.
 988                         wc().stream_register (key, title, 6 * 60 * 60)
 989                     else:
 990                         # Make sure the human readable name is up to date.
 991                         if wc()[key].human_readable_name != title:
 992                             wc()[key].human_readable_name = title
 993                         stream_ids.remove (key)
 994
 995
 996                 # Unregister any streams that are no longer subscribed to.
 997                 for id in stream_ids:
 998                     logger.debug("Unregistering %s" % (id,))
 999                     w.stream_unregister (id)
1000         except Exception:
1001             logger.exception("Registering streams with Woodchuck")
1002
1003     def importOldFormatFeeds(self):
1004         """This function loads feeds that are saved in an outdated format, and converts them to sqlite"""
1005         import rss
1006         listing = rss.Listing(self.configdir)
1007         rank = 0
1008         for id in listing.getListOfFeeds():
1009             try:
1010                 rank += 1
1011                 values = (id, listing.getFeedTitle(id) , listing.getFeedUrl(id), 0, time.time(), rank, None, "None", 1)
1012                 self.db.execute("INSERT INTO feeds (id, title, url, unread, updateTime, rank, etag, modified, widget, category) VALUES (?, ?, ? ,? ,? ,?, ?, ?, ?, 1);", values)
1013                 self.db.commit()
1014
1015                 feed = listing.getFeed(id)
1016                 new_feed = self.getFeed(id)
1017
1018                 items = feed.getIds()[:]
1019                 items.reverse()
1020                 for item in items:
1021                         if feed.isEntryRead(item):
1022                             read_status = 1
1023                         else:
1024                             read_status = 0
1025                         date = timegm(feed.getDateTuple(item))
1026                         title = feed.getTitle(item)
1027                         newId = new_feed.generateUniqueId({"date":date, "title":title})
1028                         values = (newId, title , feed.getContentLink(item), date, tuple(time.time()), feed.getExternalLink(item), read_status)
1029                         new_feed.db.execute("INSERT INTO feed (id, title, contentLink, date, updated, link, read) VALUES (?, ?, ?, ?, ?, ?, ?);", values)
1030                         new_feed.db.commit()
1031                         try:
1032                             images = feed.getImages(item)
1033                             for image in images:
1034                                 new_feed.db.execute("INSERT INTO images (id, imagePath) VALUES (?, ?);", (item, image) )
1035                                 new_feed.db.commit()
1036                         except:
1037                             pass
1038                 self.updateUnread(id)
1039             except:
1040                 logger.error("importOldFormatFeeds: %s"
1041                              % (traceback.format_exc(),))
1042         remove(self.configdir+"feeds.pickle")
1043
1044
1045     def addArchivedArticle(self, key, index):
1046         feed = self.getFeed(key)
1047         title = feed.getTitle(index)
1048         link = feed.getExternalLink(index)
1049         date = feed.getDate(index)
1050         count = self.db.execute("SELECT count(*) FROM feeds where id=?;", ("ArchivedArticles",) ).fetchone()[0]
1051         if count == 0:
1052             self.addFeed("Archived Articles", "", id="ArchivedArticles")
1053
1054         archFeed = self.getFeed("ArchivedArticles")
1055         archFeed.addArchivedArticle(title, link, date, self.configdir)
1056         self.updateUnread("ArchivedArticles")
1057
1058     def updateFeed(self, key, expiryTime=None, proxy=None, imageCache=None,
1059                    priority=0):
1060         if expiryTime is None:
1061             expiryTime = self.config.getExpiry()
1062         if not expiryTime:
1063             # Default to 24 hours
1064             expriyTime = 24
1065         if proxy is None:
1066             (use_proxy, proxy) = self.config.getProxy()
1067             if not use_proxy:
1068                 proxy = None
1069         if imageCache is None:
1070             imageCache = self.config.getImageCache()
1071
1072         feed = self.getFeed(key)
1073         (url, etag, modified) = self.db.execute("SELECT url, etag, modified FROM feeds WHERE id=?;", (key,) ).fetchone()
1074         try:
1075             modified = time.struct_time(eval(modified))
1076         except:
1077             modified = None
1078         feed.updateFeed(
1079             self.configdir, url, etag, modified, expiryTime, proxy, imageCache,
1080             priority, postFeedUpdateFunc=self._queuePostFeedUpdate)
1081
1082     def _queuePostFeedUpdate(self, *args, **kwargs):
1083         mainthread.execute (self._postFeedUpdate, async=True, *args, **kwargs)
1084
1085     def _postFeedUpdate(self, key, updateTime, etag, modified, title):
1086         if modified==None:
1087             modified="None"
1088         else:
1089             modified=str(tuple(modified))
1090         if updateTime > 0:
1091             self.db.execute("UPDATE feeds SET updateTime=?, etag=?, modified=? WHERE id=?;", (updateTime, etag, modified, key) )
1092         else:
1093             self.db.execute("UPDATE feeds SET etag=?, modified=? WHERE id=?;", (etag, modified, key) )
1094
1095         if title is not None:
1096             self.db.execute("UPDATE feeds SET title=(case WHEN title=='' THEN ? ELSE title END) where id=?;",
1097                             (title, key))
1098         self.db.commit()
1099         self.cache_invalidate('feeds')
1100         self.updateUnread(key)
1101
1102         update_server_object().ArticleCountUpdated()
1103
1104         stats = JobManager().stats()
1105         global jobs_at_start
1106         completed = stats['jobs-completed'] - jobs_at_start
1107         in_progress = stats['jobs-in-progress']
1108         queued = stats['jobs-queued']
1109
1110         try:
1111             percent = (100 * ((completed + in_progress / 2.))
1112                        / (completed + in_progress + queued))
1113         except ZeroDivisionError:
1114             percent = 100
1115
1116         update_server_object().UpdateProgress(
1117             percent, completed, in_progress, queued, 0, 0, 0, key)
1118
1119         if in_progress == 0 and queued == 0:
1120             jobs_at_start = stats['jobs-completed']
1121
1122     def getFeed(self, key):
1123         if key == "ArchivedArticles":
1124             return ArchivedArticles(self.configdir, key)
1125         return Feed(self.configdir, key)
1126
1127     def editFeed(self, key, title, url, category=None):
1128         if category:
1129             self.db.execute("UPDATE feeds SET title=?, url=?, category=? WHERE id=?;", (title, url, category, key))
1130         else:
1131             self.db.execute("UPDATE feeds SET title=?, url=? WHERE id=?;", (title, url, key))
1132         self.db.commit()
1133         self.cache_invalidate('feeds')
1134
1135         if wc().available():
1136             try:
1137                 wc()[key].human_readable_name = title
1138             except KeyError:
1139                 logger.debug("Feed %s (%s) unknown." % (key, title))
1140
1141     def getFeedUpdateTime(self, key):
1142         update_time = self.lookup('feeds', 'updateTime', key)
1143
1144         if not update_time:
1145             return "Never"
1146
1147         delta = time.time() - update_time
1148
1149         delta_hours = delta / (60. * 60.)
1150         if delta_hours < .1:
1151             return "A few minutes ago"
1152         if delta_hours < .75:
1153             return "Less than an hour ago"
1154         if delta_hours < 1.5:
1155             return "About an hour ago"
1156         if delta_hours < 18:
1157             return "About %d hours ago" % (int(delta_hours + 0.5),)
1158
1159         delta_days = delta_hours / 24.
1160         if delta_days < 1.5:
1161             return "About a day ago"
1162         if delta_days < 18:
1163             return "%d days ago" % (int(delta_days + 0.5),)
1164
1165         delta_weeks = delta_days / 7.
1166         if delta_weeks <= 8:
1167             return "%d weeks ago" % int(delta_weeks + 0.5)
1168
1169         delta_months = delta_days / 30.
1170         if delta_months <= 30:
1171             return "%d months ago" % int(delta_months + 0.5)
1172
1173         return time.strftime("%x", time.gmtime(update_time))
1174
1175     def getFeedNumberOfUnreadItems(self, key):
1176         return self.lookup('feeds', 'unread', key)
1177
1178     def getFeedTitle(self, key):
1179         title = self.lookup('feeds', 'title', key)
1180         if title:
1181             return title
1182
1183         return self.getFeedUrl(key)
1184
1185     def getFeedUrl(self, key):
1186         return self.db.execute("SELECT url FROM feeds WHERE id=?;", (key,)).fetchone()[0]
1187
1188     def getFeedCategory(self, key):
1189         return self.db.execute("SELECT category FROM feeds WHERE id=?;", (key,)).fetchone()[0]
1190
1191     def getListOfFeeds(self, category=None):
1192         if category:
1193             rows = self.db.execute("SELECT id FROM feeds WHERE category=? ORDER BY rank;", (category, ) )
1194         else:
1195             rows = self.db.execute("SELECT id FROM feeds ORDER BY rank;" )
1196         keys = []
1197         for row in rows:
1198             if row[0]:
1199                 keys.append(row[0])
1200         return keys
1201
1202     def getListOfCategories(self):
1203         return list(row[0] for row in self.db.execute(
1204                 "SELECT id FROM categories ORDER BY rank;"))
1205
1206     def getCategoryTitle(self, id):
1207         return self.lookup('categories', 'title', id)
1208
1209     def getSortedListOfKeys(self, order, onlyUnread=False, category=1):
1210         if   order == "Most unread":
1211             tmp = "ORDER BY unread DESC"
1212             #keyorder = sorted(feedInfo, key = lambda k: feedInfo[k][1], reverse=True)
1213         elif order == "Least unread":
1214             tmp = "ORDER BY unread"
1215             #keyorder = sorted(feedInfo, key = lambda k: feedInfo[k][1])
1216         elif order == "Most recent":
1217             tmp = "ORDER BY updateTime DESC"
1218             #keyorder = sorted(feedInfo, key = lambda k: feedInfo[k][2], reverse=True)
1219         elif order == "Least recent":
1220             tmp = "ORDER BY updateTime"
1221             #keyorder = sorted(feedInfo, key = lambda k: feedInfo[k][2])
1222         else: # order == "Manual" or invalid value...
1223             tmp = "ORDER BY rank"
1224             #keyorder = sorted(feedInfo, key = lambda k: feedInfo[k][0])
1225         if onlyUnread:
1226             sql = "SELECT id FROM feeds WHERE unread>0 AND category=%s " %category + tmp
1227         else:
1228             sql = "SELECT id FROM feeds WHERE category=%s " %category + tmp
1229         rows = self.db.execute(sql)
1230         keys = []
1231         for row in rows:
1232             if row[0]:
1233                 keys.append(row[0])
1234         return keys
1235
1236     def getFavicon(self, key):
1237         filename = "%s%s.d/favicon.ico" % (self.configdir, key)
1238         if isfile(filename):
1239             return filename
1240         else:
1241             return False
1242
1243     def updateUnread(self, key):
1244         feed = self.getFeed(key)
1245         self.db.execute("UPDATE feeds SET unread=? WHERE id=?;", (feed.getNumberOfUnreadItems(), key))
1246         self.db.commit()
1247         self.cache_invalidate('feeds')
1248
1249     def addFeed(self, title, url, id=None, category=1):
1250         if not id:
1251             id = getId(url)
1252         count = self.db.execute("SELECT count(*) FROM feeds WHERE id=?;", (id,) ).fetchone()[0]
1253         if count == 0:
1254             max_rank = self.db.execute("SELECT MAX(rank) FROM feeds;").fetchone()[0]
1255             if max_rank == None:
1256                 max_rank = 0
1257             values = (id, title, url, 0, 0, max_rank+1, None, "None", 1, category)
1258             self.db.execute("INSERT INTO feeds (id, title, url, unread, updateTime, rank, etag, modified, widget, category) VALUES (?, ?, ? ,? ,? ,?, ?, ?, ?,?);", values)
1259             self.db.commit()
1260             # Ask for the feed object, it will create the necessary tables
1261             self.getFeed(id)
1262
1263             if wc().available():
1264                 # Register the stream with Woodchuck.  Update approximately
1265                 # every 6 hours.
1266                 wc().stream_register(stream_identifier=id,
1267                                      human_readable_name=title,
1268                                      freshness=6*60*60)
1269
1270             return True
1271         else:
1272             return False
1273
1274     def addCategory(self, title):
1275         rank = self.db.execute("SELECT MAX(rank)+1 FROM categories;").fetchone()[0]
1276         if rank==None:
1277             rank=1
1278         id = self.db.execute("SELECT MAX(id)+1 FROM categories;").fetchone()[0]
1279         if id==None:
1280             id=1
1281         self.db.execute("INSERT INTO categories (id, title, unread, rank) VALUES (?, ?, 0, ?)", (id, title, rank))
1282         self.db.commit()
1283
1284     def removeFeed(self, key):
1285         if wc().available ():
1286             try:
1287                 del wc()[key]
1288             except KeyError:
1289                 logger.debug("Removing unregistered feed %s failed" % (key,))
1290
1291         rank = self.db.execute("SELECT rank FROM feeds WHERE id=?;", (key,) ).fetchone()[0]
1292         self.db.execute("DELETE FROM feeds WHERE id=?;", (key, ))
1293         self.db.execute("UPDATE feeds SET rank=rank-1 WHERE rank>?;", (rank,) )
1294         self.db.commit()
1295
1296         if isdir(self.configdir+key+".d/"):
1297            rmtree(self.configdir+key+".d/")
1298
1299     def removeCategory(self, key):
1300         if self.db.execute("SELECT count(*) FROM categories;").fetchone()[0] > 1:
1301             rank = self.db.execute("SELECT rank FROM categories WHERE id=?;", (key,) ).fetchone()[0]
1302             self.db.execute("DELETE FROM categories WHERE id=?;", (key, ))
1303             self.db.execute("UPDATE categories SET rank=rank-1 WHERE rank>?;", (rank,) )
1304             self.db.execute("UPDATE feeds SET category=1 WHERE category=?;", (key,) )
1305             self.db.commit()
1306
1307     #def saveConfig(self):
1308     #    self.listOfFeeds["feedingit-order"] = self.sortedKeys
1309     #    file = open(self.configdir+"feeds.pickle", "w")
1310     #    pickle.dump(self.listOfFeeds, file)
1311     #    file.close()
1312
1313     def moveUp(self, key):
1314         rank = self.db.execute("SELECT rank FROM feeds WHERE id=?;", (key,)).fetchone()[0]
1315         if rank>0:
1316             self.db.execute("UPDATE feeds SET rank=? WHERE rank=?;", (rank, rank-1) )
1317             self.db.execute("UPDATE feeds SET rank=? WHERE id=?;", (rank-1, key) )
1318             self.db.commit()
1319
1320     def moveCategoryUp(self, key):
1321         rank = self.db.execute("SELECT rank FROM categories WHERE id=?;", (key,)).fetchone()[0]
1322         if rank>0:
1323             self.db.execute("UPDATE categories SET rank=? WHERE rank=?;", (rank, rank-1) )
1324             self.db.execute("UPDATE categories SET rank=? WHERE id=?;", (rank-1, key) )
1325             self.db.commit()
1326
1327     def moveDown(self, key):
1328         rank = self.db.execute("SELECT rank FROM feeds WHERE id=?;", (key,)).fetchone()[0]
1329         max_rank = self.db.execute("SELECT MAX(rank) FROM feeds;").fetchone()[0]
1330         if rank<max_rank:
1331             self.db.execute("UPDATE feeds SET rank=? WHERE rank=?;", (rank, rank+1) )
1332             self.db.execute("UPDATE feeds SET rank=? WHERE id=?;", (rank+1, key) )
1333             self.db.commit()
1334
1335     def moveCategoryDown(self, key):
1336         rank = self.db.execute("SELECT rank FROM categories WHERE id=?;", (key,)).fetchone()[0]
1337         max_rank = self.db.execute("SELECT MAX(rank) FROM categories;").fetchone()[0]
1338         if rank<max_rank:
1339             self.db.execute("UPDATE categories SET rank=? WHERE rank=?;", (rank, rank+1) )
1340             self.db.execute("UPDATE categories SET rank=? WHERE id=?;", (rank+1, key) )
1341             self.db.commit()
1342
1343