Move download management from frontends to rss_sqlite.py.

[feedingit] / src / httpprogresshandler.py

diff --git a/src/httpprogresshandler.py b/src/httpprogresshandler.py
new file mode 100644 (file)
index 0000000..78eb39c
--- /dev/null
+++ b/src/httpprogresshandler.py
@@ -0,0 +1,143 @@
+#!/usr/bin/env python2.5
+
+# Copyright (c) 2011 Neal H. Walfield <neal@walfield.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import urllib2 
+import httplib
+import time
+
+class ProgressSocket(object):
+    """
+    Monitor what is being sent and received.
+    """
+    def __init__(self, socket, connection):
+        self.socket = socket
+        self.connection = connection
+
+    def __getattribute__(self, attr):
+        # print "%s.__getattribute__(%s)" % (self.__class__.__name__, attr)
+
+        def send(data):
+            # 100k at a time.
+            bs = 100 * 1024
+            sent = 0
+            while sent < len (data):
+                remaining = len (data) - sent
+                if remaining < bs:
+                    amount = remaining
+                else:
+                    amount = bs
+        
+                self.socket.sendall(data[sent:sent+amount])
+                sent += amount
+                self.connection.stats['sent'] += amount
+                self.connection.opener.stats['sent'] += amount
+        
+                if self.connection.callback is not None:
+                    self.connection.callback ()
+        
+        def read(*args, **kwargs):
+            data = self.socket.read (*args, **kwargs)
+            # print "GOT: %s" % (data[0:240],)
+            self.connection.stats['received'] += len (data)
+            self.connection.opener.stats['received'] += len (data)
+            if self.connection.callback is not None:
+                self.connection.callback ()
+            return data
+
+        if attr == 'send' or attr == 'sendall':
+            return send
+        if attr == 'read':
+            return read
+
+        try:
+            return super (ProgressSocket, self).__getattribute__(attr)
+        except AttributeError:
+            socket = super (ProgressSocket, self).__getattribute__('socket')
+            return socket.__getattribute__(attr)
+
+    def makefile(self, mode, bufsize):
+        return ProgressSocket (socket=self.socket.makefile(mode, bufsize),
+                               connection=self.connection)
+
+    def close(self):
+        return self.socket.close ()
+
+def HTTPProgressConnectionBuilder(callback, opener):
+    class HTTPProgressConnection(httplib.HTTPConnection):
+        def __init__(self, *args, **kwargs):
+            self.method = None
+            self.url = None
+            return httplib.HTTPConnection.__init__ (self, *args, **kwargs)
+
+        def putrequest(self, method, url, *args, **kwargs):
+            self.method = method
+            self.url = url
+            return httplib.HTTPConnection.putrequest (
+                self, method, url, *args, **kwargs)
+
+        def connect(self):
+            httplib.HTTPConnection.connect(self)
+            # Wrap the socket.
+            self.sock = ProgressSocket(socket=self.sock,
+                                       connection=self)
+
+    HTTPProgressConnection.callback = callback
+    HTTPProgressConnection.opener = opener
+    HTTPProgressConnection.stats \
+        = {'sent': 0, 'received': 0, 'started':time.time()}
+    return HTTPProgressConnection
+
+class HTTPProgressHandler(urllib2.HTTPHandler):
+    def __init__(self, callback):
+        self.callback = callback
+        self.stats = {'sent': 0, 'received': 0, 'started':time.time()}
+        return urllib2.HTTPHandler.__init__(self)
+
+    def http_open(self, request):
+        return self.do_open(
+            HTTPProgressConnectionBuilder(self.callback, self),
+            request)
+
+if __name__ == '__main__':
+    def callback(connection):
+        req = ""
+        if connection.method:
+            req += connection.method + " "
+        req += connection.host + ':' + str (connection.port)
+        if connection.url:
+            req += connection.url
+
+        cstats = connection.stats
+        ostats = connection.opener.stats
+
+        print (("%s: connection: %d sent, %d received: %d kb/s; "
+                + "opener: %d sent, %d received, %d kb/s")
+               % (req,
+                  cstats['sent'], cstats['received'],
+                  ((cstats['sent'] + cstats['received'])
+                   / (time.time() - cstats['started']) / 1024),
+                  ostats['sent'], ostats['received'],
+                  ((ostats['sent'] + ostats['received'])
+                   / (time.time() - ostats['started']) / 1024)))
+
+    opener = urllib2.build_opener(HTTPProgressHandler(callback))
+
+    data = opener.open ('http://google.com')
+    downloaded = 0
+    for d in data:
+        downloaded += len (d)
+    print "Document is %d bytes in size" % (downloaded,)