Tried out an optimization that didn't seem to work.

[doneit] / src / rtm_api.py

diff --git a/src/rtm_api.py b/src/rtm_api.py
index c30b5b9..b748b38 100644 (file)
--- a/src/rtm_api.py
+++ b/src/rtm_api.py
@@ -91,24 +91,53 @@ class RTMapi(object):
                warnings.warn("Performing download of %s" % url, stacklevel=5)
                return urllib2.urlopen(url)
 
-       def get(self, **params):
-               "Get the XML response for the passed `params`."
-               params['api_key'] = self._apiKey
-               params['format'] = 'json'
-               params['api_sig'] = self._sign(params)
+       @staticmethod
+       def read_by_length(connection, timeout):
+               # It appears that urllib uses the non-blocking variant of file objects
+               # which means reads might not always be complete, so grabbing as much
+               # of the data as possible with a sleep in between to give it more time
+               # to grab data.
+               contentLengthField = "Content-Length"
+               assert contentLengthField in connection.info(), "Connection didn't provide content length info"
+               specifiedLength = int(connection.info()["Content-Length"])
 
-               connection = self.open_url(SERVICE_URL, params)
+               actuallyRead = 0
+               chunks = []
+               chunk = connection.read()
+               while 0 < timeout:
+                       actuallyRead += len(chunk)
+                       if actuallyRead == specifiedLength:
+                               break
+                       chunks.append(chunk)
+                       time.sleep(1)
+                       timeout -= 1
+                       chunk = connection.read()
+               chunks.append(chunk)
+               json = "".join(chunks)
 
+               if "Content-Length" in connection.info():
+                       assert len(json) == int(connection.info()["Content-Length"]), "The packet header promised %s of data but only was able to read %s of data" % (
+                               connection.info()["Content-Length"],
+                               len(json),
+                       )
+
+               return json
+
+       @staticmethod
+       def read_by_guess(connection, timeout):
                # It appears that urllib uses the non-blocking variant of file objects
                # which means reads might not always be complete, so grabbing as much
                # of the data as possible with a sleep in between to give it more time
                # to grab data.
+
                chunks = []
                chunk = connection.read()
-               while chunk:
+               while chunk and 0 < timeout:
                        chunks.append(chunk)
                        time.sleep(1)
+                       timeout -= 1
                        chunk = connection.read()
+               chunks.append(chunk)
                json = "".join(chunks)
 
                if "Content-Length" in connection.info():
@@ -117,6 +146,18 @@ class RTMapi(object):
                                len(json),
                        )
 
+               return json
+
+       def get(self, **params):
+               "Get the XML response for the passed `params`."
+               params['api_key'] = self._apiKey
+               params['format'] = 'json'
+               params['api_sig'] = self._sign(params)
+
+               connection = self.open_url(SERVICE_URL, params)
+               json = self.read_by_guess(connection, 5)
+               # json = self.read_by_length(connection, 5)
+
                data = DottedDict('ROOT', parse_json(json))
                rsp = data.rsp