1 import urllib, threading, os, gzip, time, simplejson, re
2 _DUMP_URL = '''http://img.jamendo.com/data/dbdump_artistalbumtrack.xml.gz'''
3 _DUMP = os.path.expanduser('''~/.cache/jamaendo/dbdump.xml.gz''')
4 _DUMP_TMP = os.path.expanduser('''~/.cache/jamaendo/new_dbdump.xml.gz''')
7 # /get2/stream/track/m3u/radio_track_inradioplaylist/?order=numradio_asc&radio_id=283
11 os.makedirs(os.path.dirname(_DUMP))
16 return os.path.isfile(_DUMP)
18 def _file_is_old(fil, old_age):
19 return os.path.getmtime(fil) < (time.time() - old_age)
22 return not has_dump() or _file_is_old(_DUMP, 60*60*24) # 1 day
24 def refresh_dump(complete_callback, progress_callback=None, force=False):
25 if force or _dump_is_old():
26 downloader = Downloader(complete_callback, progress_callback)
29 complete_callback(True)
31 class Downloader(threading.Thread):
32 def __init__(self, complete_callback, progress_callback):
33 threading.Thread.__init__(self)
34 self.complete_callback = complete_callback
35 self.progress_callback = progress_callback
37 def actual_callback(self, numblocks, blocksize, filesize):
38 if self.progress_callback:
40 percent = min((numblocks*blocksize*100)/filesize, 100)
43 self.progress_callback(percent)
48 urllib.urlretrieve(_DUMP_URL, _DUMP_TMP, self.actual_callback)
49 if os.path.isfile(_DUMP):
51 os.rename(_DUMP_TMP, _DUMP)
54 self.complete_callback(success)
56 def fast_iter(context, func):
57 for event, elem in context:
60 while elem.getprevious() is not None:
61 del elem.getparent()[0]
64 from lxml import etree
69 if isinstance(v, basestring):
70 return v.encode('utf-8')
73 return "{%s}" % (", ".join("%s=%s"%(k.encode('utf-8'), printable(v)) \
74 for k,v in self.__dict__.iteritems() if not k.startswith('_')))
76 class LocalDB(object):
81 self.fil = gzip.open(_DUMP)
86 def make_album_brief(self, element):
90 ret['id'] = int(info.text)
91 elif info.tag == 'name':
92 ret['name'] = info.text
95 def make_artist_obj(self, element):
99 ret['id'] = int(child.text)
100 elif child.tag in ('name', 'image'):
101 ret[child.tag] = child.text
102 elif child.tag == 'Albums':
103 ret['albums'] = [self.make_album_brief(a) for a in child]
106 def make_track_obj(self, element):
112 ret['mp3'] = Query.track_mp3(_id)
113 ret['ogg'] = Query.track_ogg(_id)
114 elif info.tag in ('name', 'numalbum'):
115 ret[info.tag] = info.text
118 def make_album_obj(self, element):
120 artist = element.getparent().getparent()
121 if artist is not None:
123 if child.tag == 'name':
124 ret['artist_name'] = child.text
125 elif child.tag == 'id':
126 ret['artist_id'] = int(child.text)
127 for child in element:
128 if child.tag == 'id':
129 ret['id'] = int(child.text)
130 elif child.tag in ('name', 'image'):
132 ret[child.tag] = child.text
135 elif child.tag == 'Tracks':
136 ret['tracks'] = [self.make_track_obj(t) for t in child]
139 def artist_walker(self, name_match):
140 for event, element in etree.iterparse(self.fil, tag="artist"):
141 name = element.xpath('./name')[0].text.lower()
142 if name and name.find(name_match) > -1:
143 yield self.make_artist_obj(element)
145 while element.getprevious() is not None:
146 del element.getparent()[0]
149 def album_walker(self, name_match):
150 for event, element in etree.iterparse(self.fil, tag="album"):
151 name = element.xpath('./name')[0].text
152 if name and name.lower().find(name_match) > -1:
153 yield self.make_album_obj(element)
155 while element.getprevious() is not None:
156 del element.getparent()[0]
159 def artistid_walker(self, artistids):
160 for event, element in etree.iterparse(self.fil, tag="artist"):
161 _id = element.xpath('./id')[0].text
162 if _id and int(_id) in artistids:
163 yield self.make_artist_obj(element)
165 while element.getprevious() is not None:
166 del element.getparent()[0]
169 def albumid_walker(self, albumids):
170 for event, element in etree.iterparse(self.fil, tag="album"):
171 _id = element.xpath('./id')[0].text
172 if _id and (int(_id) in albumids):
173 yield self.make_album_obj(element)
175 while element.getprevious() is not None:
176 del element.getparent()[0]
179 def search_artists(self, substr):
180 substr = substr.lower()
181 return (artist for artist in self.artist_walker(substr))
183 def search_albums(self, substr):
184 substr = substr.lower()
185 return (album for album in self.album_walker(substr))
187 def get_artists(self, artistids):
188 return (artist for artist in self.artistid_walker(artistids))
190 def get_albums(self, albumids):
191 return (album for album in self.albumid_walker(albumids))
193 _GET2 = '''http://api.jamendo.com/get2/'''
196 last_query = time.time()
198 cache_time = 60*60*24
199 rate_limit = 1.0 # max queries per second
202 select=['id', 'name', 'image', 'artist_name', 'artist_id'],
204 track=['track_album', 'album_artist']):
205 if request == 'track':
206 self.url = "%s%s/%s/json/%s" % (_GET2, '+'.join(select), request, '+'.join(track))
208 self.url = "%s%s/%s/json/" % (_GET2, '+'.join(select), request)
210 def __call__(self, order=None, count=5, query=None, albumids=None):
211 return self.emit(order=order, count=count, query=query, albumids=albumids)
213 def emit(self, order=None, count=5, query=None, albumids=None):
214 """ratelimited query"""
216 paramdict = {'n':count}
217 if order is not None:
218 paramdict['order'] = order
219 if query is not None:
220 paramdict['searchquery'] = query
221 if albumids is not None:
222 paramdict['album_id'] = " ".join(str(_id) for _id in albumids)
223 params = urllib.urlencode(paramdict)
224 url = self.url + "?%s" % (params)
225 f = urllib.urlopen(url)
226 ret = simplejson.load(f)
230 def _ratelimit(self):
232 if now - self.last_query < self.rate_limit:
233 time.sleep(self.rate_limit - (now - self.last_query))
234 self.last_query = now
238 def album_cover(albumid, size=200):
239 to = '~/.cache/jamaendo/cover-%d-%d.jpg'%(albumid, size)
240 if not os.path.isfile(to):
241 url = _GET2+'image/album/redirect/?id=%d&imagesize=%d'%(albumid, size)
242 urllib.urlretrieve(url, to)
246 def track_ogg(trackid):
247 return _GET2+ 'stream/track/redirect/?id=%d&streamencoding=ogg2'%(trackid)
250 def track_mp3(trackid):
251 return _GET2+ 'stream/track/redirect/?id=%d&streamencoding=mp31'%(trackid)
253 # http://www.jamendo.com/get2/id+name+idstr+image/radio/json?order=starred_desc
254 #track_id/track/json/radio_track_inradioplaylist/?order=numradio_asc&radio_id=%i
255 class Queries(object):
257 def albums_this_week():
258 return Query().emit(order='ratingweek_desc')
260 def albums_all_time():
261 return Query().emit(order='ratingtotal_desc')
263 def albums_this_month():
264 return Query().emit(order='ratingmonth_desc')
267 return Query().emit(order='ratingday_desc')
269 def playlists_all_time():
270 q = Query(select=['id','name', 'user_idstr'], request='playlist')
271 return q.emit(order='ratingtotal_desc')
274 def tracks_this_month():
275 q = Query(select=['id', 'name',
277 'album_name', 'artist_name',
278 'album_id', 'artist_id'],
280 return q.emit(order='ratingmonth_desc')
283 def search_albums(query):
285 return q.emit(order='searchweight_desc', query=query)
288 def search_artists(query):
289 q = Query(request='artist', select=['id', 'name', 'image'])
290 return q.emit(order='searchweight_desc', query=query)
293 def album_tracks(albumids, select=['id',
302 #http://api.jamendo.com/get2/id+name/track/jsonpretty/?album_id=33+46
303 q = Query(select=select,
305 ret = q.emit(albumids=albumids, count=100)
307 track['mp3'] = Query.track_mp3(int(track['id']))
308 track['ogg'] = Query.track_ogg(int(track['id']))