Source code for inphosite.lib.multi_get

from threading import Thread, enumerate
from urllib2 import urlopen, Request
from time import sleep

UPDATE_INTERVAL = 0.01

[docs]class URLThread(Thread): def __init__(self,url): super(URLThread, self).__init__() self.url = url self.response = None
[docs] def run(self): self.request = Request(self.url, None, {'Referer' : 'http://inpho.cogs.indiana.edu'}) self.request = urlopen(self.request) self.response = self.request.read()
[docs]def multi_get(uris,timeout=2.5): def alive_count(lst): alive = map(lambda x : 1 if x.isAlive() else 0, lst) return reduce(lambda a,b : a + b, alive) threads = [ URLThread(uri) for uri in uris ] for thread in threads: thread.start() while alive_count(threads) > 0 and timeout > 0.0: timeout = timeout - UPDATE_INTERVAL sleep(UPDATE_INTERVAL) return [ (x.url, x.response) for x in threads ]

This Page