Twisted Parallel1#
Question#
Parallel Download using Twisted.
Solution#
from twisted.internet import defer, task
from twisted.python import log
from twisted.internet import reactor
from twisted.web import client
from twisted.internet.utils import getProcessValue
def parallel(iterable, count, callable, *args, **named):
print(args, named)
coop = task.Cooperator()
work = (callable(elem, *args, **named) for elem in iterable)
return defer.DeferredList([coop.coiterate(work) for i in range(count)])
def download(xxx_todo_changeme):
(url, fileName) = xxx_todo_changeme
return client.downloadPage(url, file(fileName, 'wb'))
urls = [(url, str(n)) for (n, url) in enumerate(file('urls.txt'))]
finished = parallel(urls, 50, download)
finished.addErrback(log.err)
finished.addCallback(lambda ign: reactor.stop())
reactor.run()