用 python 实现一个多线程网页下载器

喜欢ヅ旅行 2022-02-05 08:29 216阅读 0赞

很久没有更新博客了，今天上来分享一下昨天实现的一个多线程网页下载器。

这是一个有着真实需求的实现，我的用途是拿它来通过 HTTP 方式向服务器提交游戏数据。把它放上来也是想大家帮忙挑刺，找找 bug，让它工作得更好。

keywords:python,http,multi-threads,thread,threading,httplib,urllib,urllib2,Queue,http pool,httppool

废话少说，上源码：

\# -\*- coding:utf-8 -\*-import urllib, httplibimport threadimport timefrom Queue import Queue, Empty, FullHEADERS = \{"Content-type": "application/x-www-form-urlencoded",      'Accept-Language':'zh-cn',      'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0;Windows NT 5.0)',      "Accept": "text/plain"\}UNEXPECTED\_ERROR = -1POST = 'POST'GET = 'GET'def base\_log(msg): print msgdef base\_fail\_op(task, status, log): log('fail op. task = %s, status = %d'%(str(task), status))def get\_remote\_data(tasks, results, fail\_op = base\_fail\_op, log = base\_log): while True:  task = tasks.get()  try:   tid = task\['id'\]   hpt = task\['conn\_args'\] \# hpt <= host:port, timeout  except KeyError, e:   log(str(e))   continue  log('thread\_%s doing task %d'%(thread.get\_ident(), tid))  \#log('hpt = ' + str(hpt))  conn = httplib.HTTPConnection(\*\*hpt)     try:   params = task\['params'\]  except KeyError, e:   params = \{\}  params = urllib.urlencode(params)  \#log('params = ' + params)    try:   method = task\['method'\]  except KeyError:   method = 'GET'  \#log('method = ' + method)    try:   url = task\['url'\]  except KeyError:   url = '/'  \#log('url = ' + url)    headers = HEADERS  try:   tmp = task\['headers'\]  except KeyError, e:   tmp = \{\}  headers.update(tmp)  \#log('headers = ' + str(headers))  headers\['Content-Length'\] = len(params)    try:   if method == POST:    conn.request(method, url, params, headers)   else:    conn.request(method, url + params)   response = conn.getresponse()  except Exception, e:   log('request failed. method = %s, url = %s, params = %s headers = %s'%(      method, url, params, headers))   log(str(e))   fail\_op(task, UNEXPECTED\_ERROR, log)   continue     if response.status != httplib.OK:   fail\_op(task, response.status, log)   continue     data = response.read()  results.put((tid, data), True)  class HttpPool(object): def \_\_init\_\_(self, threads\_count, fail\_op, log):  self.\_tasks = Queue()  self.\_results = Queue()    for i in xrange(threads\_count):   thread.start\_new\_thread(get\_remote\_data,                (self.\_tasks, self.\_results, fail\_op, log))    def add\_task(self, tid, host, url, params, headers = \{\}, method = 'GET', timeout = None):  task = \{   'id' : tid,   'conn\_args' : \{'host' : host\} if timeout is None else \{'host' : host, 'timeout' : timeout\},   'headers' : headers,   'url' : url,   'params' : params,   'method' : method,   \}  try:   self.\_tasks.put\_nowait(task)  except Full:   return False  return True   def get\_results(self):  results = \[\]  while True:   try:    res = self.\_results.get\_nowait()   except Empty:    break   results.append(res)  return results  def test\_google(task\_count, threads\_count): hp = HttpPool(threads\_count, base\_fail\_op, base\_log) for i in xrange(task\_count):  if hp.add\_task(i,    'www.google.cn',    '/search?',    \{'q' : 'lai'\},\#    method = 'POST'    ):   print 'add task successed.'    while True:  results = hp.get\_results()  if not results:   time.sleep(1.0 \* random.random())  for i in results:   print i\[0\], len(i\[1\])\#   print unicode(i\[1\], 'gb18030')   if \_\_name\_\_ == '\_\_main\_\_': import sys, random task\_count, threads\_count = int(sys.argv\[1\]), int(sys.argv\[2\]) test\_google(task\_count, threads\_count)

有兴趣想尝试运行的朋友，可以把它保存为 xxxx.py，然后执行 python xxxx.py 10 4，其中 10 表示向 google.cn 请求 10 次查询，4 表示由 4 条线程来执行这些任务。