使用Python抓取美团数据存于Excel中

野性酷女 2022-09-24 04:21 191阅读 0赞

> **0.**程序是针对美团中的美食部分数据按好评排序采集。
> 
> 要抓取保存的数据为：
> 
> 商家名 类型  地理位置  评论人数  均价  最低价格
> 
> **1.**首先编写网页数据采集函数，使用request采集网页源码，具体实现如下
> 
>     def getHtml(url):
>         headers = ('User-Agent',
>         'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11')
>         opener = urllib.request.build_opener()
>         opener.addheaders = [headers]
>         htmldata = opener.open(url).read()
>         htmldata=htmldata.decode('utf-8')
>         return htmldata
> 
>   
> 
> 
> **2.**根据网页源码解析获取已上线城市的url
> 
>     class GetCityUrl(HTMLParser):
>         part = ('gaevent','changecity/build')
>         urldic = {}
>         def handle_starttag(self, tag, attrs):
>             if tag=='a' and (self.part in attrs):
>                 for att,value in attrs:
>                     if att=='href':
>                         self.urldic.__setitem__(value, value+'/category/meishi/all/rating')
>                         
>         def getUrl(self):
>             return self.urldic
> 
>   
> 
> 
> **3.**获取分页url
> 
>     class GetPages(HTMLParser):
>         pagelist = list()
>         temphref = str()
>         flg = 0
>         initurl = str()
>         def setInitUrl(self,url):
>             self.initurl = url
>         def handle_starttag(self, tag, attrs):
>             if tag=='a':
>                 for attr,value in attrs:
>                     if attr=='href' and ('page' in value):
>                         self.temphref = self.initurl + value
>                         if self.temphref not in self.pagelist:
>                             self.pagelist.append(self.temphref)
>     
>         def getList(self):
>             return self.pagelist
> 
>   
> 
> 
> **4.**解析网页源码 获取有效信息  
> 
> 
>     class MyHTMLParser(HTMLParser):
>         tempstr = str()
>         divsum = int()
>         def handle_starttag(self, tag, attrs):
>             if tag=='div':
>                 for attr,value in attrs:
>                     if attr=='class' and value.find('poi-tile-nodeal')!=-1:
>                         self.tempstr=''
>                         self.divsum = 0
>     
>         def handle_data(self, data):
>             if(data.isspace()==False):
>                 data = data.replace('·', '·')
>                 if  data=='¥':
>                     if '¥' not in self.tempstr:
>                         self.tempstr+='无' +'\t'
>                     self.tempstr+=data
>                 elif data=='¥':
>                     if '¥' not in self.tempstr:
>                         self.tempstr+='无' +'\t'
>                     self.tempstr+='¥'
>                 elif data=='人评价':
>                     self.tempstr=self.tempstr[0:-1]+data+'\t'
>                 elif data=='人均 ':
>                     self.tempstr+='人均'
>                 elif data[0]=='起':
>                     self.tempstr=self.tempstr[0:-1]+'起'
>                 else:
>                     self.tempstr+=data+'\t'
>             
>         def handle_endtag(self, tag):
>             if tag=='div':
>                 self.divsum+=1
>                 if self.divsum==6:
>                     if (self.tempstr.find('¥'))!=-1:
>                         if (re.split(r'\t', self.tempstr).__len__())==5:
>                             teststr = str()
>                             flg = 0
>                             for stmp in re.split(r'\t',self.tempstr):
>                                 if flg==2:
>                                     teststr+='无位置信息'+'\t'
>                                 teststr+=stmp+'\t'
>                                 flg+=1
>                             self.tempstr=teststr
>                         if (re.split(r'\t', self.tempstr).__len__())==6:
>                             arraystr.append(self.tempstr)
>                             self.divsum=0
>                             self.tempstr=''
> 
>   
>   
> 
> 
>   
> 
> 
> **5.**将信息存放于Excel中
> 
>     def SaveExcel(listdata):
>         head=['商家名','类型','地理位置','评论人数','均价','最低价格']
>         wbk=xlwt.Workbook()
>         sheet1=wbk.add_sheet("sheet1")
>         ii=0
>         for testhand in head:
>             sheet1.write(0,ii,testhand)
>             ii+=1
>         i=1
>         j=0
>         for stt in listdata:
>             j=0
>             lis = re.split(r'\t',stt)
>             for ls in lis:
>                 sheet1.write(i,j,ls)
>                 j=j+1
>             i+=1
>         wbk.save('test.xls')
> 
> 以下是Excel中的数据：
> 
> ![20160723223319078][]  
> 
> 
>   
> 
> 
> 附录完整代码：
> 
>     #encoding:utf-8
>     '''
>     Created on 2016年7月22日
>     python version 3.5
>     @author: baalhuo
>     '''
>     from html.parser import HTMLParser
>     import re
>     import urllib.request
>     
>     import xlwt
>     import time
>     
>     #存放采集的商家信息
>     arraystr = list()
>     
>     #解析网页源码 获取有效信息
>     class MyHTMLParser(HTMLParser):
>         tempstr = str()
>         divsum = int()
>         def handle_starttag(self, tag, attrs):
>             if tag=='div':
>                 for attr,value in attrs:
>                     if attr=='class' and value.find('poi-tile-nodeal')!=-1:
>                         self.tempstr=''
>                         self.divsum = 0
>     
>         def handle_data(self, data):
>             if(data.isspace()==False):
>                 data = data.replace('·', '·')
>                 if  data=='¥':
>                     if '¥' not in self.tempstr:
>                         self.tempstr+='无' +'\t'
>                     self.tempstr+=data
>                 elif data=='¥':
>                     if '¥' not in self.tempstr:
>                         self.tempstr+='无' +'\t'
>                     self.tempstr+='¥'
>                 elif data=='人评价':
>                     self.tempstr=self.tempstr[0:-1]+data+'\t'
>                 elif data=='人均 ':
>                     self.tempstr+='人均'
>                 elif data[0]=='起':
>                     self.tempstr=self.tempstr[0:-1]+'起'
>                 else:
>                     self.tempstr+=data+'\t'
>             
>         def handle_endtag(self, tag):
>             if tag=='div':
>                 self.divsum+=1
>                 if self.divsum==6:
>                     if (self.tempstr.find('¥'))!=-1:
>                         if (re.split(r'\t', self.tempstr).__len__())==5:
>                             teststr = str()
>                             flg = 0
>                             for stmp in re.split(r'\t',self.tempstr):
>                                 if flg==2:
>                                     teststr+='无位置信息'+'\t'
>                                 teststr+=stmp+'\t'
>                                 flg+=1
>                             self.tempstr=teststr
>                         if (re.split(r'\t', self.tempstr).__len__())==6:
>                             arraystr.append(self.tempstr)
>                             self.divsum=0
>                             self.tempstr=''
>     
>     #获取美团已上线城市的url 目前为844个城市地区
>     class GetCityUrl(HTMLParser):
>         part = ('gaevent','changecity/build')
>         urldic = {}
>         def handle_starttag(self, tag, attrs):
>             if tag=='a' and (self.part in attrs):
>                 for att,value in attrs:
>                     if att=='href':
>                         self.urldic.__setitem__(value, value+'/category/meishi/all/rating')
>                         
>         def getUrl(self):
>             return self.urldic
>     
>     #获取分页URL
>     class GetPages(HTMLParser):
>         pagelist = list()
>         temphref = str()
>         flg = 0
>         initurl = str()
>         def setInitUrl(self,url):
>             self.initurl = url
>         def handle_starttag(self, tag, attrs):
>             if tag=='a':
>                 for attr,value in attrs:
>                     if attr=='href' and ('page' in value):
>                         self.temphref = self.initurl + value
>                         if self.temphref not in self.pagelist:
>                             self.pagelist.append(self.temphref)
>     
>         def getList(self):
>             return self.pagelist
>     
>     #采集网页源码信息
>     def getHtml(url):
>         headers = ('User-Agent',
>         'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11')
>         opener = urllib.request.build_opener()
>         opener.addheaders = [headers]
>         htmldata = opener.open(url).read()
>         htmldata=htmldata.decode('utf-8')
>         return htmldata
>     
>     #将信息保存到Excel中
>     def SaveExcel(listdata):
>         head=['商家名','类型','地理位置','评论人数','均价','最低价格']
>         wbk=xlwt.Workbook()
>         sheet1=wbk.add_sheet("sheet1")
>         ii=0
>         for testhand in head:
>             sheet1.write(0,ii,testhand)
>             ii+=1
>         i=1
>         j=0
>         for stt in listdata:
>             j=0
>             lis = re.split(r'\t',stt)
>             for ls in lis:
>                 sheet1.write(i,j,ls)
>                 j=j+1
>             i+=1
>         wbk.save('e:/test3.xls')
>     
>     par = GetCityUrl()
>     par.feed(getHtml('http://www.meituan.com/index/changecity/initiative'))
>     urldic = par.getUrl()
>     
>     par = MyHTMLParser()
>     
>     print(time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time())))
>     
>     ffwait=1
>     
>     for url in urldic:
>         data = getHtml(urldic.get(url))
>         
>         getpage = GetPages()
>         getpage.setInitUrl(url)
>         getpage.feed(data)
>         pageurllist = getpage.getList()
>      
>         par.feed(data)
>         for urltemp in pageurllist:
>             par.feed(getHtml(urltemp))
>         arraystr.append('切换地区     ')
>         if ffwait ==4:#此处只抓取了4个城市数据
>             break;
>         ffwait+=1
>     SaveExcel(arraystr)
>     print(time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time())))
>     print('Done')
> 
>   
> 
> 
>   
> 
> 
> **学之，以记之。**

[20160723223319078]: /images/20220720/cfb100070d4d4d7baef53ab7aee7017d.png