全网股票爬取

ゝ一纸荒年。 2022-09-30 06:49 174阅读 0赞

注：使用了东方财富网提供的股票列表，配合百度股票的数据实现，实现上交所和深交所的股票信息爬取

import requests,re
    from bs4 import BeautifulSoup
    
    def getHTMLText(url):
        try:
            r = requests.get(url,timeout=30)
            r.raise_for_status()
            r.encoding = r.apparent_encoding
            return r.text
        except:
            return ''
        
    #获取股票编号的列表
    def getStockList(stocklist,url):
        html = getHTMLText(url)
        soup = BeautifulSoup(html , 'html.parser')
        a = soup('a',href = re.compile(r's[zh]\d{6}'))
        for i in a:
            try:
                href = i.attrs['href']
                id = re.findall(r's[zh]\d{6}',href)[0]
                stocklist.append(id)
            except:
                continue
    
    #获取股票信息
    def getStockInfo(stockList,stockSearchUrl):
        count = 1
        for i in stockList:
            searchUrl = stockSearchUrl + i + '.html'
            html = getHTMLText(searchUrl)
            try:
                if html == '':
                    continue
                
                soup = BeautifulSoup(html , 'html.parser')
                stockinfos = soup.find('div',attrs = {'class' : 'stock-bets'})
                stockDict = {}
    
                stockname = stockinfos.find_all(attrs = {'class' : 'bets-name'})[0]
                stockDict.update({'\n股票名称':stockname.text.split()[0]})
    
                keyList = stockinfos.find_all('dt')
                valueList = stockinfos.find_all('dd')
    
                for j in range(len(keyList)):
                    key= keyList[j].text
                    val = valueList[j].text
                    stockDict[key] = val
                    
                with open('股票信息.txt','a') as f:
                    for k,v in stockDict.items():
                        f.write('\n\t{}:{}'.format(k,v))
                    print("\r当前进度：{:.2f}".format(count*100/len(stockList)),end="")
                    count += 1
            except:
                print("\r当前进度：{:.2f}%".format(count*100/len(stockList)),end="")
                count += 1
                continue
       
    def main():
        stockListUrl = 'http://quote.eastmoney.com/stocklist.html'
        stockSearchUrl = 'https://gupiao.baidu.com/stock/'
        #股票编号列表
        stockList = []
        getStockList(stockList,stockListUrl)
        getStockInfo(stockList,stockSearchUrl)
        
    main()

因为股票数量太多，仅测试了一部分数据。

![SouthEast][]![SouthEast 1][]

[SouthEast]: https://img-blog.csdn.net/20170319194039055?watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQveXVhbmxhaWppa2U=/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast
[SouthEast 1]: /images/20220708/f806950a292c4e4f96c0314cd3b2cc8b.png