python3 [爬虫入门实战]爬虫之scrapy爬取游天下南京短租房存mongodb

素颜马尾好姑娘i 2021-06-10 20:42 620阅读 0赞

总结：总的来说不是很难，只是提取的字段有些多。总共获取了一个120多个南京房租信息

这里写图片描述

1 爬取的item

  1. # -*- coding: utf-8 -*-
  3. # Define here the models for your scraped items
  4. #
  5. # See documentation in:
  6. # http://doc.scrapy.org/en/latest/topics/items.html
  8. import scrapy
  11. class YoutxnanjinItem(scrapy.Item):
  12.     # define the fields for your item here like:
  13.     # name = scrapy.Field()
  14.     # pass
  16.     # 房源名称
  17.     homeName = scrapy.Field()
  18.     # 房源链接
  19.     homeLine = scrapy.Field()
  20.     # 房租单价
  21.     homeSinglePrice = scrapy.Field()
  22.     # 房租地址
  23.     homeAddress = scrapy.Field()
  24.     # 房租近期信息
  25.     homeDetai = scrapy.Field()
  26.     # 满七天价格
  27.     homeSeven = scrapy.Field()
  28.     # 满30天价格
  29.     homeThirth = scrapy.Field()
  31.     # 房东
  32.     homePerson = scrapy.Field()
  33.     # 房东头像
  34.     homePersonImg = scrapy.Field()
  35.     # 房东头像链接
  36.     homePersonLink = scrapy.Field()
  38.     # 房子大图
  39.     homePicBg = scrapy.Field()
  40.     # 房子大图链接
  41.     homePicLink = scrapy.Field()
  43.     # 品牌店铺信息
  44.     # homePinPai = scrapy.Field()
  45.     # 明星房东
  46.     # homeStarrPerson = scrapy.Field()

我就问：是不是注释很详细，。

2 spider里面的内容

  1. #encoding=utf8
  2. import scrapy
  3. from youtxNanJin.items import YoutxnanjinItem
  5. class NanJinDefault(scrapy.Spider):
  6.     name = 'youtx'
  7.     allowed_domains = ['youtx.com']
  8.     start_urls = ["http://www.youtx.com/nanjing/longrent1-page{}".format(n) for n in range(0,6)]
  9.     def parse(self, response):
  10.         # print(response.body)
  11.         node_list = response.xpath("//div[@class='duanzu houseList']/ul/li[@class='clearfix']")
  12.         # print(node_list)
  13.         for node in node_list:
  14.             item = YoutxnanjinItem()
  15.             homeName = node.xpath("./div[@class='houseInfo clearfix']/div[@class='house-tit clearfix']/h3/a/text()").extract()
  16.             homeLink = node.xpath("./div[@class='houseInfo clearfix']/div[@class='house-tit clearfix']/h3/a/@href").extract()
  17.             print(homeName)
  18.             print(homeLink)
  20.             # 单日价格
  21.             homeSinglePrice = node.xpath("./div[@class='houseInfo clearfix']/div[@class='house-tit clearfix']/div[@class='house-price mt9']/span/span[@class='housePrice']/text()").extract()
  22.             print(homeSinglePrice)
  24.             # 获取房源地址
  25.             homeAddress = node.xpath("./div[@class='houseInfo clearfix']/div[@class='houseInfo-left mt2']/p[@class='clearfix mt5']/text()").extract()
  26.             # 房租信息
  27.             homeDesc =node.xpath("./div[@class='houseInfo clearfix']/div[@class='houseInfo-left mt2']/p[@class='mt5']/text()").extract()
  28.             homeDesc2 =node.xpath("./div[@class='houseInfo clearfix']/div[@class='houseInfo-left mt2']/p[@class='mt5']/span[2]/text()").extract()
  29.             print(homeAddress)
  30.             print(homeDesc)
  31.             print(homeDesc2)
  33.             # 满30天的信息
  34.             homeThrty = node.xpath("./div[@class='houseInfo clearfix']/div[@class='house-tit clearfix']/div[@class='house-price mt9']/div[@class='mix12_5']/div[@class='discount']/div[@class='discount-price']/span//text()").extract()
  35.             print(homeThrty)
  36.             # 房东信息
  37.             homePerson = node.xpath("./div[@class='houseInfo clearfix']/div[@class='agentInfo mt16']/p[1]/a/text()").extract()
  38.             # 房东链接
  39.             homePersonLink = node.xpath("./div[@class='houseInfo clearfix']/div[@class='agentInfo mt16']/p[1]/a/@href").extract()
  40.             print(homePerson)
  41.             print(homePersonLink)
  43.             # 房源大图图片
  44.             homeBigPic = node.xpath("./div[@class='house-img']/a[1]/img/@src").extract()
  45.             homeBigPicLink = node.xpath("./div[@class='house-img']/a[1]/@href").extract()
  46.             print(homeBigPic)
  47.             print(homeBigPicLink)
  48.             # 房东头像信息
  49.             personPic = node.xpath("./div[@class='house-img']/a[2]/img/@src").extract()
  50.             # 房东头像链接地址
  51.             personPicLink = node.xpath("./div[@class='house-img']/a[2]/img/@href").extract()
  53.             print(personPic)
  54.             print(homePersonLink)
  55.             item['homeName'] ="".join(homeName)
  56.             item['homeLine'] ="".join(homeLink)
  57.             item['homeSinglePrice'] ="".join(homeSinglePrice)
  58.             item['homeAddress'] ="".join(homeAddress)
  59.             item['homeDetai'] ="".join(homeDesc)+"".join(homeDesc2)
  60.             # 这里的值暂时没有取出来
  61.             item['homeSeven'] ="".join(homeThrty)
  62.             item['homeThirth'] ="".join(homeThrty)
  64.             item['homePerson'] ="".join(homePerson)
  65.             item['homePersonImg'] ="".join(personPic)
  66.             item['homePersonLink'] ="".join(homePersonLink)
  67.             item['homePicBg'] ="".join(homeBigPic)
  68.             item['homePicLink'] ="".join(homeBigPicLink)
  69.             yield item

注意：里面xpath根据东西写的值比较长，可以提取出来，这里就暂时不进行提取了。

3 接下来是pipline管道流的地方

  1. # -*- coding: utf-8 -*-
  3. # Define your item pipelines here
  4. #
  5. # Don't forget to add your pipeline to the ITEM_PIPELINES setting
  6. # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
  7. import json
  8. from scrapy.conf import settings
  9. import pymongo
  11. class YoutxnanjinPipeline(object):
  12.     def process_item(self, item, spider):
  13.         return item
  15. class YouTXMongo(object):
  16.     def __init__(self):
  17.         self.client = pymongo.MongoClient(host=settings['MONGO_HOST'], port=settings['MONGO_PORT'])
  18.         self.db = self.client[settings['MONGO_DB']]
  19.         self.post = self.db[settings['MONGO_COLL']]
  21.     def process_item(self, item, spider):
  22.         postItem = dict(item)
  23.         self.post.insert(postItem)
  24.         return item
  26. # 写入json文件
  27. class JsonWritePipline(object):
  28.     def __init__(self):
  29.         self.file = open('游天下南京.json','w',encoding='utf-8')
  31.     def process_item(self,item,spider):
  32.         line  = json.dumps(dict(item),ensure_ascii=False)+"\n"
  33.         self.file.write(line)
  34.         return item
  36.     def spider_closed(self,spider):
  37.         self.file.close()

只要写了一次存数据方式，其他的跟着搬过来就行了，前提是不出毛病的情况下

4 settings里面的代码

这里主要是进行mongodb的配置，user-agent头信息的配置

  1. # -*- coding: utf-8 -*-
  3. # Scrapy settings for youtxNanJin project
  4. #
  5. # For simplicity, this file contains only settings considered important or
  6. # commonly used. You can find more settings consulting the documentation:
  7. #
  8. # http://doc.scrapy.org/en/latest/topics/settings.html
  9. # http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
  10. # http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
  12. BOT_NAME = 'youtxNanJin'
  14. SPIDER_MODULES = ['youtxNanJin.spiders']
  15. NEWSPIDER_MODULE = 'youtxNanJin.spiders'
  18. # Crawl responsibly by identifying yourself (and your website) on the user-agent
  19. # USER_AGENT = 'youtxNanJin (+http://www.yourdomain.com)'
  20. USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
  22. # Obey robots.txt rules
  23. ROBOTSTXT_OBEY = False
  24. # 配置mongoDB
  25. MONGO_HOST = "127.0.0.1"  # 主机IP
  26. MONGO_PORT = 27017  # 端口号
  27. MONGO_DB = "YouTianXia"  # 库名
  28. MONGO_COLL = "house_nanjin"  # collection
  29. # Configure maximum concurrent requests performed by Scrapy (default: 16)
  30. #CONCURRENT_REQUESTS = 32
  32. # Configure a delay for requests for the same website (default: 0)
  33. # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
  34. # See also autothrottle settings and docs
  35. #DOWNLOAD_DELAY = 3
  36. # The download delay setting will honor only one of:
  37. #CONCURRENT_REQUESTS_PER_DOMAIN = 16
  38. #CONCURRENT_REQUESTS_PER_IP = 16
  40. # Disable cookies (enabled by default)
  41. COOKIES_ENABLED = False
  43. # Disable Telnet Console (enabled by default)
  44. #TELNETCONSOLE_ENABLED = False
  46. # Override the default request headers:
  47. #DEFAULT_REQUEST_HEADERS = { 
  48. # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  49. # 'Accept-Language': 'en',
  50. #}
  52. # Enable or disable spider middlewares
  53. # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
  54. #SPIDER_MIDDLEWARES = { 
  55. # 'youtxNanJin.middlewares.YoutxnanjinSpiderMiddleware': 543,
  56. #}
  58. # Enable or disable downloader middlewares
  59. # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
  60. #DOWNLOADER_MIDDLEWARES = { 
  61. # 'youtxNanJin.middlewares.MyCustomDownloaderMiddleware': 543,
  62. #}
  64. # Enable or disable extensions
  65. # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
  66. #EXTENSIONS = { 
  67. # 'scrapy.extensions.telnet.TelnetConsole': None,
  68. #}
  70. # Configure item pipelines
  71. # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
  72. ITEM_PIPELINES = {
  73.    # 'youtxNanJin.pipelines.YoutxnanjinPipeline': 300,
  74.    'youtxNanJin.pipelines.YouTXMongo': 300,
  75.    'youtxNanJin.pipelines.JsonWritePipline': 300,
  76. }
  78. # Enable and configure the AutoThrottle extension (disabled by default)
  79. # See http://doc.scrapy.org/en/latest/topics/autothrottle.html
  80. #AUTOTHROTTLE_ENABLED = True
  81. # The initial download delay
  82. #AUTOTHROTTLE_START_DELAY = 5
  83. # The maximum download delay to be set in case of high latencies
  84. #AUTOTHROTTLE_MAX_DELAY = 60
  85. # The average number of requests Scrapy should be sending in parallel to
  86. # each remote server
  87. #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
  88. # Enable showing throttling stats for every response received:
  89. #AUTOTHROTTLE_DEBUG = False
  91. # Enable and configure HTTP caching (disabled by default)
  92. # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
  93. #HTTPCACHE_ENABLED = True
  94. #HTTPCACHE_EXPIRATION_SECS = 0
  95. #HTTPCACHE_DIR = 'httpcache'
  96. #HTTPCACHE_IGNORE_HTTP_CODES = []
  97. #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

最后再来一张爬取下来的截图吧。相同的爬虫做着练习，以后要学会爬其他大一些的爬取不容易乱一些的网站。

这里写图片描述

发表评论

表情：
评论列表 （有 0 条评论，620人围观）

还没有评论，来说两句吧...

相关阅读