scrapy--子类CrawlSpider&中间件

 免责声明:本文仅做分享参考~
  目次
CrawlSpider
先容
xj.py
中间件
部分middlewares.py
wyxw.py 
完整的middlewares.py

---

CrawlSpider

先容

CrawlSpider类：界说了一些规则来做数据爬取，从爬取的网页中获取链接并进行继承爬取.

1. 创建方式：scrapy genspider -t crawl 爬虫名 爬虫域名
2. scrapy genspider -t crawl zz zz.com
4. (子类就是牛,因为可以继承啊,还可以有自己的方法.)
5. spider类--》ZzscSpider类
6. spider类--》CrawLSpider类--》XjSpider类
7. CrawLSpider类：定义了一些规则来做数据爬取，从爬取的网页中获取链接并进行继续爬取

复制代码

  这个子类注意好 规则 , 有助于帮你跟进追踪类似数据(多页,类似html结构),  是这个子类最大的特点.
  --正则匹配 !!!
  新疆xxedu
xj.py

1. import scrapy
2. from scrapy.linkextractors import LinkExtractor
3. from scrapy.spiders import CrawlSpider, Rule
6. class XjSpider(CrawlSpider):
7.     name = "xj"
8.     # allowed_domains = ["xj.com"]
9.     start_urls = ["https://www.xjie.edu.cn/tzgg1.htm"]
10.     # 规则,根据正则匹配: \ 转义, ^ 开始, $ 结束, () 分组, [] 范围, | 或, + 1次或多次, * 0次或多次
11.     # 获取详情页链接:
12.     # rules = (Rule(LinkExtractor(allow=r"Items/"), callback="parse_item", follow=True),)
13.     # follow属性为True,会自动跟进详情页链接,并调用parse_item方法处理.
14.     rules = (
15.         Rule(
16.             LinkExtractor(allow=r"info/1061/.*?\.htm"),
17.             callback="parse_item",
18.             follow=False,
19.         ),
20.         Rule(LinkExtractor(allow=r"tzgg1/.*?\.htm"), follow=True),
21.     )
22.     count = 0
24.     # 匹配100次,就进入parse_item方法100次.
25.     def parse_item(self, response):
26.         # print(response.request.headers) #查看伪装的请求头
27.         self.count += 1
28.         # 基于详情页,获取标题:
29.         title = response.xpath("//h3/text()").get()
30.         print(title, self.count)  # 竟然也自动拿到了分页的url.
32.         item = {}
33.         # item["domain_id"] = response.xpath('//input[@id="sid"]/@value').get()
34.         # item["name"] = response.xpath('//div[@id="name"]').get()
35.         # item["description"] = response.xpath('//div[@id="description"]').get()
37.         return item

复制代码

---

中间件

   scrapy中有两个中间件:
  下载中间件：位于引擎和下载器中间.
  爬虫中间件：位于引擎和爬虫中间.(一般不用)
  

---

  下载中间件的作用:
  用来窜改哀求和相应，比如窜改哀求：加一些哀求头，加代理等等;
  窜改相应就是更改相应的内容.
  
  # 下载器
拦截哀求 --伪装
拦截相应 --修改返回内容

 

 

---

部分middlewares.py

settings.py里面的是全局设置,在这你可以自己界说爬虫的伪装~~

1. # Define here the models for your spider middleware
2. #
3. # See documentation in:
4. # https://docs.scrapy.org/en/latest/topics/spider-middleware.html
6. from scrapy import signals
8. # useful for handling different item types with a single interface
9. from itemadapter import is_item, ItemAdapter
11. # 引擎把请求对象交给下载器,下载器把响应对象交给引擎.
14. class ScrapyDemo1SpiderMiddleware:
15.     # Not all methods need to be defined. If a method is not defined,
16.     # scrapy acts as if the spider middleware does not modify the
17.     # passed objects.
19.     @classmethod
20.     def from_crawler(cls, crawler):
21.         # This method is used by Scrapy to create your spiders.
22.         s = cls()
23.         crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
24.         return s
26.     def process_spider_input(self, response, spider):
27.         # Called for each response that goes through the spider
28.         # middleware and into the spider.
30.         # Should return None or raise an exception.
31.         return None
33.     def process_spider_output(self, response, result, spider):
34.         # Called with the results returned from the Spider, after
35.         # it has processed the response.
37.         # Must return an iterable of Request, or item objects.
38.         for i in result:
39.             yield i
41.     def process_spider_exception(self, response, exception, spider):
42.         # Called when a spider or process_spider_input() method
43.         # (from other spider middleware) raises an exception.
45.         # Should return either None or an iterable of Request or item objects.
46.         pass
48.     def process_start_requests(self, start_requests, spider):
49.         # Called with the start requests of the spider, and works
50.         # similarly to the process_spider_output() method, except
51.         # that it doesn’t have a response associated.
53.         # Must return only requests (not items).
54.         for r in start_requests:
55.             yield r
57.     def spider_opened(self, spider):
58.         spider.logger.info("Spider opened: %s" % spider.name)
61. class ScrapyDemo1DownloaderMiddleware:
62.     # Not all methods need to be defined. If a method is not defined,
63.     # scrapy acts as if the downloader middleware does not modify the
64.     # passed objects.
66.     @classmethod
67.     def from_crawler(cls, crawler):
68.         # This method is used by Scrapy to create your spiders.
69.         s = cls()
70.         crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
71.         return s
73.     # 发起请求执行的方法
74.     def process_request(self, request, spider):
75.         # Called for each request that goes through the downloader
76.         # middleware.
78.         # Must either:
79.         # - return None: continue processing this request
80.         # - or return a Response object
81.         # - or return a Request object
82.         # - or raise IgnoreRequest: process_exception() methods of
83.         #   installed downloader middleware will be called
84.         #
85.         # if spider.name == "wyxw": # 判断,如果是指定爬虫,则修改请求头.
86.         request.headers["User-Agent"] = (
87.             "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
88.         )
89. # cookie伪装
90.         # request.cookies["name"] = "value"
91.         # 如果更换ip地址,使用代理:
92.         # request.meta['proxy'] = 'https://ip:端口'
94.         print(request)
95.         return None
97.     # 得到响应执行的方法
98.     #
99.     def process_response(self, request, response, spider):
100.         # Called with the response returned from the downloader.
102.         # Must either;
103.         # - return a Response object
104.         # - return a Request object
105.         # - or raise IgnoreRequest
106.         return response
108.     # 处理异常执行的方法
109.     def process_exception(self, request, exception, spider):
110.         # Called when a download handler or a process_request()
111.         # (from other downloader middleware) raises an exception.
113.         # Must either:
114.         # - return None: continue processing this exception
115.         # - return a Response object: stops process_exception() chain
116.         # - return a Request object: stops process_exception() chain
117.         pass
119.     def spider_opened(self, spider):
120.         spider.logger.info("Spider opened: %s" % spider.name)

复制代码

为什么要用中间件的原因：
网页是必要滚动加载数据的，所以借助自动化工具去拿到所有的页面数据，利用scrapy中 间件整合自动化工具.

# 网易消息爬取四个板块数据
# 国内 国际 军事 航空 由于它们的页面结构一致，取值方式一致.
 四个版块的数据 通过访问url 只能拿到50条数据 此中有一个版块必要手动点击 加载更多才可以继承把数据加载完整 其它的几个版块几乎都是一直滚动滚动条，直到数据完全加载为止
解决标题？--> 滚动条的操作 requests scrapy也没有，只有自动化技术才可以实现滚动滚动条 怎么样才可以scrapy结合自动化技术呢？
使用自动化技术 加载所有数据 !

---

wyxw.py 

爬虫文件

1. import scrapy
3. """
5. 新闻页面的数据是通过滚动翻页加载
6. 如果要获取所有的数据：
7. 解决办法：
8.     传统的方式：通过网络面板看分页加载的请求是什么
9.     自动化工具的方式：让自动化工具实现鼠标滚动
11. """
14. class WyxwSpider(scrapy.Spider):
15.     name = "wyxw"
16.     # allowed_domains = ["wyxw.com"]
17.     start_urls = ["https://news.163.com/"]
18.     count = 0
20.     def parse(self, response):
21.         #  解析四个版块的url
22.         lis = response.xpath('//div[@class="ns_area list"]/ul/li')
23.         # 获取四个版块的li
24.         # lis[1:3] lis[1] lis[2]
25.         # lis[4:6] lis[4] lis[5]
26.         target_li = lis[1:3] + lis[4:6]
27.         # print(target_li)
28.         # 循环li 拿到每一个，li的a标签的hraf属性值 再发起请求
29.         for li in target_li:
30.             href = li.xpath("./a/@href").get()
31.             # 发起请求
32.             yield scrapy.Request(url=href, callback=self.news_parse)
34.     # 解析每个版块的数据
35.     # HtmlResponse（元素） 还是 scrapy的response？是自己返回的HtmlResponse,所以要以元素面板为主
36.     # 因为数据是通过自动化工具获取到的,自动化获取的数据都是元素面板数据
37.     def news_parse(self, response):
38.         divs = response.xpath('//div[@class="ndi_main"]/div')  # 从元素面板看的
39.         # divs = response.xpath('//div[@class="hidden"]/div')  # 从响应内容看的
40.         for div in divs:
41.             self.count += 1
42.             title = div.xpath(".//h3/a/text()").get()  # 从元素面板看的
43.             # title = div.xpath('./a/text()').get()  # 从响应内容看的
44.             print(title, self.count)

复制代码

---

完整的middlewares.py

   process_request 函数里面 伪装哀求.
  process_response 函数里面 拦截哀求,返回我们想要的数据.

1. # Define here the models for your spider middleware
2. #
3. # See documentation in:
4. # https://docs.scrapy.org/en/latest/topics/spider-middleware.html
5. import scrapy
6. from scrapy import signals
8. # useful for handling different item types with a single interface
9. from itemadapter import is_item, ItemAdapter
10. from DrissionPage._pages.chromium_page import ChromiumPage
11. from scrapy.http import HtmlResponse
14. class Scrapy4SpiderMiddleware:
15.     # Not all methods need to be defined. If a method is not defined,
16.     # scrapy acts as if the spider middleware does not modify the
17.     # passed objects.
19.     @classmethod
20.     def from_crawler(cls, crawler):
21.         # This method is used by Scrapy to create your spiders.
22.         s = cls()
23.         crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
24.         return s
26.     def process_spider_input(self, response, spider):
27.         # Called for each response that goes through the spider
28.         # middleware and into the spider.
30.         # Should return None or raise an exception.
31.         return None
33.     def process_spider_output(self, response, result, spider):
34.         # Called with the results returned from the Spider, after
35.         # it has processed the response.
37.         # Must return an iterable of Request, or item objects.
38.         for i in result:
39.             yield i
41.     def process_spider_exception(self, response, exception, spider):
42.         # Called when a spider or process_spider_input() method
43.         # (from other spider middleware) raises an exception.
45.         # Should return either None or an iterable of Request or item objects.
46.         pass
48.     def process_start_requests(self, start_requests, spider):
49.         # Called with the start requests of the spider, and works
50.         # similarly to the process_spider_output() method, except
51.         # that it doesn’t have a response associated.
53.         # Must return only requests (not items).
54.         for r in start_requests:
55.             yield r
57.     def spider_opened(self, spider):
58.         spider.logger.info("Spider opened: %s" % spider.name)
61. class Scrapy4DownloaderMiddleware:
62.     # Not all methods need to be defined. If a method is not defined,
63.     # scrapy acts as if the downloader middleware does not modify the
64.     # passed objects.
66.     @classmethod
67.     def from_crawler(cls, crawler):
68.         # This method is used by Scrapy to create your spiders.
69.         s = cls()
70.         crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
71.         return s
73.     # 发起请求执行的方法
74.     def process_request(self, request, spider):
75.         c_dict = {'cookiesu': '291715258858774', ' device_id': '2bad6b34106f8be1d2762204c306aa5b',
76.          ' smidV2': '20240509204738adc267d3b66fca8adf0d37b8ac9a1e8800dbcffc52451ef70', ' s': 'ak145lfb3s',
77.          ' __utma': '1.1012417897.1720177183.1720177183.1720435739.2',
78.          ' __utmz': '1.1720435739.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic',
79.          ' acw_tc': '2760827c17230343015564537e9504eebc062a87e8d3ba4425778641e1164e',
80.          ' xq_a_token': 'fb0f503ef881090db449e976c330f1f2d626c371', ' xqat': 'fb0f503ef881090db449e976c330f1f2d626c371',
81.          ' xq_r_token': '967c806e113fbcb1e314d5ef2dc20f1dd8e66be3',
82.          ' xq_id_token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJ1aWQiOi0xLCJpc3MiOiJ1YyIsImV4cCI6MTcyNTMyNDc3NSwiY3RtIjoxNzIzMDM0MzAxODI4LCJjaWQiOiJkOWQwbjRBWnVwIn0.I0pylYRqjdDOE0QbAuLFr5rUCFG1Sj13kTEHAdap2fEse0i12LG7a-14rhvQKsK9G0F7VZSWDYBsI82mqHimsBbWvgefkOy-g8V_nhb-ntGBubbVHlfjv7y-3GDatzcZPfgYvu7m0wEu77PcWdKqJR-KwZsZjQVwaKiHcFuvFUmpfYN942D6YY2MIzgWJxSQCX_t4f1YujRJRsKLvrq9QbVIqvJu-SpPT1SJfSPT9e7h_ERkU0QOsmgARJVfivkAAFM2cyb7HKJsHQSqVSU6hcIq4CMs5r90IsrZ4fOL5cUqaGNT58qkjx-flta27QhCIeHxexi1K95TKZTcD8EygA',
83.          ' u': '291715258858774',
84.          ' Hm_lvt_1db88642e346389874251b5a1eded6e3': '1722350318,1722863131,1722925588,1723034326',
85.          ' Hm_lpvt_1db88642e346389874251b5a1eded6e3': '1723034326', ' HMACCOUNT': '13108745FF137EDD',
86.          ' .thumbcache_f24b8bbe5a5934237bbc0eda20c1b6e7': 'YEsBB9JFA5Q4gQHJIe1Lx6JjvpZzcuUljYTfjFKm3lmCSpRZMpoNmnSBV0UptK3ripTe4xifyqRUZO/LEPx6Iw%3D%3D',
87.          ' ssxmod_itna': 'WqIx9DgD0jD==0dGQDHWWoeeqBKorhBoC7ik8qGN6xYDZDiqAPGhDC4bUxD5poPoqWW3pi4kiYr23PZF2E2GaeBm8EXTDU4i8DCwiK=ODem=D5xGoDPxDeDAQKiTDY4DdjpNv=DEDeKDRDAQDzLdyDGfBDYP9QqDgSqDBGOdDKqGgzTxD0TxNaiqq8GKKvkd5qjbDAwGgniq9D0UdxBLxAax9+j9kaBUg8ZaPT2jx5eGuDG6DOqGmSfb3zdNPvAhWmY4sm75Ymbq4n75e8YervGPrPuDNh0wKY7DoBGp=GDLMxDfT0bD',
88.          ' ssxmod_itna2': 'WqIx9DgD0jD==0dGQDHWWoeeqBKorhBoC7ik4A=W=e4D/D0hq7P7phOF4WG2WCxjKD2WYD=='}
89.         if spider.name == 'xueqiu':
90.         # ua  Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0
91.             request.headers['referer'] = 'https://xueqiu.com/'
92.             request.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0'
93.         # 失效 request.headers['cookie'] = 'cookiesu=291715258858774; device_id=2bad6b34106f8be1d2762204c306aa5b; smidV2=20240509204738adc267d3b66fca8adf0d37b8ac9a1e8800dbcffc52451ef70; s=ak145lfb3s; __utma=1.1012417897.1720177183.1720177183.1720435739.2; __utmz=1.1720435739.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; acw_tc=276077b217228631300332345e2956486032bd992e9f60b5689d9b248c1491; xq_a_token=fb0f503ef881090db449e976c330f1f2d626c371; xq_r_token=967c806e113fbcb1e314d5ef2dc20f1dd8e66be3; xq_id_token=eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJ1aWQiOi0xLCJpc3MiOiJ1YyIsImV4cCI6MTcyNTMyNDc3NSwiY3RtIjoxNzIyODYzMDg5MDIwLCJjaWQiOiJkOWQwbjRBWnVwIn0.AdH2II1N2kGggG7ZgmP8MOgNPdxMoewCSK-gyWYBkw7zFExn6gaYV6YU8ReNkp2F5CBohxjZYyVyLtn98MJfk9dDwIe8ypTgXLkI_a5R1O1og5Fy6BeFv6FUJqgp8EVt8EvHBOYfRNl9iGtgrO3V_R0fJXq1aJTpV8lopNwEAzQbHRK58uXcbaoOwkUcX8MOv6XR-eGqnHYRSJ35P769atb6vF05LqutQphcairWpGGgWJc9fMhVBym_GkOxy4_AWaURWf8Zpge7dJQszkCo-ljPbBP94vz3zM_PTnussZV3jeTRmacaJcHTee6mlE00hrtrAFZNf7UIjnpqbdzvjw; u=291715258858774; Hm_lvt_1db88642e346389874251b5a1eded6e3=1720435739,1722235054,1722350318,1722863131; HMACCOUNT=13108745FF137EDD; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1722863143; .thumbcache_f24b8bbe5a5934237bbc0eda20c1b6e7=ha9Q037kb9An+HmEECE3+OwwxbDqAMnKP2QzosxWbRaWA89HDsKTwe/0XwLZXhS9S5OvxrWGFu6LMFW2iVI9xw%3D%3D; ssxmod_itna=QqmxgD0Cq4ciDXYDHiuYYK0=e4D5Dk3DA2LOni44qGNjKYDZDiqAPGhDC4fUerbrq5=7oC03qeiB4hvpkKrmDKijF/qqeDHxY=DUa7aeDxpq0rD74irDDxD3wxneD+D04kguOqi3DhxGQD3qGylR=DA3tDbh=uDiU/DDUOB4G2D7UyiDDli0TeA8mk7CtUCQ03xGUxIqqQ7qDMUeGX87Fe7db86PhMIwaHamPuKCiDtqD94m=DbRL3vB6lWW+r7hGrGBvNGGYQKDqNQ7eeKDmjvfhu2GDile4tKOG5nBpan/zeDDfj0bD===; ssxmod_itna2=QqmxgD0Cq4ciDXYDHiuYYK0=e4D5Dk3DA2LOni4A=c==D/QKDFxnAO9aP7QmHGcDYK4xD==='
94.         #     request.cookies['键'] = '值'
95.         #     request.cookies['cookiesu'] = '291715258858774'
96.             for key,value in c_dict.items():
97.                 request.cookies[key] = value
98.         # print(request)
99.         # 如果更换ip地址，使用代理
100.         # request.meta['proxy'] = 'https://ip:端口'
103.         return None
105.     # 得到响应执行的方法
106.     # 结合自动化工具
107.     def process_response(self, request, response, spider):
108.         # url必须是四个版块的其中一个
109.         if spider.name == 'wyxw':
110.             url = request.url
111.             # url = 'https://news.163.com/domestic/'
112.             dp = ChromiumPage()
113.             dp.get(url)
114.             # 滚动到页面底部 一次性不能完成
115.             while True:
116.                 is_block = dp.ele('.load_more_tip').attr('style')
117.                 if is_block == 'display: block;':
118.                     #     已经到最后了
119.                     break
120.                 dp.scroll.to_bottom()
121.                 #     国内页面需要额外手动点击
122.                 try:
123.                     # 只有一个页面能够点击加载更多，加入异常处理，如果出现了异常，进行下一次循环
124.                     click_tag = dp.ele('text:加载更多')
125.                     click_tag.click()
126.                 except:
127.                     continue
128.         #     创建响应对象，把自动化工具得到的html数据放入到对象中
129.         # 返回给引擎，由引擎交给spider
130.         #     HtmlResponse响应对象
131.             '''
132.            class HtmlResponse:
133.                 def __init__(self,request,url,body):
134.              HtmlResponse(url=url,request=request,body=dp.html)   
135.             '''
136.             return HtmlResponse(url=url,request=request,body=dp.html,encoding='utf-8')
137.         return response
139.     # 遇到异常执行的方法
140.     def process_exception(self, request, exception, spider):
141.         # Called when a download handler or a process_request()
142.         # (from other downloader middleware) raises an exception.
144.         # Must either:
145.         # - return None: continue processing this exception
146.         # - return a Response object: stops process_exception() chain
147.         # - return a Request object: stops process_exception() chain
148.         pass
150.     def spider_opened(self, spider):
151.         spider.logger.info("Spider opened: %s" % spider.name)

复制代码

---

免责声明：如果侵犯了您的权益，请联系站长，我们会及时删除侵权内容，谢谢合作！更多信息从访问主页：qidao123.com:ToB企服之家，中国第一个企服评测及商务社交产业平台。

继续阅读请点击广告