Python爬虫小训练 - ToB企服应用市场:ToB评测及商务社交产业平台

import requests
import os
import parsel
end=input("你想爬取多少章？（阿拉伯数字输入，大于2）:")
end= int(end)
print("正在爬取请稍等...")
#爬取第一页的内容
print("==============正在爬取第1章==============")
url = f'xxxxxxx'
response = requests.get(url=url)
response.encoding = response.apparent_encoding
html = response.text
# 解析html
selector = parsel.Selector(html)
# 获取文章标题
title = selector.css('.content h1::text').get()
#print(title)
# 获取小标题内容
content_1 = selector.css('#chaptercontent::text').get()
#print(content_1)
# passage = ''.join(selector.xpath('//div[@id="chaptercontent"]//text()').getall()).strip()
passage = ''.join(selector.css('#chaptercontent').xpath('./text()').getall()[:-4])
passage = passage.replace('　　', '\n')
#print(passage)
filename = 'xxxxxx\\'
if not os.path.exists(filename):
os.mkdir(filename)
with open(filename +title+ '.txt', mode='wb') as f:
f.write(passage.encode('utf-8'))
#后续内容
for page in range(2,end+1):
print(f"==============正在爬取第{page}章==============")
url = f'xxxxxxxxxxxxxxxxxxx'
response=requests.get(url=url)
response.encoding = response.apparent_encoding
html=response.text
# 解析html
selector=parsel.Selector(html)
# 获取文章标题
title=selector.css('.content h1::text').get()
#print(title)
# 获取小标题内容
#content_1 = selector.css('#chaptercontent::text').get()
#print(content_1)
#passage = ''.join(selector.xpath('//div[@id="chaptercontent"]//text()').getall()).strip()
passage = ''.join(selector.css('#chaptercontent').xpath('./text()').getall()[:-4])
passage=passage.replace('　　','\n')
#print(passage)
with open(filename + title+ '.txt', mode='wb') as f:
f.write(passage.encode('utf-8'))
print("爬取完成，已保存在同目录下")

复制代码