【记录】Python爬虫｜爬取空间PC版日志模板 - IT评测·应用市场-qidao123.com技术社区

# author: shandianchengzi
# description: get templates of qq diary, saving as "qq日志模板.xlsx". Result: 43 free, 4 damaged.
# status: complete
import json
import requests
import pandas as pd
import re
from time import sleep
url="https://h5.qzone.qq.com/proxy/domain/mall.qzone.qq.com/cgi-bin/v3/cgi_get_letter_paper"
headers={
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36 Edg/90.0.818.56',
}
headers['Cookie']="" #不需要填
params={
'mallIds':'',
'keyWord':'',
'vip':0,
'g_tk':'1002114705',
'pageNum':3,
'pageCount':5,
}
datalist = [] #用来存储爬取的网页信息
byYourself={
'number':5, #一次爬取的个数，建议5个，否则得到的数据会对不上
}
def LoadJson(JSON):
try:
return json.load(JSON)
except:
#找到"({"作为开始标志和"})"作为结束标志之间的所有信息[jsonp格式]
return json.loads(re.match(".*?({.*}).*",JSON,re.S).group(1))
def AddData(content):
for i in content['data']['items']:
i['name']=i['mall']['name']
i['attr']=i['mall']['attr']
datalist.append(i)
#print(i)
def export_excel(export):
try:
#将字典列表转换为DataFrame
pf = pd.DataFrame(list(export))
#指定生成的Excel表格名称
file_path = pd.ExcelWriter('qq日志模板.xlsx')
#替换空单元格
pf.fillna(' ', inplace=True)
#输出
pf.to_excel(file_path, encoding='utf-8', index=False)
#保存表格
file_path.save()
print('保存成功！')
except Exception as e:
print("[-] Error = "+str(e))
print('无法导出为excel，请检查是否未关闭同名excel文件。正在重试……')
sleep(2)
export_excel(export)
def getData(total):
try:
params['pageCount']=byYourself['number']
pageTotal=int(total/byYourself['number'])+2
print('一共要加载',pageTotal,'页，请耐心等待:')
for i in range(1,pageTotal):
params['pageNum']=i
print('第',i,'页,',end='')
res = requests.get(url, params=params, headers=headers)
content=LoadJson(res.text)
AddData(content)
except Exception as e:
print("[-] Error = "+str(e))
print(res.text)
print(len(datalist))
export_excel(datalist)
def myFunc():
datalist.clear()
res = requests.get(url, params=params, headers=headers)
#找到"({"作为开始标志和"})"作为结束标志之间的所有信息[jsonp格式]
content=LoadJson(res.text)
total=content['data']['total']
print(total)
getData(total)
myFunc()

复制代码