一个豆瓣电影Top250爬虫
一个爬虫这是我第一次接触爬虫，写的第一个爬虫实例。
https://img2022.cnblogs.com/blog/2911292/202206/2911292-20220625000408593-1866073318.png
https://movie.douban.com/top250
模块
import requests #用于发送请求
import re #使用正则表达式，用于匹配处理文本
import os #用于创建文件夹
from lxml import etree #这里我使用了Xpath表达式用于数据解析，我觉得这个模块比BeautifulSoup好用，强烈推荐
from fake_useragent import UserAgent #反爬虫，随机获取浏览器 UA 信息代码
import requests
import re
import os
from lxml import etree
from fake_useragent import UserAgent
class doubanSpider(object):
def __init__(self):
if not os.path.exists('db/douban'):
os.makedirs('db/douban')
else:
pass
self.f = open('./db/douban/douban.txt', 'a', encoding='utf-8')
def start(self):
for i in range(46):
headers = {
'User-Agent': UserAgent().random
}
url = 'https://movie.douban.com/top250?start=' + str(i * 25)
r = requests.get(url, headers=headers)
html = etree.HTML(r.text)
li_list = html.xpath('//*[@id="content"]/div/div/ol/li')
movies = []
for each in li_list:
movie = {}
order = each.xpath('.//div/div/em/text()')
movie['id'] = order
cover = each.xpath('.//div/div/a/img/@src')
movie['cover'] = cover
name = each.xpath('.//div/div/div/a/span/text()')
flag = ''
for mo in name:
flag += mo
movie['name'] = flag
info = each.xpath('.//div/div/div/p/text()').strip()
info = info.replace("\n", "")
info = info.replace("\xa0", "")
director = re.findall(r'[导演:].+[主演:]', info)
director = director
movie['director'] = director
try:
role = re.findall(r'主.+', info)
role = role
except IndexError:
role = ''
movie['role'] = role
plot = each.xpath('.//div/div/div/p/text()').strip()
plot = plot.replace("\xa0", "")
movie['plot'] = plot
star = each.xpath('.//div/div/div/div/span/text()')
movie['star'] = star
try:
comment = each.xpath('.//div/div/div/p/span/text()')
except IndexError:
comment = ''
movie['comment'] = comment
movies.append(movie)
self.f.write(str(movie)+'\n')
print(movie)
def run(self):
self.start()
self.f.close()
if __name__ == '__main__':
spider = doubanSpider()
spider.run()
免责声明：如果侵犯了您的权益，请联系站长，我们会及时删除侵权内容，谢谢合作！
页:
[1]