import scrapy
from ..items import DouBanprojectItem
class DoubanprojectSpider(scrapy.Spider):
name = "DouBanProject"
allowed_domains = ["movie.douban.cn"]
pre_urls = ["http://movie.douban.com/"]
Second_urls = ["https://movie.douban.com/subject/1292052/"]
index = 25
pre_url = 'https://movie.douban.com/top250?start='
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
'Cookie': 'bid=dJvnLS9LSZw; dbcl2="268997517
xIMwxecuoM"; ck=_Gt4; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1679986418%2C%22https%3A%2F%2Faccounts.douban.com%2Fpassport%2Flogin%3Fredir%3Dhttps%253A%252F%252Fmovie.douban.com%252Ftop250%253Fstart%253D0%22%5D; _pk_id.100001.4cf6=0c504467b6cba249.1679986418.1.1679986418.1679986418.; __utma=30149280.314202744.1679986418.1679986418.1679986418.1; __utmc=30149280; __utmz=30149280.1679986418.1.1.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/passport/login; __utma=223695111.1488601765.1679986418.1679986418.1679986418.1; __utmc=223695111; __utmz=223695111.1679986418.1.1.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/passport/login; push_noty_num=0; push_doumail_num=0; __yadk_uid=7XpTBmn419zZXYVbR23RXwxFnykicX94; __gads=ID=d32f3926a8359ffe-221cff7fc4dc00c8:T=1679986420:RT=1679986420:S=ALNI_MZTs_yGwuU--7NrVfPYG23h3f0Pdw; __gpi=UID=00000be25238883c:T=1679986420:RT=1679986420:S=ALNI_MY88r1HJI5u3zUtk_lMeFw4_aa2Lg'
}
def start_requests(self):
new_url = self.pre_url + str(0)
yield scrapy.Request(url=new_url, headers=self.headers,callback=self.parse)
def parse(self, response):
for element in response.xpath('//*[ @ id = "content"]/div/div[1]/ol/li'):
no = element.xpath('./div/div[1]/em/text()').get()
name = element.xpath('./div/div[2]/div[1]/a/span[1]/text()').get()
grade = element.xpath('./div/div[2]/div[2]/div/span[2]/text()').get()
new_url = element.xpath('./div/div[2]/div[1]/a/@href').get()
# print(no, name, grade)
yield scrapy.Request(url=new_url, headers=self.headers, callback=self.movie)
if self.index < 250:
new_url = self.pre_url + str(self.index)
# https: // movie.douban.com / top250?start = 27
self.index += 25
yield scrapy.Request(url=new_url, headers=self.headers, callback=self.parse)
# def Second_requests(self):
#
# two_url = self.pre1_url + str(0)
# yield scrapy.Request(url=two_url, headers=self.headers, callback=self.movie)
def movie(self, response):
# response.xpath('//*[@id="content"]/h1/span[1]'):
daoyan = response.xpath('//*[@id="info"]/span[1]/span[2]/a/text()').get()
biaoju = response.xpath('//*[@id="info"]/span[2]/span[2]/a/text()').get()
zhuyan = response.xpath('//*[@id="info"]/span[3]/span[2]/span/a/text()').get()
leixing = response.xpath('//*[@id="info"]/span/text()').get()
shangyintime = response.xpath('//*[@id="info"]/span[10]/text()').get()
pianchang = response.xpath('//*[@id="info"]/span[13]/text()').get()
pingfenRshu = response.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/div/div[2]/a/text()').get()
juqingjiejia = response.xpath('//*[@id="link-report-intra"]/span[1]/span/text()').get()
print(daoyan, biaoju, zhuyan, leixing, shangyintime, pianchang, pingfenRshu, juqingjiejia)
item = DouBanprojectItem()
# item['no'] = no
# item['name'] = name
item['daoyan'] = daoyan
item['biaoju'] = biaoju
item['zhuyan'] = zhuyan
item['shangyintime'] = shangyintime
item['pianchang'] = pianchang
item['pianchang'] = pianchang
item['pingfenRshu'] = pingfenRshu
item['juqingjiejia'] = juqingjiejia
yield item
