《数据采集与预处置惩罚》实验二网络爬虫低级实践 - ToB企服应用市场:ToB评测及商务社交产业平台

import scrapy
from ..items import DouBanprojectItem

class DoubanprojectSpider(scrapy.Spider):
    name = "DouBanProject"
    allowed_domains = ["movie.douban.cn"]
    pre_urls = ["http://movie.douban.com/"]
    Second_urls = ["https://movie.douban.com/subject/1292052/"]
    index = 25

    pre_url = 'https://movie.douban.com/top250?start='

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
        'Cookie': 'bid=dJvnLS9LSZw; dbcl2="268997517

xIMwxecuoM"; ck=_Gt4; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1679986418%2C%22https%3A%2F%2Faccounts.douban.com%2Fpassport%2Flogin%3Fredir%3Dhttps%253A%252F%252Fmovie.douban.com%252Ftop250%253Fstart%253D0%22%5D; _pk_id.100001.4cf6=0c504467b6cba249.1679986418.1.1679986418.1679986418.; __utma=30149280.314202744.1679986418.1679986418.1679986418.1; __utmc=30149280; __utmz=30149280.1679986418.1.1.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/passport/login; __utma=223695111.1488601765.1679986418.1679986418.1679986418.1; __utmc=223695111; __utmz=223695111.1679986418.1.1.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/passport/login; push_noty_num=0; push_doumail_num=0; __yadk_uid=7XpTBmn419zZXYVbR23RXwxFnykicX94; __gads=ID=d32f3926a8359ffe-221cff7fc4dc00c8:T=1679986420:RT=1679986420:S=ALNI_MZTs_yGwuU--7NrVfPYG23h3f0Pdw; __gpi=UID=00000be25238883c:T=1679986420:RT=1679986420:S=ALNI_MY88r1HJI5u3zUtk_lMeFw4_aa2Lg'

    }

    def start_requests(self):
        new_url = self.pre_url + str(0)
        yield scrapy.Request(url=new_url, headers=self.headers,callback=self.parse)

    def parse(self, response):
        for element in response.xpath('//*[ @ id = "content"]/div/div[1]/ol/li'):
            no = element.xpath('./div/div[1]/em/text()').get()
            name = element.xpath('./div/div[2]/div[1]/a/span[1]/text()').get()
            grade = element.xpath('./div/div[2]/div[2]/div/span[2]/text()').get()
            new_url = element.xpath('./div/div[2]/div[1]/a/@href').get()

            # print(no, name, grade)
            yield scrapy.Request(url=new_url, headers=self.headers, callback=self.movie)

        if self.index < 250:
            new_url = self.pre_url + str(self.index)
            # https: // movie.douban.com / top250?start = 27
            self.index += 25
            yield scrapy.Request(url=new_url, headers=self.headers, callback=self.parse)

    # def Second_requests(self):
    #
    #     two_url = self.pre1_url + str(0)
    #     yield scrapy.Request(url=two_url, headers=self.headers, callback=self.movie)

    def movie(self, response):
        # response.xpath('//*[@id="content"]/h1/span[1]'):
        daoyan = response.xpath('//*[@id="info"]/span[1]/span[2]/a/text()').get()
        biaoju = response.xpath('//*[@id="info"]/span[2]/span[2]/a/text()').get()
        zhuyan = response.xpath('//*[@id="info"]/span[3]/span[2]/span/a/text()').get()
        leixing = response.xpath('//*[@id="info"]/span/text()').get()
        shangyintime = response.xpath('//*[@id="info"]/span[10]/text()').get()
        pianchang = response.xpath('//*[@id="info"]/span[13]/text()').get()
        pingfenRshu = response.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/div/div[2]/a/text()').get()
        juqingjiejia = response.xpath('//*[@id="link-report-intra"]/span[1]/span/text()').get()
        print(daoyan, biaoju, zhuyan, leixing, shangyintime, pianchang, pingfenRshu, juqingjiejia)
        item = DouBanprojectItem()

        # item['no'] = no
        # item['name'] = name
        item['daoyan'] = daoyan
        item['biaoju'] = biaoju
        item['zhuyan'] = zhuyan
        item['shangyintime'] = shangyintime
        item['pianchang'] = pianchang
        item['pianchang'] = pianchang
        item['pingfenRshu'] = pingfenRshu
        item['juqingjiejia'] = juqingjiejia
        yield item