爬虫学习案例3

爬取玉人图片

精美图库地点
一页图片
安装依靠库文件

1. pip install selenium requests beautifulsoup4
4. import time
5. import requests
6. import random
7. from bs4 import BeautifulSoup
8. from selenium import webdriver
9. from selenium.webdriver.chrome.service import Service
10. from selenium.webdriver.chrome.options import Options
11. # 设置Chrome选项
12. chrome_options = Options()
13. chrome_options.add_argument("--headless")  # 无头模式，不打开浏览器窗口
14. chrome_options.add_argument("--disable-gpu")
15. chrome_options.add_argument("--no-sandbox")
17. # 设置ChromeDriver路径
18. service = Service('D:\env\python3\chromedriver.exe')
19. url = 'https://www.umei.cc/touxiangtupian/nvshengtouxiang/'
20. baseUrl = "https://www.umei.cc"
21. # 初始化WebDriver
22. driver = webdriver.Chrome(service=service, options=chrome_options)
23. driver.get(url)
24. time.sleep(random.uniform(5, 10))  # 等待页面加载
25. html = driver.page_source # 原页面
26. soup = BeautifulSoup(html, 'html.parser')
27. # print(soup)
28. # BeautifulSoup分析提取元素
29. divList = soup.find_all("div",class_= "item masonry_brick")
30. # print(divList)
31. # 一个美女信息
32. for divItem in divList:
33.     linkImage = divItem.find("div",class_ = "item_t").find("div",class_ = "img").find("a")["href"]
34.     linkImage = baseUrl + linkImage
35.     # 拿去子页面的大图
36.     driver.get(linkImage)
37.     time.sleep(random.uniform(5, 10))
38.     html = driver.page_source
39.     sonSoup = BeautifulSoup(html, 'html.parser')
40.     imgUrl = sonSoup.find("div",class_ = "tsmaincont-main-cont-txt").find("img")["src"]
41.     print(f"准备下载图片{imgUrl}")
42.     # 下载图片
43.     img_response = requests.get(imgUrl)
44.     img_name = imgUrl.split('/')[-1]
45.     with open("img"+img_name, "wb") as f:
46.         f.write(img_response.content)
47.     print(f"图片{img_name}下载完成")
48. print("第一页图片全部下载到当前目录了.....")
49. driver.quit()  # 关闭浏览器

复制代码

爬取多页

1. import time
2. import requests
3. import random
4. from bs4 import BeautifulSoup
5. from selenium import webdriver
6. from selenium.webdriver.chrome.service import Service
7. from selenium.webdriver.chrome.options import Options
8. # 设置Chrome选项
9. chrome_options = Options()
10. chrome_options.add_argument("--headless")  # 无头模式，不打开浏览器窗口
11. chrome_options.add_argument("--disable-gpu")
12. chrome_options.add_argument("--no-sandbox")
14. # 设置ChromeDriver路径
15. service = Service('D:\env\python3\chromedriver.exe')
16. url = 'https://www.umei.cc/touxiangtupian/nvshengtouxiang/'
17. baseUrl = "https://www.umei.cc"
18. # 初始化WebDriver
19. driver = webdriver.Chrome(service=service, options=chrome_options)
21. def getImage(url,page):
22.     driver.get(url)
23.     print(f"正在爬取第{page}页图片资源源...")
24.     print(url)
25.     time.sleep(random.uniform(5, 10))  # 等待页面加载
26.     html = driver.page_source # 原页面
27.     soup = BeautifulSoup(html, 'html.parser')
28.     # BeautifulSoup分析提取元素
29.     divList = soup.find_all("div",class_= "item masonry_brick")
30.     for divItem in divList:
31.         linkImage = divItem.find("div",class_ = "item_t").find("div",class_ = "img").find("a")["href"]
32.         linkImage = baseUrl + linkImage
33.         # 拿取子页面的大图
34.         driver.get(linkImage)
35.         time.sleep(random.uniform(5, 10))
36.         html = driver.page_source
37.         sonSoup = BeautifulSoup(html, 'html.parser')
38.         imgUrl = sonSoup.find("div",class_ = "tsmaincont-main-cont-txt").find("img")["src"]
39.         print(f"准备下载图片{imgUrl}")
40.         # 下载图片
41.         img_response = requests.get(imgUrl)
42.         img_name = imgUrl.split('/')[-1]
43.         with open("img"+img_name, "wb") as f:
44.             f.write(img_response.content)
45.         print(f"图片{img_name}下载完成")
46.     print(f"第{page}页图片全部下载到当前img目录了.....")
48. # 爬取1-10页
49. # 控制爬取的页面数
50. for page in range(1, 11):
51.     if page == 1:
52.         getImage(url,page)
53.     else:
54.         pageUrl = f"{url}index_{page}.htm"
55.         getImage(pageUrl,page)
56. driver.quit()  # 关闭浏览器

复制代码

免责声明：如果侵犯了您的权益，请联系站长，我们会及时删除侵权内容，谢谢合作！更多信息从访问主页：qidao123.com:ToB企服之家，中国第一个企服评测及商务社交产业平台。