selenium 知网爬虫之根据【关键词】获取文献信息 - ToB企服应用市场:ToB评测及商务社交产业平台

# 关键词 Xpath 路径或 CSS 选择器
//*[@id="gradetxt"]/dd[1]/div[2]/div[1]/div[2]/ul/li[3]
li[data-val="KY"]

复制代码

# 输入框
//*[@id="gradetxt"]/dd[1]/div[2]/input

复制代码

# 检索
/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[2]/input

复制代码

# 文献条数
/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[1]/span[1]/em

复制代码

import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.action_chains import ActionChains

复制代码

def webserver():
# get直接返回，不再等待界面加载完成
desired_capabilities = DesiredCapabilities.EDGE
desired_capabilities["pageLoadStrategy"] = "none"
# 设置微软驱动器的环境
options = webdriver.EdgeOptions()
# 设置浏览器不加载图片，提高加载速度
options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
# 创建一个微软驱动器
driver = webdriver.Edge(options=options)
return driver

复制代码

driver.get("https://kns.cnki.net/kns8/AdvSearch")
time.sleep(2)

复制代码

# 修改属性，使下拉框显示
opt = driver.find_element(By.CSS_SELECTOR, 'div.sort-list') # 定位下拉框
# 执行 js 脚本进行属性的修改; arguments[0]代表第一个属性
driver.execute_script("arguments[0].setAttribute('style', 'display: block;')", opt)

复制代码

# 【通讯作者】定位
/html/body/div[2]/div/div[2]/div/div[2]/div[1]/div[1]/div[2]/ul/li[8]
li[data-val="RP"]

复制代码

# 鼠标移动到下拉框
ActionChains(driver).move_to_element(driver.find_element(By.CSS_SELECTOR, 'li[data-val="RP"]')).perform()
# 找到[关键词]选项并点击
WebDriverWait(driver, 100).until(
EC.visibility_of_element_located((By.CSS_SELECTOR, 'li[data-val="KY"]'))).click()

复制代码

# 传入关键字
WebDriverWait(driver, 100).until(
EC.presence_of_element_located((By.XPATH, '''//*[@id="gradetxt"]/dd[1]/div[2]/input'''))
).send_keys(keyword)
# 点击搜索
WebDriverWait(driver, 100).until(
EC.presence_of_element_located((By.XPATH, "/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[2]/input"))
).click()

复制代码

# 获取总文献数和页数
res_unm = WebDriverWait(driver, 100).until(EC.presence_of_element_located(
(By.XPATH, "/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[1]/span[1]/em"))
).text

复制代码

import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.action_chains import ActionChainsdef webserver(): # get直接返回，不再等待界面加载完成 desired_capabilities = DesiredCapabilities.EDGE desired_capabilities["pageLoadStrategy"] = "none" # 设置微软驱动器的环境 options = webdriver.EdgeOptions() # 设置浏览器不加载图片，提高速度 options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2}) # 创建一个微软驱动器 driver = webdriver.Edge(options=options) return driverdef open_page(driver, keyword): # 打开页面，等待两秒 driver.get("https://kns.cnki.net/kns8/AdvSearch")
time.sleep(2) # 修改属性，使下拉框显示 opt = driver.find_element(By.CSS_SELECTOR, 'div.sort-list') # 定位元素 driver.execute_script("arguments[0].setAttribute('style', 'display: block;')", opt) # 执行 js 脚本进行属性的修改；arguments[0]代表第一个属性 # 鼠标移动到下拉框中的[通讯作者] ActionChains(driver).move_to_element(driver.find_element(By.CSS_SELECTOR, 'li[data-val="RP"]')).perform() # 找到[关键词]选项并点击 WebDriverWait(driver, 100).until( EC.visibility_of_element_located((By.CSS_SELECTOR, 'li[data-val="KY"]'))).click() # 传入关键字
WebDriverWait(driver, 100).until(
EC.presence_of_element_located((By.XPATH, '''//*[@id="gradetxt"]/dd[1]/div[2]/input'''))
).send_keys(keyword)
# 点击搜索
WebDriverWait(driver, 100).until(
EC.presence_of_element_located((By.XPATH, "/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[2]/input"))
).click() # 点击切换中文文献 WebDriverWait(driver, 100).until( EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[1]/div/div/div/a[1]")) ).click() # 获取总文献数和页数
res_unm = WebDriverWait(driver, 100).until(EC.presence_of_element_located(
(By.XPATH, "/html/body/div[3]/div[2]/div[2]/div[2]/form/div/div[1]/div[1]/span[1]/em"))
).text # 去除千分位里的逗号 res_unm = int(res_unm.replace(",", '')) page_unm = int(res_unm / 20) + 1 print(f"共找到 {res_unm} 条结果, {page_unm} 页。")if __name__ == '__main__': keyword = "数字普惠金融" driver = webserver() open_page(driver, keyword)

复制代码