NSTL国家科技图书文献中心 2017 机械 仪表工业 所有期刊论文信息
代码比较随意，不要介意
第一步，爬取所有期刊链接
#coding=utf-8 import time
from selenium import webdriver
from lxml import etree
from pymongo import MongoClient client = MongoClient("IP", 27017)
db = client["nstl"]
collection=db["journal_urls"]
db.authenticate("","") driver = webdriver.Chrome(executable_path=r"D:\chromedriver_win32\chromedriver.exe")
driver.get('https://www.nstl.gov.cn/facade/search/clcSearch.do?&lan=eng&clc=TH') html = driver.page_source
tree = etree.HTML(html)
count = int(tree.xpath("//span[@id='totalPages1']/text()")[0]) # 共47页
for i in range(count): html = driver.page_source
tree = etree.HTML(html) # 提取当前页所有期刊链接并存储
table = tree.xpath("//div[@class='s2listtd2']/span/a/@href")
for j in table:
bson = {}
bson['url'] = j
collection.insert(bson) # i等于46时终止
if i==(count-1):
break # 点击接下来一页按钮
driver.find_element_by_xpath('//div[@id="page"]/div//a[text()="%s"]'%str(i+2)).click() # 判断翻页成功后跳出while
while True:
time.sleep(1)
if driver.page_source!=html:
break driver.close()
第二步，爬取每个期刊中所有2017年论文链接
#coding=utf-8
import requests
from pymongo import MongoClient
from lxml import etree
from selenium import webdriver
import time client = MongoClient("IP", 27017)
db = client["nstl"]
collection1=db["journal_urls"]
collection2=db["journalArticle2017_urls"]
db.authenticate("","")
driver = webdriver.Chrome(executable_path=r"D:\chromedriver_win32\chromedriver.exe")
# 循环所有期刊链接
for item in collection1.find({}, {"url":1, "_id":0}):
driver.get(item['url'][29:-4])
html = driver.page_source
tree = etree.HTML(html)
# 判断如果有18年论文，需要点击出17年论文
table_2018 = tree.xpath("//div[@id='year_2018']")
if table_2018!=[]:
driver.find_element_by_xpath("//div[@id='year_2017']").click()
time.sleep(1)
driver.find_element_by_xpath("//div[@id='volumeUl_2017']/div[@class='ltreebom2']").click()
# 获取17年期的个数并循环
table = tree.xpath("//div[@id='volumeUl_2017']//div[@class='ltreebom3']/a")
for i in range(1, len(table)+1):
wen_html = driver.page_source
wen_tree = etree.HTML(wen_html)
# 获取当前一期的所有论文链接
wen_table = tree.xpath("//div[@class='s2listtd2']/a/@href")
for j in wen_table:
bson = {}
bson['url'] = j
collection2.insert(bson)
# 判断结束循环
if i==len(table):
break
# 点击出下一期论文
try:
driver.find_element_by_xpath("//div[@id='volumeUl_2017']//div[@class='ltreebom3'][%s]"%str(i+1)).click()
except:
break
# 判断是否点击成功
while True:
time.sleep(1)
if driver.page_source!=wen_html:
break driver.close()
第三步，爬取论文信息详情页源码
#coding=utf-8
import requests
from pymongo import MongoClient
from lxml import etree
from selenium import webdriver
import time client = MongoClient("IP", 27017)
db = client["nstl"]
collection=db["journalArticle2017_urls"]
collection1=db["journalArticle2017_codes"]
db.authenticate("","") driver = webdriver.Chrome(executable_path=r"D:\chromedriver_win32\chromedriver.exe") # 循环所有论文并构造链接
for item in collection.find({}, {"url":1, "_id":0}): url = "https://www.nstl.gov.cn/facade/search/toFullView.do?checkedSEQNO="+item['url'][23:-11]+"&subDocType="+item['url'][-8:-3] # # post方法获取当前页源码
# for i in range(100):
# try:
# result = requests.post(url, verify = False)
# except:
# time.sleep(1)
# continue # html = result.text
# if html:
# break # 模拟浏览器获取源码, 得到含有文献数据的源码后跳出循环
driver.get(url)
for i in range(100):
time.sleep(1)
if driver.page_source!=html:
break # 存储
bson = {}
html1 = driver.page_source
bson['html'] = html1
collection1.insert(bson) driver.close()
第四步，解析源码
#coding=utf-8
from pymongo import MongoClient
from lxml import etree client = MongoClient("IP", 27017)
db = client["nstl"]
collection1 = db["journalArticle2017_codes"]
collection2 = db["journalArticle2017_data"]
db.authenticate("","") zzdw, km, ma, cbn, j, q, qy, zy, zys, flh, gjc, yz, wz = u'【作者单位】：', u'【刊名】：', u'【ISSN】：', u'【出版年】：', u'【卷】：', u'【期】：', u'【起页】：', u'【止页】：', u'【总页数】：', u'【分类号】：', u'【关键词】：', u'【语种】：', u'【文摘】：' # 循环所有论文并构造链接
n = 0
for item in collection1.find({}, {"html":1, "_id":0}):
html = item["html"]
tree = etree.HTML(html) title = tree.xpath("//span[@name='title']/text()")
author = tree.xpath("//a[starts-with(@href,'javascript:searchByAuthor')]/text()")
organization = tree.xpath("//div[text()='%s']/following-sibling::*/text()"%zzdw)
journal_name = tree.xpath("//div[text()='%s']/following-sibling::*/text()"%km)
issn = tree.xpath("//div[text()='%s']/following-sibling::*/text()"%ma)
publication_year = tree.xpath("//div[text()='%s']/following-sibling::*/text()"%cbn)
volume = tree.xpath("//div[text()='%s']/following-sibling::*/text()"%j)
issue = tree.xpath("//div[text()='%s']/following-sibling::*/text()"%q)
page_start = tree.xpath("//div[text()='%s']/following-sibling::*/text()"%qy)
page_end = tree.xpath("//div[text()='%s']/following-sibling::*/text()"%zy)
page_count = tree.xpath("//div[text()='%s']/following-sibling::*/text()"%zys)
clc = tree.xpath("//div[text()='%s']/following-sibling::*/text()"%flh)
keywords = tree.xpath("//div[text()='%s']/following-sibling::*/span/a/text()"%gjc)
language = tree.xpath("//div[text()='%s']/following-sibling::*/text()"%yz)
summary = tree.xpath("//div[text()='%s']/following-sibling::*/text()"%wz) dc = {}
dc['title'] = title[0]
if author: dc['author'] = author
if organization: dc['organization'] = organization[0]
if journal_name: dc['journal_name'] = journal_name[0]
if issn: dc['issn'] = issn[0]
if publication_year: dc['publication_year'] = publication_year[0]
if volume: dc['volume'] = volume[0]
if issue: dc['issue'] = issue[0]
if page_start: dc['page_start'] = page_start[0]
if page_end: dc['page_end'] = page_end[0]
if page_count: dc['page_count'] = page_count[0]
if clc: dc['clc'] = clc[0]
if keywords: dc['keywords'] = keywords[0]
if language: dc['language'] = language[0]
if summary: dc['summary'] = summary[0] collection2.insert(dc)