python requests + xpath 获取分页详情页数据存入到txt文件中

时间:2022-10-31 20:12:07
直接代码,如有不懂请加群讨论
# *-* coding:utf-8 *-* #
import json
import requests
import pytesseract
import time
import datetime
from PIL import Image
from bs4 import BeautifulSoup
import urllib3
import random
import os from lxml import etree headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
} session = requests.session() url = 'https://www.saikr.com/' def get_index():
'''
直接访问活动页面
:return:
'''
response = session.get(url+'activity',headers=headers)
response.encoding = 'utf8'
return response.content def get_page(res = ''):
'''
获取数据进行存储
:return:
'''
if res:
html = etree.HTML(res)
else:
html = etree.HTML(get_index())
#获取尾页数据
lastpage = html.xpath('//li[@class="last"]/a/@data-ci-pagination-page')[0]
#从第一页开始爬取
for p in range(1, int(lastpage)):
net_url = url + 'activity/' + str(p)
response = session.get(net_url, headers=headers)
response.encoding = 'utf8'
get_page(response.content)
#得到详情页数据
items = html.xpath('//div[@id="activeLoadContentBox"]/ul/li')
title = ''
for item in items:
at_url = item.xpath('./div/h3/a/@href')[0]
at_title = item.xpath('./div/h3/a/text()')[0]
title = at_title.strip() + '\n'
get_content(at_url,title) def get_content(u,t):
res = session.get(u, headers=headers)
html = etree.HTML(res.content)
items = html.xpath('//div[@class="new-active-box"]/ul/li') result = t for item in items:
title = item.xpath('./span/text()')[0]
cont = item.xpath('./div/p[1]/text()')
#结束时间
cont2 = item.xpath('./div/p[2]/text()')
if cont2:
endtime = cont2[0]
else:
endtime = ''
if cont:
conts = cont
else:
conts = item.xpath('./p/text()')
if conts[0].strip() == '':
conts = item.xpath('./p/span/text()') result += title.strip() +conts[0].strip() + endtime + '\n' print(result)
file = os.getcwd() + '/active.txt'
output = open(file, 'a')
output.write(result)
output.close()
if __name__ == '__main__':
get_page() 运行:

python requests + xpath 获取分页详情页数据存入到txt文件中

python requests + xpath 获取分页详情页数据存入到txt文件中