爬虫--XPATH解析

时间：2023-03-09 19:37:32
爬虫--XPATH解析

今天说一下关于爬取数据解析的方式---->XPATH,XPATH是解析方式中最重要的一种方式

1.安装：pip install lxml

 2.原理

　　1. 获取页面源码数据

　　2.实例化一个etree的对象，并且将页面源码数据加载到该对象中

　　3.调用该对象的xpath方法进行指定标签的定位

　　4.注意：xpath函数必须结合着xpath表达式进行标签定位和内容捕获

说了也不明白，直接上例子！！！！

1.解析58二手房的相关数据

#引用requests

import requests

#引用lxml

from lxml import etree

#地址

url = 'https://bj.58.com/ershoufang/sub/l16/s2242/?utm_source=market&spm=u-2d2yxv86y3v43nkddh1.bdpcpz_bt&PGTID=0d30000c-0000-1139-b00c-643d0d315a04&ClickID=1'

#伪装的请求头，证明我是浏览器

headers = {

    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'

}

#获取当前整个页面

page_text = requests.get(url,headers=headers).text

#作用于当前页面之后再解析数据

tree = etree.HTML(page_text)

#//ul[@class="house-list-wrap"]/li这就是lxml解析，//代表前面的层次

li_list = tree.xpath('//ul[@class="house-list-wrap"]/li')

# print(li_list)#得到每一个<Element li at 0x202a8c62288>这玩意

#再次循环

for li in li_list:

#再次解析得到准确的数据！！！

    title = li.xpath('./div[2]/h2[1]/a/text()')[0]

    print(title)

2.福利福利！！！！下载彼岸图网中的图片数据

import os

import requests

from lxml import etree

#这里注意，这是python3中的写法！！！

import urllib.request

url = 'http://pic.netbian.com/4kmeinv/'

headers = {

    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'

}

#二话不说直接发情再说

response = requests.get(url,headers=headers)

#如果当前根目录下没有imgs文件夹，就创建！！！

if not os.path.exists('./imgs'):

    os.mkdir('./imgs')

#得到请求数据

page_text = response.text

#作用当前页面

tree = etree.HTML(page_text)

#lxml解析

li_list = tree.xpath('//div[@class="slist"]/ul/li')

#循环得到准确的数据

for li in li_list:

    img_name = li.xpath('./a/b/text()')[0]

    # 处理中文乱码！不要理解记住就ok

    img_name = img_name.encode('ISO-8859-1').decode('gbk')

    #拼接完整的地址

    img_url = 'http://pic.netbian.com' + li.xpath('./a/img/@src')[0]

    #图片的名字

    img_path = './imgs/' + img_name + '.jpg'

    #这里避免打开文件就用urllib直接写入

    urllib.request.urlretrieve(url=img_url,filename=img_path)

3.解析所有城市名称（https://www.aqistudy.cn/historydata/）

import requests

from lxml import etree

url = 'https://www.aqistudy.cn/historydata/'

headers = {

    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'

}

res = requests.get(url=url, headers=headers).text

tree = etree.HTML(res)

city_list = tree.xpath('//div[@class="bottom"]/ul/li/a/text() | //div[@class="bottom"]/ul/div[2]/li/a/text()')        # 逻辑运算符，这里 | 表示或的关系

city = ''.join(city_list)

4.煎蛋网的爬去图片

# 煎蛋网图片

import requests

from lxml import etree

#base对于加魔数据进行解密

import base64

import os

import urllib.request

if not os.path.exists('./jiandan'):

    os.mkdir('./jiandan')

headers = {

    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'

}

url = 'http://jandan.net/ooxx'

res1 = requests.get(url, headers=headers).text

tree = etree.HTML(res1)

span_list = tree.xpath('//span[@class="img-hash"]/text()')

for span_hash in span_list:

    #对于加密数据进行解密，编码是utf-8并且拼接完整的url

    img_url = 'http:' + base64.b64decode(span_hash).decode('utf8')

    #得到具体的数据

    img_data = requests.get(url=img_url, headers=headers).content

    filepath = './jiandan/' + img_url.split('/')[-1]

    urllib.request.urlretrieve(url=img_url, filename=filepath)

    print(filepath, '下载完成！')

print('over')

5.爬去简历模板

import requests

from lxml import etree

import random

import os

headers = {

    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'

}

if not os.path.exists('./jianli'):

    os.mkdir('./jianli')

#爬去前4页的数据

for i in range(1, 4):

    if i == 1:

        #第一页数据

        url = 'http://sc.chinaz.com/jianli/free.html'

    else:

        #不是第一页的数据，进行数据的拼接

        url = 'http://sc.chinaz.com/jianli/free_%s.html' % (i)

    response = requests.get(url=url, headers=headers)

    #字符编码改一下,否则出现这种问题：求职电子版简历免费下载 下载完成！

    response.encoding = 'utf8'

    res = response.text

    tree = etree.HTML(res)

    a_list = tree.xpath('//a[@class="title_wl"]')

    for a in a_list:

        name = a.xpath('./text()')[0]

        jl_url = a.xpath('./@href')[0]

        response = requests.get(url=jl_url, headers=headers)

        response.encoding = 'utf8'

        res1 = response.text

        tree = etree.HTML(res1)

        download_url_list = tree.xpath('//div[@class="clearfix mt20 downlist"]/ul/li/a/@href')

        download_url = random.choice(download_url_list)

        res3 = requests.get(url=download_url, headers=headers).content

        filepath = './jianli/' + name + '.rar'

        #如果上边是content，写入的时候记得’wb'

        with open(filepath, 'wb') as f:

            f.write(res3)

        print(name, '下载完成！')

print('over')

6.站长直接图片下载，图片懒加载

import requests

from lxml import etree

import os

import urllib

import urllib.request

if not os.path.exists('./tupian'):

    os.mkdir('./tupian')

url = 'http://sc.chinaz.com/tupian/'

headers = {

    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'

}

response = requests.get(url=url, headers=headers)

response.encoding = 'utf8'

res = response.text

tree = etree.HTML(res)

url_list = tree.xpath('//div[@id="container"]/div/div/a/img/@src2')  # img标签是伪属性src2，当图片滚动到视野内时变为 src

for url in url_list:

    filepath = './tupian/' + url.rsplit('/', 1)[-1]

    urllib.request.urlretrieve(url, filepath)

    print(filepath, '下载完成！')

print('over')

相关文章