1.豆瓣
爬取单个页面数据
import requests
from lxml import etree
#import os url = "https://movie.douban.com/cinema/nowplaying/yongzhou/"
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
req = requests.get(url=url,headers=headers)
text = req.text
dics = []
#将抓取下来的数据根据一定的规则进行提取
html = etree.HTML(text)
ul = html.xpath("//ul[@class='lists']")[0]
#print(etree.tostring(ul,encoding='utf-8').decode('utf-8'))
lis = ul.xpath("./li")
for li in lis:
title = li.xpath("@data-title")[0]
score = li.xpath("@data-actors")[0]
adress = li.xpath("@data-region")[0]
img_hai = li.xpath(".//img/@src")[0]
dic = {
'title':title,
'score':score,
'adress':adress,
'img':img_hai
}
dics.append(dic)
print(dics)
2.电影天堂
爬取多个页面数据
import requests
import json
from lxml import etree
url = "http://www.dytt8.net"
HEADERS = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
'Referer':'http://www.dytt8.net/html/gndy/dyzz/list_23_1.html'
} def get_url(urls):
response = requests.get(urls,headers=HEADERS)
text = response.text #请求页面
html = etree.HTML(text) #解析为HTML文档 html为Element对象 （可以执行xpath语法）
detail_urls = html.xpath("//table[@class='tbspan']//a/@href") #获取页面下的href
detail_urls = map(lambda urls:url+urls,detail_urls) #将detail_urls这个列表中每个url都扔给lambda这个函数合并 再将整个修改后的赋给detail_urls
return detail_urls def parse_detail_url(de_ur):
movie = {}
response = requests.get(de_ur,headers=HEADERS)
text = response.content.decode('gbk')
html = etree.HTML(text)
title = html.xpath("//div[@class='title_all']//font[@color='#07519a']/text()")[0] #获取标题
movie['title'] = title #放入字典
zoomE = html.xpath("//div[@id='Zoom']")[0]
img_hb = zoomE.xpath(".//img/@src")
cover = img_hb[0] #海报
#sst = img_hb[1] #电影截图
movie['cover'] = cover
#movie['sst'] = sst def parse_info(info,rule):
return info.replace(rule,"").strip() #.strip()把前后空格删掉
infos = zoomE.xpath(".//text()")
for index,info in enumerate(infos): #enumerate 索引序列（0 str 1 str 2 str）
if info.startswith("◎片 名"): #判断 以。。开始
info = parse_info(info,"◎片 名") #调用parse_info将"◎片 名"替换为无（没有）
movie['pian'] = info
elif info.startswith("◎年 代"):
info = parse_info(info, "◎年 代")
movie['year'] = info
elif info.startswith("◎产 地"):
info = parse_info(info, "◎产 地")
movie['adress'] = info
elif info.startswith("◎导 演"):
info = parse_info(info, "◎导 演")
movie['actor'] = info
elif info.startswith("◎类 别"):
info = parse_info(info, "◎类 别")
movie['lb'] = info
elif info.startswith("◎豆瓣评分"):
info = parse_info(info, "◎豆瓣评分")
movie['db'] = info
elif info.startswith("◎主 演"):
info = parse_info(info, "◎主 演")
actors = []
for x in range(index+1,len(infos)):
actor = infos[x]
if actor.startswith("◎"): #过滤简介部分
break
actors.append(actor)
movie['actors'] = actors
elif info.startswith("◎简 介"):
info = parse_info(info,"◎简 介")
for x in range(index+1,len(infos)):
profile = infos[x].strip()
if profile.startswith("【"): #过滤下载地址部分
break
movie['profile'] = profile
download_url = html.xpath("//td[@bgcolor='#fdfddf']/a/@href")[0] #下载地址
movie['download_url'] = download_url
return movie def write_to_file(content):
with open('result.txt','a',encoding='utf-8') as f:
f.write(json.dumps(content,ensure_ascii=False)+'\n') #ensure_ascii=False 输出为中文
f.close() def dianying():
urld = "http://www.dytt8.net/html/gndy/dyzz/list_23_{}.html" #这里用到了{} .format()的用法
movies = [] #定义一个列表
for x in range(1,8):
#第一个for循环用来控制7个页面
print(x)
urls = urld.format(x)
if x==5: #这里因为第5个页面出现报错信息 可能是编码问题 解决不了 所以我就过滤了第5页
continue
detail_ur = get_url(urls) #解析每页的详细信息
write_to_file("第%s页" % x)
for detail_url in detail_ur:
#第二个for循环用来遍历每个页
movie = parse_detail_url(detail_url)
movies.append(movie)
write_to_file(movie) if __name__ == '__main__':
dianying()
3.腾讯招聘
跟上一个电影天堂的代码差不多
import requests
import json
from lxml import etree
url = "https://hr.tencent.com/"
HEADERS = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
def get_url(urld):
response = requests.get(urld,headers=HEADERS)
text = response.text
html = etree.HTML(text)
detail_url = html.xpath("//tr[@class='even' or @class='odd']//a/@href")
detail_url = map(lambda x:url+x,detail_url) return detail_url def prease_url(detail_url):
dic = {}
#print(detail_url)
response = requests.get(detail_url,headers=HEADERS)
text =response.text
html = etree.HTML(text)
title = html.xpath("//tr[@class='h']//td[@class='l2 bold size16']//text()")[0]
dic['title'] = title #方法一 (死板)
adress = html.xpath("//tr[@class='c bottomline']//td//text()")[1]
dic['adress'] = adress
# 方法二 (简洁)
str = html.xpath("//tr[@class='c bottomline']//td")
leibie = str[1].xpath(".//text()")[1]
dic['leibie'] = leibie
nums = str[2].xpath(".//text()")[1]
dic['nums'] = nums
gz = html.xpath("//ul[@class='squareli']")
gzzz = gz[0].xpath(".//text()")
gzyq = gz[1].xpath(".//text()")
dic['工作职责'] = gzzz
dic['工作要求'] = gzyq
#print(dic)
return dic def write_to_file(content):
with open('tengxun.txt','a',encoding='utf-8') as f:
f.write(json.dumps(content,ensure_ascii=False)+'\n') #ensure_ascii=False 输出为中文
f.close()
def tengxun():
movies = []
urls = "https://hr.tencent.com/position.php?keywords=python&lid=0&tid=87&start={}#a"
for x in range(0,501,10): #步长为10
print(x)
urld = urls.format(x)
detail_urls = get_url(urld)
for detail_url in detail_urls:
movie = prease_url(detail_url)
movies.append(movie)
write_to_file(x)
write_to_file(movies) if __name__ == '__main__':
tengxun()