本文目的：介绍如何抓取微博内容，利用requests包+cookies实现登陆微博，lxml包的xpath语法解析网页，抓取目标内容。
所需python包：requests、lxml 皆使用pip安装即可
XPath即为XML路径语言，它是一种用来确定XML（标准通用标记语言的子集）文档中某部分位置的语言。XPath基于XML的树状结构，提供在数据结构树中找寻节点的能力。 xPath 同样也支持HTML.
XPath 是一门小型的查询语言，这里我们将它与 python 爬虫相结合来介绍。
xpath使用方法简介：
step1: 安装 lxml 库。
step2: from lxml import etree
step3: Selector = etree.HTML(网页源代码)
step4: Selector.xpath(一段神奇的符号)
具体的xpath解析方法大家可以参考之后下边的代码或者参考这篇博文http://cighao.com/2016/03/01/introduction-of-xPath/
言归正传，接下来从头看一下抓取的思路，首先找到微博的网址，为了方便抓取，准备抓取3g版微博，网址为 https://weibo.cn/，进而找到登陆的网址为 https://passport.weibo.cn/signin/login，按F12调出开发者工具，点击最上边一栏的network，然后输入自己的账号密码进行登录，登录之后可以看到弹出了很多元素，找到m.weibo.cn这一项点击，如图所示，里面三个红色框就是我们需要的
Request URL 代表了我们刚刚请求的网址
cookies代表了刚刚登录我们的账号里面包含的密码信息，待会登录会用到
user-Agent代表了我们使用的浏览器的头部信息，用来对我们的爬虫进行伪装，降低被封禁的风险
接下来上一段代码
#coding=utf-8
import re
import time
import string
import os
import pickleimport requests
from lxml import etree
import traceback
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
class weibo:
cookie = {"Cookie": "~~~~~~~~~~"} #将your cookie替换成自己的cookie
header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'}#这里就是浏览器的头部信息
def __init__(self,user_id,filter = 0):
self.user_id = user_id #用户id，即需要我们输入的数字，如昵称为“Dear-迪丽热巴”的id为1669879400
self.filter = filter #取值范围为0、1，程序默认值为0，代表要爬取用户的全部微博，1代表只爬取用户的原创微博
self.userName = '' #用户名，如“Dear-迪丽热巴”
self.weiboNum = 0 #用户全部微博数
self.weiboNum2 = 0 #爬取到的微博数
self.following = 0 #用户关注数
self.followers = 0 #用户粉丝数
self.weibos = [] #微博内容
self.num_zan = [] #微博对应的点赞数
self.num_forwarding = [] #微博对应的转发数
self.num_comment = [] #微博对应的评论数
self.weibo_detail_urls=[]#pickle.load(open("weibourl1.pkl", "r"))#微博评论
self.weibourls=[]#pickle.load(open('weibourl2.pkl','r'))#每一条微博链接，用于断点续爬
def getUserName(self):
try:
url = 'http://weibo.cn/%d/info'%(self.user_id)
html = requests.get(url, cookies = weibo.cookie,headers=weibo.header).content #这一句里面的cookies与headers就实现了登录
selector = etree.HTML(html)
userName = selector.xpath("//title/text()")[0]
self.userName = userName[:-3]
#print '用户昵称：' + self.userName
except Exceptione,e:
print "Error: ",e
traceback.print_exc()
上边的代码就实现了利用requests包+cookies登录微博，接下来就是找到想要抓取的用户id，然后使用xpath解析网页进行信息抓取，代码如下：
def getUserInfo(self):
try:
url = 'http://weibo.cn/u/%d?filter=%d&page=1'%(self.user_id,self.filter)
html = requests.get(url, cookies = weibo.cookie,headers=weibo.header).content
selector = etree.HTML(html)
pattern = r"\d+\.?\d*"
str_wb = selector.xpath("//div[@class='tip2']/span[@class='tc']/text()")
guid = re.findall(pattern, str_wb[0], re.S|re.M)
for value in guid:
num_wb = int(value)
break
self.weiboNum = num_wb
print '微博数: ' + str(self.weiboNum)
str_gz = selector.xpath("//div[@class='tip2']/a/text()")[0]
guid = re.findall(pattern, str_gz, re.M)
self.following = int(guid[0])
print '关注数: ' + str(self.following)
str_fs = selector.xpath("//div[@class='tip2']/a/text()")[1]
guid = re.findall(pattern, str_fs, re.M)
self.followers = int(guid[0])
print '粉丝数: ' + str(self.followers)
except Exception,e:
print "Error: ",e
traceback.print_exc()
def getWeiboInfo(self):
try:
url = 'http://weibo.cn/u/%d?filter=%d&page=1'%(self.user_id,self.filter)
html = requests.get(url, cookies = weibo.cookie,headers=weibo.header).content
selector = etree.HTML(html)
if selector.xpath('//input[@name="mp"]')==[]:
pageNum = 1
else:
pageNum = (int)(selector.xpath('//input[@name="mp"]')[0].attrib['value'])
pattern = r"\d+\.?\d*"
f = open("./%s.txt"%self.user_id, "wb")
for page in range(1,pageNum+1):
if page % 10 == 0:
print('[ATTEMPTING] rest for 5 minutes to cheat weibo site, avoid being banned.')
time.sleep(60*5)
url2 = 'http://weibo.cn/u/%d?filter=%d&page=%d'%(self.user_id,self.filter,page)
html2 = requests.get(url2, cookies = weibo.cookie,headers=weibo.header).content
selector2 = etree.HTML(html2)
info = selector2.xpath("//div[@class='c']")
if len(info) > 3:
for i in range(0,len(info)-2):
detail = info[i].xpath("@id")[0]
url3='http://weibo.cn/comment/{}?uid={}&rl=0'.format(detail.split('_')[-1], self.user_id)
if url3 not in self.weibo_detail_urls:
self.weiboNum2 = self.weiboNum2 + 1
#print self.weibo_detail_urls
str_t = info[i].xpath("div/span[@class='ctt']")
weibos = str_t[0].xpath('string(.)')
self.weibos.append(weibos)
#print '微博内容：'+ weibos+'***'+'No.%s'%self.weiboNum2
str_zan = info[i].xpath("div/a/text()")[-4]
guid = re.findall(pattern, str_zan, re.M)
num_zan = int(guid[0])
self.num_zan.append(num_zan)
#print '点赞数: ' + str(num_zan)
forwarding = info[i].xpath("div/a/text()")[-3]
guid = re.findall(pattern, forwarding, re.M)
num_forwarding = int(guid[0])
self.num_forwarding.append(num_forwarding)
#print '转发数: ' + str(num_forwarding)
comment = info[i].xpath("div/a/text()")[-2]
guid = re.findall(pattern, comment, re.M)
num_comment = int(guid[0])
self.num_comment.append(num_comment)
#print '评论数: ' + str(num_comment)
self.weibo_detail_urls.append(url3)
text=str(self.weiboNum2)+':'+weibos + '\t'+'点赞数：' + str(num_zan) + '\t'+ ' 转发数：' + str(num_forwarding) +'\t'+ ' 评论数：' + str(num_comment) + '\n'
f.write(text)
pickle.dump(self.weibo_detail_urls, open("weibourl1.pkl", "w"))
else:
print url3+'这条微博已经爬取过，忽略'
if self.filter == 0:
print '共'+str(self.weiboNum2)+'条微博'
else:
print '共'+str(self.weiboNum)+'条微博，其中'+str(self.weiboNum2)+'条为原创微博'
except Exception,e:
print "Error: ",e
traceback.print_exc()
def get_weibo_detail_comment(self):
weibo_comments_save_path = './weibo/{}.txt'.format(self.user_id)
with open(weibo_comments_save_path, 'a') as f:
for i, url in enumerate(self.weibo_detail_urls):
if url not in self.weibourls:
self.weibourls.append(url)
pickle.dump(self.weibourls, open("weibourl2.pkl", "w"))
print('solving weibo detail from {}'.format(url))
html_detail = requests.get(url, cookies=weibo.cookie, headers=weibo.header).content
selector = etree.HTML(html_detail)
str1='id="pagelist"'
if str1 in html_detail:
all_comment_pages = selector.xpath('//*[@id="pagelist"]/form/div/input[1]/@value')[0]
else:
all_comment_pages = 1
print('\n这是 {} 的微博：'.format(self.userName))
#print('微博内容： {}'.format(self.weibos[i]))
#print('接下来是下面的评论：\n\n')
f.writelines('微博内容： {}'.format(self.weibos[i])+'\n')
f.writelines('接下来是下面的评论:\n')
for page in range(1,int(all_comment_pages) + 1):
if page % 10 == 0:
print('[ATTEMPTING] rest for 5 minutes to cheat weibo site, avoid being banned.')
time.sleep(60*5)
detail_comment_url = url + '&page=' + str(page)
try:
html_detail_page = requests.get(detail_comment_url, cookies=weibo.cookie,headers=weibo.header).content
selector = etree.HTML(html_detail_page)
comment_div_element = selector.xpath('//div[starts-with(@id, "C_")]')
for child in comment_div_element:
single_comment_user_name = child.xpath('a[1]/text()')[0]
if child.xpath('span[1][count(*)=0]'):
single_comment_content = child.xpath('span[1][count(*)=0]/text()')[0]
else:
span_element = child.xpath('span[1]')[0]
at_user_name = span_element.xpath('a/text()')[0]
at_user_name = '$' + at_user_name.split('@')[-1] + '$'
single_comment_content = span_element.xpath('/text()')
single_comment_content.insert(1, at_user_name)
single_comment_content = ' '.join(single_comment_content)
full_single_comment = '<' + single_comment_user_name + '>' + ': ' + single_comment_content
#print(full_single_comment)
f.writelines(full_single_comment + '\n')
#f.writelines('F\n')
except etree.XMLSyntaxError as e:
print('user id {} all done!'.format(self.user_id))
print('all weibo content and comments saved into {}'.format(weibo_comments_save_path))
f.writelines('F\n')
else:
print 'has alredy'
def writeTxt(self):
try:
if self.filter == 1:
resultHeader = '\n\n原创微博内容：\n'
else:
resultHeader = '\n\n微博内容：\n'
result = '用户信息\n用户昵称：' + self.userName + '\n用户id：' + str(self.user_id) + '\n微博数：' + str(self.weiboNum) + '\n关注数：' + str(self.following) + '\n粉丝数：' + str(self.followers) + resultHeader
if os.path.isdir('weibo') == False:
os.mkdir('weibo')
f = open("./%s.txt"%self.user_id, "wb")
f.write(result)
f.close()
except Exception,e:
print "Error: ",e
traceback.print_exc()
def start(self):
try:
weibo.getUserName(self)
weibo.getUserInfo(self)
weibo.writeTxt(self)
weibo.getWeiboInfo(self)
weibo.get_weibo_detail_comment(self)
print '信息抓取完毕'
print '==========================================================================='
except Exception,e:
print "Error: ",e
user_id =~~~~~~~~~~~~~ #可以改成任意合法的用户id（爬虫的微博id除外）
filter = 0 #值为0表示爬取全部的微博信息（原创微博+转发微博），值为1表示只爬取原创微博
#open('./weibourl1.pkl','w')
#open('./weibourl2.pkl','w')
wb = weibo(user_id,filter) #调用weibo类，创建微博实例wb
wb.start() #爬取微博信息
print '用户名：' + wb.userName
print '全部微博数：' + str(wb.weiboNum)
print '关注数：' + str(wb.following)
print '粉丝数：' + str(wb.followers)
print '最新一条微博为：' + wb.weibos[0] #若filter=1则为最新的原创微博，如果该用户微博数为0，即len(wb.weibos)==0,打印会出错，下同
print '最新一条微博获得的点赞数：' + str(wb.num_zan[0])
print '最新一条微博获得的转发数：' + str(wb.num_forwarding[0])
print '最新一条微博获得的评论数：' + str(wb.num_comment[0])
wb.writeTxt() #wb.writeTxt()只是把信息写到文件里，大家可以根据自己的需要重新编写writeTxt()函数
这两段代码就实现了对微博用户的用户名、微博数、微博转发、评论、点赞数、以及微博及对微博评论的抓取。
参考博文：http://cighao.com/2016/03/01/introduction-of-xPath/