python版本 python2.7
爬取知乎流程:
一 、分析 在访问知乎首页的时候（https://www.zhihu.com），在没有登录的情况下，会进行重定向到（https://www.zhihu.com/signup?next=%2F）这个页面，
爬取知乎，首先要完成登录操作,登陆的时候观察往那个页面发送了post或者get请求。可以利用抓包工具来获取登录时密码表单等数据的提交地址。
1、利用抓包工具，查看用户名密码数据的提交地址页就是post请求,将表单数据提交的网址,经过查看。是这个网址 'https://www.zhihu.com/api/v3/oauth/sign_in'。
2、通过抓取上述登录地址,在其请求的contenr字段中,发现post请求服务器不止包含用户名,密码,还有timetamp,lang,client_id,sihnature等表单数据,需要知道每一个表单数据的特点,而特点是我们数据变化 在每次登录的时候的变化来查找数据的规律。
3、经过多次登录观察,这些表单数据中只有timetamp,和signature是变化的,其他的值是不变的。
4、通过js发现 signature字段的值是有多个字段组合加密而成,其实timetamp时间戳是核心,每次根据时间的变化,生成不同的signature值。
5、考虑到signature的值加密较为复杂,直接将浏览器登陆成功后的时间戳timetamp和signature 复制到请求数据中,然后进行登录。
6、表单数据田中完毕,发送post请求时,出现了缺少验证码票据的错误(capsion_ticket) 经过分析验证码票据是为了获取验证码而提供的一种验证方式,
而抓包装工具中关于验证码的请求有两次, 一次获取的是:
{'show_captcha':true}
而同时第二次获取的是:{'img_base_64':Rfadausifpoauerfae}。
7、经过分析{'show_captcha':true} 是获取验证码的关键信息,再抓包信息中发现第一次请求相应的set-cookie中,包含了capsion_ticket验证码票据信息。
8、在此模拟登陆又出现了错误'ERR_xxx_AUTH_TOKEN'错误信息,而她出现在我们很根据验证码票据获取验证码图片时,
我们从抓包中查看关于Authorization:oauth ce30dasjfsdjhfkiswdnf.所以将其在headers当中进行配置。
验证码问题：
验证码问题
-对于知乎的验证码，有两种情况，一种是英文的图片验证码，一种是点击倒立文字的验证码，当登录需要验证码的时候，回向这两个网站发送数据
倒立文字验证码：https://www.zhihu.com/api/v3/oauth/captcha?lang=cn
英文图片验证码：https://www.zhihu.com/api/v3/oauth/captcha?lang=en
-英文验证码得到数据是四个英文字母。可采用云打码在线识别。
-倒立文字验证码是得到的是每个汉字有一定的范围，当登陆的时候点击验证码的时候，
从https://www.zhihu.com/api/v3/oauth/captcha?lang=cn该网站获取到的一个像素点（x,y),比如倒立文字在第三个和第五个，就会有一个可选范围，只要输入合适的像素点 就可以登录。
-只对倒立文字进行验证
-只是简单地爬取第一页的问题及回答
二、创建scrapy项目
scrapy startproject ZhiHuSpider
scrapy genspider zhihu zhihu.com
三、代码
在zhihu.py中代码如下：
# -*- coding: utf-8 -*-
import base64
import json
import urlparse
import re
from datetime import datetime
import scrapy
from scrapy.loader import ItemLoader
from ..items import ZhiHuQuestionItem, ZhiHuAnswerItem class ZhihuSpider(scrapy.Spider):
name = 'zhihu'
allowed_domains = ['www.zhihu.com']
start_urls = ['https://www.zhihu.com']
start_answer_url = "https://www.zhihu.com/api/v4/questions/{}/answers?include=data%5B%2A%5D.is_normal%2Cadmin_closed_comment%2Creward_info%2Cis_collapsed%2Cannotation_action%2Cannotation_detail%2Ccollapse_reason%2Cis_sticky%2Ccollapsed_by%2Csuggest_edit%2Ccomment_count%2Ccan_comment%2Ccontent%2Ceditable_content%2Cvoteup_count%2Creshipment_settings%2Ccomment_permission%2Ccreated_time%2Cupdated_time%2Creview_info%2Crelevant_info%2Cquestion%2Cexcerpt%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%2Cupvoted_followees%3Bdata%5B%2A%5D.mark_infos%5B%2A%5D.url%3Bdata%5B%2A%5D.author.follower_count%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics&limit=20&offset={}&sort_by=default" headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0',
'Referer': 'https://www.zhihu.com',
'HOST': 'www.zhihu.com',
'Authorization': 'oauth c3cef7c66a1843f8b3a9e6a1e3160e20'
}
points_list = [[20, 27], [42, 25], [65, 20], [90, 25], [115, 32], [140, 25], [160, 25]] def start_requests(self):
"""
重写父类的start_requests()函数，在这里设置爬虫的起始url为登录页面的url。
:return:
"""
yield scrapy.Request(
url='https://www.zhihu.com/api/v3/oauth/captcha?lang=cn',
callback=self.captcha,
headers=self.headers,
) def captcha(self, response):
show_captcha = json.loads(response.body)['show_captcha']
if show_captcha:
print u'有验证码'
yield scrapy.Request(
url='https://www.zhihu.com/api/v3/oauth/captcha?lang=cn',
method='PUT',
headers=self.headers,
callback=self.shi_bie
)
else:
print u'没有验证码'
# 直接进行登录的操作
post_url = 'https://www.zhihu.com/api/v3/oauth/sign_in'
post_data = {
'client_id': 'c3cef7c66a1843f8b3a9e6a1e3160e20',
'grant_type': 'password',
'timestamp': '',
'source': 'com.zhihu.web',
'signature': '6d1d179e50a06d1c17d6e8b5c89f77db34f406ac',
'username': '',#账号
'password': '',#密码
'captcha': '',
'lang': 'cn',
'ref_source': 'homepage',
'utm_source': ''
} yield scrapy.FormRequest(
url=post_url,
headers=self.headers,
formdata=post_data,
callback=self.index_page
) def shi_bie(self, response):
try:
img= json.loads(response.body)['img_base64']
except Exception, e:
print '获取img_base64的值失败，原因：%s'%e
else:
print '成功获取加密后的图片地址'
# 将加密后的图片进行解密，同时保存到本地
img = img.encode('utf-8')
img_data = base64.b64decode(img)
with open('zhihu_captcha.GIF', 'wb') as f:
f.write(img_data) captcha = raw_input('请输入倒立汉字的位置：')
if len(captcha) == 2:
# 说明有两个倒立的汉字
pass
first_char = int(captcha[0]) - 1 # 第一个汉字对应列表中的索引
second_char = int(captcha[1]) - 1 # 第二个汉字对应列表中的索引
captcha = '{"img_size":[200,44],"input_points":[%s,%s]}' % (self.points_list[first_char], self.points_list[second_char])
else:
# 说明只有一个倒立的汉字
pass
first_char = int(captcha[0]) - 1
captcha = '{"img_size":[200,44],"input_points":[%s]}' % (
self.points_list[first_char]) data = {
'input_text': captcha
}
yield scrapy.FormRequest(
url='https://www.zhihu.com/api/v3/oauth/captcha?lang=cn',
headers=self.headers,
formdata=data,
callback=self.get_result
) def get_result(self, response):
try:
yan_zheng_result = json.loads(response.body)['success']
except Exception, e:
print '关于验证码的POST请求响应失败，原因：{}'.format(e)
else:
if yan_zheng_result:
print u'验证成功'
post_url = 'https://www.zhihu.com/api/v3/oauth/sign_in'
post_data = {
'client_id': 'c3cef7c66a1843f8b3a9e6a1e3160e20',
'grant_type': 'password',
'timestamp': '',
'source': 'com.zhihu.web',
'signature': '6d1d179e50a06d1c17d6e8b5c89f77db34f406ac',
'username': '',#账号
'password': '',#密码
'captcha': '',
'lang': 'cn',
'ref_source': 'homepage',
'utm_source': ''
}
#以上数据需要在抓包中获取 yield scrapy.FormRequest(
url=post_url,
headers=self.headers,
formdata=post_data,
callback=self.index_page
)
else:
print u'是错误的验证码！' def index_page(self, response):
for url in self.start_urls:
yield scrapy.Request(
url=url,
headers=self.headers
) def parse(self, response):
"""
提取首页中的所有问题的url，并对这些url进行进一步的追踪，爬取详情页的数据。
:param response:
:return:
"""
# /question/19618276/answer/267334062
all_urls = response.xpath('//a[@data-za-detail-view-element_name="Title"]/@href').extract()
all_urls = [urlparse.urljoin(response.url, url) for url in all_urls]
for url in all_urls:
# https://www.zhihu.com/question/19618276/answer/267334062
# 同时提取：详情的url；文章的ID；
result = re.search('(.*zhihu.com/question/(\d+))', url)
if result:
detail_url = result.group(1)
question_id = result.group(2)
# 将详情url交由下载器去下载网页源码
yield scrapy.Request(
url=detail_url,
headers=self.headers,
callback=self.parse_detail_question,
meta={
'question_id': question_id,
}
) # 在向详情url发送请求的同时，根据问题的ID，同时向问题的url发送请求。由于问题和答案是两个独立的url。而答案其实是一个JSON的API接口，直接请求即可，不需要和问题url产生联系。
yield scrapy.Request(
# 参数：问题ID，偏移量。默认偏移量为0，从第一个答案开始请求
url=self.start_answer_url.format(question_id, 0),
headers=self.headers,
callback=self.parse_detail_answer,
meta={
'question_id': question_id
}
) break def parse_detail_question(self, response):
"""
用于处理详情页面关于question问题的数据，比如：问题名称，简介，浏览数，关注者数等
:param response:
:return:
"""
item_loader = ItemLoader(item=ZhiHuQuestionItem(), response=response)
item_loader.add_value('question_id', response.meta['question_id'])
item_loader.add_xpath('question_title', '//div[@class="QuestionHeader"]//h1/text()')
item_loader.add_xpath('question_topic', '//div[@class="QuestionHeader-topics"]//div[@class="Popover"]/div/text()')
# 获取的问题中，可能会不存在简介
item_loader.add_xpath('question_content', '//span[@class="RichText"]/text()')
item_loader.add_xpath('question_watch_num', '//button[contains(@class, "NumberBoard-item")]//strong/text()')
item_loader.add_xpath('question_click_num', '//div[@class="NumberBoard-item"]//strong/text()')
item_loader.add_xpath('question_answer_num', '//h4[@class="List-headerText"]/span/text()')
item_loader.add_xpath('question_comment_num', '//div[@class="QuestionHeader-Comment"]/button/text()')
item_loader.add_value('question_url', response.url)
item_loader.add_value('question_crawl_time', datetime.now()) question_item = item_loader.load_item()
yield question_item def parse_detail_answer(self, response):
"""
用于解析某一个问题ID对应的所有答案。
:param response:
:return:
"""
answer_dict = json.loads(response.body)
is_end = answer_dict['paging']['is_end']
next_url = answer_dict['paging']['next'] for answer in answer_dict['data']:
answer_item = ZhiHuAnswerItem()
answer_item['answer_id'] = answer['id']
answer_item['answer_question_id'] = answer['question']['id']
answer_item['answer_author_id'] = answer['author']['id']
answer_item['answer_url'] = answer['url']
answer_item['answer_comment_num'] = answer['comment_count']
answer_item['answer_praise_num'] = answer['voteup_count']
answer_item['answer_create_time'] = answer['created_time']
answer_item['answer_content'] = answer['content']
answer_item['answer_crawl_time'] = datetime.now()
answer_item['answer_update_time'] = answer['updated_time'] yield answer_item # 判断is_end如果值为False，说明还有下一页
if not is_end:
yield scrapy.Request(
url=next_url,
headers=self.headers,
callback=self.parse_detail_answer
)
item.py中代码:
# -*- coding: utf-8 -*- # Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html from datetime import datetime
import scrapy
from utils.common import extract_num class ZhihuspiderItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass class ZhiHuQuestionItem(scrapy.Item):
question_id=scrapy.Field() # 问题ID
question_title = scrapy.Field() # 问题标题
question_topic = scrapy.Field() # 问题分类
question_content = scrapy.Field() # 问题内容
question_watch_num = scrapy.Field() # 关注者数量
question_click_num = scrapy.Field() # 浏览者数量
question_answer_num = scrapy.Field() # 回答总数
question_comment_num = scrapy.Field() # 评论数量
question_crawl_time = scrapy.Field() # 爬取时间
question_url = scrapy.Field() # 问题详情url def get_insert_sql(self):
insert_sql = "insert into zhihu_question(question_id, question_title, question_topic, question_content, question_watch_num, question_click_num, question_answer_num, question_comment_num, question_crawl_time, question_url) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE question_id=VALUES(question_id),question_title=VALUES(question_title),question_topic=VALUES(question_topic),question_content=VALUES(question_content),question_watch_num=VALUES(question_watch_num),question_click_num=VALUES(question_click_num),question_answer_num=VALUES(question_answer_num),question_comment_num=VALUES(question_comment_num),question_crawl_time=VALUES(question_crawl_time),question_url=VALUES(question_url)" # 整理字段对应的数据
question_id = str(self['question_id'][0])
question_title = ''.join(self['question_title'])
question_topic = ",".join(self['question_topic']) try:
question_content = ''.join(self['question_content'])
except Exception,e:
question_content = 'question_content内容为空' question_watch_num = ''.join(self['question_watch_num']).replace(',', '')
question_watch_num = extract_num(question_watch_num) question_click_num = ''.join(self['question_click_num']).replace(',', '')
question_click_num = extract_num(question_click_num)
# '86 回答'
question_answer_num = ''.join(self['question_answer_num'])
question_answer_num = extract_num(question_answer_num)
# '100 条评论'
question_comment_num = ''.join(self['question_comment_num'])
question_comment_num = extract_num(question_comment_num) question_crawl_time = self['question_crawl_time'][0]
question_url = self['question_url'][0] args_tuple = (question_id, question_title, question_topic, question_content, question_watch_num, question_click_num, question_answer_num, question_comment_num, question_crawl_time, question_url) return insert_sql, args_tuple class ZhiHuAnswerItem(scrapy.Item):
answer_id = scrapy.Field() # 答案的ID (zhihu_answer表的主键)
answer_question_id = scrapy.Field() # 问题的ID (zhihu_question表的主键)
answer_author_id = scrapy.Field() # 回答用户的ID
answer_url = scrapy.Field() # 回答的url
answer_comment_num = scrapy.Field() # 该回答的总评论数
answer_praise_num = scrapy.Field() # 该回答的总点赞数
answer_create_time = scrapy.Field() # 该回答的创建时间
answer_content = scrapy.Field() # 回答的内容
answer_update_time = scrapy.Field() # 回答的更新时间 answer_crawl_time = scrapy.Field() # 爬虫的爬取时间 def get_insert_sql(self):
insert_sql = "insert into zhihu_answer(answer_id, answer_question_id, answer_author_id, answer_url, answer_comment_num, answer_praise_num, answer_create_time, answer_content, answer_update_time, answer_crawl_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE answer_id=VALUES(answer_id),answer_question_id=VALUES(answer_question_id),answer_author_id=VALUES(answer_author_id),answer_url=VALUES(answer_url),answer_comment_num=VALUES(answer_comment_num),answer_praise_num=VALUES(answer_praise_num),answer_create_time=VALUES(answer_create_time),answer_content=VALUES(answer_content),answer_update_time=VALUES(answer_update_time),answer_crawl_time=VALUES(answer_crawl_time)" # 处理answer_item中的数据
# fromtimestamp(timestamp)：将一个时间戳数据转化为一个date日期类型的数据
answer_id = self['answer_id']
answer_question_id = self['answer_question_id']
answer_author_id = self['answer_author_id']
answer_url = self['answer_url']
answer_comment_num = self['answer_comment_num']
answer_praise_num = self['answer_praise_num']
answer_content = self['answer_content']
answer_create_time = datetime.fromtimestamp(self['answer_create_time'])
answer_update_time = datetime.fromtimestamp(self['answer_update_time'])
answer_crawl_time = self['answer_crawl_time'] args_tuple = (answer_id, answer_question_id, answer_author_id, answer_url, answer_comment_num, answer_praise_num, answer_create_time, answer_content, answer_update_time, answer_crawl_time) return insert_sql, args_tuple
pipeline,py代码如下:
# -*- coding: utf-8 -*- # Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html import MySQLdb
import MySQLdb.cursors
from twisted.enterprise import adbapi # 数据库的异步写入操作。因为execute()及commit()提交数据库的方式是同步插入数据，一旦数据量比较大，scrapy的解析是异步多线程的方式，解析速度非常快，而数据库的写入速度比较慢，可能会导致item中的数据插入数据库不及时，造成数据库写入的阻塞，最终导致数据库卡死或者数据丢失。 class ZhihuspiderPipeline(object):
def process_item(self, item, spider):
return item class MySQLTwistedPipeline(object):
def __init__(self, dbpool):
self.dbpool = dbpool @classmethod
def from_settings(cls, settings):
args = dict(
host=settings['MYSQL_HOST'],
db=settings['MYSQL_DB'],
user=settings['MYSQL_USER'],
passwd=settings['MYSQL_PASSWD'],
charset=settings['MYSQL_CHARSET'],
cursorclass=MySQLdb.cursors.DictCursor
)
# 创建一个线程池对象
# 参数1：用于连接MySQL数据库的驱动
# 参数2：数据库的链接信息（host, port, user等）
dbpool = adbapi.ConnectionPool('MySQLdb', **args)
return cls(dbpool) def process_item(self, item, spider):
# 在线程池dbpool中通过调用runInteraction()函数，来实现异步插入数据的操作。runInteraction()会insert_sql交由线程池中的某一个线程执行具体的插入操作。
query = self.dbpool.runInteraction(self.insert, item)
# addErrorback()数据库异步写入失败时，会执行addErrorback()内部的函数调用。
query.addErrback(self.handle_error, item) def handle_error(self, failure, item):
print u'插入数据失败，原因：{}，错误对象：{}'.format(failure, item) def insert(self, cursor, item):
pass
# 当存在多张表时，每一个表对应的数据，解析时间是不确定的，不太可能保证问题，答案同时能够解析完成，并且同时进入到pipeline中执行Insert的操作。
# 所以，不能再这个函数中，对所有的表执行execute()的操作。
# 解决办法：将sql语句在每一个Item类中实现。
# insert_question = ''
# insert_answer = ''
# insert_user = ''
insert_sql, args = item.get_insert_sql()
cursor.execute(insert_sql, args)
setting.py代码如下:
# -*- coding: utf-8 -*- # Scrapy settings for ZhiHuSpider project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html BOT_NAME = 'ZhiHuSpider' SPIDER_MODULES = ['ZhiHuSpider.spiders']
NEWSPIDER_MODULE = 'ZhiHuSpider.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'ZhiHuSpider (+http://www.yourdomain.com)' # Obey robots.txt rules
ROBOTSTXT_OBEY = False # Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default)
#COOKIES_ENABLED = False # Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False # Override the default request headers:
# DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
# } # Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'ZhiHuSpider.middlewares.ZhihuspiderSpiderMiddleware': 543,
#} # Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'ZhiHuSpider.middlewares.ZhihuspiderDownloaderMiddleware': 543,
#} # Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#} # Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
# 'ZhiHuSpider.pipelines.ZhihuspiderPipeline': 300,
'ZhiHuSpider.pipelines.MySQLTwistedPipeline':1,
} # Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' MYSQL_HOST = 'localhost'# 本机端口,
MYSQL_DB = '' #数据库名字
MYSQL_USER = '' #数据库用户名
MYSQL_PASSWD = '' #密码
MYSQL_CHARSET = 'utf8'
另外设置了一个工具模块新建了一个python package.用来过滤item数据
需要在item中导入模块
代码如下:
import re def extract_num(value):
result = re.search(re.compile('(\d+)'), value)
res = int(result.group(1))
return res