python+SQLAlchemy+爬虫

时间:2021-05-24 22:10:06
python+SQLAlchemy+爬虫

python+SQLAlchemy+爬虫

前面分享了SQLAlchemy的知识,这次我共享一下学习用python开发爬虫再把爬出来的数据放到用SQLAlchemy的数据库上面的知识,当然我这个是带测试,后面我和附带用TDD写的测试。

"""
一个简单的豆瓣音乐前250爬虫
Author: Jar.guo email:1418229458@qq.com
Date: 2016-08-27
Language: Python2.7.10 """ import urllib2
import sys from lxml import html
from MusicORM import Music,MusicORMHelper reload(sys)
sys.setdefaultencoding('utf8')

类的简要说明
    本类主要用于抓取豆瓣图书Top前250的书籍的名称

Attributes:
        cur_url: 用于表示当前争取抓取页面的url
        datas: 存储处理好的抓取到的图书名称

class MusicPicker(object):
def __init__(self):
self.cur_url = "https://music.douban.com/chart"
self.datas = []
# 储存的数据库
self.db=MusicORMHelper("flaskr.db")
self.db.create_db() print "豆瓣音乐爬虫准备就绪, 准备爬取数据..."

Returns:
            返回抓取到整个页面的HTML(unicode编码)
        Raises:
            URLError:url引发的异常

    def Acquire_music_open(self):
try:
html_string = urllib2.urlopen(self.cur_url).read().decode("utf-8") except urllib2.URLError, e:
if hasattr(e, "code"):
print "The server couldn't fulfill the request."
print "Error code: %s" % e.code
elif hasattr(e, "reason"):
print "We failed to reach a server. Please check your url and read the Reason"
print "Reason: %s" % e.reason
return html_string

通过返回的整个网页HTML, 正则匹配前250的书籍名称
        Args:
            content_items: 传入页面的HTML文本进行匹配,这里用的不是正则

    def select_music_Content(self, html_string):
tree = html.fromstring(html_string)
content_items = tree.xpath('//a[@href="javascript:;"]/text()')
return content_items

下面是简单的内容处理的问题

    def form_music_Content(self, content_items):
top_num = 1
temp_data = []
for index, item in enumerate(content_items):
if (item.find("&nbsp") == -1 and top_num<=10):
temp_data.append("第" + str(top_num) + "名 " + item)
top_num += 1
self.datas.extend(temp_data)
return self.datas

爬虫入口, 并控制爬虫抓取页面的范围

    def start_music_spider(self):
my_page = self.Acquire_music_open()
content_items = self.select_music_Content(my_page)
self.form_music_Content(content_items)

这里我们来写一个插入数据库的方法

    def exportData(self, music):
return self.db.addmusic(music)

最后我们对前面的进行一下处理

def main():
print """
###############################
一个简单的豆瓣音乐前250爬虫
Jar.guo email:1418229458@qq.com
Date: 2016-08-27
###############################
"""
my_spider = MusicPicker()
my_spider.start_music_spider()
# 在这里对爬出来的内容进行迭代并且插入数据库,并且在这里我们把结果打印出来,方便检查爬虫爬出来的内容是不是自己想要的
for item in my_spider.datas:
item_unicode = unicode(item)
my_spider.exportData(Music(item_unicode,item_unicode)) print item print "spider is done..."
if __name__ == '__main__':
main()

下面我发一下相关测试

from MusicORM import Music
from musicPicker import MusicPicker import unittest # 包含单元测试模块
import sys reload(sys)
sys.setdefaultencoding('utf8') class filmReptileTests(unittest.TestCase): def setUp(self): # 单元测试环境配置
self.spider= MusicPicker() def tearDown(self): # 单元测试环境清除
self.spider =None def testInit(self):
self.assertIsNotNone(self.spider)
self.assertIsNotNone(self.spider.cur_url)
self.assertEqual(self.spider.cur_url,"https://music.douban.com/chart")
self.assertEqual(self.spider.datas,[]) def testGet_page_string(self):
self.assertIsNotNone(self.spider.Acquire_music_open()) def testFind_title(self):
html_string=self.spider.Acquire_music_open()
titles=self.spider.select_music_Content(html_string)
self.assertIsNotNone(titles)
titles_length=len(titles)
model = self.spider.form_music_Content(titles)
model_length=len(model)
self.assertGreater(titles_length, 0)
self.assertEqual(titles_length,20)
self.assertIsNotNone(model)
self.assertEqual(model_length, 10) def testExportData(self):
html_string=self.spider.Acquire_music_open()
titles=self.spider.select_music_Content(html_string)
self.assertIsNotNone(titles)
titles_length=len(titles)
model = self.spider.form_music_Content(titles)
model_length=len(model)
self.assertGreater(titles_length, 0)
self.assertEqual(titles_length,20)
self.assertIsNotNone(model)
self.assertEqual(model_length, 10)
for item in model:
isSuccess=self.spider.exportData(Music(unicode( item), unicode( item)))
self.assertTrue(isSuccess)