'''
Created on 2017年4月5日
@author: Admin
'''
import requests
from bs4 import BeautifulSoup as bsp
# 网站链接
site = 'http://www.runoob.com'
lineNo = 1
class Movie:
def __init__(self, name, url):
self.name = name
self.url = url
def __str__(self):
return '%s,\t%s分,\t%s' % (self.name, self.url)
__repr__ = __str__
def getSoup(url):
r = requests.get(url)
# r.encoding = 'gb18030'
return bsp(r.text, "html.parser")
# 解析指定url,获取其中指定内容
def filterMovie(url):
resultList = []
# url处理,注意页面的编码集
soup = getSoup(url)
# 查找所有target=_top的a标签
atags = soup.find_all('a', target='_top')
for atag in atags:
# 取到指定a标签的title属性
titleA = atag['title']
if atag is not None:
try:
# 根据取到的href属性,拼接上页面域名 获得完整的跳转url
url = site + atag['href']
print('url:', url)
print('title:', titleA)
# 根据主方法里面 _init_ 中定义的内容,将获取到的字段进行组合,然后放到集合中
movie = Movie(titleA, url)
resultList.append(movie)
except:
print('error !!')
return resultList
# 根据传入的结果结合 按照指定格式保存到 文本文件中
def saveInfo(atagList):
# 注意指定编码,否则输入到txt文件的时候会乱码
fileObj = open('vueJS.txt', 'a',encoding='utf8')
for atag in atagList:
atagName = str(atag.name)
print('url info:', atagName)
global lineNo
fileObj.write('(' + str(lineNo) + ') ' + atagName )
fileObj.write('\t')
fileObj.write(atag.url)
fileObj.write(
'————————————————————————————————————————————————————————————————————————————————————————————————')
fileObj.write('\n')
lineNo += 1
fileObj.close()
# 获取到根据url整理出的一个结果集合,然后将结果集合保存的物理文件中
def getPageResource(url):
resultList = filterMovie(url)
if len(resultList) > 0:
saveInfo(resultList)
# 方法入口,定义一个页面url,然后去解析
if __name__ == '__main__':
url = 'http://www.runoob.com/vue2/vue-tutorial.html'
getPageResource(url)