# -*- coding:UTF-8 -*-
from urllib import request
from bs4 import BeautifulSoup
import re
import sys if __name__ == "__main__":
#创建txt文件
file = open('一念永恒.txt', 'w', encoding='utf-8')
#一念永恒小说目录地址
target_url = 'http://www.biqukan.com/1_1094/'
#User-Agent
head = {}
head['User-Agent'] = 'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19'
target_req = request.Request(url = target_url, headers = head)
target_response = request.urlopen(target_req)
target_html = target_response.read().decode('gbk','ignore')
#创建BeautifulSoup对象
listmain_soup = BeautifulSoup(target_html,'lxml') #搜索文档树,找出div标签中class为listmain的所有子标签
chapters = listmain_soup.find_all('div',class_ = 'listmain')
#使用查询结果再创建一个BeautifulSoup对象,对其继续进行解析
download_soup = BeautifulSoup(str(chapters), 'lxml')
#计算章节个数
numbers = (len(download_soup.dl.contents) - 1) / 2 - 8
index = 1
#开始记录内容标志位,只要正文卷下面的链接,最新章节列表链接剔除
begin_flag = False
#遍历dl标签下所有子节点
for child in download_soup.dl.children:
#滤除回车
if child != '\n':
#找到《一念永恒》正文卷,使能标志位
if child.string == u"《一念永恒》正文卷":
begin_flag = True
#爬取链接并下载链接内容
if begin_flag == True and child.a != None:
download_url = "http://www.biqukan.com" + child.a.get('href')
download_req = request.Request(url = download_url, headers = head)
download_response = request.urlopen(download_req)
download_html = download_response.read().decode('gbk','ignore')
download_name = child.string
soup_texts = BeautifulSoup(download_html, 'lxml')
texts = soup_texts.find_all(id = 'content', class_ = 'showtxt')
soup_text = BeautifulSoup(str(texts), 'lxml')
write_flag = True
file.write(download_name + '\n\n')
#将爬取内容写入文件
for each in soup_text.div.text.replace('\xa0',''):
if each == 'h':
write_flag = False
if write_flag == True and each != ' ':
file.write(each)
if write_flag == True and each == '\r':
file.write('\n')
file.write('\n\n')
#打印爬取进度
sys.stdout.write("已下载:%.3f%%" % float(index/numbers) + '\r')
sys.stdout.flush()
index += 1
file.close()
>>> for link in soup.find_all('a'):
... print(link.get('href'))
#用于爬取a标签的链接
Beautiful Soup 4.4.0 文档链接：http://beautifulsoup.readthedocs.io/zh_CN/v4.4.0/
已下文章来自博客园大佬：http://www.cnblogs.com/sakura3/p/8460224.html（为了复习方便，搬一下，谢谢）
爬小说：
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import requests
from bs4 import BeautifulSoup
# get_url_list 获取所有章节的URL，在一个list里
def get_url_list(url):
content = requests.get(url).content #获取页面内容
soup = BeautifulSoup(content,'lxml') #Beautifulsoup 实例化对象
url_list = [] #空的url_list 数组
# urls = soup.find('div',{'id':'list'}).find('dl').find_all('dd')
urls = soup.select('#list > dl > dd > a') # 根据页面选择到URL ，还可以urls = soup.find('div',{'id':'list'}).find('dl').find_all('dd')
for i in urls: #遍历里面的每一章的URL
i = i.get('href') #获取URL
# print(i)
i = 'http://www.biquge.com.tw' + i #分析文章组成，形成最终的URL
url_list.append(i) #添加到url_list 里面去
# print (url_list)
return url_list
# 获取这一章的内容
def get_data(url):
content = requests.get(url).content
soup = BeautifulSoup(content, 'lxml')
f = open(r'C:\Users\HBX\Documents\staudy\HMXX.txt','a+',encoding='utf-8') #不加utf-8 会有编码报错
text_name = soup.find('div',{'class':'bookname'}).find('h1').text #获得章节名字
# text_content = soup.select('#content')
text_content = soup.find('div',{'id':'content'}).get_text() #获得章节内容 ，还有一种select css 选择的获取章节内容的方式
book =text_name+ '\r\n' + text_content #整体的一章
# print(book)
f.write((book)+'\r\n') #换行写入
f.close() #关闭文件
# for x in text_content:
# a = x.text.replace('readx();', '')
# print(a) if __name__ =='__main__':
url = 'http://www.biquge.com.tw/18_18049/' #笔趣阁的小说目录页面
url_list = get_url_list(url) #获取了所有的url
for i in url_list: # 循环一章url
get_data(i) #获取文章内容