这个作业的要求来自于：https://edu.cnblogs.com/campus/gzcc/GZCC-16SE2/homework/2941。
由于存在多次请求，所以稍微将请求封装如下
def tranfrom_dom_tree(url):
'''
将获取的html文本转化为dom树
'''
response = requests.get(url);
response.encoding = "utf-8";
return BeautifulSoup(response.text, "html.parser");
将具体新闻内容封装如下
class News(object):
'''
广商校园新闻数据模型
'''
def __init__(self, url):
self._url = url; #新闻网页地址
self._dom_tree = tranfrom_dom_tree(url);
self._show_infos = self._dom_tree.select(".show-info")[0].text.split();
self._update_time = "";
self._auditor = "";
self._auothor = "";
self._origin = ""; for index, args in enumerate(self._show_infos):
if args.startswith("发布时间"):
self._update_time = args[5:] + " " + self._show_infos[index+1];
continue;
elif args.startswith("作者"):
self._auothor = args[3:];
continue;
elif args.startswith("审核"):
self._auditor = args[3:];
continue;
elif args.startswith("来源"):
self._origin = args[3:];
continue; @property
def title(self):
'''
:return: 新闻标题
'''
return self._dom_tree.select(".show-title")[0].text; @property
def auothor(self):
'''
:return: 新闻作者
'''
return self._auothor; @property
def auditor(self):
'''
:return: 新闻审核
'''
return self._auditor; @property
def origin(self):
'''
:return: 新闻发布单位
'''
return self._origin; @property
def update_time(self):
'''
:return: 新闻最后更新时间
'''
return self._update_time; @update_time.setter
def update_time(self, time):
'''
设置最后更新时间
:param time: 时间
'''
self._update_time = time; @property
def times(self):
'''
:return: 点击次数
'''
clickUrl = 'http://oa.gzcc.cn/api.php?op=count&id={}&modelid=80'.format(self.news_id);
response = requests.get(clickUrl);
click = re.findall('(\d+)', response.text)[-1];
return click; @property
def news_id(self):
'''
:return: 新闻标识
'''
time = datetime.strptime(self._update_time, '%Y-%m-%d %H:%M:%S');
time = time.strftime("%m%d");
return re.match('http://news.gzcc.cn/html/.*/.*/(\d+).html', self._url).group(1); @property
def summary(self):
'''
:return: 新闻摘要内容
'''
return self._summary; @summary.setter
def summary(self, text):
'''
设置新闻摘要
:param text: 新闻摘要
'''
self._summary = text; def to_dict(self):
'''
将此类实例转换为字典
:return: 转换后的字典
'''
dict = {};
dict["news_url"] = self._url;
dict["news_id"] = self.news_id
dict["news_title"] = self.title;
dict["news_summary"] = self.summary;
dict["news_update_time"] = self.update_time;
dict["news_times"] = self.times;
dict["news_auothor"] = self.auothor;
dict["news_auditor"] = self.auditor;
dict["news_origin"] = self.origin;
return dict;
对新闻进行批操作代码如下
class GZCCNewsReptile(object):
'''
广州商学院校园新闻获取工具
'''
def __init__(self):
self._news_type = "dict";
self._root_url = "http://news.gzcc.cn/html/xiaoyuanxinwen/";
self._url = self.page_url();
self._dom_tree = tranfrom_dom_tree(self._url); def page_url(self, page=1):
'''
将指定页面解析数值解析对应url
:param page: 指定的一个新页面
'''
if page == 1:
self._now_page = "index";
else:
self._now_page = page;
return self._root_url+str(self._now_page)+".html"; @property
def count(self):
'''
:return: 返回校园新闻总条数
'''
count = self._dom_tree.select(".a1")[0].text;
return int(count[0:-1]); @property
def page(self):
'''
:return: 获取校园新闻总页数
'''
page = int(self.count) / 10;
int_page = int(page);
if page > int_page:
return ( int_page + 1);
else:
return int_page; def get_news_from_news_page_size(self, start_page, end_page):
'''
设置爬取页数范围,
start_page 小于 1 抛出 异常
start_page 大于 end_page 抛出 异常
end_page 大于 总页数 抛出 异常
:param start_page: 要爬取范围的开始页
:param end_page: 结束爬取范围的页面(不包括该页面)
'''
if start_page < 1:
raise IndexError("start_page不在指定范围内");
if start_page > end_page:
raise IndexError("start_page大于end_page");
if end_page > self.page:
raise IndexError("end_page不在指定范围内");
news_list = [];
times = (index for index in range(start_page, end_page));
# 爬取指定范围数据
for index in times:
# news_page_list = self.get_news_from_page_url(self.page_url(index));
news_page_list = self.get_page_news(index); # 较上句更利于封装
news_list.append(news_page_list);
news_list = sum(news_list, []);
return news_list; def get_page_news(self, page):
'''
获取指定页数
若指定的页数在可爬取页数的范围之外，则抛出运行异常异常
:param pages: 指定的页数
:return: 返回指定页的新闻列表
'''
if page < 1 or page > self.count:
raise IndexError("page不在指定范围内");
else:
print("\r当前正在%d页" % page, end="");
return self.get_news_from_page_url(self.page_url(page)); def get_news_from_page_url(self, url):
'''
获取指定url的所有新闻列表
:param pages: 指定的页面url
:return: news_page_list的列表
'''
dom_tree = tranfrom_dom_tree(url);
news_ui = dom_tree.select(".news-list li a");
news_page_list = [];
for index in range(0, 10):
try:
a_tag = news_ui[index];
href = a_tag.get("href"); news = News(href);
news.summary = a_tag.select(".news-list-description")[0].text;
news_page_list.append(eval("news.to_"+self.news_type+"()"));
except Exception:
error_log = "此页面不正常: %s"%href;
print("此页面不正常: %s"%href, end="");
with open("./logger.txt", "a", encoding="utf-8") as file:
now_time = time.time();
file.write(error_log + " 错误时间:" + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(now_time)) ); return news_page_list; @property
def news_type(self):
'''
新闻默认类型设置
:return: 返回默认类型
'''
return self._news_type;
@news_type.setter
def news_type(self, type="dict"):
'''
新闻可选类型
:param type: 选择的类型
'''
if type == "dict":
self._news_type = type;
else :
raise Exception("未能匹配该类型");
测试代码如下
# 测试GZCCNewsReptil可用性
start_page = 106
# news_list = GZCCNewsReptile().get_news_from_news_page_size(start_page,start_page+10);
news_list = GZCCNewsReptile().get_news_from_news_page_size(1, 256);
pandas_date = pandas.DataFrame(news_list);
print(news_list);
# #csv
pandas_date.to_csv(".\pandas_date.csv", encoding="utf-8_sig");
# #sql
# with sqlite3.connect("test.sqlite") as db:
# pandas_date.to_sql("test", db);
# date = pandas_date.read_sql_query('SELECT * FROM gzccnewsdb5', con=db)
# date[date["news_times"]>380];