github完整项目地址：
https://github.com/doubunv/phantomJS-weibo
'''
Created on 2017年2月22日@author: test
'''
import time
#phantomjs导入包
from selenium import webdriver
#xpath
from lxml.html.clean import Cleaner
import random
if __name__ == '__main__':
driver = webdriver.PhantomJS()
driver.maximize_window() #设置浏览器窗口最大话
url="https://login.sina.com.cn/signup/signin.php?entry=sso"
#logger.info("程序开始启动")
#登录
while(True):
driver.get(url)
# logger.info("登录页面:"+driver.current_url)
username = driver.find_element_by_id('username')
password = driver.find_element_by_id('password')
username.send_keys(config.get("login").get("name"))
password.send_keys(config.get("login").get("pwd"))
sbtn = driver.find_element_by_css_selector('.btn_mod .W_btn_a')
sbtn.submit()
time.sleep(20)
if driver.current_url != url:
break
logger.info("个人主页:"+driver.current_url)
#跳转到微博页面
weibo = driver.find_element_by_css_selector(".l_pdt a")
url=weibo.get_attribute('href')
driver.get(url)
logger.info("微博页面:"+driver.current_url)
#跳转到'car'类
car = driver.find_element_by_partial_link_text('car')
url=car.get_attribute("href")
driver.get(url)
logger.info("专栏页面:"+driver.current_url)
#跳转到原创
yy = driver.find_element_by_partial_link_text('原创')
url=yy.get_attribute("href")
driver.get(url)
logger.info("原创页面:"+driver.current_url)
# with open('temp.html','rb') as file:
# html = file.read()
times = time.strftime("%Y-%m-%d %H:%M", time.localtime())
#cs = CrawlService()
while(True):
#清理html
page = driver.page_source
cleaner = Cleaner(style=True, javascript=True,scripts=True,comments=True)
html = cleaner.clean_html(page)
logger.info('爬取时间:'+time.strftime("%Y-%m-%d %H:%M", time.localtime()))
#爬取内容
# temptime = cs.crawl_data(html,times)
if temptime:
times = temptime
logger.info("最后一次发表时间为:"+times)
#如果配置文件有配置值,则配置文件优先
steps = config.get('steps')
if steps:
logger.info("间隔%s分钟抓取一次网页"%steps)
time.sleep(steps*60)
else:
logger.info("间隔(3-5)分钟抓取一次网页")
time.sleep(random.randint(3,5)*60)
#刷新网页
driver.refresh()
driver.quit()
logger.info("程序已关闭")