数据来源网站http://baidu.lecai.com/lottery/draw/list/50?d=2013-01-01
HTML解析器http://pythonhosted.org/pyquery/ (可以像JQuery那样使用)
源码:
import MySQLdb as mysql
from pyquery import PyQuery as pq create_table_sql = '''
create table union_lotto(
issue int primary key,
lottery_date date,
lottery_number varchar(30)
)''' sql = "insert into union_lotto values(%(issue)s, %(date)s, %(number)s)"
conn = mysql.connect(host='localhost', db='caipiao', user='root', passwd='')
cur = conn.cursor() def inserts(rows):
cur.executemany(sql, rows)
conn.commit() def close():
conn.close() def handler_row(row):
children = row.getchildren()
date = children[0].text_content()
issue = children[1].getchildren()[0].text_content()
spans = children[2].getchildren()[0].getchildren()
numbers = []
for span in spans:
numbers.append(span.text_content())
lottery_number = '-'.join(numbers)
return {'issue': int(issue.strip()), 'date': date, 'number': lottery_number} def grab_data(url):
d = pq(url=url)
rows = d("#draw_list > tbody > tr")
result = []
for row in rows:
result.append(handler_row(row))
return result def main():
years = [(2003 + i) for i in range(0, 11)]
url = 'http://baidu.lecai.com/lottery/draw/list/50?d=%d-01-01'
print '.......star.........'
for year in years:
result = grab_data(url % year)
inserts(result)
close()
print '.......end..........' if __name__ == '__main__':
main()