爬取python岗位智联招聘
- 这里爬取北京地区岗位招聘python岗位，并存入EXECEL文件内，代码如下：
import json
import xlwt
import requests
from queue import Queue
from threading import Thread
def producer(q,path):
res1 = requests.get(path,
headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3676.400 QQBrowser/10.4.3469.400'},
)
result = json.loads(res1.text)
for item in result["data"]["results"]:
jobName = item["jobName"]
companyname = item["company"]["name"]
type = item["company"]["type"]["name"]
size = item["company"]["size"]["name"]
url = item["company"]['url']
city = item['city']["items"][0]["name"]
updateDate = item["updateDate"]
salary = item["salary"]
eduLevel = item["eduLevel"]["name"]
workingExp = item["workingExp"]["name"]
emplType = item["emplType"]
condition = item["timeState"]
msg = "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s"%(jobName,companyname,type,size,url,city,updateDate,salary,eduLevel,workingExp,emplType,condition)
q.put(msg)
def consumer(q):
count = 1
workbook = xlwt.Workbook(encoding="utf-8")
worksheet = workbook.add_sheet(r"智联招聘")
pattern = xlwt.Pattern()
pattern.pattern = xlwt.Pattern.SOLID_PATTERN
pattern.pattern_fore_colour = 2
style = xlwt.XFStyle()
style.pattern = pattern
worksheet.write(0,0,r"工作名称", style)
worksheet.write(0,1,r"公司",style)
worksheet.write(0,2,r"企业性质",style)
worksheet.write(0,3,r"人数",style)
worksheet.write(0,4,r"链接网站",style)
worksheet.write(0,5,r"所在地",style)
worksheet.write(0,6,r"发布日期",style)
worksheet.write(0,7,r"薪资",style)
worksheet.write(0,8,r"学历",style)
worksheet.write(0,9,r"工作年限",style)
worksheet.write(0,10,r"职位",style)
worksheet.write(0,11,r"状态",style)
while True:
msg = q.get()
if not msg:
workbook.save("智联招聘python信息.xls")
break
msg_list = msg.split("|")
for index in range(len(msg_list)):
worksheet.write(count, index, label=msg_list[index])
count += 1
def cp(c_count,p_count):
q = Queue(10)
for i in range(c_count):
Thread(target=consumer, args=(q,)).start()
p_l = []
for i in range(p_count,):
path = "https://fe-api.zhaopin.com/c/i/sou?start=%d&pageSize=90&cityId=530&salary=0,0&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=python&kt=3&=0&_v=0.55208065&x-zp-page-request-id=53d546acdd2a4e369735f9cb3a372e05-1560873187327-646177&x-zp-client-id=dc9fd9ff-68eb-42ab-8bc3-796640e734b3" % (
i * 90)
p1 = Thread(target=producer, args=(q,path))
p1.start()
p_l.append(p1)
for p in p_l: p.join()
for i in range(c_count):q.put(None)
if __name__ == '__main__':
cp(1,12)
