import urllib.request
import re
import time
import random def getResponse(url):
req = urllib.request.Request(url)
req.add_header("User-Agent","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36") resp = urllib.request.urlopen(req)
return resp.read() def getHtml(url,charSet = "utf-8"):
return getResponse(url).decode(charSet) def createOpenner(ipList):
m_proxy = urllib.request.ProxyHandler({"http":random.choice(ipList)})
openner = urllib.request.build_opener(m_proxy)
urllib.request.install_opener(openner) #
def getProxyList(url,iPage = 10):
ipList = []
for i in range(1,iPage+1):
html_str = getHtml(url+str(i))
ip = re.findall("IP\">((?:\d{1,3}\.){3}(?:\d{1,3}))(?:[\s\S]{0,50})\"PORT\">(\d{2,4})", html_str)
for addr in ip:
ipList.append(addr[0]+":"+addr[1])
time.sleep(2) return ipList ipList = getProxyList("http://www.kuaidaili.com/free/outha/",1)#爬取1页 print(ipList)
快代理："IP\">((?:\d{1,3}\.){3}(?:\d{1,3}))(?:[\s\S]*?)\"PORT\">(\d{2,4})" #下划线处原来是[\s\S]*，不带问号，后果是默认的贪婪模式。
只能取到一个地址，加上问号开启非贪婪模式。
西刺："((?:\d{1,3}\.){3}(?:\d{1,3}))(?:[\s\S]*?)(\d{2,4})"
返回ipList列表：['46.101.3.126:8118', '177.207.234.14:80', '113.255.49.49:80', '52.59.18.222:80', '36.81.0.138:8080', '54.165.24.194:80', '115.252.35.104:8080', '136.169.58.21:8080', '51.254.106.65:80', '178.238.213.246:8080', '49.205.212.243:8080', '137.135.166.225:8131', '168.63.24.174:8138', '179.243.46.131:8080', '186.90.160.245:8080']