Agent = client.Agent
class ScrapyAgent(object):
_Agent = Agent#为twisted的client.Agent类
_ProxyAgent = ProxyAgent
_TunnelingAgent = TunnelingAgent
def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
maxsize=0, warnsize=0, fail_on_dataloss=True):
self._contextFactory = contextFactory
self._connectTimeout = connectTimeout
self._bindAddress = bindAddress
self._pool = pool
self._maxsize = maxsize
self._warnsize = warnsize
self._fail_on_dataloss = fail_on_dataloss
self._txresponse = None
def _get_agent(self, request, timeout):#获得代理
bindaddress = request.meta.get('bindaddress') or self._bindAddress
proxy = request.meta.get('proxy')
if proxy:
_, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
if scheme == b'https' and not omitConnectTunnel:
proxyConf = (proxyHost, proxyPort,
request.headers.get(b'Proxy-Authorization', None))
return self._TunnelingAgent(reactor, proxyConf,
contextFactory=self._contextFactory, connectTimeout=timeout,
bindAddress=bindaddress, pool=self._pool)
else:
endpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort,
timeout=timeout, bindAddress=bindaddress)
return self._ProxyAgent(endpoint)
return self._Agent(reactor, contextFactory=self._contextFactory,
connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
def download_request(self, request):
timeout = request.meta.get('download_timeout') or self._connectTimeout
agent = self._get_agent(request, timeout)
# request details
url = urldefrag(request.url)[0]
method = to_bytes(request.method)
headers = TxHeaders(request.headers)
if isinstance(agent, self._TunnelingAgent):
headers.removeHeader(b'Proxy-Authorization')
if request.body:
bodyproducer = _RequestBodyProducer(request.body)
elif method == b'POST':
# Setting Content-Length: 0 even for POST requests is not a
# MUST per HTTP RFCs, but it's common behavior, and some
# servers require this, otherwise returning HTTP 411 Length required
#
# RFC 7230#section-3.3.2:
# "a Content-Length header field is normally sent in a POST
# request even when the value is 0 (indicating an empty payload body)."
#
# Twisted < 17 will not add "Content-Length: 0" by itself;
# Twisted >= 17 fixes this;
# Using a producer with an empty-string sends `0` as Content-Length
# for all versions of Twisted.
bodyproducer = _RequestBodyProducer(b'')
else:
bodyproducer = None
start_time = time()
d = agent.request(#调用代理的请求
method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
# set download latency
d.addCallback(self._cb_latency, request, start_time)
# response body is ready to be consumed
d.addCallback(self._cb_bodyready, request)
d.addCallback(self._cb_bodydone, request, url)
# check download timeout
self._timeout_cl = reactor.callLater(timeout, d.cancel)
d.addBoth(self._cb_timeout, req
class Agent(_AgentBase):
def __init__(self, reactor,
contextFactory=BrowserLikePolicyForHTTPS(),
connectTimeout=None, bindAddress=None,
pool=None):
if not IPolicyForHTTPS.providedBy(contextFactory):
warnings.warn(
repr(contextFactory) +
" was passed as the HTTPS policy for an Agent, but it does "
"not provide IPolicyForHTTPS. Since Twisted 14.0, you must "
"pass a provider of IPolicyForHTTPS.",
stacklevel=2, category=DeprecationWarning
)
contextFactory = _DeprecatedToCurrentPolicyForHTTPS(contextFactory)
endpointFactory = _StandardEndpointFactory(
reactor, contextFactory, connectTimeout, bindAddress)
self._init(reactor, endpointFactory, pool)
@classmethod
def usingEndpointFactory(cls, reactor, endpointFactory, pool=None):
"""
Create a new L{Agent} that will use the endpoint factory to figure
out how to connect to the server.
"""
agent = cls.__new__(cls)
agent._init(reactor, endpointFactory, pool)
return agent
def _init(self, reactor, endpointFactory, pool):
_AgentBase.__init__(self, reactor, pool)
self._endpointFactory = endpointFactory
def _getEndpoint(self, uri):
return self._endpointFactory.endpointForURI(uri)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a request to the server indicated by the given C{uri}.
An existing connection from the connection pool may be used or a new
one may be created.
I{HTTP} and I{HTTPS} schemes are supported in C{uri}.
@see: L{twisted.web.iweb.IAgent.request}
"""
parsedURI = URI.fromBytes(uri)
try:
endpoint = self._getEndpoint(parsedURI)
except SchemeNotSupported:
return defer.fail(Failure())
key = (parsedURI.scheme, parsedURI.host, parsedURI.port)#key值的计算
return self._requestWithEndpoint(key, endpoint, method, parsedURI,
headers, bodyProducer,
parsedURI.originForm)
#从class HTTPConnectionPool(object)中取得一个连接
def getConnection(self, key, endpoint): # Try to get cached version:
connections = self._connections.get(key)
while connections:
connection = connections.pop(0)
# Cancel timeout:
self._timeouts[connection].cancel()
del self._timeouts[connection]
if connection.state == "QUIESCENT":#该连接为空闲状态
if self.retryAutomatically:
newConnection = lambda: self._newConnection(key, endpoint)
connection = _RetryingHTTP11ClientProtocol(
connection, newConnection)
return defer.succeed(connection)#成功 return self._newConnection(key, endpoint)