已经快一个月了。python小有进展。现在研究scrapy项目。

时间:2021-10-26 23:41:24

刚刚研究学习了一个新的技能,觉得自己水平又升了一级。就是跨页面item 抓取的问题。以前一直不明白。代码如下!

item申明如下:

import scrapy


class QuotesItem(scrapy.Item):
quote
= scrapy.Field()
author
= scrapy.Field()
tags
= scrapy.Field()

author_born_date
= scrapy.Field()
author_born_location
= scrapy.Field()
author_description
= scrapy.Field()
author_full_url
= scrapy.Field()

spider.py如下

import scrapy
from quotes_2.items import QuotesItem

class QuotesSpider(scrapy.Spider):
name
= 'quotes_2_6'
start_urls
= [
'http://quotes.toscrape.com',
]
allowed_domains
= [
'toscrape.com',
]

def parse(self,response):
for quote in response.css('div.quote'):
item
= QuotesItem()
item[
'quote'] = quote.css('span.text::text').extract_first()
item[
'author'] = quote.css('small.author::text').extract_first()
item[
'tags'] = quote.css('div.tags a.tag::text').extract()
author_page
= response.css('small.author+a::attr(href)').extract_first()
item[
'author_full_url'] = response.urljoin(author_page)



yield scrapy.Request(url=item['authro_full_url'], meta={'item':item},callback=self.parse_author,dont_filter=True)

next_page
= response.css('li.next a::attr("href")').extract_first()
if next_page is not None:
next_full_url
= response.urljoin(next_page)
yield scrapy.Request(next_full_url, callback=self.parse)



def parse_author(self,response):
item
= response.meta['item']
item[
'author_born_date'] = response.css('.author-born-date::text').extract_first()
item[
'author_born_location'] = response.css('.author-born-location::text').extract_first()
item[
'author_description'] = response.css('.author-born-location::text').extract_first()
yield item
  """通过meta参数,把item这个字典,赋值给meta中的'item'键(记住meta本身也是一个字典)。
Scrapy.Request请求url后生成一个"Request对象",这个meta字典(含有键值'key','key'的值也是一个字典,即item)
会被“放”在"Request对象"里一起发送给parse2()函数 """
item = response.meta['item']
# """这个response已含有上述meta字典,此句将这个字典赋值给item,
dont_filter=True  将去重关闭。