I am trying to download Amazon reviews using the ASIN number. However, I am only able to download the first page and not all the pages.
The CSS selector for "Next page" is not working. Thank you.
I am facing difficulties with the following code:
import scrapy from urllib.parse import urljoin
class AmazonReviewsSpider(scrapy.Spider): name = "amazon_reviews"
custom_settings = {
'FEEDS': { 'data/%(name)s_%(time)s.csv': { 'format': 'csv',}}
}
def start_requests(self):
asin_list = ['B08GKK7NMH']
for asin in asin_list:
amazon_reviews_url = f'https://www.amazon.com/product-reviews/{asin}/'
yield scrapy.Request(url=amazon_reviews_url, callback=self.parse_reviews, meta={'asin': asin, 'retry_count': 0})
def parse_reviews(self, response):
asin = response.meta['asin']
retry_count = response.meta['retry_count']
next_page_relative_url = response.css(".a-pagination .a-last>a::attr(href)::after").get()
if next_page_relative_url is not None:
retry_count = 0
next_page = urljoin('https://www.amazon.com/', next_page_relative_url)
yield scrapy.Request(url=next_page, callback=self.parse_reviews, meta={'asin': asin, 'retry_count': retry_count})
## Adding this retry_count here so we retry any amazon js rendered review pages
elif retry_count < 3:
retry_count = retry_count+1
yield scrapy.Request(url=response.url, callback=self.parse_reviews, dont_filter=True, meta={'asin': asin, 'retry_count': retry_count})
## Parse Product Reviews
review_elements = response.css("#cm_cr-review_list div.review")
for review_element in review_elements:
yield {
"asin": asin,
"text": "".join(review_element.css("span[data-hook=review-body] ::text").getall()).strip(),
"title": review_element.css("*[data-hook=review-title]>span::text").get(),
"location_and_date": review_element.css("span[data-hook=review-date] ::text").get(),
"verified": bool(review_element.css("span[data-hook=avp-badge] ::text").get()),
"rating": review_element.css("[data-hook=review-star-rating] ::text").re(r"(d+.d) out")[0],
}
2
Answers
https://github.com/ShivNandan-28/Amazon-Selenium-Scraper All infos there. It is no more working via scrapy
it seems like you have to get href attribute by
.attrib
method.references