mirror of
https://github.com/timberjoegithub/GoogleScrape.git
synced 2026-07-22 00:19:48 +00:00
80 lines
3.3 KiB
Python
80 lines
3.3 KiB
Python
import scrapy
|
|
from scrapy.http.request import Request
|
|
import re
|
|
|
|
|
|
class GoogleSpider(scrapy.Spider):
|
|
name = 'google'
|
|
|
|
HEADERS = {
|
|
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
|
|
'referer': None
|
|
}
|
|
|
|
def start_requests(self):
|
|
urls = [
|
|
"https://www.google.com/maps/contrib/109274792898041753066/reviews"
|
|
# "https://www.google.com/search?q=concord+honda&oq=conc&aqs=chrome.0.69i59j69i60l3j69i57j0l2.788j0j7&sourceid=chrome&ie=UTF-8#lrd=0x808566dd35d0d82b:0x3bca18b3f8745548,1,,,"
|
|
]
|
|
print ('URLS: ',urls)
|
|
|
|
for url in urls:
|
|
# async_id = url.split("lrd=")[1].split(",")[0]
|
|
# ajax_url = "https://www.google.com/async/reviewDialog?async=feature_id:" + str(
|
|
# async_id) + ",start_index:0,_fmt:pc,sort_by:newestFirst"
|
|
|
|
ajax_url = url
|
|
|
|
yield Request(url=ajax_url, headers=self.HEADERS, callback=self.get_total_iteration)
|
|
|
|
print ('Ajax URL: ',ajax_url)
|
|
|
|
def get_total_iteration(self, response):
|
|
# total_reviews_text = response.css('.z5jxId::text').extract_first()
|
|
# print (total_reviews_text)
|
|
# total_reviews = int(re.sub(r'[^0-9]', '', total_reviews_text))
|
|
|
|
# temp = total_reviews / 10 # since
|
|
# new_num = int(temp)
|
|
# if temp > new_num:
|
|
# new_num += 1
|
|
# iteration_number = new_num
|
|
|
|
# j = 0
|
|
# print(iteration_number)
|
|
# if total_reviews > 10:
|
|
# for _ in range(0, iteration_number + 1):
|
|
# yield Request(url=response.request.url.replace('start_index:0', f'start_index:{j}'),
|
|
# headers=self.HEADERS, callback=self.parse_reviews, dont_filter=True)
|
|
# j += 10
|
|
# else:
|
|
# yield Request(url=response.request.url, headers=self.HEADERS, callback=self.parse_reviews, dont_filter=True)
|
|
yield Request(url=response.request.url, headers=self.HEADERS, callback=self.parse_reviews, dont_filter=True)
|
|
|
|
def parse_reviews(self, response):
|
|
all_reviews = response.xpath('//*[@id="reviewSort"]/div/div[2]/div')
|
|
|
|
for review in all_reviews:
|
|
print('Review:',review,"\n\n")
|
|
# reviewer = review.xpath('.//div[@class="TSUbDb"]/a/text()').extract_first()
|
|
reviewer = review.css('div.TSUbDb a::text').extract_first()
|
|
description = review.xpath('.//span[@class="review-full-text"]').extract_first()
|
|
if description is None:
|
|
description = review.css('.Jtu6Td span::text').extract_first()
|
|
if description is None:
|
|
description = ''
|
|
|
|
review_rating = float(
|
|
review.xpath('.//span[@class="Fam1ne EBe2gf"]/@aria-label').extract_first().split(" ")[1])
|
|
review_date = review.xpath('.//span[@class="dehysf lTi8oc"]/text()').extract_first()
|
|
print(reviewer)
|
|
print(description)
|
|
print(review_rating)
|
|
print(review_date)
|
|
yield {
|
|
"reviewer": reviewer,
|
|
"description": description,
|
|
"rating": review_rating,
|
|
"review_date": review_date
|
|
}
|