added new files

2026-07-22 00:19:48 +00:00 · 2024-02-10 15:34:13 -05:00
parent 1e1fd423f5
commit 9ef1133263
2 changed files with 107 additions and 0 deletions
@@ -0,0 +1,79 @@
+import scrapy
+from scrapy.http.request import Request
+import re
+
+
+class GoogleSpider(scrapy.Spider):
+    name = 'google'
+
+    HEADERS = {
+        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
+        'referer': None
+    }
+
+    def start_requests(self):
+        urls = [
+            "https://www.google.com/maps/contrib/109274792898041753066/reviews"
+        #    "https://www.google.com/search?q=concord+honda&oq=conc&aqs=chrome.0.69i59j69i60l3j69i57j0l2.788j0j7&sourceid=chrome&ie=UTF-8#lrd=0x808566dd35d0d82b:0x3bca18b3f8745548,1,,,"
+        ]
+        print ('URLS: ',urls)
+
+        for url in urls:
+#            async_id = url.split("lrd=")[1].split(",")[0]
+#            ajax_url = "https://www.google.com/async/reviewDialog?async=feature_id:" + str(
+#                async_id) + ",start_index:0,_fmt:pc,sort_by:newestFirst"
+
+            ajax_url = url
+
+            yield Request(url=ajax_url, headers=self.HEADERS, callback=self.get_total_iteration)        
+
+            print ('Ajax URL: ',ajax_url)
+
+    def get_total_iteration(self, response):
+        # total_reviews_text = response.css('.z5jxId::text').extract_first()
+        # print (total_reviews_text)
+        # total_reviews = int(re.sub(r'[^0-9]', '', total_reviews_text))
+
+        # temp = total_reviews / 10  # since
+        # new_num = int(temp)
+        # if temp > new_num:
+        #     new_num += 1
+        # iteration_number = new_num
+
+        # j = 0
+        # print(iteration_number)
+        # if total_reviews > 10:
+        #     for _ in range(0, iteration_number + 1):
+        #         yield Request(url=response.request.url.replace('start_index:0', f'start_index:{j}'),
+        #                       headers=self.HEADERS, callback=self.parse_reviews, dont_filter=True)
+        #         j += 10
+        # else:
+        #     yield Request(url=response.request.url, headers=self.HEADERS, callback=self.parse_reviews, dont_filter=True)
+        yield Request(url=response.request.url, headers=self.HEADERS, callback=self.parse_reviews, dont_filter=True)
+
+    def parse_reviews(self, response):
+        all_reviews = response.xpath('//*[@id="reviewSort"]/div/div[2]/div')
+
+        for review in all_reviews:
+            print('Review:',review,"\n\n")
+            # reviewer = review.xpath('.//div[@class="TSUbDb"]/a/text()').extract_first()
+            reviewer = review.css('div.TSUbDb a::text').extract_first()
+            description = review.xpath('.//span[@class="review-full-text"]').extract_first()
+            if description is None:
+                description = review.css('.Jtu6Td span::text').extract_first()
+                if description is None:
+                    description = ''
+
+            review_rating = float(
+                review.xpath('.//span[@class="Fam1ne EBe2gf"]/@aria-label').extract_first().split(" ")[1])
+            review_date = review.xpath('.//span[@class="dehysf lTi8oc"]/text()').extract_first()
+            print(reviewer)
+            print(description)
+            print(review_rating)
+            print(review_date)
+            yield {
+                "reviewer": reviewer,
+                "description": description,
+                "rating": review_rating,
+                "review_date": review_date
+            }