From 9ef11332637dc30f91916543b56865580354626a Mon Sep 17 00:00:00 2001
From: Joe Steele <jsteele@mathworks.com>
Date: Sat, 10 Feb 2024 15:34:13 -0500
Subject: [PATCH] added new files

---
 google.py  | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 google2.py | 28 +++++++++++++++++++
 2 files changed, 107 insertions(+)
 create mode 100644 google.py
 create mode 100644 google2.py

diff --git a/google.py b/google.py
new file mode 100644
index 0000000..d700da6
--- /dev/null
+++ b/google.py
@@ -0,0 +1,79 @@
+import scrapy
+from scrapy.http.request import Request
+import re
+
+
+class GoogleSpider(scrapy.Spider):
+    name = 'google'
+
+    HEADERS = {
+        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
+        'referer': None
+    }
+
+    def start_requests(self):
+        urls = [
+            "https://www.google.com/maps/contrib/109274792898041753066/reviews"
+        #    "https://www.google.com/search?q=concord+honda&oq=conc&aqs=chrome.0.69i59j69i60l3j69i57j0l2.788j0j7&sourceid=chrome&ie=UTF-8#lrd=0x808566dd35d0d82b:0x3bca18b3f8745548,1,,,"
+        ]
+        print ('URLS: ',urls)
+
+        for url in urls:
+#            async_id = url.split("lrd=")[1].split(",")[0]
+#            ajax_url = "https://www.google.com/async/reviewDialog?async=feature_id:" + str(
+#                async_id) + ",start_index:0,_fmt:pc,sort_by:newestFirst"
+
+            ajax_url = url
+
+            yield Request(url=ajax_url, headers=self.HEADERS, callback=self.get_total_iteration)        
+
+            print ('Ajax URL: ',ajax_url)
+
+    def get_total_iteration(self, response):
+        # total_reviews_text = response.css('.z5jxId::text').extract_first()
+        # print (total_reviews_text)
+        # total_reviews = int(re.sub(r'[^0-9]', '', total_reviews_text))
+
+        # temp = total_reviews / 10  # since
+        # new_num = int(temp)
+        # if temp > new_num:
+        #     new_num += 1
+        # iteration_number = new_num
+
+        # j = 0
+        # print(iteration_number)
+        # if total_reviews > 10:
+        #     for _ in range(0, iteration_number + 1):
+        #         yield Request(url=response.request.url.replace('start_index:0', f'start_index:{j}'),
+        #                       headers=self.HEADERS, callback=self.parse_reviews, dont_filter=True)
+        #         j += 10
+        # else:
+        #     yield Request(url=response.request.url, headers=self.HEADERS, callback=self.parse_reviews, dont_filter=True)
+        yield Request(url=response.request.url, headers=self.HEADERS, callback=self.parse_reviews, dont_filter=True)
+
+    def parse_reviews(self, response):
+        all_reviews = response.xpath('//*[@id="reviewSort"]/div/div[2]/div')
+
+        for review in all_reviews:
+            print('Review:',review,"\n\n")
+            # reviewer = review.xpath('.//div[@class="TSUbDb"]/a/text()').extract_first()
+            reviewer = review.css('div.TSUbDb a::text').extract_first()
+            description = review.xpath('.//span[@class="review-full-text"]').extract_first()
+            if description is None:
+                description = review.css('.Jtu6Td span::text').extract_first()
+                if description is None:
+                    description = ''
+
+            review_rating = float(
+                review.xpath('.//span[@class="Fam1ne EBe2gf"]/@aria-label').extract_first().split(" ")[1])
+            review_date = review.xpath('.//span[@class="dehysf lTi8oc"]/text()').extract_first()
+            print(reviewer)
+            print(description)
+            print(review_rating)
+            print(review_date)
+            yield {
+                "reviewer": reviewer,
+                "description": description,
+                "rating": review_rating,
+                "review_date": review_date
+            }
diff --git a/google2.py b/google2.py
new file mode 100644
index 0000000..97be592
--- /dev/null
+++ b/google2.py
@@ -0,0 +1,28 @@
+
+# Import the beautifulsoup  
+# and request libraries of python. 
+import requests 
+import bs4 
+  
+# Make two strings with default google search URL 
+# 'https://google.com/search?q=' and 
+# our customized search keyword. 
+# Concatenate them 
+text= "joe"
+url = "https://www.google.com/maps/contrib/109274792898041753066/reviews"
+  
+# Fetch the URL data using requests.get(url), 
+# store it in a variable, request_result. 
+request_result=requests.get( url ) 
+  
+# Creating soup from the fetched request 
+soup = bs4.BeautifulSoup(request_result.text, 
+                         "html.parser") 
+#print(soup) 
+heading_object=soup.find_all( 'h3' ) 
+  
+# Iterate through the object  
+# and print it as a string. 
+for info in heading_object: 
+    print(info.getText()) 
+    print("------") 
\ No newline at end of file