Major rewrite

app.py

@@ -1,83 +1,157 @@
 import time
-
+import os
 from selenium import webdriver
 from selenium.webdriver.chrome.webdriver import WebDriver
-
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+import re
+import urllib3
+from urllib.request import urlretrieve
 from openpyxl import Workbook
 import pandas as pd
-
+import urllib3
 from env import URL, DriverLocation
+from datetime import datetime
+
+today = datetime.today().strftime('%Y-%m-%d')
+
+# Rotate User Agent would be helpful
 
 def get_data(driver):
     """
     this function get main text, score, name
     """
     print('get data...')
-    more_elemets = driver.find_elements_by_class_name('w8nwRe kyuRq')
+    # Click on more botton on each text reviews
+    more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq')
     for list_more_element in more_elemets:
         list_more_element.click()
-    
+    # Find Pictures that have the expansion indicator to see the rest of the pictures under them and click it to expose them all
-    elements = driver.find_elements_by_class_name(
+    more_pics = driver.find_elements(By.CLASS_NAME, 'Tya61d') 
-        'jftiEf')
+    for list_more_pics in more_pics:
+        if 'showMorePhotos' in  list_more_pics.get_attribute("jsaction") :
+            print('Found extra pics')
+            list_more_pics.click()
+    elements = driver.find_elements(By.CLASS_NAME, 'jftiEf')
     lst_data = []
     for data in elements:
-        name = data.find_element_by_xpath(
+        name = data.find_element(By.CSS_SELECTOR, 'div.d4r55.YJxk2d').text 
-            './/a/div[@class="d4r55"]/span').text
+        print ('Name of location: ',name)
-        text = data.find_element_by_xpath(
+        try: text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text 
-            './/div[@class="MyEned"]/span[2]').text
+        except: text = '' 
-        score = data.find_element_by_xpath(
+        score = data.find_element(By.CSS_SELECTOR, 'span.kvMYJc').get_attribute("aria-label")  #find_element(By.CSS_SELECTOR,'aria-label').text #)  ##QA0Szd > div > div > div.w6VYqd > div:nth-child(2) > div > div.e07Vkf.kA9KIf > div > div > div.m6QErb.DxyBCb.kA9KIf.dS8AEf > div.m6QErb > div:nth-child(3) > div:nth-child(2) > div > div:nth-child(4) > div.DU9Pgb > span.kvMYJc
-            './/span[@class="kvMYJc"]').get_attribute("aria-label")
+        more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d') 
-
+        pics= [] 
-        lst_data.append([name + " from GoogleMaps", text, score[1]])
+        pics2 = []
-
+        # check to see if folder for pictures and videos already exists, if not, create it
+        cleanname = re.sub( r'[^a-zA-Z0-9]','', name)
+        if not os.path.exists('./Output/Pics/'+cleanname):
+            os.makedirs('./Output/Pics/'+cleanname)
+        # Walk through all the pictures and videos for a given review
+        for lmpics in more_specific_pics:  
+            # Grab URL from style definiton (long multivalue string), and remove the -p-k so that it is full size 
+            urlmedia = re.sub('=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]", lmpics.get_attribute("style")))[0])
+            print ('URL : ',urlmedia)
+            pics.append(urlmedia)
+#            time.sleep(2)
+#            photoindex = str(lmpics.get_attribute("data-photo-index"))
+            # Grab the name of the file and remove all spaces and special charecters to name the folder
+            filename = re.sub( r'[^a-zA-Z0-9]','', str(lmpics.get_attribute("aria-label")))
+#            filename = re.sub( r'[^a-zA-Z0-9]','', filename) 
+            # Check to see if it has a sub div, which represents the label with the video length displayed, this will be done 
+            # because videos are represented by pictures in the main dialogue, so we need to click through and grab the video URL
+            if (lmpics.find_elements(By.CSS_SELECTOR,'div.fontLabelMedium.e5A3N')) :
+                ext='.mp4'
+                lmpics.click()
+                time.sleep(2)
+                # After we click the right side is rendered in an inframe, Store iframe web element
+                iframe = driver.find_element(By.TAG_NAME, "iframe")
+                # switch to selected iframe
+                driver.switch_to.frame(iframe)
+                # Now find button and click on button
+                video_elements = driver.find_elements(By.XPATH ,'//video') #.get_attribute('src')
+                urlmedia = str((video_elements[0]).get_attribute("src"))
+                # return back away from iframe
+                driver.switch_to.default_content()
+            else:
+                # The default path if it is not a video link
+                ext='.jpg' 
+            # Add the correct extension to the file name
+            filename = filename+ext
+            # Test to see if file already exists, and if it does not grab the media and store it in location folder
+            if not os.path.isfile('./Output/Pics/'+cleanname+'/'+filename):
+                urlretrieve(urlmedia, './Output/Pics/'+cleanname+'/'+filename)
+            # Store the local path to be used in the excel document
+            picsLocalpath = "./Output/Pics/"+cleanname+"/"+filename
+            pics2.append(picsLocalpath)
+        lst_data.append([name , text, score,pics,pics2,"GoogleMaps",today])
     return lst_data
 
-
+# Grab a count of how far we need to scroll
 def counter():
-    result = driver.find_element_by_class_name('jANrlb').find_element_by_class_name('fontBodySmall').text
+    result = driver.find_element(By.CLASS_NAME,'Qha3nb').text
     result = result.replace(',', '')
     result = result.split(' ')
     result = result[0].split('\n')
     return int(int(result[0])/10)+1
 
-
+# Do the scrolling
 def scrolling(counter):
     print('scrolling...')
-    scrollable_div = driver.find_element_by_xpath(
+    time.sleep(3)
-        '//div[@class="lXJj5c Hk4XGb"]')
+    scrollable_div = driver.find_element(By.XPATH,
+        '//*[@id="QA0Szd"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[5]/div[2]')
+#        '//*[@id="QA0Szd"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[2]/div[10]/div')
     for _i in range(counter):
-        scrolling = driver.execute_script(
+        try:
-            'document.getElementsByClassName("dS8AEf")[0].scrollTop = document.getElementsByClassName("dS8AEf")[0].scrollHeight',
+            scrolling = driver.execute_script(
-            scrollable_div
+                'document.getElementsByClassName("dS8AEf")[0].scrollTop = document.getElementsByClassName("dS8AEf")[0].scrollHeight',
-        )
+                scrollable_div
-        time.sleep(3)
+            )
+            time.sleep(3)
 
+        except Exception as e:
+            print(f"Error while scrolling: {e}")
+            break
 
 def write_to_xlsx(data):
     print('write to excel...')
-    cols = ["name", "comment", 'rating']
+    cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date']
     df = pd.DataFrame(data, columns=cols)
-    df.to_excel('out.xlsx')
+    df.to_excel('./Output/reviews.xlsx')
 
 
 if __name__ == "__main__":
-
     print('starting...')
     options = webdriver.ChromeOptions()
-    options.add_argument("--headless")  # show browser or not
+    options.add_argument("--ignore-certificate-error")
+    options.add_argument("--ignore-ssl-errors")
+#    options.add_argument("--headless")  # show browser or not ||| HEAD =>  43.03 ||| No Head => 39 seg
     options.add_argument("--lang=en-US")
-    options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'})
+    # caps = webdriver.DesiredCapabilities.CHROME.copy()
-    DriverPath = DriverLocation
+    # caps['acceptInsecureCerts'] = True
-    driver = webdriver.Chrome(DriverPath, options=options)
+    # caps['acceptSslCerts'] = True
-
+    # driver = webdriver.Chrome(desired_capabilities=caps)
+# options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'})
+    # DriverPath = Service(DriverLocation)
+    # Adding argument to disable the AutomationControlled flag 
+    options.add_argument("--disable-blink-features=AutomationControlled") 
+    # Exclude the collection of enable-automation switches 
+    options.add_experimental_option("excludeSwitches", ["enable-automation"]) 
+    # Turn-off userAutomationExtension 
+    options.add_experimental_option("useAutomationExtension", False) 
+    # Setting the driver path and requesting a page 
+    driver = webdriver.Chrome(options=options) # Firefox(options=options)  
+    # Changing the property of the navigator value for webdriver to undefined 
+    driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})") 
     driver.get(URL)
     time.sleep(5)
 
-    counter = counter()
+#    counter = counter()
-    scrolling(counter)
+#    scrolling(counter)
 
     data = get_data(driver)
     driver.close()
 
     write_to_xlsx(data)
-    print('Done!')
+    print('Done!')