import time import os from selenium import webdriver from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.by import By import re import urllib3 from urllib.request import urlretrieve from openpyxl import Workbook import pandas as pd from env import URL, DriverLocation from datetime import datetime from review_workflow_utils import ( build_google_chrome_options, build_review_media_path, count_google_review_pages, normalize_google_media_url, sanitize_review_name, scroll_google_reviews, set_stealth_driver, ) today = datetime.today().strftime('%Y-%m-%d') # Rotate User Agent would be helpful def get_data(driver): """ this function get main text, score, name """ print('get data...') # Click on more botton on each text reviews more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq') for list_more_element in more_elemets: list_more_element.click() # Find Pictures that have the expansion indicator to see the rest of the pictures under them and click it to expose them all more_pics = driver.find_elements(By.CLASS_NAME, 'Tya61d') for list_more_pics in more_pics: if 'showMorePhotos' in list_more_pics.get_attribute("jsaction") : print('Found extra pics') list_more_pics.click() elements = driver.find_elements(By.CLASS_NAME, 'jftiEf') lst_data = [] for data in elements: name = data.find_element(By.CSS_SELECTOR, 'div.d4r55.YJxk2d').text try: address = data.find_element(By.CSS_SELECTOR, 'div.RfnDt.xJVozb').text except: address = 'Unknonwn' print ('Name of location: ',name, ' Address:',address) try: visitdate = data.find_element(By.CSS_SELECTOR, 'span.rsqaWe').text except: visitdate = "Unknown" print('Visited: ',visitdate) try: text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text except: text = '' try: score = data.find_element(By.CSS_SELECTOR, 'span.kvMYJc').get_attribute("aria-label") #find_element(By.CSS_SELECTOR,'aria-label').text #) ##QA0Szd > div > div > div.w6VYqd > div:nth-child(2) > div > div.e07Vkf.kA9KIf > div > div > div.m6QErb.DxyBCb.kA9KIf.dS8AEf > div.m6QErb > div:nth-child(3) > div:nth-child(2) > div > div:nth-child(4) > div.DU9Pgb > span.kvMYJc except: score = "Unknown" more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d') pics= [] pics2 = [] # check to see if folder for pictures and videos already exists, if not, create it cleanname = sanitize_review_name(name) if not os.path.exists('./Output/Pics/'+cleanname): os.makedirs('./Output/Pics/'+cleanname) # Walk through all the pictures and videos for a given review for lmpics in more_specific_pics: # Grab URL from style definiton (long multivalue string), and remove the -p-k so that it is full size urlmedia = normalize_google_media_url(lmpics.get_attribute("style")) print ('URL : ',urlmedia) pics.append(urlmedia) # time.sleep(2) # photoindex = str(lmpics.get_attribute("data-photo-index")) # Grab the name of the file and remove all spaces and special charecters to name the folder filename = re.sub( r'[^a-zA-Z0-9]','', str(lmpics.get_attribute("aria-label"))) # filename = re.sub( r'[^a-zA-Z0-9]','', filename) if lmpics == more_specific_pics[0]: lmpics.click() time.sleep(2) #iframe = driver.find_element(By.TAG_NAME, "iframe") tempdate = str((driver.find_element(By.CLASS_NAME,'mqX5ad')).text).rsplit("-",1) visitdate = re.sub( r'[^a-zA-Z0-9]','',tempdate[1]) print ('Visited: ',visitdate) # driver.switch_to.default_content() # Check to see if it has a sub div, which represents the label with the video length displayed, this will be done # because videos are represented by pictures in the main dialogue, so we need to click through and grab the video URL if (lmpics.find_elements(By.CSS_SELECTOR,'div.fontLabelMedium.e5A3N')) : ext='.mp4' lmpics.click() time.sleep(2) # After we click the right side is rendered in an inframe, Store iframe web element iframe = driver.find_element(By.TAG_NAME, "iframe") # switch to selected iframe driver.switch_to.frame(iframe) # Now find button and click on button video_elements = driver.find_elements(By.XPATH ,'//video') #.get_attribute('src') urlmedia = str((video_elements[0]).get_attribute("src")) # return back away from iframe driver.switch_to.default_content() else: # The default path if it is not a video link ext='.jpg' # Add the correct extension to the file name filename = filename+ext # Test to see if file already exists, and if it does not grab the media and store it in location folder picsLocalpath = build_review_media_path(name, visitdate, filename) if not os.path.isfile(picsLocalpath): urlretrieve(urlmedia, picsLocalpath) # Store the local path to be used in the excel document pics2.append(picsLocalpath) dictPostComplete= {'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,'Instagram':0,'tiktok':0} lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address,dictPostComplete]) return lst_data # Grab a count of how far we need to scroll def counter(driver): return count_google_review_pages(driver) # Do the scrolling def scrolling(driver, review_count): return scroll_google_reviews(driver, review_count, label='scrolling...') def write_to_xlsx(data): print('write to excel...') cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address','dictPostComplete'] df = pd.DataFrame(data, columns=cols) df.to_excel('./Output/reviews.xlsx') if __name__ == "__main__": print('starting...') options = build_google_chrome_options( ignore_certificate_errors=True, ignore_ssl_errors=True, ) # Setting the driver path and requesting a page driver = webdriver.Chrome(options=options) # Firefox(options=options) # Changing the property of the navigator value for webdriver to undefined set_stealth_driver(driver) driver.get(URL) time.sleep(5) review_count = counter(driver) scrolling(driver, review_count) data = get_data(driver) driver.close() write_to_xlsx(data) print('Done!')