mirror of
https://github.com/timberjoegithub/GoogleScrape.git
synced 2026-07-22 00:19:48 +00:00
timberjoegithub
13ae301667
- Remove dead imports/duplication from Google2xls.py and social.py - Centralize shared logic in review_workflow_utils.py - Add focused posting-flow tests in tests/test_social.py (all pass) - Update .gitignore for secrets and local artifacts - Trim requirements.txt to direct deps - Improve CI: compile all scripts, run focused tests - Prep repo for clean commit (no secrets, no local artifacts)
152 lines
6.8 KiB
Python
152 lines
6.8 KiB
Python
import time
|
|
import os
|
|
from selenium import webdriver
|
|
from selenium.webdriver.chrome.webdriver import WebDriver
|
|
from selenium.webdriver.chrome.service import Service
|
|
from selenium.webdriver.common.by import By
|
|
import re
|
|
import urllib3
|
|
from urllib.request import urlretrieve
|
|
from openpyxl import Workbook
|
|
import pandas as pd
|
|
from env import URL, DriverLocation
|
|
from datetime import datetime
|
|
|
|
from review_workflow_utils import (
|
|
build_google_chrome_options,
|
|
build_review_media_path,
|
|
count_google_review_pages,
|
|
normalize_google_media_url,
|
|
sanitize_review_name,
|
|
scroll_google_reviews,
|
|
set_stealth_driver,
|
|
)
|
|
|
|
today = datetime.today().strftime('%Y-%m-%d')
|
|
|
|
# Rotate User Agent would be helpful
|
|
|
|
def get_data(driver):
|
|
"""
|
|
this function get main text, score, name
|
|
"""
|
|
print('get data...')
|
|
# Click on more botton on each text reviews
|
|
more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq')
|
|
for list_more_element in more_elemets:
|
|
list_more_element.click()
|
|
# Find Pictures that have the expansion indicator to see the rest of the pictures under them and click it to expose them all
|
|
more_pics = driver.find_elements(By.CLASS_NAME, 'Tya61d')
|
|
for list_more_pics in more_pics:
|
|
if 'showMorePhotos' in list_more_pics.get_attribute("jsaction") :
|
|
print('Found extra pics')
|
|
list_more_pics.click()
|
|
elements = driver.find_elements(By.CLASS_NAME, 'jftiEf')
|
|
lst_data = []
|
|
for data in elements:
|
|
name = data.find_element(By.CSS_SELECTOR, 'div.d4r55.YJxk2d').text
|
|
try: address = data.find_element(By.CSS_SELECTOR, 'div.RfnDt.xJVozb').text
|
|
except: address = 'Unknonwn'
|
|
print ('Name of location: ',name, ' Address:',address)
|
|
try: visitdate = data.find_element(By.CSS_SELECTOR, 'span.rsqaWe').text
|
|
except: visitdate = "Unknown"
|
|
print('Visited: ',visitdate)
|
|
try: text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text
|
|
except: text = ''
|
|
try: score = data.find_element(By.CSS_SELECTOR, 'span.kvMYJc').get_attribute("aria-label") #find_element(By.CSS_SELECTOR,'aria-label').text #) ##QA0Szd > div > div > div.w6VYqd > div:nth-child(2) > div > div.e07Vkf.kA9KIf > div > div > div.m6QErb.DxyBCb.kA9KIf.dS8AEf > div.m6QErb > div:nth-child(3) > div:nth-child(2) > div > div:nth-child(4) > div.DU9Pgb > span.kvMYJc
|
|
except: score = "Unknown"
|
|
more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d')
|
|
pics= []
|
|
pics2 = []
|
|
# check to see if folder for pictures and videos already exists, if not, create it
|
|
cleanname = sanitize_review_name(name)
|
|
if not os.path.exists('./Output/Pics/'+cleanname):
|
|
os.makedirs('./Output/Pics/'+cleanname)
|
|
# Walk through all the pictures and videos for a given review
|
|
for lmpics in more_specific_pics:
|
|
# Grab URL from style definiton (long multivalue string), and remove the -p-k so that it is full size
|
|
urlmedia = normalize_google_media_url(lmpics.get_attribute("style"))
|
|
print ('URL : ',urlmedia)
|
|
pics.append(urlmedia)
|
|
# time.sleep(2)
|
|
# photoindex = str(lmpics.get_attribute("data-photo-index"))
|
|
# Grab the name of the file and remove all spaces and special charecters to name the folder
|
|
filename = re.sub( r'[^a-zA-Z0-9]','', str(lmpics.get_attribute("aria-label")))
|
|
# filename = re.sub( r'[^a-zA-Z0-9]','', filename)
|
|
|
|
if lmpics == more_specific_pics[0]:
|
|
lmpics.click()
|
|
time.sleep(2)
|
|
#iframe = driver.find_element(By.TAG_NAME, "iframe")
|
|
tempdate = str((driver.find_element(By.CLASS_NAME,'mqX5ad')).text).rsplit("-",1)
|
|
visitdate = re.sub( r'[^a-zA-Z0-9]','',tempdate[1])
|
|
print ('Visited: ',visitdate)
|
|
# driver.switch_to.default_content()
|
|
|
|
# Check to see if it has a sub div, which represents the label with the video length displayed, this will be done
|
|
# because videos are represented by pictures in the main dialogue, so we need to click through and grab the video URL
|
|
if (lmpics.find_elements(By.CSS_SELECTOR,'div.fontLabelMedium.e5A3N')) :
|
|
ext='.mp4'
|
|
lmpics.click()
|
|
time.sleep(2)
|
|
# After we click the right side is rendered in an inframe, Store iframe web element
|
|
iframe = driver.find_element(By.TAG_NAME, "iframe")
|
|
# switch to selected iframe
|
|
driver.switch_to.frame(iframe)
|
|
# Now find button and click on button
|
|
video_elements = driver.find_elements(By.XPATH ,'//video') #.get_attribute('src')
|
|
urlmedia = str((video_elements[0]).get_attribute("src"))
|
|
# return back away from iframe
|
|
driver.switch_to.default_content()
|
|
else:
|
|
# The default path if it is not a video link
|
|
ext='.jpg'
|
|
# Add the correct extension to the file name
|
|
filename = filename+ext
|
|
# Test to see if file already exists, and if it does not grab the media and store it in location folder
|
|
picsLocalpath = build_review_media_path(name, visitdate, filename)
|
|
if not os.path.isfile(picsLocalpath):
|
|
urlretrieve(urlmedia, picsLocalpath)
|
|
# Store the local path to be used in the excel document
|
|
pics2.append(picsLocalpath)
|
|
dictPostComplete= {'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,'Instagram':0,'tiktok':0}
|
|
lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address,dictPostComplete])
|
|
return lst_data
|
|
|
|
# Grab a count of how far we need to scroll
|
|
def counter(driver):
|
|
return count_google_review_pages(driver)
|
|
|
|
# Do the scrolling
|
|
def scrolling(driver, review_count):
|
|
return scroll_google_reviews(driver, review_count, label='scrolling...')
|
|
|
|
def write_to_xlsx(data):
|
|
print('write to excel...')
|
|
|
|
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address','dictPostComplete']
|
|
df = pd.DataFrame(data, columns=cols)
|
|
df.to_excel('./Output/reviews.xlsx')
|
|
|
|
|
|
if __name__ == "__main__":
|
|
print('starting...')
|
|
options = build_google_chrome_options(
|
|
ignore_certificate_errors=True,
|
|
ignore_ssl_errors=True,
|
|
)
|
|
# Setting the driver path and requesting a page
|
|
driver = webdriver.Chrome(options=options) # Firefox(options=options)
|
|
# Changing the property of the navigator value for webdriver to undefined
|
|
set_stealth_driver(driver)
|
|
driver.get(URL)
|
|
time.sleep(5)
|
|
|
|
review_count = counter(driver)
|
|
scrolling(driver, review_count)
|
|
|
|
data = get_data(driver)
|
|
driver.close()
|
|
|
|
write_to_xlsx(data)
|
|
print('Done!') |