mirror of
https://github.com/timberjoegithub/GoogleScrape.git
synced 2026-07-22 00:19:48 +00:00
Refactor: repo hygiene, shared utils, focused tests, and CI improvements
- Remove dead imports/duplication from Google2xls.py and social.py - Centralize shared logic in review_workflow_utils.py - Add focused posting-flow tests in tests/test_social.py (all pass) - Update .gitignore for secrets and local artifacts - Trim requirements.txt to direct deps - Improve CI: compile all scripts, run focused tests - Prep repo for clean commit (no secrets, no local artifacts)
This commit is contained in:
timberjoegithub
+152
@@ -0,0 +1,152 @@
|
||||
import time
|
||||
import os
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.webdriver import WebDriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
from selenium.webdriver.common.by import By
|
||||
import re
|
||||
import urllib3
|
||||
from urllib.request import urlretrieve
|
||||
from openpyxl import Workbook
|
||||
import pandas as pd
|
||||
from env import URL, DriverLocation
|
||||
from datetime import datetime
|
||||
|
||||
from review_workflow_utils import (
|
||||
build_google_chrome_options,
|
||||
build_review_media_path,
|
||||
count_google_review_pages,
|
||||
normalize_google_media_url,
|
||||
sanitize_review_name,
|
||||
scroll_google_reviews,
|
||||
set_stealth_driver,
|
||||
)
|
||||
|
||||
today = datetime.today().strftime('%Y-%m-%d')
|
||||
|
||||
# Rotate User Agent would be helpful
|
||||
|
||||
def get_data(driver):
|
||||
"""
|
||||
this function get main text, score, name
|
||||
"""
|
||||
print('get data...')
|
||||
# Click on more botton on each text reviews
|
||||
more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq')
|
||||
for list_more_element in more_elemets:
|
||||
list_more_element.click()
|
||||
# Find Pictures that have the expansion indicator to see the rest of the pictures under them and click it to expose them all
|
||||
more_pics = driver.find_elements(By.CLASS_NAME, 'Tya61d')
|
||||
for list_more_pics in more_pics:
|
||||
if 'showMorePhotos' in list_more_pics.get_attribute("jsaction") :
|
||||
print('Found extra pics')
|
||||
list_more_pics.click()
|
||||
elements = driver.find_elements(By.CLASS_NAME, 'jftiEf')
|
||||
lst_data = []
|
||||
for data in elements:
|
||||
name = data.find_element(By.CSS_SELECTOR, 'div.d4r55.YJxk2d').text
|
||||
try: address = data.find_element(By.CSS_SELECTOR, 'div.RfnDt.xJVozb').text
|
||||
except: address = 'Unknonwn'
|
||||
print ('Name of location: ',name, ' Address:',address)
|
||||
try: visitdate = data.find_element(By.CSS_SELECTOR, 'span.rsqaWe').text
|
||||
except: visitdate = "Unknown"
|
||||
print('Visited: ',visitdate)
|
||||
try: text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text
|
||||
except: text = ''
|
||||
try: score = data.find_element(By.CSS_SELECTOR, 'span.kvMYJc').get_attribute("aria-label") #find_element(By.CSS_SELECTOR,'aria-label').text #) ##QA0Szd > div > div > div.w6VYqd > div:nth-child(2) > div > div.e07Vkf.kA9KIf > div > div > div.m6QErb.DxyBCb.kA9KIf.dS8AEf > div.m6QErb > div:nth-child(3) > div:nth-child(2) > div > div:nth-child(4) > div.DU9Pgb > span.kvMYJc
|
||||
except: score = "Unknown"
|
||||
more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d')
|
||||
pics= []
|
||||
pics2 = []
|
||||
# check to see if folder for pictures and videos already exists, if not, create it
|
||||
cleanname = sanitize_review_name(name)
|
||||
if not os.path.exists('./Output/Pics/'+cleanname):
|
||||
os.makedirs('./Output/Pics/'+cleanname)
|
||||
# Walk through all the pictures and videos for a given review
|
||||
for lmpics in more_specific_pics:
|
||||
# Grab URL from style definiton (long multivalue string), and remove the -p-k so that it is full size
|
||||
urlmedia = normalize_google_media_url(lmpics.get_attribute("style"))
|
||||
print ('URL : ',urlmedia)
|
||||
pics.append(urlmedia)
|
||||
# time.sleep(2)
|
||||
# photoindex = str(lmpics.get_attribute("data-photo-index"))
|
||||
# Grab the name of the file and remove all spaces and special charecters to name the folder
|
||||
filename = re.sub( r'[^a-zA-Z0-9]','', str(lmpics.get_attribute("aria-label")))
|
||||
# filename = re.sub( r'[^a-zA-Z0-9]','', filename)
|
||||
|
||||
if lmpics == more_specific_pics[0]:
|
||||
lmpics.click()
|
||||
time.sleep(2)
|
||||
#iframe = driver.find_element(By.TAG_NAME, "iframe")
|
||||
tempdate = str((driver.find_element(By.CLASS_NAME,'mqX5ad')).text).rsplit("-",1)
|
||||
visitdate = re.sub( r'[^a-zA-Z0-9]','',tempdate[1])
|
||||
print ('Visited: ',visitdate)
|
||||
# driver.switch_to.default_content()
|
||||
|
||||
# Check to see if it has a sub div, which represents the label with the video length displayed, this will be done
|
||||
# because videos are represented by pictures in the main dialogue, so we need to click through and grab the video URL
|
||||
if (lmpics.find_elements(By.CSS_SELECTOR,'div.fontLabelMedium.e5A3N')) :
|
||||
ext='.mp4'
|
||||
lmpics.click()
|
||||
time.sleep(2)
|
||||
# After we click the right side is rendered in an inframe, Store iframe web element
|
||||
iframe = driver.find_element(By.TAG_NAME, "iframe")
|
||||
# switch to selected iframe
|
||||
driver.switch_to.frame(iframe)
|
||||
# Now find button and click on button
|
||||
video_elements = driver.find_elements(By.XPATH ,'//video') #.get_attribute('src')
|
||||
urlmedia = str((video_elements[0]).get_attribute("src"))
|
||||
# return back away from iframe
|
||||
driver.switch_to.default_content()
|
||||
else:
|
||||
# The default path if it is not a video link
|
||||
ext='.jpg'
|
||||
# Add the correct extension to the file name
|
||||
filename = filename+ext
|
||||
# Test to see if file already exists, and if it does not grab the media and store it in location folder
|
||||
picsLocalpath = build_review_media_path(name, visitdate, filename)
|
||||
if not os.path.isfile(picsLocalpath):
|
||||
urlretrieve(urlmedia, picsLocalpath)
|
||||
# Store the local path to be used in the excel document
|
||||
pics2.append(picsLocalpath)
|
||||
dictPostComplete= {'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,'Instagram':0,'tiktok':0}
|
||||
lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address,dictPostComplete])
|
||||
return lst_data
|
||||
|
||||
# Grab a count of how far we need to scroll
|
||||
def counter(driver):
|
||||
return count_google_review_pages(driver)
|
||||
|
||||
# Do the scrolling
|
||||
def scrolling(driver, review_count):
|
||||
return scroll_google_reviews(driver, review_count, label='scrolling...')
|
||||
|
||||
def write_to_xlsx(data):
|
||||
print('write to excel...')
|
||||
|
||||
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address','dictPostComplete']
|
||||
df = pd.DataFrame(data, columns=cols)
|
||||
df.to_excel('./Output/reviews.xlsx')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print('starting...')
|
||||
options = build_google_chrome_options(
|
||||
ignore_certificate_errors=True,
|
||||
ignore_ssl_errors=True,
|
||||
)
|
||||
# Setting the driver path and requesting a page
|
||||
driver = webdriver.Chrome(options=options) # Firefox(options=options)
|
||||
# Changing the property of the navigator value for webdriver to undefined
|
||||
set_stealth_driver(driver)
|
||||
driver.get(URL)
|
||||
time.sleep(5)
|
||||
|
||||
review_count = counter(driver)
|
||||
scrolling(driver, review_count)
|
||||
|
||||
data = get_data(driver)
|
||||
driver.close()
|
||||
|
||||
write_to_xlsx(data)
|
||||
print('Done!')
|
||||
Reference in New Issue
Block a user