admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

linting

Browse Source
This commit is contained in:
Joe Steele
2024-11-03 12:32:21 -05:00
parent e6aa6b6339
commit 3e400170b0
1 changed files with 54 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+50 -46
View File
@@ -175,12 +175,12 @@ def get_auth_connect():
c_column = sheet.max_column c_column = sheet.max_column
#out_data=[] #out_data=[]
#import inspect #import inspect
local_dict=Posts #local_dict=Posts
column_list = [] column_list = []
my_list = [] my_list = []
my_list_data = [] my_list_data = []
my_post = Posts #my_post = Posts
my_dict = {} #my_dict = {}
for x in inspect.getmembers(Posts): for x in inspect.getmembers(Posts):
if not (x[0].startswith('_') or 'metadata' in x[0]): if not (x[0].startswith('_') or 'metadata' in x[0]):
column_list.append(x[0]) column_list.append(x[0])
@@ -296,9 +296,9 @@ def get_hastags(address, name, hashtype):
else: else:
defaulttags = "\n\n\n#"+name_no_spaces+\ defaulttags = "\n\n\n#"+name_no_spaces+\
" #foodie #music #food #travel #drinks #instagood #feedme #joeeatswhat @timberjoe" " #foodie #music #food #travel #drinks #instagood #feedme #joeeatswhat @timberjoe"
citytag = "#"+city citytag = f"#{city}"
statetag = "#"+state statetag = f"#{state}"
ziptag = "#"+zip_code ziptag = f"#{zip_code}"
if statetag in ['FL', 'fl','Fl']: if statetag in ['FL', 'fl','Fl']:
statetag += '#Florida' statetag += '#Florida'
if statetag == 'OR': if statetag == 'OR':
@@ -549,9 +549,9 @@ def get_google_data(driver, local_outputs):
else: else:
if env.forcegoogleupdate: if env.forcegoogleupdate:
database_update_row(name,"google",True,"forceall",local_outputs) database_update_row(name,"google",True,"forceall",local_outputs)
if len(local_outputs['postssession'].query(Posts).filter(Posts.name == name,Posts.google\ if len(local_outputs['postssession'].query(Posts).filter(Posts.name == name,\
is not True).all()) > 0: Posts.google is not True).all()) > 0:
print (' Post was already in database, skipping update unless you activate override') print (' Post was already in database, skipping update unless overriden')
# else: # else:
# database_update_row(name,"google",True,"forceall",local_outputs) # database_update_row(name,"google",True,"forceall",local_outputs)
pics= [] pics= []
@@ -566,8 +566,8 @@ def get_google_data(driver, local_outputs):
# Walk through all the pictures and videos for a given review # Walk through all the pictures and videos for a given review
lmpics = '' lmpics = ''
for lmpics in more_specific_pics: for lmpics in more_specific_pics:
# Grab URL from style definiton (long multivalue string), and remove the -p-k so that # Grab URL from style definiton (long multivalue string), and remove the -p-k
# it is full size # so that it is full size
urlmedia = re.sub(r'=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]", urlmedia = re.sub(r'=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]",
lmpics.get_attribute("style")))[0]) lmpics.get_attribute("style")))[0])
print (' Pic URL : ',urlmedia) print (' Pic URL : ',urlmedia)
@@ -584,13 +584,14 @@ def get_google_data(driver, local_outputs):
#print (' Visited: ',visitdate) #print (' Visited: ',visitdate)
# Check to see if it has a sub div, which represents the label with the video length # Check to see if it has a sub div, which represents the label with the video length
# displayed, this will be done # displayed, this will be done
# because videos are represented by pictures in the main dialogue, so we need to click # because videos are represented by pictures in the main dialogue, so we need
# through and grab the video URL # to click through and grab the video URL
if lmpics.find_elements(By.CSS_SELECTOR,'div.fontLabelMedium.e5A3N') : if lmpics.find_elements(By.CSS_SELECTOR,'div.fontLabelMedium.e5A3N') :
ext='.mp4' ext='.mp4'
lmpics.click() lmpics.click()
time.sleep(2) time.sleep(2)
# After we click the right side is rendered in an inframe, Store iframe web element # After we click the right side is rendered in an iframe Store iframe
# web element
iframe = driver.find_element(By.TAG_NAME, "iframe") iframe = driver.find_element(By.TAG_NAME, "iframe")
# switch to selected iframe # switch to selected iframe
driver.switch_to.frame(iframe) driver.switch_to.frame(iframe)
@@ -614,8 +615,17 @@ def get_google_data(driver, local_outputs):
pics_local_path = "./Output/Pics/"+cleanname+"/"+visitdate+'/'+filename pics_local_path = "./Output/Pics/"+cleanname+"/"+visitdate+'/'+filename
pics2.append(pics_local_path) pics2.append(pics_local_path)
if env.forcegoogleupdate: if env.forcegoogleupdate:
database_update_row(name,"review_id",current_url,"forceall",local_outputs)# Add the correct extension to the file name database_update_row(name,"review_id",current_url,"forceall",local_outputs)
if len(pics2) > 0 and not os.path.isfile("./Output/Pics/"+cleanname+"/"+visitdate+'/'+cleanname+'-montage.mp4'): # Add the correct extension to the file name
if pics2 and not os.path.isfile(
"./Output/Pics/"
+ cleanname
+ "/"
+ visitdate
+ '/'
+ cleanname
+ '-montage.mp4'
):
make_montage_video_from_google(pics2,cleanname) make_montage_video_from_google(pics2,cleanname)
for loop in pics2: for loop in pics2:
if "montage.mp4" in loop: if "montage.mp4" in loop:
@@ -803,7 +813,8 @@ def write_to_database(data, local_outputs):
# # Using the DataFrame.append() function # # Using the DataFrame.append() function
try: try:
getdata2 = local_outputs['postssession'].query(Posts).filter(Posts.name == processrow[0]) getdata2 = local_outputs['postssession'].query(Posts).filter(Posts.name \
== processrow[0])
if getdata2.count() >0: if getdata2.count() >0:
print (' Row ',processrow[0],' already in Database') print (' Row ',processrow[0],' already in Database')
if env.forcegoogleupdate: if env.forcegoogleupdate:
@@ -813,8 +824,8 @@ def write_to_database(data, local_outputs):
'source':processrow[5],'date':processrow[6],'address':processrow[7],\ 'source':processrow[5],'date':processrow[6],'address':processrow[7],\
'dictPostComplete':str(processrow[8])} 'dictPostComplete':str(processrow[8])}
print (' Row ',processrow[0],' updated in Database') print (' Row ',processrow[0],' updated in Database')
local_outputs['postssession'].query(Posts).filter(Posts.name == processrow[0]).update\ local_outputs['postssession'].query(Posts).filter(Posts.name == processrow[0]).\
(d2_dict) update(d2_dict)
local_outputs['postssession'].commit() local_outputs['postssession'].commit()
else: else:
d2_row = Posts(name=processrow[0] ,comment=processrow[1],rating=processrow[2]\ d2_row = Posts(name=processrow[0] ,comment=processrow[1],rating=processrow[2]\
@@ -1104,21 +1115,24 @@ def post_to_x2(title, content, headers,date, rating, address, picslist,local_out
imgs_pic.append(img.strip()) imgs_pic.append(img.strip())
if imgs_vid: if imgs_vid:
try: try:
video_path = imgs_vid[0]
# Message to post along with the video # Message to post along with the video
attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\ attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\
.all() .all()
business_url = attrib_list[0].businessurl if wpurl := attrib_list[0].wpurl:
wpurl = attrib_list[0].wpurl if business_url := attrib_list[0].businessurl:
if wpurl: # Don't post of website URL does not exist yet status_message = (
if business_url: #Sometimes Business URL does not exist, so account for it f'{str(title)}: My Review - '
status_message=str(title)+': My Review - '+wpurl+'\n Business website: '+\ + wpurl
business_url + '\n' + '\n Business website: '
+ business_url
+ '\n'
)
else: else:
status_message = str(title) + ': My Review - '+ wpurl + '\n' status_message = f'{str(title)}: My Review - ' + wpurl + '\n'
status_message2 = status_message +' '+str(get_hastags(address, title, 'short'))\ status_message2 = status_message +' '+str(get_hastags(address, title, 'short'))\
+' ' +' '
status_message_short = status_message2[:279] status_message_short = status_message2[:279]
video_path = imgs_vid[0]
# Upload video # Upload video
media = client_v1.media_upload(filename=video_path) media = client_v1.media_upload(filename=video_path)
# Post tweet with video # Post tweet with video
@@ -1152,9 +1166,7 @@ def post_to_threads(title, content,headers, date, rating, address, picslist, loc
wpurl = attrib_list[0].wpurl wpurl = attrib_list[0].wpurl
auth_token = env.threads_access_token auth_token = env.threads_access_token
group_id = env.threads_page_id group_id = env.threads_page_id
imgs_id = []
imgs_vid = [] imgs_vid = []
imgs_pic = []
if wpurl and (attrib_list[0].picsLocalpath != '[]'): if wpurl and (attrib_list[0].picsLocalpath != '[]'):
if business_url: if business_url:
status_message = ( status_message = (
@@ -1162,12 +1174,14 @@ def post_to_threads(title, content,headers, date, rating, address, picslist, loc
+ ' \n\n') + content + ' \n\n') + content
else: else:
status_message = f'{str(title)}: My Review - ' + wpurl + ' \n\n' + content status_message = f'{str(title)}: My Review - ' + wpurl + ' \n\n' + content
imgs_pic = []
for img in img_list: for img in img_list:
if 'montage.mp4' in img: if 'montage.mp4' in img:
imgs_vid.append(img.strip()) imgs_vid.append(img.strip())
else: else:
imgs_pic.append(img.strip()) imgs_pic.append(img.strip())
if imgs_vid: if imgs_vid:
imgs_id = []
try: try:
post_id = post_threads_video(group_id, imgs_vid,auth_token,title, status_message, post_id = post_threads_video(group_id, imgs_vid,auth_token,title, status_message,
date, rating, address) date, rating, address)
@@ -1204,9 +1218,7 @@ def post_facebook3(title, content,headers, date, rating, address, picslist, loca
pics = ((picslist[1:-1]).replace("'","")).split(",") pics = ((picslist[1:-1]).replace("'","")).split(",")
group_id = env.facebookpageID group_id = env.facebookpageID
auth_token = env.facebookpass auth_token = env.facebookpass
imgs_id = []
imgs_vid = [] imgs_vid = []
imgs_pic = []
img_list = pics img_list = pics
attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all() attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()
business_url = attrib_list[0].businessurl business_url = attrib_list[0].businessurl
@@ -1218,12 +1230,14 @@ def post_facebook3(title, content,headers, date, rating, address, picslist, loca
+ ' \n\n') + content + ' \n\n') + content
else: else:
status_message = f'{str(title)}: My Review - ' + wpurl + ' \n\n' + content status_message = f'{str(title)}: My Review - ' + wpurl + ' \n\n' + content
imgs_pic = []
for img in img_list: for img in img_list:
if 'montage.mp4' in img: if 'montage.mp4' in img:
imgs_vid.append(img.strip()) imgs_vid.append(img.strip())
else: else:
imgs_pic.append(img.strip()) imgs_pic.append(img.strip())
if imgs_vid: if imgs_vid:
imgs_id = []
try: try:
post_id = post_facebook_video(group_id, imgs_vid,auth_token,title, status_message, post_id = post_facebook_video(group_id, imgs_vid,auth_token,title, status_message,
date, rating, address) date, rating, address)
@@ -1331,23 +1345,13 @@ def post_to_tiktok(title, content, headers2, date, rating, address, picslist, lo
# If you want to schedule your video, replace 'schedule_timestamp' with the Unix timestamp. # If you want to schedule your video, replace 'schedule_timestamp' with the Unix timestamp.
# Leave it as None if you want to upload immediately. # Leave it as None if you want to upload immediately.
schedule_time = None # or Unix timestamp (e.g., 1672592400) #schedule_time = None # or Unix timestamp (e.g., 1672592400)
pics = ((picslist[1:-1]).replace("'","").replace(" ","")).split(",") pics = ((picslist[1:-1]).replace("'","").replace(" ","")).split(",")
if any(element.endswith("montage.mp4") for element in pics): if not any(element.endswith("montage.mp4") for element in pics):
for pic in pics:
if 'montage.mp4' in pic:
file_path = pic
else:
return False return False
#content = content + get_hastags(address, title) for pic in pics:
#pics = ((picslist[1:-1].replace(",","")).replace("'","")).split(" ") if 'montage.mp4' in pic:
# video, outputmontage = make_montage_video_from_google(pics) file_path = pic
# try:
# instasession.video_upload(outputmontage, data)
# except AttributeError as error:
# print(" An error occurred uploading video to Instagram:", type(error).__name__)
# return False
# return True
# """ # """
# client_key string The unique identification key # client_key string The unique identification key
# provisioned to the partner. # provisioned to the partner.