admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

continue to lint

Browse Source
This commit is contained in:
timberjoegithub
2024-05-28 11:03:05 -04:00
parent 12a65c5999
commit 63955c7807
1 changed files with 91 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+71 -65
View File
@@ -102,7 +102,8 @@ def authconnect():
if mariadb: if mariadb:
print('Connecting to MariaDB for configuration and storage') print('Connecting to MariaDB for configuration and storage')
from sqlalchemy import create_engine from sqlalchemy import create_engine
engine = sqlalchemy.create_engine("mysql+mysqldb://"+mariadbuser+":"+mariadbpass+"@"+mariadbserver+"/"+mariadbdb+"?charset=utf8mb4", echo=False) engine = sqlalchemy.create_engine("mysql+mysqldb://"+mariadbuser+":"+mariadbpass+
"@"+mariadbserver+"/"+mariadbdb+"?charset=utf8mb4", echo=False)
Session = sqlalchemy.orm.sessionmaker() Session = sqlalchemy.orm.sessionmaker()
Session.configure(bind=engine) Session.configure(bind=engine)
session = Session() session = Session()
@@ -116,11 +117,11 @@ def authconnect():
connections.update({'postssession':session}) connections.update({'postssession':session})
if data: if data:
print(' loading XLS content data source ...') print(' loading XLS content data source ...')
if (os.path.exists(xls)): if os.path.exists(xls):
wb = load_workbook(filename = xls) wb = load_workbook(filename = xls)
xlswbDF = pd.read_excel(xls) xlswbDF = pd.read_excel(xls)
else: else:
if (os.path.exists('./GoogleScrape/'+ xls)): if os.path.exists('./GoogleScrape/'+ xls):
wb = load_workbook(filename = './GoogleScrape/'+ xls) wb = load_workbook(filename = './GoogleScrape/'+ xls)
xlswbDF = pd.read_excel('./GoogleScrape/'+ xls) xlswbDF = pd.read_excel('./GoogleScrape/'+ xls)
else: else:
@@ -137,7 +138,6 @@ def authconnect():
connections.update({'instagram':instasessionclient}) connections.update({'instagram':instasessionclient})
if facebook : if facebook :
print(' Connecting to facebook ...') print(' Connecting to facebook ...')
# msg = 'Purple Ombre Bob Lace Wig Natural Human Hair now available on https://lace-wigs.co.za/'
# page_id_1 = facebookpageID # page_id_1 = facebookpageID
# facebook_access_token = 'paste-your-page-access-token-here' # facebook_access_token = 'paste-your-page-access-token-here'
# image_url = 'https://graph.facebook.com/{}/photos'.format(page_id_1) # image_url = 'https://graph.facebook.com/{}/photos'.format(page_id_1)
@@ -165,7 +165,7 @@ def authconnect():
connections.update({'web' : headers}) connections.update({'web' : headers})
if tiktok : if tiktok :
print(' Connecting to Instagram ...') print(' Connecting to Instagram ...')
return connections return (connections)
################################################################################################## ##################################################################################################
@@ -214,7 +214,7 @@ def postImage(group_id, img,auth_token):
print(" An error getting date occurred:", type(error).c) # An error occurred: print(" An error getting date occurred:", type(error).c) # An error occurred:
r = False r = False
time.sleep(facebooksleep) time.sleep(facebooksleep)
return r return (r)
################################################################################################## ##################################################################################################
@@ -228,16 +228,15 @@ def postVideo(group_id, video_path,auth_token,title, content, date, rating, addr
# my_dict['key'].append(1) # my_dict['key'].append(1)
files.update({eachfile: open(eachfile, 'rb')}) files.update({eachfile: open(eachfile, 'rb')})
data = { "title":title,"description" : title + "\n"+ address+"\nGoogle map to destination: " data = { "title":title,"description" : title + "\n"+ address+"\nGoogle map to destination: "
r"https://www.google.com/maps/dir/?api=1&destination="+addresshtml +"\n\n"+ content + "\n"+rating+"\n"+date+"\n\n"+ hastags(address, title)+"\n\nhttps://www.joeeatswhat.com"+"\n\n","published" : True r"https://www.google.com/maps/dir/?api=1&destination="+addresshtml +"\n\n"+ content +
"\n"+rating+"\n"+date+"\n\n"+ hastags(address, title)+
"\n\nhttps://www.joeeatswhat.com"+"\n\n","published" : True
} }
#try: r = requests.post(url, files=files, data=data)
try: r = requests.post(url, files=files, data=data).json() try: r = requests.post(url, files=files, data=data).json()
except Exception as error: except Exception as error:
print(" An error getting date occurred:", type(error).c) # An error occurred: print(" An error getting date occurred:", type(error).c) # An error occurred:
r = False r = False
time.sleep(facebooksleep) time.sleep(facebooksleep)
print (r)
print ('r id = ',r['id'])
return r return r
################################################################################################## ##################################################################################################
@@ -271,7 +270,6 @@ def post_facebook2(title, content, date, rating, address, picslist, instasession
imgs_id.append(post_id['id']) imgs_id.append(post_id['id'])
except Exception as error: except Exception as error:
print(" An error occurred:", type(error)) # An error occurred: print(" An error occurred:", type(error)) # An error occurred:
# try: # try:
# imgs_id.append(post_id['id']) # imgs_id.append(post_id['id'])
# except Exception as error: # except Exception as error:
@@ -307,14 +305,15 @@ def post_facebook3(title, content, date, rating, address, picslist, instasession
imgs_pic = [] imgs_pic = []
img_list = pics img_list = pics
for img in img_list: for img in img_list:
if ('montage.mp4' in img ): if 'montage.mp4' in img:
imgs_vid.append(img.strip()) imgs_vid.append(img.strip())
else: else:
imgs_pic.append(img.strip()) imgs_pic.append(img.strip())
if (imgs_vid ): if imgs_vid:
# print ("loop") # print ("loop")
try: try:
post_id = postVideo(group_id, imgs_vid,auth_token,title, content, date, rating, address) post_id = postVideo(group_id, imgs_vid,auth_token,title, content,
date, rating, address)
imgs_id.append(post_id['id']) imgs_id.append(post_id['id'])
except Exception as error: except Exception as error:
print(" An error occurred:", type(error).c) # An error occurred: print(" An error occurred:", type(error).c) # An error occurred:
@@ -327,7 +326,9 @@ def post_facebook3(title, content, date, rating, address, picslist, instasession
def get_data(driver,outputs ): def get_data(driver,outputs ):
# curl -X GET -H 'Content-Type: application/json' -H "X-Goog-Api-Key: API_KEY" -H "X-Goog-FieldMask: id,displayName,formattedAddress,plusCode" https://places.googleapis.com/v1/places/ChIJj61dQgK6j4AR4GeTYWZsKWw #placeId #websiteUri AIzaSyB_hglzM7N8HzjXwi1dF1E8WYTql3akS7Q # curl -X GET -H 'Content-Type: application/json' -H "X-Goog-Api-Key: API_KEY" -H
# "X-Goog-FieldMask: id,displayName,formattedAddress,plusCode"
# https://places.googleapis.com/v1/places/ChIJj61dQgK6j4AR4GeTYWZsKWw #placeId #websiteUri
""" """
this function get main text, score, name this function get main text, score, name
""" """
@@ -336,7 +337,8 @@ def get_data(driver,outputs ):
more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq') more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq')
for list_more_element in more_elemets: for list_more_element in more_elemets:
list_more_element.click() list_more_element.click()
# Find Pictures that have the expansion indicator to see the rest of the pictures under them and click it to expose them all # Find Pictures that have the expansion indicator to see the rest of the pictures under
# them and click it to expose them all
more_pics = driver.find_elements(By.CLASS_NAME, 'Tya61d') more_pics = driver.find_elements(By.CLASS_NAME, 'Tya61d')
for list_more_pics in more_pics: for list_more_pics in more_pics:
if 'showMorePhotos' in list_more_pics.get_attribute("jsaction") : if 'showMorePhotos' in list_more_pics.get_attribute("jsaction") :
@@ -354,7 +356,11 @@ def get_data(driver,outputs ):
print('Visited: ',visitdate) print('Visited: ',visitdate)
try: text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text try: text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text
except: text = '' except: text = ''
try: score = data.find_element(By.CSS_SELECTOR, 'span.kvMYJc').get_attribute("aria-label") #find_element(By.CSS_SELECTOR,'aria-label').text #) ##QA0Szd > div > div > div.w6VYqd > div:nth-child(2) > div > div.e07Vkf.kA9KIf > div > div > div.m6QErb.DxyBCb.kA9KIf.dS8AEf > div.m6QErb > div:nth-child(3) > div:nth-child(2) > div > div:nth-child(4) > div.DU9Pgb > span.kvMYJc try: score = data.find_element(By.CSS_SELECTOR, 'span.kvMYJc').get_attribute("aria-label")
#find_element(By.CSS_SELECTOR,'aria-label').text #) ##QA0Szd > div > div > div.w6VYqd >
# div:nth-child(2) > div > div.e07Vkf.kA9KIf > div > div > div.m6QErb.DxyBCb.kA9KIf.dS8AEf
# > div.m6QErb > div:nth-child(3) > div:nth-child(2) > div > div:nth-child(4) > div.DU9Pgb
# > span.kvMYJc
except: score = "Unknown" except: score = "Unknown"
more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d') more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d')
pics= [] pics= []
@@ -365,13 +371,14 @@ def get_data(driver,outputs ):
os.makedirs('./Output/Pics/'+cleanname) os.makedirs('./Output/Pics/'+cleanname)
# Walk through all the pictures and videos for a given review # Walk through all the pictures and videos for a given review
for lmpics in more_specific_pics: for lmpics in more_specific_pics:
# if pics2 and not pics: # Grab URL from style definiton (long multivalue string), and remove the -p-k so that
# pics2 = [] # it is full size
# Grab URL from style definiton (long multivalue string), and remove the -p-k so that it is full size urlmedia = re.sub('=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]",
urlmedia = re.sub('=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]", lmpics.get_attribute("style")))[0]) lmpics.get_attribute("style")))[0])
print ('URL : ',urlmedia) print ('URL : ',urlmedia)
pics.append(urlmedia) pics.append(urlmedia)
# Grab the name of the file and remove all spaces and special charecters to name the folder # Grab the name of the file and remove all spaces and special charecters to name the
# folder
filename = re.sub( r'[^a-zA-Z0-9]','', str(lmpics.get_attribute("aria-label"))) filename = re.sub( r'[^a-zA-Z0-9]','', str(lmpics.get_attribute("aria-label")))
if lmpics == more_specific_pics[0]: if lmpics == more_specific_pics[0]:
lmpics.click() lmpics.click()
@@ -380,8 +387,10 @@ def get_data(driver,outputs ):
tempdate = str((driver.find_element(By.CLASS_NAME,'mqX5ad')).text).rsplit("-",1) tempdate = str((driver.find_element(By.CLASS_NAME,'mqX5ad')).text).rsplit("-",1)
visitdate = re.sub( r'[^a-zA-Z0-9]','',tempdate[1]) visitdate = re.sub( r'[^a-zA-Z0-9]','',tempdate[1])
print ('Visited: ',visitdate) print ('Visited: ',visitdate)
# Check to see if it has a sub div, which represents the label with the video length displayed, this will be done # Check to see if it has a sub div, which represents the label with the video length
# because videos are represented by pictures in the main dialogue, so we need to click through and grab the video URL # displayed, this will be done
# because videos are represented by pictures in the main dialogue, so we need to click
# through and grab the video URL
if (lmpics.find_elements(By.CSS_SELECTOR,'div.fontLabelMedium.e5A3N')) : if (lmpics.find_elements(By.CSS_SELECTOR,'div.fontLabelMedium.e5A3N')) :
ext='.mp4' ext='.mp4'
lmpics.click() lmpics.click()
@@ -400,7 +409,8 @@ def get_data(driver,outputs ):
ext='.jpg' ext='.jpg'
# Add the correct extension to the file name # Add the correct extension to the file name
filename = filename+ext filename = filename+ext
# Test to see if file already exists, and if it does not grab the media and store it in location folder # Test to see if file already exists, and if it does not grab the media and store it
# in location folder
if not os.path.exists('./Output/Pics/'+cleanname+'/'+visitdate): if not os.path.exists('./Output/Pics/'+cleanname+'/'+visitdate):
os.makedirs('./Output/Pics/'+cleanname+'/'+visitdate) os.makedirs('./Output/Pics/'+cleanname+'/'+visitdate)
if not os.path.isfile('./Output/Pics/'+cleanname+'/'+visitdate+'/'+filename): if not os.path.isfile('./Output/Pics/'+cleanname+'/'+visitdate+'/'+filename):
@@ -411,9 +421,10 @@ def get_data(driver,outputs ):
if pics2: if pics2:
make_video(pics2) make_video(pics2)
pics2.append("./Output/Pics/"+cleanname+"/"+visitdate+'/'+'montage.mp4') pics2.append("./Output/Pics/"+cleanname+"/"+visitdate+'/'+'montage.mp4')
dictPostComplete= {'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,'instagram':0,'tiktok':0} dictPostComplete= {'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,
lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address,dictPostComplete]) 'instagram':0,'tiktok':0}
lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address,
dictPostComplete])
return lst_data return lst_data
################################################################################################## ##################################################################################################
@@ -428,7 +439,8 @@ def scrolling(counter,driver):
for _i in range(counter): for _i in range(counter):
try: try:
scrolling = driver.execute_script( scrolling = driver.execute_script(
'document.getElementsByClassName("dS8AEf")[0].scrollTop = document.getElementsByClassName("dS8AEf")[0].scrollHeight', 'document.getElementsByClassName("dS8AEf")[0].scrollTop = /
document.getElementsByClassName("dS8AEf")[0].scrollHeight',
scrollable_div scrollable_div
) )
time.sleep(3) time.sleep(3)
@@ -440,7 +452,8 @@ def scrolling(counter,driver):
def write_to_xlsx(webdata, outputs): def write_to_xlsx(webdata, outputs):
print('Start to write to excel...') print('Start to write to excel...')
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address','dictPostComplete'] cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
'dictPostComplete']
# rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row))) # rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)))
rows = list(webdata) rows = list(webdata)
if needreversed: if needreversed:
@@ -452,10 +465,13 @@ def write_to_xlsx(webdata, outputs):
else: else:
if (processrow[0] != None): if (processrow[0] != None):
# processrow.append({'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,'Instagram':0,'tiktok':0}) # processrow.append({'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,'Instagram':0,'tiktok':0})
outputs['data'].append([processrow[0],processrow[1],processrow[2],str(processrow[3]),str(processrow[4]),str(processrow[5]),str(processrow[6]),str(processrow[7]),str(processrow[8])]) # sheet_obj.append([col1, col2]) outputs['data'].append([processrow[0],processrow[1],processrow[2],str(
processrow[3]),str(processrow[4]),str(processrow[5]),str(processrow[6]),
str(processrow[7]),str(processrow[8])]) # sheet_obj.append([col1, col2])
#outputs['data'].parent.save(xls) #outputs['data'].parent.save(xls)
#outputs['datawb'].save(xls) #outputs['datawb'].save(xls)
cols = ['num',"name", "comment", 'rating','picsURL','picsLocalpath','source','date','address','dictPostComplete', 'test'] cols = ['num',"name", "comment", 'rating','picsURL','picsLocalpath','source','date',
'address','dictPostComplete', 'test']
df = pd.DataFrame(outputs['data'], columns=cols) df = pd.DataFrame(outputs['data'], columns=cols)
df.to_excel(xls) df.to_excel(xls)
return True return True
@@ -483,9 +499,6 @@ def write_to_xlsx2(data, outputs):
if (processrow[1] in df.values): if (processrow[1] in df.values):
print (' Row ',processrow[0],' ', processrow[1],' already in XLS sheet') print (' Row ',processrow[0],' ', processrow[1],' already in XLS sheet')
d2_row = Posts(name=processrow[1],comment=processrow[2],rating=processrow[3],picsURL=processrow[4],picsLocalpath=processrow[5],source=processrow[6],date=processrow[7],address=processrow[8],dictPostComplete=processrow[9]) d2_row = Posts(name=processrow[1],comment=processrow[2],rating=processrow[3],picsURL=processrow[4],picsLocalpath=processrow[5],source=processrow[6],date=processrow[7],address=processrow[8],dictPostComplete=processrow[9])
# if not (processrow[1] in outputs['posts']) :
# outputs['postssession'].add(d2_row)
# outputs['postssession'].commit()
else: else:
if (processrow[1] != None): if (processrow[1] != None):
# Create a Python dictionary object with all the column values # Create a Python dictionary object with all the column values
@@ -516,7 +529,7 @@ def database_read(data):
db_connection_str = 'mysql+pymysql://mysql_user:mysql_password@mysql_host/mysql_db' db_connection_str = 'mysql+pymysql://mysql_user:mysql_password@mysql_host/mysql_db'
db_connection = create_engine(db_connection_str) db_connection = create_engine(db_connection_str)
df = pd.read_sql('SELECT * FROM table_name', con=db_connection) df = pd.read_sql('SELECT * FROM table_name', con=db_connection)
return df return (df)
################################################################################################## ##################################################################################################
@@ -531,7 +544,7 @@ def check_web_media(filename,headers):
return file_id, link return file_id, link
except Exception as error: except Exception as error:
print(' No existing media with same name in Wordpress media folder: ' + filename) print(' No existing media with same name in Wordpress media folder: ' + filename)
return False, False return (False, False)
################################################################################################## ##################################################################################################
@@ -644,7 +657,6 @@ def post_x(title, content, date, rating, address, picslist, instasession):
################################################################################################## ##################################################################################################
def post_to_wp(title, content, headers,date, rating,address, picslist): def post_to_wp(title, content, headers,date, rating,address, picslist):
# post # post
NewPost = False NewPost = False
@@ -655,10 +667,8 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
picl = picslist[1:-1] picl = picslist[1:-1]
pic2 = picl.replace(",","")#re.sub(r',','',picl) #re.sub( r'[^a-zA-Z0-9]','',tempdate[1]) pic2 = picl.replace(",","")#re.sub(r',','',picl) #re.sub( r'[^a-zA-Z0-9]','',tempdate[1])
pic3= pic2.replace("'","") pic3= pic2.replace("'","")
# print (pic3)
pidchop = pic3.split(" ") pidchop = pic3.split(" ")
linkslist=[] linkslist=[]
# linkslist.clear
print (' Figuring out date of Post : ',title) print (' Figuring out date of Post : ',title)
format = '%b/%Y/%d' #specifify the format of the date_string. format = '%b/%Y/%d' #specifify the format of the date_string.
date_string = date date_string = date
@@ -740,7 +750,7 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
post_id = check_post(title,newdate2,headers) post_id = check_post(title,newdate2,headers)
except Exception as error : except Exception as error :
print ('Could not check to see post already exists', type(error).c) print ('Could not check to see post already exists', type(error).c)
if ( post_id == False): if post_id == False:
googleadress = r"<a href=https://www.google.com/maps/dir/?api=1&destination="+addresshtml + r">"+address+r"</a>" googleadress = r"<a href=https://www.google.com/maps/dir/?api=1&destination="+addresshtml + r">"+address+r"</a>"
post_data = { post_data = {
"title": title, "title": title,
@@ -775,7 +785,7 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
print (' Found Picture: ',picname) print (' Found Picture: ',picname)
file_id, link = check_media(picname, headers) file_id, link = check_media(picname, headers)
# link = linknew['rendered'] # link = linknew['rendered']
if (file_id) is False: if file_id is False:
print (' '+picname+' was not already found in library, adding it') print (' '+picname+' was not already found in library, adding it')
countreview = True countreview = True
image = { image = {
@@ -788,7 +798,7 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
image_response = requests.post(wpAPI + "/media", headers=headers, files=image) image_response = requests.post(wpAPI + "/media", headers=headers, files=image)
except Exception as error: except Exception as error:
print(" An error uploading picture ' + picname+ ' occurred:", type(error).__name__) # An error occurred: print(" An error uploading picture ' + picname+ ' occurred:", type(error).__name__) # An error occurred:
if ( image_response.status_code != 201 ): if image_response.status_code != 201 :
print (' Error- Image ',picname,' was not successfully uploaded. response: ',image_response) print (' Error- Image ',picname,' was not successfully uploaded. response: ',image_response)
else: else:
PicDict=image_response.json() PicDict=image_response.json()
@@ -808,7 +818,7 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
print (' Error- Image ',picname,' was not attached to post. response: ',image_response) print (' Error- Image ',picname,' was not attached to post. response: ',image_response)
try: try:
post_response = requests.post(wpAPI + "/posts/" + str(post_id), headers=headers) post_response = requests.post(wpAPI + "/posts/" + str(post_id), headers=headers)
if (link in str(post_response.text)): if link in str(post_response.text):
print (' Image link for ', picname, 'already in content of post: ',post_id, post_response.text, link) print (' Image link for ', picname, 'already in content of post: ',post_id, post_response.text, link)
else: else:
linkslist.append({'file_id' : file_id , 'link' : link}) linkslist.append({'file_id' : file_id , 'link' : link})
@@ -821,9 +831,9 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
#for loop in linkslist: #for loop in linkslist:
print (' Adding ', piclink['link'], ' to posting') print (' Adding ', piclink['link'], ' to posting')
try: try:
ext = (piclink['link'].split( '.')[-1] ) ext = piclink['link'].split( '.')[-1]
if (ext == 'mp4'): if ext == 'mp4':
if (firstMP4): if firstMP4:
contentpics += '\n' +r'[evp_embed_video url="' + piclink['link'] + r'" autoplay="true"]' contentpics += '\n' +r'[evp_embed_video url="' + piclink['link'] + r'" autoplay="true"]'
firstMP4 = False firstMP4 = False
else: else:
@@ -850,13 +860,13 @@ def make_video(inphotos):
dir = inphotos[0].rsplit(r'/', 1) dir = inphotos[0].rsplit(r'/', 1)
folder = dir[0] folder = dir[0]
output = folder+"/montage.mp4" output = folder+"/montage.mp4"
if ((not os.path.exists(output)) and (len(inphotos) >1)): if (not os.path.exists(output) and (len(inphotos) >1)):
video = VideoFileClip(inphotos[0]) video = VideoFileClip(inphotos[0])
for photo in inphotos : for photo in inphotos :
#clip = VideoFileClip("myHolidays.mp4").subclip(50,60) #clip = VideoFileClip("myHolidays.mp4").subclip(50,60)
clip = VideoFileClip(photo) clip = VideoFileClip(photo)
# Concatenate the photos into a clip # Concatenate the photos into a clip
if (".jpg" in photo): if ".jpg" in photo:
clip.duration = 2 clip.duration = 2
try: try:
video = concatenate_videoclips([video, clip], method="compose") video = concatenate_videoclips([video, clip], method="compose")
@@ -875,9 +885,6 @@ def make_video(inphotos):
################################################################################################## ##################################################################################################
##################################################################################################
def post_to_instagram2 (title, content, date, rating, address, picslist, instasession): def post_to_instagram2 (title, content, date, rating, address, picslist, instasession):
#post_to_instagram2(processrow[1].value, processrow[2].value ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['instagram']) #post_to_instagram2(processrow[1].value, processrow[2].value ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['instagram'])
#montageexists = "montage.mp4" in picslist #montageexists = "montage.mp4" in picslist
@@ -948,7 +955,7 @@ def hastags (address, name):
fulltag = defaulttags+" "+citytag+" "+statetag+" "+ziptag fulltag = defaulttags+" "+citytag+" "+statetag+" "+ziptag
# 153 Sugar Belle Dr, Winter Garden, FL 34787 # 153 Sugar Belle Dr, Winter Garden, FL 34787
# inphotos[0].rsplit(r'/', 1) # inphotos[0].rsplit(r'/', 1)
return fulltag return (fulltag)
################################################################################################## ##################################################################################################
@@ -956,7 +963,7 @@ def process_reviews(outputs):
# Process # Process
webcount = xtwittercount = instagramcount = yelpcount = threadscount = facebookcount= tiktokcount = 0 webcount = xtwittercount = instagramcount = yelpcount = threadscount = facebookcount= tiktokcount = 0
rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row))) rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)))
if (google): if google:
print('Configuration says to update google Reviews prior to processing them') print('Configuration says to update google Reviews prior to processing them')
options = webdriver.ChromeOptions() options = webdriver.ChromeOptions()
options.add_argument("--log-level=3") options.add_argument("--log-level=3")
@@ -980,7 +987,6 @@ def process_reviews(outputs):
driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})") driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
driver.get(URL) driver.get(URL)
time.sleep(5) time.sleep(5)
# counter = counter()
scrolling(counter(driver), driver) scrolling(counter(driver), driver)
webdata = get_data(driver,outputs) webdata = get_data(driver,outputs)
write_to_xlsx2(webdata, outputs) write_to_xlsx2(webdata, outputs)
@@ -999,8 +1005,8 @@ def process_reviews(outputs):
# Check to see if the website has already been written to according to the xls sheet, if it has not... then process # Check to see if the website has already been written to according to the xls sheet, if it has not... then process
if ((writtento["web"]) == 0 or writtento["instagram"]==0 or writtento["facebook"]==0 or writtento["xtwitter"]==0 or writtento["yelp"]==0 or writtento["tiktok"]==0 or writtento["threads"]==0 ) and (is_port_open(wpAPI, 443)) and (web or instagram or yelp or xtwitter or tiktok or facebook or threads or google)and (processrow[2].value != None) : if ((writtento["web"]) == 0 or writtento["instagram"]==0 or writtento["facebook"]==0 or writtento["xtwitter"]==0 or writtento["yelp"]==0 or writtento["tiktok"]==0 or writtento["threads"]==0 ) and (is_port_open(wpAPI, 443)) and (web or instagram or yelp or xtwitter or tiktok or facebook or threads or google)and (processrow[2].value != None) :
if web : if web :
if (writtento["web"] == 0) : if writtento["web"] == 0 :
if (webcount <= postsperrun): if webcount <= postsperrun:
try: try:
#NewWebPost = post_to_wp(processrow[1].value, processrow[2].value, processrow[2].value ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value) #NewWebPost = post_to_wp(processrow[1].value, processrow[2].value, processrow[2].value ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value)
NewWebPost = post_to_wp(processrow[1].value, processrow[2].value, outputs['web'] ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value) NewWebPost = post_to_wp(processrow[1].value, processrow[2].value, outputs['web'] ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value)
@@ -1025,8 +1031,8 @@ def process_reviews(outputs):
else: else:
print (' Website: Skipping posting for ',processrow[1].value,' previously written') print (' Website: Skipping posting for ',processrow[1].value,' previously written')
if instagram: if instagram:
if (writtento["instagram"] == 0 ): if writtento["instagram"] == 0:
if (instagramcount <= postsperrun): if instagramcount <= postsperrun:
try: try:
print(' Starting to generate Instagram post') print(' Starting to generate Instagram post')
NewInstagramPost = post_to_instagram2(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['instagram'] ) NewInstagramPost = post_to_instagram2(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['instagram'] )
@@ -1054,8 +1060,8 @@ def process_reviews(outputs):
else: else:
print (' Instagram: Skipping posting for ',processrow[1].value,' previously written') print (' Instagram: Skipping posting for ',processrow[1].value,' previously written')
if facebook: if facebook:
if (writtento["facebook"] == 0 ): if writtento["facebook"] == 0:
if (facebookcount <= postsperrun): if facebookcount <= postsperrun:
try: try:
print(' Starting to generate Facebook post') print(' Starting to generate Facebook post')
NewFacebookPost = post_facebook3(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['facebook'] ) NewFacebookPost = post_facebook3(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['facebook'] )
@@ -1083,8 +1089,8 @@ def process_reviews(outputs):
else: else:
print (' Facebook: Skipping posting for ',processrow[1].value,' previously written') print (' Facebook: Skipping posting for ',processrow[1].value,' previously written')
if False: if False:
if (writtento["xtwitter"] == 0 ): if writtento["xtwitter"] == 0:
if (xtwittercount <= postsperrun): if xtwittercount <= postsperrun:
try: try:
print(' Starting to generate xtwitter post') print(' Starting to generate xtwitter post')
NewxtwitterPost = post_x(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['xtwitter'] ) NewxtwitterPost = post_x(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['xtwitter'] )
@@ -1138,9 +1144,9 @@ def socials(name, namedict,outputs, writtento, processrow,funct):# namecount, na
#namedict{'name':name, 'namecount':namecount, 'namepost':namepost, 'subroutine':subroutine} #namedict{'name':name, 'namecount':namecount, 'namepost':namepost, 'subroutine':subroutine}
#namedict{'name':xtwitter, 'namecount':xtwiteercount, 'namepost':NewxtwitterPost, 'subroutine':postx} #namedict{'name':xtwitter, 'namecount':xtwiteercount, 'namepost':NewxtwitterPost, 'subroutine':postx}
if name: if name:
if (writtento[name] == 0 ): if writtento[name] == 0 :
print (namedict['namecount']) print (namedict['namecount'])
if (int(namedict['namecount']) <= postsperrun): if int(namedict['namecount']) <= postsperrun:
try: try:
print(' Starting to generate xtwitter post : ',namedict['subroutine']) print(' Starting to generate xtwitter post : ',namedict['subroutine'])
postoutput = funct(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[4].value,outputs ) postoutput = funct(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[4].value,outputs )