admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

formatting for linting

Browse Source
This commit is contained in:
Joe Steele
2024-06-17 16:57:48 -04:00
parent 9f480185cd
commit 058477d87f
1 changed files with 143 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+123 -155
View File
@@ -1,15 +1,11 @@
import time import time
import os import os
from pathlib import Path from pathlib import Path
import pathlib
#from selenium.webdriver.chrome.webdriver import WebDriver #from selenium.webdriver.chrome.webdriver import WebDriver
#from selenium.webdriver.chrome.service import Service #from selenium.webdriver.chrome.service import Service
import re import re
from urllib.request import urlretrieve
import urllib3
#from openpyxl import Workbook, load_workbook #from openpyxl import Workbook, load_workbook
from openpyxl import load_workbook
import pandas as pd import pandas as pd
from datetime import datetime from datetime import datetime
from dateutil.relativedelta import relativedelta from dateutil.relativedelta import relativedelta
@@ -19,24 +15,25 @@ import requests
import datetime as dt import datetime as dt
#import json #import json
import jsonpickle import jsonpickle
from urllib.request import urlretrieve
import urllib3
#Instgram #Instgram
#from instapy import InstaPy #from instapy import InstaPy
#import instapy #import instapy
#from instabot import Bot #from instabot import Bot
import pathlib from openpyxl import load_workbook
import instagrapi import instagrapi
#from instagrapi.types import StoryMention, StoryMedia, StoryLink, StoryHashtag #from instagrapi.types import StoryMention, StoryMedia, StoryLink, StoryHashtag
#from instagrapi.story import StoryBuilder #from instagrapi.story import StoryBuilder
from moviepy.editor import VideoFileClip, concatenate_videoclips from moviepy.editor import VideoFileClip, concatenate_videoclips
#import moviepy #import moviepy
from selenium import webdriver from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException
#from selenium.webdriver.chrome.webdriver import WebDriver #from selenium.webdriver.chrome.webdriver import WebDriver
#from selenium.webdriver.chrome.service import Service #from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
#twitter #twitter
import tweepy import tweepy
from selenium.common.exceptions import NoSuchElementException
#import asyncio #import asyncio
#import aiohttp #import aiohttp
@@ -93,11 +90,11 @@ class Posts(Base):
comment = sqlalchemy.Column(sqlalchemy.String(length=4096, collation="utf8")) comment = sqlalchemy.Column(sqlalchemy.String(length=4096, collation="utf8"))
rating = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8")) rating = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8"))
picsURL = sqlalchemy.Column(sqlalchemy.String(length=4096, collation="utf8")) picsURL = sqlalchemy.Column(sqlalchemy.String(length=4096, collation="utf8"))
picsLocalpath = sqlalchemy.Column(sqlalchemy.String(length=4096, collation="utf8")) pics_local_path = sqlalchemy.Column(sqlalchemy.String(length=4096, collation="utf8"))
source = sqlalchemy.Column(sqlalchemy.String(length=64, collation="utf8")) source = sqlalchemy.Column(sqlalchemy.String(length=64, collation="utf8"))
date = sqlalchemy.Column(sqlalchemy.String(length=64, collation="utf8")) date = sqlalchemy.Column(sqlalchemy.String(length=64, collation="utf8"))
address = sqlalchemy.Column(sqlalchemy.String(length=256, collation="utf8")) address = sqlalchemy.Column(sqlalchemy.String(length=256, collation="utf8"))
dictPostComplete = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8")) dict_post_complete = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8"))
#googleurl = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8")) #googleurl = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8"))
wpurl = sqlalchemy.Column(sqlalchemy.String(length=512, collation="utf8")) wpurl = sqlalchemy.Column(sqlalchemy.String(length=512, collation="utf8"))
businessurl = sqlalchemy.Column(sqlalchemy.String(length=2048, collation="utf8")) businessurl = sqlalchemy.Column(sqlalchemy.String(length=2048, collation="utf8"))
@@ -266,15 +263,16 @@ def get_hastags(address, name, hashtype):
str: The generated hashtags based on the input parameters. str: The generated hashtags based on the input parameters.
""" """
nameNoSpaces = re.sub( r'[^a-zA-Z]','',name) name_no_spaces = re.sub( r'[^a-zA-Z]','',name)
addressdict = address.rsplit(r' ',3) addressdict = address.rsplit(r' ',3)
zip_code = addressdict[3] zip_code = addressdict[3]
state = addressdict[2] state = addressdict[2]
city = re.sub( r'[^a-zA-Z]','',addressdict[1]) city = re.sub( r'[^a-zA-Z]','',addressdict[1])
if 'short' in hashtype: if 'short' in hashtype:
defaulttags = '#'+nameNoSpaces+' #foodie #food #joeeatswhat @timberjoe' defaulttags = '#'+name_no_spaces+' #foodie #food #joeeatswhat @timberjoe'
else: else:
defaulttags = "\n\n\n#"+nameNoSpaces+" #foodie #music #food #travel #drinks #instagood #feedme #joeeatswhat @timberjoe" defaulttags = "\n\n\n#"+name_no_spaces+\
" #foodie #music #food #travel #drinks #instagood #feedme #joeeatswhat @timberjoe"
citytag = "#"+city citytag = "#"+city
statetag = "#"+state statetag = "#"+state
ziptag = "#"+zip_code ziptag = "#"+zip_code
@@ -396,25 +394,18 @@ def post_facebook_video(group_id, video_path, auth_token, title, content, date,
################################################################################################## ##################################################################################################
def get_google_data(driver, outputs): def get_google_data(driver, local_outputs):
""" """
Retrieves data from Google Maps including name, address, and review details. Retrieves data from Google Maps including name, address, and review details.
Args: Args:
driver: The Selenium WebDriver instance. driver: The Selenium WebDriver instance.
outputs: Output information. local_outputs: Output information.
Returns: Returns:
list: A list of data extracted from Google Maps. list: A list of data extracted from Google Maps.
""" """
print('get google data...')
# curl -X GET -H 'Content-Type: application/json' -H "X-Goog-Api-Key: API_KEY" -H
# "X-Goog-FieldMask: id,displayName,formattingtedAddress,plusCode"
# https://places.googleapis.com/v1/places/ChIJj61dQgK6j4AR4GeTYWZsKWw #placeId #websiteUri
"""
this function gets main text, score, name
"""
print('get data...')
# Click on more botton on each text reviews # Click on more botton on each text reviews
more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq') more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq')
for list_more_element in more_elemets: for list_more_element in more_elemets:
@@ -455,7 +446,9 @@ def get_google_data(driver, outputs):
print ('Error: ',error) print ('Error: ',error)
more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d') more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d')
# Grab more info from google maps entry on this particular review # Grab more info from google maps entry on this particular review
if len(outputs['postssession'].query(Posts).filter(Posts.name == name,Posts.google is not True).all()) == 0 or env.forcegoogleupdate or env.block_google_maps is not True: if len(local_outputs['postssession'].query(Posts).filter(Posts.name == name,Posts.google\
is not True).all()) == 0 or env.forcegoogleupdate or env.block_google_maps is not\
True:
gmaps = googlemaps.Client(env.googleapipass) gmaps = googlemaps.Client(env.googleapipass)
place_ids = gmaps.find_place(name+address, input_type = 'textquery', fields='') place_ids = gmaps.find_place(name+address, input_type = 'textquery', fields='')
if len(place_ids['candidates']) == 1 : if len(place_ids['candidates']) == 1 :
@@ -468,19 +461,19 @@ def get_google_data(driver, outputs):
longitude = details['result']['geometry']['location']['lng'] longitude = details['result']['geometry']['location']['lng']
pluscode = details['result']['plus_code']['compound_code'] pluscode = details['result']['plus_code']['compound_code']
googleurl = details['result']['url'] googleurl = details['result']['url']
database_update_row(name,"businessurl",businessurl,"onlyempty",outputs) database_update_row(name,"businessurl",businessurl,"onlyempty",local_outputs)
database_update_row(name,"latitude",latitude,"onlyempty",outputs) database_update_row(name,"latitude",latitude,"onlyempty",local_outputs)
database_update_row(name,"longitude",longitude,"onlyempty",outputs) database_update_row(name,"longitude",longitude,"onlyempty",local_outputs)
database_update_row(name,"pluscode",pluscode,"onlyempty",outputs) database_update_row(name,"pluscode",pluscode,"onlyempty",local_outputs)
database_update_row(name,"googleurl",googleurl,"onlyempty",outputs) database_update_row(name,"googleurl",googleurl,"onlyempty",local_outputs)
database_update_row(name,"place_id",place_id,"onlyempty",outputs) database_update_row(name,"place_id",place_id,"onlyempty",local_outputs)
database_update_row(name,"googledetails",details,"onlyempty",outputs) database_update_row(name,"googledetails",details,"onlyempty",local_outputs)
database_update_row(name,"google",True,"onlyempty",outputs) database_update_row(name,"google",True,"onlyempty",local_outputs)
except KeyError as error: except KeyError as error:
print('Error writing business details from google maps : ',error) print('Error writing business details from google maps : ',error)
else: else:
print (' Post was already in database, skipping update unless you activate override') print (' Post was already in database, skipping update unless you activate override')
database_update_row(name,"google",True,"forceall",outputs) database_update_row(name,"google",True,"forceall",local_outputs)
pics= [] pics= []
pics2 = [] pics2 = []
# check to see if folder for pictures and videos already exists, if not, create it # check to see if folder for pictures and videos already exists, if not, create it
@@ -534,15 +527,15 @@ def get_google_data(driver, outputs):
if not os.path.isfile('./Output/Pics/'+cleanname+'/'+visitdate+'/'+filename): if not os.path.isfile('./Output/Pics/'+cleanname+'/'+visitdate+'/'+filename):
urlretrieve(urlmedia, './Output/Pics/'+cleanname+'/'+visitdate+'/'+filename) urlretrieve(urlmedia, './Output/Pics/'+cleanname+'/'+visitdate+'/'+filename)
# Store the local path to be used in the excel document # Store the local path to be used in the excel document
picsLocalpath = "./Output/Pics/"+cleanname+"/"+visitdate+'/'+filename pics_local_path = "./Output/Pics/"+cleanname+"/"+visitdate+'/'+filename
pics2.append(picsLocalpath) pics2.append(pics_local_path)
if pics2: if pics2:
make_montage_video_from_google(pics2) make_montage_video_from_google(pics2)
pics2.append("./Output/Pics/"+cleanname+"/"+visitdate+'/'+'montage.mp4') pics2.append("./Output/Pics/"+cleanname+"/"+visitdate+'/'+'montage.mp4')
dictPostComplete= {'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0, dict_post_complete= {'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,
'instagram':0,'tiktok':0} 'instagram':0,'tiktok':0}
lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address, lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address,
dictPostComplete]) dict_post_complete])
return lst_data return lst_data
################################################################################################## ##################################################################################################
@@ -582,13 +575,13 @@ def google_scroll(counter_google_scroll,driver):
################################################################################################## ##################################################################################################
def write_to_xlsx2(data, outputs): def write_to_xlsx2(data, local_outputs):
""" """
Writes data to an Excel file and updates the database with new entries. Writes data to an Excel file and updates the database with new entries.
Args: Args:
data: Data to be written to the Excel file. data: Data to be written to the Excel file.
outputs: Output information. local_outputs: Output information.
Returns: Returns:
Data: The data that was written to the Excel file. Data: The data that was written to the Excel file.
@@ -596,48 +589,51 @@ def write_to_xlsx2(data, outputs):
print('write to excel...') print('write to excel...')
sqlalchemy.null() sqlalchemy.null()
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address', cols = ["name", "comment", 'rating','picsURL','pics_local_path','source','date','address',
'dictPostComplete'] 'dict_post_complete']
cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date', cols2 = ["num","name", "comment", 'rating','picsURL','pics_local_path','source','date',
'address','dictPostComplete'] 'address','dict_post_complete']
df = pd.DataFrame(data, columns=cols) df = pd.DataFrame(data, columns=cols)
df2 = pd.DataFrame(outputs['xlsdf'].values, columns=cols2) df2 = pd.DataFrame(local_outputs['xlsdf'].values, columns=cols2)
#df2 = df1.where((pd.notnull(df)), None) # take out NAN problems #df2 = df1.where((pd.notnull(df)), None) # take out NAN problems
#df3.astype(object).where(pd.notnull(df2), None) #df3.astype(object).where(pd.notnull(df2), None)
print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True)) print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True))
rows = list(data) rows = list(data)
if env.needreversed: if env.needreversed:
rows = reversed(rows) rows = reversed(rows)
#jsonposts = json.dumps(outputs['posts'], default=Posts) #jsonposts = json.dumps(local_outputs['posts'], default=Posts)
print("Encode Object into JSON formatted Data using jsonpickle") print("Encode Object into JSON formatted Data using jsonpickle")
jsonposts = jsonpickle.encode(outputs['posts'], unpicklable=False) jsonposts = jsonpickle.encode(local_outputs['posts'], unpicklable=False)
for processrow in df2.values: for processrow in df2.values:
if (processrow[1] in df.values): if (processrow[1] in df.values):
print (' Row ',processrow.id,' ', processrow.name ,' already in XLS sheet') print (' Row ',processrow.id,' ', processrow.name ,' already in XLS sheet')
d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=processrow.rating, d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=\
picsURL=processrow.picsURL,picsLocalpath=processrow.picsLocalpath,source=processrow.source, processrow.rating,picsURL=processrow.picsURL,pics_local_path=processrow.\
date=processrow.date,address=processrow.address,dictPostComplete=processrow.dictPostComplete) pics_local_path,source=processrow.source,date=processrow.date,address=processrow.\
address,dict_post_complete=processrow.dict_post_complete)
elif processrow.name is not None: elif processrow.name is not None:
# Create a Python dictionary object with all the column values # Create a Python dictionary object with all the column values
# d_row = {'name':processrow.name ,'comment':processrow.comment,'rating':processrow.rating, # d_row = {'name':processrow.name ,'comment':processrow.comment,'rating':processrow.rating,
# 'picsURL':processrow.picsURL,'picsLocalpath':processrow.picsLocalpath, 'source':processrow.source, # 'picsURL':processrow.picsURL,'pics_local_path':processrow.pics_local_path, 'source':\
# 'date':processrow.date,'address':processrow.address,'dictPostComplete':processrow.dictPostComplete} # processrow.source,'date':processrow.date,'address':processrow.address,'dict_post_complete'\
d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=processrow.rating, # :processrow.dict_post_complete}
picsURL=processrow.picsURL,picsLocalpath=processrow.picsLocalpath,source=processrow.source, d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=processrow.\
date=processrow.date,address=processrow.address,dictPostComplete=processrow.dictPostComplete) rating,picsURL=processrow.picsURL,pics_local_path=processrow.pics_local_path,\
source=processrow.source,date=processrow.date,address=processrow.address,\
dict_post_complete=processrow.dict_post_complete)
print (' Row ',processrow[0],' ', processrow.name ,' added to XLS sheet') print (' Row ',processrow[0],' ', processrow.name ,' added to XLS sheet')
# Append the above Python dictionary object as a row to the existing pandas DataFrame # Append the above Python dictionary object as a row to the existing pandas DataFrame
# Using the DataFrame.append() function # Using the DataFrame.append() function
try: try:
if processrow.name in jsonposts : #outputs['posts']): if processrow.name in jsonposts : #local_outputs['posts']):
print (' Row ',processrow[0],' ', processrow.name ,' already in Database') print (' Row ',processrow[0],' ', processrow.name ,' already in Database')
else: else:
outputs['postssession'].add(d2_row) local_outputs['postssession'].add(d2_row)
outputs['postssession'].commit() local_outputs['postssession'].commit()
print (' Row ',processrow[0],' ', processrow.name ,' added to Database') print (' Row ',processrow[0],' ', processrow.name ,' added to Database')
except AttributeError as error: except AttributeError as error:
print(' Not able to write to post data table: ' , type(error)) print(' Not able to write to post data table: ' , type(error))
outputs['postssession'].rollback() local_outputs['postssession'].rollback()
raise raise
df.to_excel(env.xls) df.to_excel(env.xls)
return data return data
@@ -657,10 +653,10 @@ def write_to_database(data, local_outputs):
""" """
print('write to database ...') print('write to database ...')
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address', cols = ["name", "comment", 'rating','picsURL','pics_local_path','source','date','address',
'dictPostComplete'] 'dict_post_complete']
# cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date', # cols2 = ["num","name", "comment", 'rating','picsURL','pics_local_path','source','date',
# 'address','dictPostComplete'] # 'address','dict_post_complete']
df = pd.DataFrame(data, columns=cols) df = pd.DataFrame(data, columns=cols)
# df2 = pd.DataFrame(local_outputs['xlsdf'].values, columns=cols2) # df2 = pd.DataFrame(local_outputs['xlsdf'].values, columns=cols2)
# print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True)) # print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True))
@@ -673,17 +669,20 @@ def write_to_database(data, local_outputs):
for processrow in data: for processrow in data:
if (processrow.name in df.values): if (processrow.name in df.values):
print (' Row ',processrow.id,' ', processrow.name ,' already in database') print (' Row ',processrow.id,' ', processrow.name ,' already in database')
d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=processrow.rating, d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=processrow.\
picsURL=processrow.picsURL,picsLocalpath=processrow.picsLocalpath,source=processrow.source, rating,picsURL=processrow.picsURL,pics_local_path=processrow.pics_local_path,\
date=processrow.date,address=processrow.address,dictPostComplete=processrow.dictPostComplete) source=processrow.source,date=processrow.date,address=processrow.address,\
dict_post_complete=processrow.dict_post_complete)
elif processrow.name is not None: elif processrow.name is not None:
# Create a Python dictionary object with all the column values # Create a Python dictionary object with all the column values
# d_row = {'name':processrow.name ,'comment':processrow.comment,'rating':processrow.rating, # d_row = {'name':processrow.name ,'comment':processrow.comment,'rating':processrow.rating,
# 'picsURL':processrow.picsURL,'picsLocalpath':processrow.picsLocalpath, 'source':processrow.source, # 'picsURL':processrow.picsURL,'pics_local_path':processrow.pics_local_path, 'source':
# 'date':processrow.date,'address':processrow.address,'dictPostComplete':processrow.dictPostComplete} # processrow.source,'date':processrow.date,'address':processrow.address,'dict_post_complete'
d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=processrow.rating, # :processrow.dict_post_complete}
picsURL=processrow.picsURL,picsLocalpath=processrow.picsLocalpath,source=processrow.source, d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=processrow.\
date=processrow.date,address=processrow.address,dictPostComplete=processrow.dictPostComplete) rating,picsURL=processrow.picsURL,pics_local_path=processrow.pics_local_path,\
source=processrow.source,date=processrow.date,address=processrow.address,\
dict_post_complete=processrow.dict_post_complete)
print (' Row ',processrow[0],' ', processrow.name ,' added to XLS sheet') print (' Row ',processrow[0],' ', processrow.name ,' added to XLS sheet')
# Append the above Python dictionary object as a row to the existing pandas DataFrame # Append the above Python dictionary object as a row to the existing pandas DataFrame
# Using the DataFrame.append() function # Using the DataFrame.append() function
@@ -705,7 +704,8 @@ def write_to_database(data, local_outputs):
def database_update_row(review_name, column_name, column_value, update_style, local_outputs): def database_update_row(review_name, column_name, column_value, update_style, local_outputs):
""" """
Updates a row in the database based on the specified review name, column name, value, and update style. Updates a row in the database based on the specified review name, column name, value,
and update style.
Args: Args:
review_name (str): The name of the review to update. review_name (str): The name of the review to update.
@@ -757,7 +757,8 @@ def check_wordpress_media(filename, headers):
headers: Additional headers for the request. headers: Additional headers for the request.
Returns: Returns:
tuple: A tuple containing the ID and link of the media file if found, otherwise (False, False). tuple: A tuple containing the ID and link of the media file if found, otherwise
(False, False).
""" """
file_name_minus_extension = filename file_name_minus_extension = filename
@@ -830,7 +831,8 @@ def check_wordpress_post(postname, postdate, headers2):
headers2: Additional headers for the request. headers2: Additional headers for the request.
Returns: Returns:
tuple: A tuple containing the ID and link of the existing post if found, otherwise (False, False). tuple: A tuple containing the ID and link of the existing post if found,
otherwise (False, False).
""" """
response = requests.get(env.wpAPI+"/posts?search="+postname, headers=headers2,timeout=40) response = requests.get(env.wpAPI+"/posts?search="+postname, headers=headers2,timeout=40)
@@ -930,11 +932,14 @@ def post_to_x2(title, content, headers,date, rating, address, picslist,local_out
# Upload video # Upload video
media = client_v1.media_upload(filename=video_path) media = client_v1.media_upload(filename=video_path)
# Post tweet with video # Post tweet with video
tweetlat = (local_outputs['postssession'].query(Posts).filter(Posts.name == title).all())[0].latitude # tweetlat = (local_outputs['postssession'].query(Posts).filter(Posts.name == \
tweetlong = (local_outputs['postssession'].query(Posts).filter(Posts.name == title).all())[0].longitude # title).all())[0].latitude
# tweetlong = (local_outputs['postssession'].query(Posts).filter(Posts.name == \
# title).all())[0].longitude
if media.processing_info['state'] != 'failed': if media.processing_info['state'] != 'failed':
client_v2.create_tweet(text=status_message_short,media_ids=[media.media_id]) client_v2.create_tweet(text=status_message_short,media_ids=[media.media_id])
# client_v2.create_tweet(text=status_message_short, lat=tweetlat , long=tweetlong ,media_ids=[media.media_id]) # client_v2.create_tweet(text=status_message_short, lat=tweetlat , \
# long=tweetlong ,media_ids=[media.media_id])
else: else:
print ('Problem uploading video to twitter: ',media.processing_info['error']) print ('Problem uploading video to twitter: ',media.processing_info['error'])
return False return False
@@ -1037,57 +1042,13 @@ def post_to_threads2(title, content, headers, date, rating, address, picslist, l
else: else:
return False return False
##################################################################################################
def post_to_threads2 (title, content, headers,date, rating, address, picslist,local_outputs):
if picslist != '[]' and "montage.mp4" in picslist:
outputmontage = ''
addresshtml = re.sub(" ", ".",address)
pics = ((picslist[1:-1].replace(",","")).replace("'","")).split(" ")
video, outputmontage = make_montage_video_from_google(pics)
try:
data = title + "\n"+ address+"\nGoogle map to destination: " r"https://www.google.com/maps/dir/?api=1&destination="+addresshtml +"\n\n"+ content + "\n"+rating+"\n"+date+"\n\n"+ get_hastags(address, title,'long')+"\n\nhttps://www.joeeatswhat.com"+"\n\n"
local_outputs['instasession'].video_upload(outputmontage, data)
except AttributeError as error:
print(" An error occurred uploading video to Instagram:", type(error).__name__)
return False
#media_pk = instasession.media_pk_from_url('https://www.instagram.com/p/CGgDsi7JQdS/')
#media_path = instasession.video_download(media_pk)
# joeeatswhat = instasession.user_info_by_username('timberjoe')
# try: buildout = instagrapi.story.StoryBuilder(outputmontage,'Credits @timberjoe',[StoryMention(user=joeeatswhat)]).video(40) # seconds
# except AttributeError as error:
# print(" An error occurred uploading video to Instagram:", type(error).__name__) # An error occurred:
# try: instasession.video_upload_to_story(buildout.path,"Credits @example",mentions=buildout.mentions,links=[StoryLink(webUri='https://www.joeeatswhat.com')],medias=[StoryMedia(media_pk=outputmontage)])
# try:
# instasession.video_upload_to_story(
# outputmontage,
# "Credits @joeeatswhat",
# # mentions=[StoryMention(user='timberjoe', x=0.49892962, y=0.703125, width=0.8333333333333334, height=0.125)],
# links=[StoryLink(webUri='https://www.joeeatswhat.com')],
# # hashtags=[StoryHashtag(hashtag=get_hastags(address,title), x=0.23, y=0.32, width=0.5, height=0.22)],
# #medias=[StoryMedia(media_pk=media_pk, x=0.5, y=0.5, width=0.6, height=0.8)],
# )
# story = instasession.story_photo("path/to/photo.jpg")
# instasession.video_elements.add_link("https://www.joeeatswhat.com")
# #story.add_link("https://www.joeeatswhat.com")
# instasession.video_elements.add_hashtags(get_hastags)
# story = instasession.video_upload_to_story(outputmontage)
# story.upload()
# #instasession.video_upload_to_story(path:outputmontage,caption:content, mentions:r'@timberjoe',links:'https://www.joeeatswhat.com',hashtags: hastag) ( path: outputmontage, caption: content, mentions:['@timberjoe'], links: ['https://www.joeeatswhat.com'], hashtags: get_hastags )
# # temp = dict()
# # temp = instasession.video_upload_to_story(path=outputmontage,caption=content,mentions=r'@timberjoe',links='https://www.joeeatswhat.com',hashtags=get_hastags)
# except AttributeError as error:
# print(" An error occurred uploading video to Instagram:", type(error).__name__) # An error occurred:
# return False
return True
else:
return False
####################################################################################################### #######################################################################################################
def post_to_tiktok(title, content, headers, date, rating, address, picslist, local_outputs): def post_to_tiktok(title, content, headers, date, rating, address, picslist, local_outputs):
""" """
Posts content to TikTok with specified session ID, video file, title, hashtags, and optional scheduling. Posts content to TikTok with specified session ID, video file, title, hashtags, and optional
scheduling.
Args: Args:
title (str): The title of the video. title (str): The title of the video.
@@ -1140,7 +1101,7 @@ def post_to_tiktok(title, content, headers, date, rating, address, picslist, loc
# response = upload_video(session_id, file_path, title, tags, schedule_time) # response = upload_video(session_id, file_path, title, tags, schedule_time)
# print(response) # print(response)
####################################################################################################### ###################################################################################################
def post_to_instagram2(title, content, headers,date, rating, address, picslist,local_outputs): def post_to_instagram2(title, content, headers,date, rating, address, picslist,local_outputs):
outputmontage = '' outputmontage = ''
@@ -1150,10 +1111,17 @@ def post_to_instagram2(title, content, headers,date, rating, address, picslist,l
wpurl = attrib_list[0].wpurl wpurl = attrib_list[0].wpurl
if wpurl: if wpurl:
if business_url: if business_url:
data = title + "\n"+ address+"\n"+business_url+"\n"+"Review: "+wpurl+"\nGoogle map to destination: " r"https://www.google.com/maps/dir/?api=1&destination="+addresshtml +"\n"+"Review: "+wpurl+"\n\n"+ content + "\n"+rating+"\n"+date+"\n\n"+ get_hastags(address, title,'long')+"\n\nhttps://www.joeeatswhat.com"+"\n\n" data = title + "\n"+ address+"\n"+business_url+"\n"+"Review: "+wpurl+\
"\nGoogle map to destination: " \
r"https://www.google.com/maps/dir/?api=1&destination="\
+addresshtml +"\n"+"Review: "+wpurl+"\n\n"+ content + "\n"+rating+"\n"+date+"\n\n"\
+get_hastags(address, title,'long')+"\n\nhttps://www.joeeatswhat.com"+"\n\n"
else: else:
print (" Missing business url for : "+title+" not using it in intagram post") print (" Missing business url for : "+title+" not using it in intagram post")
data = title + "\n"+ address+"\n"+"Review: "+wpurl+"\nGoogle map to destination: " r"https://www.google.com/maps/dir/?api=1&destination="+addresshtml +"\nReview: "+wpurl+"\n\n"+ content + "\n"+rating+"\n"+date+"\n\n"+ get_hastags(address, title,'long')+"\n\nhttps://www.joeeatswhat.com"+"\n\n" data = title + "\n"+ address+"\n"+"Review: "+wpurl+"\nGoogle map to destination: " \
r"https://www.google.com/maps/dir/?api=1&destination="+addresshtml +"\nReview: "\
+wpurl+"\n\n"+ content + "\n"+rating+"\n"+date+"\n\n"+ get_hastags(address, title,\
'long')+"\n\nhttps://www.joeeatswhat.com"+"\n\n"
instasession = local_outputs['instagram'] instasession = local_outputs['instagram']
if picslist != '[]' and "montage.mp4" in picslist: if picslist != '[]' and "montage.mp4" in picslist:
#content = content + get_hastags(address, title) #content = content + get_hastags(address, title)
@@ -1227,12 +1195,14 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
addresshtml + r">"+str(address)+r"</a>" addresshtml + r">"+str(address)+r"</a>"
contentpics = '' contentpics = ''
picl = picslist[1:-1] picl = picslist[1:-1]
pic2 = picl.replace(",","")#re.sub(r',','',picl) #re.sub( r'[^a-zA-Z0-9]','',tempdate[1]) pic2 = picl.replace(",","")
#re.sub(r',','',picl) #re.sub( r'[^a-zA-Z0-9]','',tempdate[1])
pic3= pic2.replace("'","") pic3= pic2.replace("'","")
pidchop = pic3.split(" ") pidchop = pic3.split(" ")
linkslist=[] linkslist=[]
print (' Figuring out date of Post : ',title) print (' Figuring out date of Post : ',title)
formatting = '%b/%Y/%d' #specifify the formatting of the date_string. #specifify the formatting of the date_string.
# formatting = '%b/%Y/%d'
date_string = date date_string = date
if "a day" in date_string: if "a day" in date_string:
date = dt.timedelta(days=-1) date = dt.timedelta(days=-1)
@@ -1285,17 +1255,20 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
except AttributeError as error: except AttributeError as error:
print(" An error getting date occurred:",error) print(" An error getting date occurred:",error)
else: else:
formatting = '%Y-%b-%d' #specifify the formatting of the date_string. #specifify the formatting of the date_string.
formatting = '%Y-%b-%d'
month = date[:3] month = date[:3]
year = date[3:] year = date[3:]
day = '01' day = '01'
date_string = year+'-'+ month+'-'+day date_string = year+'-'+ month+'-'+day
try: try:
newdate = dt.datetime.strptime(date_string, formatting).date() newdate = dt.datetime.strptime(date_string, formatting)\
.date()
except AttributeError as error: except AttributeError as error:
print(" An error getting date occurred:",error) print(" An error getting date occurred:",error)
# try: # try:
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting)\
# .date()
# except AttributeError as error: # except AttributeError as error:
# print(" An error getting date occurred:", error) # print(" An error getting date occurred:", error)
newdate = str(newdate) newdate = str(newdate)
@@ -1434,16 +1407,17 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
else: else:
fmedia = file_id fmedia = file_id
# print ('featured_media2 = ',file_id) # print ('featured_media2 = ',file_id)
business_url_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all() business_url_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\
.all()
business_url = str(business_url_list[0].businessurl) business_url = str(business_url_list[0].businessurl)
# wpurllist = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all() # wpurllist = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()
# wpurl = wpurllist[0].wpurl # wpurl = wpurllist[0].wpurl
# if business_url or business_url is False: # if business_url or business_url is False:
status_message = str(title) + ': Business website: '+ business_url status_message = str(title) + ': Business website: '+ business_url
response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), \ response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), \
data={"content" : title+' = '+status_message+'\n\n'+content+'\n'+googleadress+'\n'+rating + contentpics,\ data={"content" : title+' = '+status_message+'\n\n'+content+'\n'+googleadress+'\n'+\
"featured_media" : fmedia,"rank_math_focus_keyword" : title }, headers=headers,\ rating+contentpics,"featured_media":fmedia,"rank_math_focus_keyword":title},\
timeout=30) headers=headers,timeout=30)
print (' ',response_piclinks) print (' ',response_piclinks)
except AttributeError as error: except AttributeError as error:
print(" An error writing images to the post " + post_response.title + ' occurred:',\ print(" An error writing images to the post " + post_response.title + ' occurred:',\
@@ -1460,17 +1434,14 @@ def process_reviews2(outputs,headers):
if env.datasource == 'xls': if env.datasource == 'xls':
rows_orig = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row))) rows_orig = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)))
rows_orig2 = (outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)) rows_orig2 = (outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row))
# rows = [({0:p.id},{1:p.name}, { 2:p.comment}, {3: p.rating}, {4:p.picsURL}, {5:p.picsLocalpath},{6:p.source},{7:p.date},{8:p.address},{9:p.dictPostComplete}) for p in rows_orig] # rows = [({0:p.id},{1:p.name}, { 2:p.comment}, {3: p.rating}, {4:p.picsURL},\
# {5:p.pics_local_path},{6:p.source},{7:p.date},{8:p.address},{9:p.dict_post_complete})\
# for p in rows_orig]
# rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row))) # rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)))
rows = outputs['data'] rows = outputs['data']
else: else:
rows_orig = (outputs['posts'])# rows = [{(p.name), ( p.comment), ( p.rating), (p.picsURL), (p.picsLocalpath),(p.source),(p.date),(p.address),(p.dictPostComplete)} for p in rows_orig]# rows = [{1: p.name, 2: p.comment, 3: p.rating, 4:p.picsURL, 5:p.picsLocalpath,6:p.source,7:p.date,8:p.address,9:p.dictPostComplete} for p in rows_orig] rows_orig = (outputs['posts'])
rows = rows_orig rows = rows_orig
# rows = [({0:p.id},{1:p.name}, { 2:p.comment}, {3: p.rating}, {4:p.picsURL}, {5:p.picsLocalpath},{6:p.source},{7:p.date},{8:p.address},{9:p.dictPostComplete}) for p in rows_orig]
#rows = list ((outputs['postsession'].name))
# ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address','dictPostComplete']
# print ('xls rows: ',list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row))))
# print ('Database rows: ', list(outputs['posts']))
if env.google: if env.google:
print('Configuration says to update google Reviews prior to processing them') print('Configuration says to update google Reviews prior to processing them')
options = webdriver.ChromeOptions() options = webdriver.ChromeOptions()
@@ -1519,7 +1490,7 @@ def process_reviews2(outputs,headers):
if processrow.name != "name": # Skip header line of xls sheet if processrow.name != "name": # Skip header line of xls sheet
outputs['postssession'].query(Posts).filter(Posts.name == processrow.name).update({"google" : 1}) outputs['postssession'].query(Posts).filter(Posts.name == processrow.name).update({"google" : 1})
print ("Processing : ",processrow.name) print ("Processing : ",processrow.name)
writtento = (ast.literal_eval(processrow.dictPostComplete)) writtento = (ast.literal_eval(processrow.dict_post_complete))
# Check to see if the website has already been written to according to the xls sheet,\ # Check to see if the website has already been written to according to the xls sheet,\
# if it has not... then process # if it has not... then process
if (writtento["web"] == 0 or writtento["instagram"]==0 or writtento["facebook"]==0 or \ if (writtento["web"] == 0 or writtento["instagram"]==0 or writtento["facebook"]==0 or \
@@ -1541,11 +1512,11 @@ def process_reviews2(outputs,headers):
webcount = process_socials("web",processrow,headers,"post_to_wordpress",\ webcount = process_socials("web",processrow,headers,"post_to_wordpress",\
webcount, outputs) webcount, outputs)
if env.instagram: if env.instagram:
instagramcount = process_socials("instagram",processrow,headers,"post_to_instagram2",\ instagramcount = process_socials("instagram",processrow,headers,\
instagramcount, outputs) "post_to_instagram2",instagramcount, outputs)
if env.facebook: if env.facebook:
facebookcount = process_socials("facebook",processrow,headers,"post_facebook3",\ facebookcount = process_socials("facebook",processrow,headers,"post_facebook3"\
facebookcount, outputs) ,facebookcount, outputs)
if env.xtwitter: if env.xtwitter:
xtwittercount = process_socials("xtwitter",processrow,headers,"post_to_x2",\ xtwittercount = process_socials("xtwitter",processrow,headers,"post_to_x2",\
xtwittercount, outputs) xtwittercount, outputs)
@@ -1570,7 +1541,7 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
Returns: Returns:
Count of the social that was selected Count of the social that was selected
""" """
writtento = (ast.literal_eval(social_post.dictPostComplete)) writtento = (ast.literal_eval(social_post.dict_post_complete))
if (len(local_outputs['postssession'].query(Posts).filter(Posts.name == social_post.name,getattr\ if (len(local_outputs['postssession'].query(Posts).filter(Posts.name == social_post.name,getattr\
(Posts, social_name) is True).all())==0) and ((local_outputs['postssession'].query(Posts).\ (Posts, social_name) is True).all())==0) and ((local_outputs['postssession'].query(Posts).\
filter(Posts.name == social_post.name).all()[0].wpurl != None)or social_name \ filter(Posts.name == social_post.name).all()[0].wpurl != None)or social_name \
@@ -1580,11 +1551,11 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
print(' Starting to generate ',social_name,' post') print(' Starting to generate ',social_name,' post')
new_social_post = eval(sub_process)(social_post.name, social_post.comment,\ new_social_post = eval(sub_process)(social_post.name, social_post.comment,\
headers, social_post.date, social_post.rating, social_post.address,\ headers, social_post.date, social_post.rating, social_post.address,\
social_post.picsLocalpath,local_outputs ) social_post.pics_local_path,local_outputs )
try: try:
print (' Start generating content to post to : ',social_post.name) print (' Start generating content to post to : ',social_post.name)
writtento[social_name] = 1 writtento[social_name] = 1
social_post.dictPostComplete = str(writtento) social_post.dict_post_complete = str(writtento)
except AttributeError as error: except AttributeError as error:
print(" An error occurred setting value to go into Excel file:", type(error)\ print(" An error occurred setting value to go into Excel file:", type(error)\
.__name__) .__name__)
@@ -1595,7 +1566,7 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
print(' write to xls for :',social_name) print(' write to xls for :',social_name)
local_outputs['datawb'].save(env.xls) local_outputs['datawb'].save(env.xls)
print(' Successfully wrote to xls for social - ',social_name) print(' Successfully wrote to xls for social - ',social_name)
# local_outputs['postssession'].update('dictPostComplete = '+str(writtento)+\ # local_outputs['postssession'].update('dict_post_complete = '+str(writtento)+\
# ' where name == '+social_post.name) # ' where name == '+social_post.name)
# local_outputs['postssession'].commit() # local_outputs['postssession'].commit()
except AttributeError as error: except AttributeError as error:
@@ -1611,7 +1582,7 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
except AttributeError as error: except AttributeError as error:
print(' Error writing social - ',social_name,' post : ',error,social_post.name,\ print(' Error writing social - ',social_name,' post : ',error,social_post.name,\
social_post.comment, local_outputs,social_post.date, social_post.rating,\ social_post.comment, local_outputs,social_post.date, social_post.rating,\
social_post.address, social_post.picsLocalpath, writtento[social_name],\ social_post.address, social_post.pics_local_path, writtento[social_name],\
type(error).__name__ ) type(error).__name__ )
else: else:
print (' Exceeded the number of social - ',social_name,' posts per run, skipping',\ print (' Exceeded the number of social - ',social_name,' posts per run, skipping',\
@@ -1627,9 +1598,6 @@ if __name__ == "__main__":
preload() preload()
print('making connections ...') print('making connections ...')
outputs,headers = get_auth_connect() outputs,headers = get_auth_connect()
if env.datasource == 'db':
process_reviews2(outputs,headers)
else:
process_reviews2(outputs,headers) process_reviews2(outputs,headers)
print('Done!') print('Done!')