admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed problems with closing database and fixed Instagram

Browse Source
This commit is contained in:
Joe Steele
2024-12-19 11:01:53 -05:00
parent e5b8f829c7
commit 778ce37c93
1 changed files with 39 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+38 -80
View File
@@ -141,10 +141,8 @@ def get_auth_connect():
if env.mariadb:
print('Connecting to MariaDB for configuration and storage')
#from sqlalchemy import create_engine
engine = sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+
# engine = sqlalchemy.create_engine("mysql+mysqldb://"+env.mariadbuser+":"+env.mariadbpass+
"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)
# "@"+env.mariadbserver+"/"+env.mariadbdb+"?charset=utf8mb4", echo=False)
engine = sqlalchemy.create_engine("mysql+mysqldb://"+env.mariadbuser+":"+env.mariadbpass+
"@"+env.mariadbserver+"/"+env.mariadbdb+"?charset=utf8mb4", echo=False)
Session = sqlalchemy.orm.sessionmaker()
Session.configure(bind=engine)
session = Session()
@@ -156,8 +154,6 @@ def get_auth_connect():
posts = session.query(Posts).all()
connections['posts'] = posts
connections['postssession'] = session
# sqlalchemy.orm.sessionmaker().configure(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
# connections['postssession'] = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
if env.data:
print(' loading XLS content data source ...')
if os.path.exists(env.xls):
@@ -480,12 +476,6 @@ def get_google_data(driver, local_outputs):
Returns:
list: A list of data extracted from Google Maps.
"""
engine = sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+
"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False, pool_recycle=3600)
Session = sqlalchemy.orm.sessionmaker()
Session.configure(bind=engine)
sqlconnection = Session()
#sqlconnection = sqlalchemy.orm.sessionmaker().configure(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
print(' get google data...', end ="")
# Click on more botton on each text reviews
more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq')
@@ -535,7 +525,7 @@ def get_google_data(driver, local_outputs):
score = "Unknown"
print ('Error: ',error)
# Grab more info from google maps entry on this particular review
if (len(sqlconnection.query(Posts).filter(Posts.name == name,Posts.google\
if (len(local_outputs['postssession'].query(Posts).filter(Posts.name == name,Posts.google\
is not True).all()) == 0 or env.forcegoogleupdate) and env.block_google_maps is not\
True:
gmaps = googlemaps.Client(env.googleapipass)
@@ -563,7 +553,7 @@ def get_google_data(driver, local_outputs):
else:
if env.forcegoogleupdate:
database_update_row(name,"google",True,"forceall",local_outputs)
if len(sqlconnection.query(Posts).filter(Posts.name == name,\
if len(local_outputs['postssession'].query(Posts).filter(Posts.name == name,\
Posts.google is not True).all()) > 0:
print (' Post was already in database, skipping update unless overriden')
# else:
@@ -653,7 +643,6 @@ def get_google_data(driver, local_outputs):
dict_post_complete])
if env.force_web_create:
database_update_row(name,"visitdate",visitdate,"forceall",local_outputs)
sqlconnection.close()
return lst_data
##################################################################################################
@@ -773,13 +762,6 @@ def write_to_database(data, local_outputs):
"""
column_list=[]
print('write to database ...')
#sqlconnection = sqlalchemy.orm.sessionmaker().configure(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
engine = sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+
"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)
Session = sqlalchemy.orm.sessionmaker()
Session.configure(bind=engine)
sqlconnection = Session()
for x in inspect.getmembers(Posts):
# if not (x[0].startswith('_') or 'metadata' in x[0] or 'registry' in x[0] or 'id' in x[0]):
if not (x[0].startswith('_') or 'metadata' in x[0] or 'registry' in x[0]):
@@ -835,7 +817,7 @@ def write_to_database(data, local_outputs):
# # Using the DataFrame.append() function
try:
getdata2 = sqlconnection.query(Posts).filter(Posts.name \
getdata2 = local_outputs['postssession'].query(Posts).filter(Posts.name \
== processrow[0])
if getdata2.count() >0:
print (' Row ',processrow[0],' already in Database')
@@ -846,24 +828,23 @@ def write_to_database(data, local_outputs):
'source':processrow[5],'date':processrow[6],'address':processrow[7],\
'dictPostComplete':str(processrow[8])}
print (' Row ',processrow[0],' updated in Database')
sqlconnection.query(Posts).filter(Posts.name == processrow[0]).\
local_outputs['postssession'].query(Posts).filter(Posts.name == processrow[0]).\
update(d2_dict)
sqlconnection.commit()
local_outputs['postssession'].commit()
else:
d2_row = Posts(name=processrow[0] ,comment=processrow[1],rating=processrow[2]\
,picsURL=str(processrow[3]),picsLocalpath=str(processrow[4]),\
source=processrow[5],date=processrow[6],address=processrow[7],\
dictPostComplete=str(processrow[8]))
print (' Row ',processrow[0],' added to Database')
sqlconnection.add(d2_row)
sqlconnection.commit()
local_outputs['postssession'].add(d2_row)
local_outputs['postssession'].commit()
print (' Row ',processrow[0],' added to Database')
except AttributeError as error:
print(' Not able to write to post data table: ' , type(error))
sqlconnection.rollback()
local_outputs['postssession'].rollback()
raise
processrow = []
sqlconnection.close()
df.to_excel(env.xls)
return data
@@ -884,38 +865,34 @@ def database_update_row(review_name, column_name, column_value, update_style, lo
Returns:
bool: True if the row was successfully updated, False otherwise.
"""
# connections['postssession'] = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
# sqlconnection.commit()
#sqlconnection.close()
try:
if update_style == "forceall" and column_value is not False:
sqlconnection.query(Posts).filter(Posts.name == review_name).update\
local_outputs['postssession'].query(Posts).filter(Posts.name == review_name).update\
({column_name : column_value})
print (' Force Updated ',column_name, ' to: ',column_value)
elif update_style == "onlyempty" and column_value is not False:
postval = sqlconnection.query(Posts).filter(Posts.name == review_name,\
postval = local_outputs['postssession'].query(Posts).filter(Posts.name == review_name,\
getattr(Posts,column_name).is_not(null())).all()
if len(postval) == 0 :
sqlconnection.query(Posts).filter(Posts.name == review_name).\
local_outputs['postssession'].query(Posts).filter(Posts.name == review_name).\
update({column_name : column_value})
print (' Updated blank ',postval ,' on value',column_name, ' to: ',\
column_value)
elif update_style == "toggletrue":
postval = sqlconnection.query(Posts).filter(Posts.name == review_name\
postval = local_outputs['postssession'].query(Posts).filter(Posts.name == review_name\
,getattr(Posts,column_name).is_not(1)).all()
sqlconnection.query(Posts).filter(Posts.name == review_name).update\
local_outputs['postssession'].query(Posts).filter(Posts.name == review_name).update\
({column_name : "1"})
print (' Updated ',column_name, ' on value: ',postval[0].column_value, ' to: ',\
column_value)
except AttributeError as error:
print(" Not able to write to post data table to update ",review_name," ",column_name,"\
to: ",column_value , type(error), error)
sqlconnection.rollback()
local_outputs['postssession'].rollback()
raise
# else:
sqlconnection.commit()
sqlconnection.close()
local_outputs['postssession'].commit()
#time.sleep(env.facebooksleep)
return True
@@ -1147,10 +1124,8 @@ def post_to_x2(title, content, headers,date, rating, address, picslist,local_out
if imgs_vid:
try:
# Message to post along with the video
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
attrib_list = sqlconnection.query(Posts).filter(Posts.name == title)\
attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\
.all()
sqlconnection.close()
if wpurl := attrib_list[0].wpurl:
if business_url := attrib_list[0].businessurl:
status_message = (
@@ -1194,9 +1169,7 @@ def post_to_x2(title, content, headers,date, rating, address, picslist,local_out
def post_to_threads(title, content,headers, date, rating, address, picslist, local_outputs):
img_list = ((picslist[1:-1]).replace("'","")).split(",")
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
attrib_list = sqlconnection.query(Posts).filter(Posts.name == title).all()
sqlconnection.close()
attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()
business_url = attrib_list[0].businessurl
wpurl = attrib_list[0].wpurl
auth_token = env.threads_access_token
@@ -1255,9 +1228,7 @@ def post_facebook3(title, content,headers, date, rating, address, picslist, loca
auth_token = env.facebookpass
imgs_vid = []
img_list = pics
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
attrib_list = sqlconnection.query(Posts).filter(Posts.name == title).all()
sqlconnection.close()
attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()
business_url = attrib_list[0].businessurl
wpurl = attrib_list[0].wpurl
if wpurl and (attrib_list[0].picsLocalpath != '[]'):
@@ -1554,9 +1525,7 @@ def post_to_instagram2(title, content, headers, date, rating, address, picslist,
outputmontage = ''
addresshtml = re.sub(" ", ".",address)
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
attrib_list = sqlconnection.query(Posts).filter(Posts.name == title).all()
sqlconnection.close()
attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()
if wpurl := attrib_list[0].wpurl:
if business_url := attrib_list[0].businessurl:
data = title + "\n"+ address+"\n"+business_url+"\n"+"Review: "+wpurl+\
@@ -1579,8 +1548,10 @@ def post_to_instagram2(title, content, headers, date, rating, address, picslist,
if "montage.mp4" in loop:
outputmontage = loop
break
pathoutputmontage = outputmontage[1:-1]
if pathlib.Path.exists(pathoutputmontage):
pathoutputmontage = outputmontage
# pathoutputmontage = outputmontage[1:-1]
# if pathlib.Path.exists(pathoutputmontage):
if Path(pathoutputmontage).exists():
try:
duration, fps, (width, height)=with_moviepy(outputmontage)
if duration < 60:
@@ -1733,10 +1704,8 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
#newdate2 = dt.datetime.strptime(str(newdate), formatting).date()
dateparts = (str(newdate)).split("-")
dateparts2 = dateparts[2].split(" ")
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
visitdate2 = sqlconnection.query(Posts).filter(Posts.name == title)\
visitdate2 = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\
.all()[0].visitdate
sqlconnection.close()
if visitdate != visitdate2:
database_update_row(title,"visitdate",visitdate,"forceall",local_outputs)
print (f'UPDATED: {visitdate2} to {visitdate} for {title}')
@@ -1886,10 +1855,8 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
if linkslist:
fmedia = linkslist[0]['file_id']
# print ('featured_media2 = ',file_id)
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
business_url_list = sqlconnection.query(Posts).filter(Posts.name == \
business_url_list = local_outputs['postssession'].query(Posts).filter(Posts.name == \
title).all()
sqlconnection.close()
business_url = "<a href="+str(business_url_list[0].businessurl)+">"+\
title+"</a>"
# wpurllist = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()
@@ -2185,10 +2152,8 @@ def create_wordpress_post(newdate2, picchop, title, address, headers, post_id, l
else:
if linkslist:
fmedia = linkslist[0]['file_id']
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
business_url_list = sqlconnection.query(Posts).filter(Posts.name == title)\
business_url_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\
.all()
sqlconnection.close()
business_url = f"<a href={str(business_url_list[0].businessurl)}>{title}</a>"
status_message = f'{str(title)}: Business website: {business_url}'
response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), \
@@ -2425,12 +2390,6 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
Returns:
Count of the social that was selected
"""
#sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
engine = sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+
"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)
Session = sqlalchemy.orm.sessionmaker()
Session.configure(bind=engine)
sqlconnection = Session()
writtento = ast.literal_eval(social_post.dictPostComplete)
if (len(local_outputs['postssession'].query(Posts).filter(Posts.name == social_post.name,\
getattr(Posts, social_name) is True).all())==0) and ((local_outputs['postssession'].\
@@ -2450,7 +2409,7 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
except AttributeError as error:
print(" An error occurred setting value to go into Excel file:", type(error)\
.__name__)
print (' Success Posting to : '+social_post.name)
print (' Success Posting to xtwitter: '+social_post.name)
if new_social_post:
social_count +=1
try:
@@ -2468,21 +2427,20 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
except AttributeError as error:
print(" An error occurred writing database", type(error).__name__)
except AttributeError as error:
# print(' Error writing social - ',social_name,' post : ',error,social_post.name,\
# social_post.comment, local_outputs,social_post.date, social_post.rating,\
# social_post.address, social_post.picsLocalpath, writtento[social_name],\
# type(error).__name__ )
print(' Error writing social - ',social_name,' post : ',error,social_post.name,\
type(error).__name__ )
# social_post.comment, local_outputs,social_post.date, social_post.rating,\
# social_post.address, social_post.picsLocalpath, writtento[social_name],\
# type(error).__name__ )
# social_post.comment, local_outputs,social_post.date, social_post.rating,\
# social_post.address, social_post.picsLocalpath, writtento[social_name],\
type(error).__name__ )
# except AttributeError as error:
# print(' Error writing social - ',social_name,' post : ',error,social_post.name,\
# social_post.comment, local_outputs,social_post.date, social_post.rating,\
# social_post.address, social_post.picsLocalpath, writtento[social_name],\
# type(error).__name__ )
else:
print (' Exceeded the number of social - ',social_name,' posts per run, skipping',\
social_post.name)
else:
print (' ',social_name,': Skipping posting for ',social_post.name,' previously written')
sqlconnection.close()
return social_count
##################################################################################################