admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed problems with closing database and fixed Instagram

Browse Source
This commit is contained in:
Joe Steele
2024-12-19 11:01:53 -05:00
parent e5b8f829c7
commit 778ce37c93
1 changed files with 39 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+33 -75
View File
@@ -141,10 +141,8 @@ def get_auth_connect():
if env.mariadb: if env.mariadb:
print('Connecting to MariaDB for configuration and storage') print('Connecting to MariaDB for configuration and storage')
#from sqlalchemy import create_engine #from sqlalchemy import create_engine
engine = sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+ engine = sqlalchemy.create_engine("mysql+mysqldb://"+env.mariadbuser+":"+env.mariadbpass+
# engine = sqlalchemy.create_engine("mysql+mysqldb://"+env.mariadbuser+":"+env.mariadbpass+ "@"+env.mariadbserver+"/"+env.mariadbdb+"?charset=utf8mb4", echo=False)
"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)
# "@"+env.mariadbserver+"/"+env.mariadbdb+"?charset=utf8mb4", echo=False)
Session = sqlalchemy.orm.sessionmaker() Session = sqlalchemy.orm.sessionmaker()
Session.configure(bind=engine) Session.configure(bind=engine)
session = Session() session = Session()
@@ -156,8 +154,6 @@ def get_auth_connect():
posts = session.query(Posts).all() posts = session.query(Posts).all()
connections['posts'] = posts connections['posts'] = posts
connections['postssession'] = session connections['postssession'] = session
# sqlalchemy.orm.sessionmaker().configure(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
# connections['postssession'] = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
if env.data: if env.data:
print(' loading XLS content data source ...') print(' loading XLS content data source ...')
if os.path.exists(env.xls): if os.path.exists(env.xls):
@@ -480,12 +476,6 @@ def get_google_data(driver, local_outputs):
Returns: Returns:
list: A list of data extracted from Google Maps. list: A list of data extracted from Google Maps.
""" """
engine = sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+
"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False, pool_recycle=3600)
Session = sqlalchemy.orm.sessionmaker()
Session.configure(bind=engine)
sqlconnection = Session()
#sqlconnection = sqlalchemy.orm.sessionmaker().configure(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
print(' get google data...', end ="") print(' get google data...', end ="")
# Click on more botton on each text reviews # Click on more botton on each text reviews
more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq') more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq')
@@ -535,7 +525,7 @@ def get_google_data(driver, local_outputs):
score = "Unknown" score = "Unknown"
print ('Error: ',error) print ('Error: ',error)
# Grab more info from google maps entry on this particular review # Grab more info from google maps entry on this particular review
if (len(sqlconnection.query(Posts).filter(Posts.name == name,Posts.google\ if (len(local_outputs['postssession'].query(Posts).filter(Posts.name == name,Posts.google\
is not True).all()) == 0 or env.forcegoogleupdate) and env.block_google_maps is not\ is not True).all()) == 0 or env.forcegoogleupdate) and env.block_google_maps is not\
True: True:
gmaps = googlemaps.Client(env.googleapipass) gmaps = googlemaps.Client(env.googleapipass)
@@ -563,7 +553,7 @@ def get_google_data(driver, local_outputs):
else: else:
if env.forcegoogleupdate: if env.forcegoogleupdate:
database_update_row(name,"google",True,"forceall",local_outputs) database_update_row(name,"google",True,"forceall",local_outputs)
if len(sqlconnection.query(Posts).filter(Posts.name == name,\ if len(local_outputs['postssession'].query(Posts).filter(Posts.name == name,\
Posts.google is not True).all()) > 0: Posts.google is not True).all()) > 0:
print (' Post was already in database, skipping update unless overriden') print (' Post was already in database, skipping update unless overriden')
# else: # else:
@@ -653,7 +643,6 @@ def get_google_data(driver, local_outputs):
dict_post_complete]) dict_post_complete])
if env.force_web_create: if env.force_web_create:
database_update_row(name,"visitdate",visitdate,"forceall",local_outputs) database_update_row(name,"visitdate",visitdate,"forceall",local_outputs)
sqlconnection.close()
return lst_data return lst_data
################################################################################################## ##################################################################################################
@@ -773,13 +762,6 @@ def write_to_database(data, local_outputs):
""" """
column_list=[] column_list=[]
print('write to database ...') print('write to database ...')
#sqlconnection = sqlalchemy.orm.sessionmaker().configure(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
engine = sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+
"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)
Session = sqlalchemy.orm.sessionmaker()
Session.configure(bind=engine)
sqlconnection = Session()
for x in inspect.getmembers(Posts): for x in inspect.getmembers(Posts):
# if not (x[0].startswith('_') or 'metadata' in x[0] or 'registry' in x[0] or 'id' in x[0]): # if not (x[0].startswith('_') or 'metadata' in x[0] or 'registry' in x[0] or 'id' in x[0]):
if not (x[0].startswith('_') or 'metadata' in x[0] or 'registry' in x[0]): if not (x[0].startswith('_') or 'metadata' in x[0] or 'registry' in x[0]):
@@ -835,7 +817,7 @@ def write_to_database(data, local_outputs):
# # Using the DataFrame.append() function # # Using the DataFrame.append() function
try: try:
getdata2 = sqlconnection.query(Posts).filter(Posts.name \ getdata2 = local_outputs['postssession'].query(Posts).filter(Posts.name \
== processrow[0]) == processrow[0])
if getdata2.count() >0: if getdata2.count() >0:
print (' Row ',processrow[0],' already in Database') print (' Row ',processrow[0],' already in Database')
@@ -846,24 +828,23 @@ def write_to_database(data, local_outputs):
'source':processrow[5],'date':processrow[6],'address':processrow[7],\ 'source':processrow[5],'date':processrow[6],'address':processrow[7],\
'dictPostComplete':str(processrow[8])} 'dictPostComplete':str(processrow[8])}
print (' Row ',processrow[0],' updated in Database') print (' Row ',processrow[0],' updated in Database')
sqlconnection.query(Posts).filter(Posts.name == processrow[0]).\ local_outputs['postssession'].query(Posts).filter(Posts.name == processrow[0]).\
update(d2_dict) update(d2_dict)
sqlconnection.commit() local_outputs['postssession'].commit()
else: else:
d2_row = Posts(name=processrow[0] ,comment=processrow[1],rating=processrow[2]\ d2_row = Posts(name=processrow[0] ,comment=processrow[1],rating=processrow[2]\
,picsURL=str(processrow[3]),picsLocalpath=str(processrow[4]),\ ,picsURL=str(processrow[3]),picsLocalpath=str(processrow[4]),\
source=processrow[5],date=processrow[6],address=processrow[7],\ source=processrow[5],date=processrow[6],address=processrow[7],\
dictPostComplete=str(processrow[8])) dictPostComplete=str(processrow[8]))
print (' Row ',processrow[0],' added to Database') print (' Row ',processrow[0],' added to Database')
sqlconnection.add(d2_row) local_outputs['postssession'].add(d2_row)
sqlconnection.commit() local_outputs['postssession'].commit()
print (' Row ',processrow[0],' added to Database') print (' Row ',processrow[0],' added to Database')
except AttributeError as error: except AttributeError as error:
print(' Not able to write to post data table: ' , type(error)) print(' Not able to write to post data table: ' , type(error))
sqlconnection.rollback() local_outputs['postssession'].rollback()
raise raise
processrow = [] processrow = []
sqlconnection.close()
df.to_excel(env.xls) df.to_excel(env.xls)
return data return data
@@ -884,38 +865,34 @@ def database_update_row(review_name, column_name, column_value, update_style, lo
Returns: Returns:
bool: True if the row was successfully updated, False otherwise. bool: True if the row was successfully updated, False otherwise.
""" """
# connections['postssession'] = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
# sqlconnection.commit()
#sqlconnection.close()
try: try:
if update_style == "forceall" and column_value is not False: if update_style == "forceall" and column_value is not False:
sqlconnection.query(Posts).filter(Posts.name == review_name).update\ local_outputs['postssession'].query(Posts).filter(Posts.name == review_name).update\
({column_name : column_value}) ({column_name : column_value})
print (' Force Updated ',column_name, ' to: ',column_value) print (' Force Updated ',column_name, ' to: ',column_value)
elif update_style == "onlyempty" and column_value is not False: elif update_style == "onlyempty" and column_value is not False:
postval = sqlconnection.query(Posts).filter(Posts.name == review_name,\ postval = local_outputs['postssession'].query(Posts).filter(Posts.name == review_name,\
getattr(Posts,column_name).is_not(null())).all() getattr(Posts,column_name).is_not(null())).all()
if len(postval) == 0 : if len(postval) == 0 :
sqlconnection.query(Posts).filter(Posts.name == review_name).\ local_outputs['postssession'].query(Posts).filter(Posts.name == review_name).\
update({column_name : column_value}) update({column_name : column_value})
print (' Updated blank ',postval ,' on value',column_name, ' to: ',\ print (' Updated blank ',postval ,' on value',column_name, ' to: ',\
column_value) column_value)
elif update_style == "toggletrue": elif update_style == "toggletrue":
postval = sqlconnection.query(Posts).filter(Posts.name == review_name\ postval = local_outputs['postssession'].query(Posts).filter(Posts.name == review_name\
,getattr(Posts,column_name).is_not(1)).all() ,getattr(Posts,column_name).is_not(1)).all()
sqlconnection.query(Posts).filter(Posts.name == review_name).update\ local_outputs['postssession'].query(Posts).filter(Posts.name == review_name).update\
({column_name : "1"}) ({column_name : "1"})
print (' Updated ',column_name, ' on value: ',postval[0].column_value, ' to: ',\ print (' Updated ',column_name, ' on value: ',postval[0].column_value, ' to: ',\
column_value) column_value)
except AttributeError as error: except AttributeError as error:
print(" Not able to write to post data table to update ",review_name," ",column_name,"\ print(" Not able to write to post data table to update ",review_name," ",column_name,"\
to: ",column_value , type(error), error) to: ",column_value , type(error), error)
sqlconnection.rollback() local_outputs['postssession'].rollback()
raise raise
# else: # else:
sqlconnection.commit() local_outputs['postssession'].commit()
sqlconnection.close()
#time.sleep(env.facebooksleep) #time.sleep(env.facebooksleep)
return True return True
@@ -1147,10 +1124,8 @@ def post_to_x2(title, content, headers,date, rating, address, picslist,local_out
if imgs_vid: if imgs_vid:
try: try:
# Message to post along with the video # Message to post along with the video
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\
attrib_list = sqlconnection.query(Posts).filter(Posts.name == title)\
.all() .all()
sqlconnection.close()
if wpurl := attrib_list[0].wpurl: if wpurl := attrib_list[0].wpurl:
if business_url := attrib_list[0].businessurl: if business_url := attrib_list[0].businessurl:
status_message = ( status_message = (
@@ -1194,9 +1169,7 @@ def post_to_x2(title, content, headers,date, rating, address, picslist,local_out
def post_to_threads(title, content,headers, date, rating, address, picslist, local_outputs): def post_to_threads(title, content,headers, date, rating, address, picslist, local_outputs):
img_list = ((picslist[1:-1]).replace("'","")).split(",") img_list = ((picslist[1:-1]).replace("'","")).split(",")
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()
attrib_list = sqlconnection.query(Posts).filter(Posts.name == title).all()
sqlconnection.close()
business_url = attrib_list[0].businessurl business_url = attrib_list[0].businessurl
wpurl = attrib_list[0].wpurl wpurl = attrib_list[0].wpurl
auth_token = env.threads_access_token auth_token = env.threads_access_token
@@ -1255,9 +1228,7 @@ def post_facebook3(title, content,headers, date, rating, address, picslist, loca
auth_token = env.facebookpass auth_token = env.facebookpass
imgs_vid = [] imgs_vid = []
img_list = pics img_list = pics
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()
attrib_list = sqlconnection.query(Posts).filter(Posts.name == title).all()
sqlconnection.close()
business_url = attrib_list[0].businessurl business_url = attrib_list[0].businessurl
wpurl = attrib_list[0].wpurl wpurl = attrib_list[0].wpurl
if wpurl and (attrib_list[0].picsLocalpath != '[]'): if wpurl and (attrib_list[0].picsLocalpath != '[]'):
@@ -1554,9 +1525,7 @@ def post_to_instagram2(title, content, headers, date, rating, address, picslist,
outputmontage = '' outputmontage = ''
addresshtml = re.sub(" ", ".",address) addresshtml = re.sub(" ", ".",address)
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()
attrib_list = sqlconnection.query(Posts).filter(Posts.name == title).all()
sqlconnection.close()
if wpurl := attrib_list[0].wpurl: if wpurl := attrib_list[0].wpurl:
if business_url := attrib_list[0].businessurl: if business_url := attrib_list[0].businessurl:
data = title + "\n"+ address+"\n"+business_url+"\n"+"Review: "+wpurl+\ data = title + "\n"+ address+"\n"+business_url+"\n"+"Review: "+wpurl+\
@@ -1579,8 +1548,10 @@ def post_to_instagram2(title, content, headers, date, rating, address, picslist,
if "montage.mp4" in loop: if "montage.mp4" in loop:
outputmontage = loop outputmontage = loop
break break
pathoutputmontage = outputmontage[1:-1] pathoutputmontage = outputmontage
if pathlib.Path.exists(pathoutputmontage): # pathoutputmontage = outputmontage[1:-1]
# if pathlib.Path.exists(pathoutputmontage):
if Path(pathoutputmontage).exists():
try: try:
duration, fps, (width, height)=with_moviepy(outputmontage) duration, fps, (width, height)=with_moviepy(outputmontage)
if duration < 60: if duration < 60:
@@ -1733,10 +1704,8 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
#newdate2 = dt.datetime.strptime(str(newdate), formatting).date() #newdate2 = dt.datetime.strptime(str(newdate), formatting).date()
dateparts = (str(newdate)).split("-") dateparts = (str(newdate)).split("-")
dateparts2 = dateparts[2].split(" ") dateparts2 = dateparts[2].split(" ")
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) visitdate2 = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\
visitdate2 = sqlconnection.query(Posts).filter(Posts.name == title)\
.all()[0].visitdate .all()[0].visitdate
sqlconnection.close()
if visitdate != visitdate2: if visitdate != visitdate2:
database_update_row(title,"visitdate",visitdate,"forceall",local_outputs) database_update_row(title,"visitdate",visitdate,"forceall",local_outputs)
print (f'UPDATED: {visitdate2} to {visitdate} for {title}') print (f'UPDATED: {visitdate2} to {visitdate} for {title}')
@@ -1886,10 +1855,8 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
if linkslist: if linkslist:
fmedia = linkslist[0]['file_id'] fmedia = linkslist[0]['file_id']
# print ('featured_media2 = ',file_id) # print ('featured_media2 = ',file_id)
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) business_url_list = local_outputs['postssession'].query(Posts).filter(Posts.name == \
business_url_list = sqlconnection.query(Posts).filter(Posts.name == \
title).all() title).all()
sqlconnection.close()
business_url = "<a href="+str(business_url_list[0].businessurl)+">"+\ business_url = "<a href="+str(business_url_list[0].businessurl)+">"+\
title+"</a>" title+"</a>"
# wpurllist = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all() # wpurllist = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()
@@ -2185,10 +2152,8 @@ def create_wordpress_post(newdate2, picchop, title, address, headers, post_id, l
else: else:
if linkslist: if linkslist:
fmedia = linkslist[0]['file_id'] fmedia = linkslist[0]['file_id']
sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) business_url_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\
business_url_list = sqlconnection.query(Posts).filter(Posts.name == title)\
.all() .all()
sqlconnection.close()
business_url = f"<a href={str(business_url_list[0].businessurl)}>{title}</a>" business_url = f"<a href={str(business_url_list[0].businessurl)}>{title}</a>"
status_message = f'{str(title)}: Business website: {business_url}' status_message = f'{str(title)}: Business website: {business_url}'
response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), \ response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), \
@@ -2425,12 +2390,6 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
Returns: Returns:
Count of the social that was selected Count of the social that was selected
""" """
#sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False))
engine = sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+
"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)
Session = sqlalchemy.orm.sessionmaker()
Session.configure(bind=engine)
sqlconnection = Session()
writtento = ast.literal_eval(social_post.dictPostComplete) writtento = ast.literal_eval(social_post.dictPostComplete)
if (len(local_outputs['postssession'].query(Posts).filter(Posts.name == social_post.name,\ if (len(local_outputs['postssession'].query(Posts).filter(Posts.name == social_post.name,\
getattr(Posts, social_name) is True).all())==0) and ((local_outputs['postssession'].\ getattr(Posts, social_name) is True).all())==0) and ((local_outputs['postssession'].\
@@ -2450,7 +2409,7 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
except AttributeError as error: except AttributeError as error:
print(" An error occurred setting value to go into Excel file:", type(error)\ print(" An error occurred setting value to go into Excel file:", type(error)\
.__name__) .__name__)
print (' Success Posting to : '+social_post.name) print (' Success Posting to xtwitter: '+social_post.name)
if new_social_post: if new_social_post:
social_count +=1 social_count +=1
try: try:
@@ -2468,12 +2427,12 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
except AttributeError as error: except AttributeError as error:
print(" An error occurred writing database", type(error).__name__) print(" An error occurred writing database", type(error).__name__)
except AttributeError as error: except AttributeError as error:
# print(' Error writing social - ',social_name,' post : ',error,social_post.name,\ print(' Error writing social - ',social_name,' post : ',error,social_post.name,\
# social_post.comment, local_outputs,social_post.date, social_post.rating,\ # social_post.comment, local_outputs,social_post.date, social_post.rating,\
# social_post.address, social_post.picsLocalpath, writtento[social_name],\ # social_post.address, social_post.picsLocalpath, writtento[social_name],\
# type(error).__name__ )
print(' Error writing social - ',social_name,' post : ',error,social_post.name,\
type(error).__name__ ) type(error).__name__ )
# except AttributeError as error:
# print(' Error writing social - ',social_name,' post : ',error,social_post.name,\
# social_post.comment, local_outputs,social_post.date, social_post.rating,\ # social_post.comment, local_outputs,social_post.date, social_post.rating,\
# social_post.address, social_post.picsLocalpath, writtento[social_name],\ # social_post.address, social_post.picsLocalpath, writtento[social_name],\
# type(error).__name__ ) # type(error).__name__ )
@@ -2482,7 +2441,6 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
social_post.name) social_post.name)
else: else:
print (' ',social_name,': Skipping posting for ',social_post.name,' previously written') print (' ',social_name,': Skipping posting for ',social_post.name,' previously written')
sqlconnection.close()
return social_count return social_count
################################################################################################## ##################################################################################################