From 778ce37c93bc967bf39f75dea409ff1a420b78e6 Mon Sep 17 00:00:00 2001 From: Joe Steele Date: Thu, 19 Dec 2024 11:01:53 -0500 Subject: [PATCH] fixed problems with closing database and fixed Instagram --- social.py | 120 ++++++++++++++++++------------------------------------ 1 file changed, 39 insertions(+), 81 deletions(-) diff --git a/social.py b/social.py index bd3d04c..8112343 100644 --- a/social.py +++ b/social.py @@ -141,10 +141,8 @@ def get_auth_connect(): if env.mariadb: print('Connecting to MariaDB for configuration and storage') #from sqlalchemy import create_engine - engine = sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+ -# engine = sqlalchemy.create_engine("mysql+mysqldb://"+env.mariadbuser+":"+env.mariadbpass+ - "@"+env.mariadbserver+"/"+env.mariadbdb, echo=False) -# "@"+env.mariadbserver+"/"+env.mariadbdb+"?charset=utf8mb4", echo=False) + engine = sqlalchemy.create_engine("mysql+mysqldb://"+env.mariadbuser+":"+env.mariadbpass+ + "@"+env.mariadbserver+"/"+env.mariadbdb+"?charset=utf8mb4", echo=False) Session = sqlalchemy.orm.sessionmaker() Session.configure(bind=engine) session = Session() @@ -156,8 +154,6 @@ def get_auth_connect(): posts = session.query(Posts).all() connections['posts'] = posts connections['postssession'] = session - # sqlalchemy.orm.sessionmaker().configure(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) - # connections['postssession'] = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) if env.data: print(' loading XLS content data source ...') if os.path.exists(env.xls): @@ -480,12 +476,6 @@ def get_google_data(driver, local_outputs): Returns: list: A list of data extracted from Google Maps. """ - engine = sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+ - "@"+env.mariadbserver+"/"+env.mariadbdb, echo=False, pool_recycle=3600) - Session = sqlalchemy.orm.sessionmaker() - Session.configure(bind=engine) - sqlconnection = Session() - #sqlconnection = sqlalchemy.orm.sessionmaker().configure(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) print(' get google data...', end ="") # Click on more botton on each text reviews more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq') @@ -535,7 +525,7 @@ def get_google_data(driver, local_outputs): score = "Unknown" print ('Error: ',error) # Grab more info from google maps entry on this particular review - if (len(sqlconnection.query(Posts).filter(Posts.name == name,Posts.google\ + if (len(local_outputs['postssession'].query(Posts).filter(Posts.name == name,Posts.google\ is not True).all()) == 0 or env.forcegoogleupdate) and env.block_google_maps is not\ True: gmaps = googlemaps.Client(env.googleapipass) @@ -563,7 +553,7 @@ def get_google_data(driver, local_outputs): else: if env.forcegoogleupdate: database_update_row(name,"google",True,"forceall",local_outputs) - if len(sqlconnection.query(Posts).filter(Posts.name == name,\ + if len(local_outputs['postssession'].query(Posts).filter(Posts.name == name,\ Posts.google is not True).all()) > 0: print (' Post was already in database, skipping update unless overriden') # else: @@ -653,7 +643,6 @@ def get_google_data(driver, local_outputs): dict_post_complete]) if env.force_web_create: database_update_row(name,"visitdate",visitdate,"forceall",local_outputs) - sqlconnection.close() return lst_data ################################################################################################## @@ -773,13 +762,6 @@ def write_to_database(data, local_outputs): """ column_list=[] print('write to database ...') - - #sqlconnection = sqlalchemy.orm.sessionmaker().configure(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) - engine = sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+ - "@"+env.mariadbserver+"/"+env.mariadbdb, echo=False) - Session = sqlalchemy.orm.sessionmaker() - Session.configure(bind=engine) - sqlconnection = Session() for x in inspect.getmembers(Posts): # if not (x[0].startswith('_') or 'metadata' in x[0] or 'registry' in x[0] or 'id' in x[0]): if not (x[0].startswith('_') or 'metadata' in x[0] or 'registry' in x[0]): @@ -835,7 +817,7 @@ def write_to_database(data, local_outputs): # # Using the DataFrame.append() function try: - getdata2 = sqlconnection.query(Posts).filter(Posts.name \ + getdata2 = local_outputs['postssession'].query(Posts).filter(Posts.name \ == processrow[0]) if getdata2.count() >0: print (' Row ',processrow[0],' already in Database') @@ -846,24 +828,23 @@ def write_to_database(data, local_outputs): 'source':processrow[5],'date':processrow[6],'address':processrow[7],\ 'dictPostComplete':str(processrow[8])} print (' Row ',processrow[0],' updated in Database') - sqlconnection.query(Posts).filter(Posts.name == processrow[0]).\ + local_outputs['postssession'].query(Posts).filter(Posts.name == processrow[0]).\ update(d2_dict) - sqlconnection.commit() + local_outputs['postssession'].commit() else: d2_row = Posts(name=processrow[0] ,comment=processrow[1],rating=processrow[2]\ ,picsURL=str(processrow[3]),picsLocalpath=str(processrow[4]),\ source=processrow[5],date=processrow[6],address=processrow[7],\ dictPostComplete=str(processrow[8])) print (' Row ',processrow[0],' added to Database') - sqlconnection.add(d2_row) - sqlconnection.commit() + local_outputs['postssession'].add(d2_row) + local_outputs['postssession'].commit() print (' Row ',processrow[0],' added to Database') except AttributeError as error: print(' Not able to write to post data table: ' , type(error)) - sqlconnection.rollback() + local_outputs['postssession'].rollback() raise processrow = [] - sqlconnection.close() df.to_excel(env.xls) return data @@ -884,38 +865,34 @@ def database_update_row(review_name, column_name, column_value, update_style, lo Returns: bool: True if the row was successfully updated, False otherwise. """ -# connections['postssession'] = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) - sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) - # sqlconnection.commit() - #sqlconnection.close() + try: if update_style == "forceall" and column_value is not False: - sqlconnection.query(Posts).filter(Posts.name == review_name).update\ + local_outputs['postssession'].query(Posts).filter(Posts.name == review_name).update\ ({column_name : column_value}) print (' Force Updated ',column_name, ' to: ',column_value) elif update_style == "onlyempty" and column_value is not False: - postval = sqlconnection.query(Posts).filter(Posts.name == review_name,\ + postval = local_outputs['postssession'].query(Posts).filter(Posts.name == review_name,\ getattr(Posts,column_name).is_not(null())).all() if len(postval) == 0 : - sqlconnection.query(Posts).filter(Posts.name == review_name).\ + local_outputs['postssession'].query(Posts).filter(Posts.name == review_name).\ update({column_name : column_value}) print (' Updated blank ',postval ,' on value',column_name, ' to: ',\ column_value) elif update_style == "toggletrue": - postval = sqlconnection.query(Posts).filter(Posts.name == review_name\ + postval = local_outputs['postssession'].query(Posts).filter(Posts.name == review_name\ ,getattr(Posts,column_name).is_not(1)).all() - sqlconnection.query(Posts).filter(Posts.name == review_name).update\ + local_outputs['postssession'].query(Posts).filter(Posts.name == review_name).update\ ({column_name : "1"}) print (' Updated ',column_name, ' on value: ',postval[0].column_value, ' to: ',\ column_value) except AttributeError as error: print(" Not able to write to post data table to update ",review_name," ",column_name,"\ to: ",column_value , type(error), error) - sqlconnection.rollback() + local_outputs['postssession'].rollback() raise # else: - sqlconnection.commit() - sqlconnection.close() + local_outputs['postssession'].commit() #time.sleep(env.facebooksleep) return True @@ -1147,10 +1124,8 @@ def post_to_x2(title, content, headers,date, rating, address, picslist,local_out if imgs_vid: try: # Message to post along with the video - sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) - attrib_list = sqlconnection.query(Posts).filter(Posts.name == title)\ + attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\ .all() - sqlconnection.close() if wpurl := attrib_list[0].wpurl: if business_url := attrib_list[0].businessurl: status_message = ( @@ -1194,9 +1169,7 @@ def post_to_x2(title, content, headers,date, rating, address, picslist,local_out def post_to_threads(title, content,headers, date, rating, address, picslist, local_outputs): img_list = ((picslist[1:-1]).replace("'","")).split(",") - sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) - attrib_list = sqlconnection.query(Posts).filter(Posts.name == title).all() - sqlconnection.close() + attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all() business_url = attrib_list[0].businessurl wpurl = attrib_list[0].wpurl auth_token = env.threads_access_token @@ -1255,9 +1228,7 @@ def post_facebook3(title, content,headers, date, rating, address, picslist, loca auth_token = env.facebookpass imgs_vid = [] img_list = pics - sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) - attrib_list = sqlconnection.query(Posts).filter(Posts.name == title).all() - sqlconnection.close() + attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all() business_url = attrib_list[0].businessurl wpurl = attrib_list[0].wpurl if wpurl and (attrib_list[0].picsLocalpath != '[]'): @@ -1554,9 +1525,7 @@ def post_to_instagram2(title, content, headers, date, rating, address, picslist, outputmontage = '' addresshtml = re.sub(" ", ".",address) - sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) - attrib_list = sqlconnection.query(Posts).filter(Posts.name == title).all() - sqlconnection.close() + attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all() if wpurl := attrib_list[0].wpurl: if business_url := attrib_list[0].businessurl: data = title + "\n"+ address+"\n"+business_url+"\n"+"Review: "+wpurl+\ @@ -1579,8 +1548,10 @@ def post_to_instagram2(title, content, headers, date, rating, address, picslist, if "montage.mp4" in loop: outputmontage = loop break - pathoutputmontage = outputmontage[1:-1] - if pathlib.Path.exists(pathoutputmontage): + pathoutputmontage = outputmontage +# pathoutputmontage = outputmontage[1:-1] + # if pathlib.Path.exists(pathoutputmontage): + if Path(pathoutputmontage).exists(): try: duration, fps, (width, height)=with_moviepy(outputmontage) if duration < 60: @@ -1733,10 +1704,8 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o #newdate2 = dt.datetime.strptime(str(newdate), formatting).date() dateparts = (str(newdate)).split("-") dateparts2 = dateparts[2].split(" ") - sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) - visitdate2 = sqlconnection.query(Posts).filter(Posts.name == title)\ + visitdate2 = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\ .all()[0].visitdate - sqlconnection.close() if visitdate != visitdate2: database_update_row(title,"visitdate",visitdate,"forceall",local_outputs) print (f'UPDATED: {visitdate2} to {visitdate} for {title}') @@ -1886,10 +1855,8 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o if linkslist: fmedia = linkslist[0]['file_id'] # print ('featured_media2 = ',file_id) - sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) - business_url_list = sqlconnection.query(Posts).filter(Posts.name == \ + business_url_list = local_outputs['postssession'].query(Posts).filter(Posts.name == \ title).all() - sqlconnection.close() business_url = ""+\ title+"" # wpurllist = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all() @@ -2185,10 +2152,8 @@ def create_wordpress_post(newdate2, picchop, title, address, headers, post_id, l else: if linkslist: fmedia = linkslist[0]['file_id'] - sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) - business_url_list = sqlconnection.query(Posts).filter(Posts.name == title)\ + business_url_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\ .all() - sqlconnection.close() business_url = f"{title}" status_message = f'{str(title)}: Business website: {business_url}' response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), \ @@ -2425,12 +2390,6 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo Returns: Count of the social that was selected """ - #sqlconnection = sqlalchemy.orm.sessionmaker(bind=sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+"@"+env.mariadbserver+"/"+env.mariadbdb, echo=False)) - engine = sqlalchemy.create_engine("mariadb+mariadbconnector://"+env.mariadbuser+":"+env.mariadbpass+ - "@"+env.mariadbserver+"/"+env.mariadbdb, echo=False) - Session = sqlalchemy.orm.sessionmaker() - Session.configure(bind=engine) - sqlconnection = Session() writtento = ast.literal_eval(social_post.dictPostComplete) if (len(local_outputs['postssession'].query(Posts).filter(Posts.name == social_post.name,\ getattr(Posts, social_name) is True).all())==0) and ((local_outputs['postssession'].\ @@ -2450,7 +2409,7 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo except AttributeError as error: print(" An error occurred setting value to go into Excel file:", type(error)\ .__name__) - print (' Success Posting to : '+social_post.name) + print (' Success Posting to xtwitter: '+social_post.name) if new_social_post: social_count +=1 try: @@ -2468,21 +2427,20 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo except AttributeError as error: print(" An error occurred writing database", type(error).__name__) except AttributeError as error: - # print(' Error writing social - ',social_name,' post : ',error,social_post.name,\ - # social_post.comment, local_outputs,social_post.date, social_post.rating,\ - # social_post.address, social_post.picsLocalpath, writtento[social_name],\ - # type(error).__name__ ) print(' Error writing social - ',social_name,' post : ',error,social_post.name,\ - type(error).__name__ ) - # social_post.comment, local_outputs,social_post.date, social_post.rating,\ - # social_post.address, social_post.picsLocalpath, writtento[social_name],\ - # type(error).__name__ ) + # social_post.comment, local_outputs,social_post.date, social_post.rating,\ + # social_post.address, social_post.picsLocalpath, writtento[social_name],\ + type(error).__name__ ) + # except AttributeError as error: + # print(' Error writing social - ',social_name,' post : ',error,social_post.name,\ + # social_post.comment, local_outputs,social_post.date, social_post.rating,\ + # social_post.address, social_post.picsLocalpath, writtento[social_name],\ + # type(error).__name__ ) else: print (' Exceeded the number of social - ',social_name,' posts per run, skipping',\ social_post.name) else: print (' ',social_name,': Skipping posting for ',social_post.name,' previously written') - sqlconnection.close() return social_count ################################################################################################## @@ -2495,4 +2453,4 @@ if __name__ == "__main__": process_reviews2(outputs) print('Done!') -################################################################################################## +################################################################################################## \ No newline at end of file