From db5878aca47da48255767e170568aabf04da1ad7 Mon Sep 17 00:00:00 2001 From: Joe Steele Date: Sun, 9 Jun 2024 19:21:49 -0400 Subject: [PATCH] see previous --- social.py | 97 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 87 insertions(+), 10 deletions(-) diff --git a/social.py b/social.py index 93afaf9..97b0d97 100644 --- a/social.py +++ b/social.py @@ -1,5 +1,3 @@ -import sqlalchemy - #data import time import os @@ -41,6 +39,8 @@ import tweepy from pathlib import Path from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base +import sqlalchemy +import googlemaps #import mysqlclient #import mysql-connector-python import env @@ -98,9 +98,9 @@ class Posts(Base): date = sqlalchemy.Column(sqlalchemy.String(length=64, collation="utf8")) address = sqlalchemy.Column(sqlalchemy.String(length=256, collation="utf8")) dictPostComplete = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8")) - googleurl = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8")) - weburl = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8")) - businessurl = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8")) + #googleurl = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8")) + wpurl = sqlalchemy.Column(sqlalchemy.String(length=512, collation="utf8")) + businessurl = sqlalchemy.Column(sqlalchemy.String(length=2048, collation="utf8")) longitude = sqlalchemy.Column(sqlalchemy.Float()) latitude = sqlalchemy.Column(sqlalchemy.Float()) google = sqlalchemy.Column(sqlalchemy.Boolean, default=False) @@ -112,6 +112,14 @@ class Posts(Base): web = sqlalchemy.Column(sqlalchemy.Boolean, default=False) tiktok = sqlalchemy.Column(sqlalchemy.Boolean, default=False) #active = sqlalchemy.Column(sqlalchemy.Boolean, default=True) + place_id = sqlalchemy.Column(sqlalchemy.String(length=126, collation="utf8")) + googledetails = sqlalchemy.Column(sqlalchemy.String(length=4096, collation="utf8")) + #businessurl = sqlalchemy.Column(sqlalchemy.String(length=512, collation="utf8")) + pluscode = sqlalchemy.Column(sqlalchemy.String(length=64, collation="utf8")) + googleurl = sqlalchemy.Column(sqlalchemy.String(length=512, collation="utf8")) + + + ################################################################################################## @@ -373,6 +381,9 @@ def get_data(driver,outputs ): print('Found extra pics') list_more_pics.click() elements = driver.find_elements(By.CLASS_NAME, 'jftiEf') + + + lst_data = [] for data in elements: name = data.find_element(By.CSS_SELECTOR, 'div.d4r55.YJxk2d').text @@ -399,6 +410,33 @@ def get_data(driver,outputs ): except Exception: score = "Unknown" more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d') + + # Grab more info from google maps entry on this particular review + if outputs['postssession'].query(Posts).filter(Posts.name == name,Posts.google != 1) or env.forcegoogleupdate: + gmaps = googlemaps.Client(env.googleapipass) + place_ids = gmaps.find_place(name+address, input_type = 'textquery', fields='') + if len(place_ids['candidates']) == 1 : + place_id = place_ids['candidates'][0]['place_id'] + details = gmaps.place(place_id) + #place_id = "ChIJh2OwH6KA54gRZLcx1Cjk8ic" + # Get place details + # googledetails = gmaps.place(place_id) + try: + businessurl = (details['result']['website']) + latitude = (details['result']['geometry']['location']['lat']) + longitude = (details['result']['geometry']['location']['lng']) + pluscode = (details['result']['plus_code']['compound_code']) + googleurl = (details['result']['url']) + update_db_row(name,"businessurl",businessurl,"onlyempty",outputs) + update_db_row(name,"latitude",latitude,"onlyempty",outputs) + update_db_row(name,"longitude",longitude,"onlyempty",outputs) + update_db_row(name,"pluscode",pluscode,"onlyempty",outputs) + update_db_row(name,"googleurl",googleurl,"onlyempty",outputs) + update_db_row(name,"place_id",place_id,"onlyempty",outputs) + update_db_row(name,"googledetails",details,"onlyempty",outputs) + update_db_row(name,"google",1,"forceall",outputs) + except Exception as error: + print('Error writing business details from google maps : ',error) pics= [] pics2 = [] # check to see if folder for pictures and videos already exists, if not, create it @@ -411,7 +449,7 @@ def get_data(driver,outputs ): # it is full size urlmedia = re.sub('=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]", lmpics.get_attribute("style")))[0]) - print ('URL : ',urlmedia) + print ('Pic URL : ',urlmedia) pics.append(urlmedia) # Grab the name of the file and remove all spaces and special charecters to name the # folder @@ -569,6 +607,45 @@ def write_to_xlsx2(data, outputs): ################################################################################################## +def update_db_row(review_name,column_name,column_value,update_style,outputs): + #kwargs = {Posts.column_name : ''} + #setattr(Posts, attribute_name, column_name) + #randdd = Posts.column_name + #filter_db = text('Posts.name == review_name,Posts.column_name == \'\'') + try: + if update_style == "forceall": + outputs['postssession'].query(Posts).filter(Posts.name == review_name).update({column_name : column_value}) + print (' Force Updated ',column_name, ' to: ',column_value) + elif update_style == "onlyempty": + #print('Posts.name == review_name,**kwargs') + postval = outputs['postssession'].query(Posts).filter(Posts.name == review_name,getattr(Posts,column_name).is_(None)).all() + # print ('Postval : ',postval) + temp = getattr(Posts,column_name) + # print ('temp = ',temp) + # print (getattr(Posts,column_name)) + # db_session.query(Notice).filter(getattr(Notice, col_name).like("%" + query + "%")) + if postval: +# if outputs['postssession'].query(Posts).filter(Posts.name == review_name,temp.is_(None)).all: + # if outputs['postssession'].query(Posts).filter(Posts.name == review_name,Posts.(eval(column_name ))): + outputs['postssession'].query(Posts).filter(Posts.name == review_name).update({column_name : column_value}) + print (' Updated blank ',postval ,' on value',column_name, ' to: ',column_value) + elif update_style == "toggletrue": + postval = outputs['postssession'].query(Posts).filter(Posts.name == review_name,getattr(Posts,column_name).is_(None)).all() + print ('Postval : ',postval) + if postval: + outputs['postssession'].query(Posts).filter(Posts.name == review_name).update({column_name : column_value}) + print (' Updated blank on value',column_name, ' to: ',column_value) + except Exception as error: + print(" Not able to write to post data table to update ",review_name," ",column_name," to: ",column_value , type(error), error) + outputs['postssession'].rollback() + raise + else: + outputs['postssession'].commit() + return True + + +################################################################################################## + def database_read(data): db_connection_str = 'mysql+pymysql://mysql_user:mysql_password@mysql_host/mysql_db' db_connection = create_engine(db_connection_str) @@ -1333,7 +1410,7 @@ def process_reviews(outputs): if env.web : #if writtento["web"] == 0 : if outputs['postssession'].query(Posts).filter(Posts.name == processrow[1].value,Posts.web != 1): - if webcount <= env.postsperrun: + if webcount < env.postsperrun: try: #NewWebPost = post_to_wp(processrow[1].value, processrow[2].value, processrow[2].value ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value) NewWebPost = post_to_wp(processrow[1].value, processrow[2].value, outputs['web'] ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value) @@ -1367,7 +1444,7 @@ def process_reviews(outputs): print (' Website: Skipping posting for ',processrow[1].value,' previously written') if env.instagram: if outputs['postssession'].query(Posts).filter(Posts.name == processrow[1].value,Posts.instagram != 1): - if instagramcount <= env.postsperrun: + if instagramcount < env.postsperrun: try: print(' Starting to generate Instagram post') NewInstagramPost = post_to_instagram2(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['instagram'] ) @@ -1401,7 +1478,7 @@ def process_reviews(outputs): print (' Instagram: Skipping posting for ',processrow[1].value,' previously written') if env.facebook: if outputs['postssession'].query(Posts).filter(Posts.name == processrow[1].value,Posts.facebook != 1): - if facebookcount <= env.postsperrun: + if facebookcount < env.postsperrun: try: print(' Starting to generate Facebook post') NewFacebookPost = post_facebook3(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['facebook'] ) @@ -1437,7 +1514,7 @@ def process_reviews(outputs): #if writtento["xtwitter"] == 0: # if Posts.query.filter(Posts.name.xtwitter.op('!=')(1)).first() if outputs['postssession'].query(Posts).filter(Posts.name == processrow[1].value,Posts.xtwitter != 1): - if xtwittercount <= env.postsperrun: + if xtwittercount < env.postsperrun: try: print(' Starting to generate xtwitter post') NewxtwitterPost = post_x2(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['posts'] )