continue to lint

2026-07-22 00:19:48 +00:00 · 2024-05-28 11:03:05 -04:00
parent 12a65c5999
commit 63955c7807
1 changed files with 91 additions and 85 deletions
@@ -102,7 +102,8 @@ def authconnect():
    if mariadb:
        print('Connecting to MariaDB for configuration and storage')
        from sqlalchemy import create_engine
-        engine = sqlalchemy.create_engine("mysql+mysqldb://"+mariadbuser+":"+mariadbpass+"@"+mariadbserver+"/"+mariadbdb+"?charset=utf8mb4", echo=False)        
+        engine = sqlalchemy.create_engine("mysql+mysqldb://"+mariadbuser+":"+mariadbpass+
+            "@"+mariadbserver+"/"+mariadbdb+"?charset=utf8mb4", echo=False)
        Session = sqlalchemy.orm.sessionmaker()
        Session.configure(bind=engine)
        session = Session()
@@ -116,11 +117,11 @@ def authconnect():
        connections.update({'postssession':session})
    if data:
        print('  loading XLS content data source ...')
-        if (os.path.exists(xls)):
+        if os.path.exists(xls):
            wb = load_workbook(filename = xls)
            xlswbDF = pd.read_excel(xls)
        else:
-            if (os.path.exists('./GoogleScrape/'+ xls)):
+            if os.path.exists('./GoogleScrape/'+ xls):
                wb = load_workbook(filename = './GoogleScrape/'+ xls)
                xlswbDF = pd.read_excel('./GoogleScrape/'+ xls)
            else:
@@ -137,7 +138,6 @@ def authconnect():
        connections.update({'instagram':instasessionclient})
    if facebook :
        print('  Connecting to facebook ...')
-        # msg = 'Purple Ombre Bob Lace Wig Natural Human Hair now available on https://lace-wigs.co.za/'
        # page_id_1 = facebookpageID
        # facebook_access_token = 'paste-your-page-access-token-here'
        # image_url = 'https://graph.facebook.com/{}/photos'.format(page_id_1)
@@ -165,7 +165,7 @@ def authconnect():
        connections.update({'web' : headers})
    if tiktok :
        print('  Connecting to Instagram ...')
-    return connections
+    return (connections)

 ##################################################################################################

@@ -214,7 +214,7 @@ def postImage(group_id, img,auth_token):
        print("    An error getting date occurred:", type(error).c) # An error occurred:
        r = False
    time.sleep(facebooksleep)
-    return r
+    return (r)

 ##################################################################################################

@@ -228,16 +228,15 @@ def postVideo(group_id, video_path,auth_token,title, content, date, rating, addr
       # my_dict['key'].append(1)
        files.update({eachfile: open(eachfile, 'rb')})
    data = { "title":title,"description" : title + "\n"+ address+"\nGoogle map to destination: "
-             r"https://www.google.com/maps/dir/?api=1&destination="+addresshtml +"\n\n"+ content + "\n"+rating+"\n"+date+"\n\n"+ hastags(address, title)+"\n\nhttps://www.joeeatswhat.com"+"\n\n","published" : True
+             r"https://www.google.com/maps/dir/?api=1&destination="+addresshtml +"\n\n"+ content +
+             "\n"+rating+"\n"+date+"\n\n"+ hastags(address, title)+
+             "\n\nhttps://www.joeeatswhat.com"+"\n\n","published" : True
    }
-    #try: r = requests.post(url, files=files, data=data)
    try: r = requests.post(url, files=files, data=data).json()
    except Exception as error:
        print("    An error getting date occurred:", type(error).c) # An error occurred:
        r = False
    time.sleep(facebooksleep)
-    print (r)
-    print ('r id  = ',r['id'])
    return r

 ##################################################################################################
@@ -271,7 +270,6 @@ def post_facebook2(title, content, date, rating, address, picslist, instasession
            imgs_id.append(post_id['id'])
        except Exception as error:
            print("    An error occurred:", type(error)) # An error occurred:
-
    # try:
    #     imgs_id.append(post_id['id'])
    # except Exception as error:
@@ -307,14 +305,15 @@ def post_facebook3(title, content, date, rating, address, picslist, instasession
    imgs_pic = []
    img_list = pics
    for img in img_list:
-        if ('montage.mp4' in img ):
+        if 'montage.mp4' in img:
            imgs_vid.append(img.strip())
        else:
            imgs_pic.append(img.strip())
-    if (imgs_vid ):
+    if imgs_vid:
       # print ("loop")
        try:
-            post_id = postVideo(group_id, imgs_vid,auth_token,title, content, date, rating, address)
+            post_id = postVideo(group_id, imgs_vid,auth_token,title, content,
+                date, rating, address)
            imgs_id.append(post_id['id'])
        except Exception as error:
            print("    An error occurred:", type(error).c) # An error occurred:
@@ -327,7 +326,9 @@ def post_facebook3(title, content, date, rating, address, picslist, instasession

 def get_data(driver,outputs ):

-#     curl -X GET -H 'Content-Type: application/json' -H "X-Goog-Api-Key: API_KEY" -H "X-Goog-FieldMask: id,displayName,formattedAddress,plusCode" https://places.googleapis.com/v1/places/ChIJj61dQgK6j4AR4GeTYWZsKWw #placeId #websiteUri  AIzaSyB_hglzM7N8HzjXwi1dF1E8WYTql3akS7Q
+# curl -X GET -H 'Content-Type: application/json' -H "X-Goog-Api-Key: API_KEY" -H 
+#   "X-Goog-FieldMask: id,displayName,formattedAddress,plusCode" 
+#   https://places.googleapis.com/v1/places/ChIJj61dQgK6j4AR4GeTYWZsKWw #placeId #websiteUri
    """
    this function get main text, score, name
    """
@@ -336,7 +337,8 @@ def get_data(driver,outputs ):
    more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq')
    for list_more_element in more_elemets:
        list_more_element.click()
-    # Find Pictures that have the expansion indicator to see the rest of the pictures under them and click it to expose them all
+    # Find Pictures that have the expansion indicator to see the rest of the pictures under
+    #    them and click it to expose them all
    more_pics = driver.find_elements(By.CLASS_NAME, 'Tya61d')
    for list_more_pics in more_pics:
        if 'showMorePhotos' in  list_more_pics.get_attribute("jsaction") :
@@ -354,7 +356,11 @@ def get_data(driver,outputs ):
        print('Visited: ',visitdate)
        try: text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text
        except: text = ''
-        try: score = data.find_element(By.CSS_SELECTOR, 'span.kvMYJc').get_attribute("aria-label")  #find_element(By.CSS_SELECTOR,'aria-label').text #)  ##QA0Szd > div > div > div.w6VYqd > div:nth-child(2) > div > div.e07Vkf.kA9KIf > div > div > div.m6QErb.DxyBCb.kA9KIf.dS8AEf > div.m6QErb > div:nth-child(3) > div:nth-child(2) > div > div:nth-child(4) > div.DU9Pgb > span.kvMYJc
+        try: score = data.find_element(By.CSS_SELECTOR, 'span.kvMYJc').get_attribute("aria-label")
+        #find_element(By.CSS_SELECTOR,'aria-label').text #)  ##QA0Szd > div > div > div.w6VYqd >
+        #  div:nth-child(2) > div > div.e07Vkf.kA9KIf > div > div > div.m6QErb.DxyBCb.kA9KIf.dS8AEf
+        #  > div.m6QErb > div:nth-child(3) > div:nth-child(2) > div > div:nth-child(4) > div.DU9Pgb
+        #  > span.kvMYJc
        except: score = "Unknown"
        more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d')
        pics= []
@@ -365,13 +371,14 @@ def get_data(driver,outputs ):
            os.makedirs('./Output/Pics/'+cleanname)
        # Walk through all the pictures and videos for a given review
        for lmpics in more_specific_pics:
-            # if pics2 and not pics:
-            #     pics2 = []
-            # Grab URL from style definiton (long multivalue string), and remove the -p-k so that it is full size 
-            urlmedia = re.sub('=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]", lmpics.get_attribute("style")))[0])
+            # Grab URL from style definiton (long multivalue string), and remove the -p-k so that
+            #   it is full size
+            urlmedia = re.sub('=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]",
+                lmpics.get_attribute("style")))[0])
            print ('URL : ',urlmedia)
            pics.append(urlmedia)
-            # Grab the name of the file and remove all spaces and special charecters to name the folder
+            # Grab the name of the file and remove all spaces and special charecters to name the
+            #    folder
            filename = re.sub( r'[^a-zA-Z0-9]','', str(lmpics.get_attribute("aria-label")))
            if lmpics == more_specific_pics[0]:
                lmpics.click()
@@ -380,8 +387,10 @@ def get_data(driver,outputs ):
                tempdate = str((driver.find_element(By.CLASS_NAME,'mqX5ad')).text).rsplit("-",1)
                visitdate = re.sub( r'[^a-zA-Z0-9]','',tempdate[1])
                print ('Visited: ',visitdate)
-            # Check to see if it has a sub div, which represents the label with the video length displayed, this will be done 
-            # because videos are represented by pictures in the main dialogue, so we need to click through and grab the video URL
+            # Check to see if it has a sub div, which represents the label with the video length
+            # displayed, this will be done
+            # because videos are represented by pictures in the main dialogue, so we need to click
+            # through and grab the video URL
            if (lmpics.find_elements(By.CSS_SELECTOR,'div.fontLabelMedium.e5A3N')) :
                ext='.mp4'
                lmpics.click()
@@ -400,7 +409,8 @@ def get_data(driver,outputs ):
                ext='.jpg'
            # Add the correct extension to the file name
            filename = filename+ext
-            # Test to see if file already exists, and if it does not grab the media and store it in location folder
+            # Test to see if file already exists, and if it does not grab the media and store it
+            #   in location folder
            if not os.path.exists('./Output/Pics/'+cleanname+'/'+visitdate):
                os.makedirs('./Output/Pics/'+cleanname+'/'+visitdate)
            if not os.path.isfile('./Output/Pics/'+cleanname+'/'+visitdate+'/'+filename):
@@ -411,9 +421,10 @@ def get_data(driver,outputs ):
        if pics2:
            make_video(pics2)
            pics2.append("./Output/Pics/"+cleanname+"/"+visitdate+'/'+'montage.mp4')
-        dictPostComplete= {'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,'instagram':0,'tiktok':0}
-        lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address,dictPostComplete])
-  
+        dictPostComplete= {'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,
+            'instagram':0,'tiktok':0}
+        lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address,
+            dictPostComplete])
    return lst_data

 ##################################################################################################
@@ -428,7 +439,8 @@ def scrolling(counter,driver):
    for _i in range(counter):
        try:
            scrolling = driver.execute_script(
-                'document.getElementsByClassName("dS8AEf")[0].scrollTop = document.getElementsByClassName("dS8AEf")[0].scrollHeight',
+                'document.getElementsByClassName("dS8AEf")[0].scrollTop = /
+                document.getElementsByClassName("dS8AEf")[0].scrollHeight',
                scrollable_div
            )
            time.sleep(3)
@@ -440,7 +452,8 @@ def scrolling(counter,driver):

 def write_to_xlsx(webdata, outputs):
    print('Start to write to excel...')
-    cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address','dictPostComplete']
+    cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
+        'dictPostComplete']
    # rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)))
    rows = list(webdata)
    if needreversed:
@@ -452,10 +465,13 @@ def write_to_xlsx(webdata, outputs):
        else:
            if (processrow[0] != None):
 #                processrow.append({'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,'Instagram':0,'tiktok':0})
-                outputs['data'].append([processrow[0],processrow[1],processrow[2],str(processrow[3]),str(processrow[4]),str(processrow[5]),str(processrow[6]),str(processrow[7]),str(processrow[8])]) # sheet_obj.append([col1, col2])
+                outputs['data'].append([processrow[0],processrow[1],processrow[2],str(
+                    processrow[3]),str(processrow[4]),str(processrow[5]),str(processrow[6]),
+                    str(processrow[7]),str(processrow[8])]) # sheet_obj.append([col1, col2])
                #outputs['data'].parent.save(xls)
    #outputs['datawb'].save(xls)
-    cols = ['num',"name", "comment", 'rating','picsURL','picsLocalpath','source','date','address','dictPostComplete', 'test']
+    cols = ['num',"name", "comment", 'rating','picsURL','picsLocalpath','source','date',
+        'address','dictPostComplete', 'test']
    df = pd.DataFrame(outputs['data'], columns=cols)
    df.to_excel(xls)
    return True
@@ -483,9 +499,6 @@ def write_to_xlsx2(data, outputs):
        if  (processrow[1] in df.values):
            print ('  Row ',processrow[0],' ', processrow[1],'  already in XLS sheet')
            d2_row = Posts(name=processrow[1],comment=processrow[2],rating=processrow[3],picsURL=processrow[4],picsLocalpath=processrow[5],source=processrow[6],date=processrow[7],address=processrow[8],dictPostComplete=processrow[9])
-            # if not (processrow[1] in outputs['posts']) : 
-            #     outputs['postssession'].add(d2_row)
-            #     outputs['postssession'].commit()
        else:
            if (processrow[1] != None):
                # Create a Python dictionary object with all the column values
@@ -516,7 +529,7 @@ def database_read(data):
    db_connection_str = 'mysql+pymysql://mysql_user:mysql_password@mysql_host/mysql_db'
    db_connection = create_engine(db_connection_str)
    df = pd.read_sql('SELECT * FROM table_name', con=db_connection)
-    return df
+    return (df)

 ##################################################################################################

@@ -531,7 +544,7 @@ def check_web_media(filename,headers):
        return file_id, link
    except Exception as error:
        print('    No existing media with same name in Wordpress media folder: ' + filename)
-        return False, False
+        return (False, False)
 
 ##################################################################################################
       
@@ -644,7 +657,6 @@ def post_x(title, content, date, rating, address, picslist, instasession):

 ##################################################################################################

-
 def post_to_wp(title, content,  headers,date, rating,address, picslist):
    # post
    NewPost = False
@@ -655,10 +667,8 @@ def post_to_wp(title, content,  headers,date, rating,address, picslist):
    picl = picslist[1:-1] 
    pic2 = picl.replace(",","")#re.sub(r',','',picl) #re.sub( r'[^a-zA-Z0-9]','',tempdate[1])
    pic3= pic2.replace("'","")
-#    print (pic3)
    pidchop = pic3.split(" ")
    linkslist=[]
-    # linkslist.clear
    print ('    Figuring out date of Post : ',title)
    format = '%b/%Y/%d' #specifify the format of the date_string.
    date_string = date
@@ -740,7 +750,7 @@ def post_to_wp(title, content,  headers,date, rating,address, picslist):
        post_id = check_post(title,newdate2,headers)
    except  Exception as error :
        print ('Could not check to see post already exists', type(error).c)
-    if ( post_id == False):        
+    if post_id == False:        
        googleadress =  r"<a href=https://www.google.com/maps/dir/?api=1&destination="+addresshtml + r">"+address+r"</a>"
        post_data = {
            "title": title,
@@ -775,7 +785,7 @@ def post_to_wp(title, content,  headers,date, rating,address, picslist):
        print ('  Found Picture: ',picname)
        file_id, link = check_media(picname, headers)
 #        link = linknew['rendered']
-        if (file_id) is False:
+        if file_id is False:
            print ('    '+picname+' was not already found in library, adding it')
            countreview = True
            image = {
@@ -788,7 +798,7 @@ def post_to_wp(title, content,  headers,date, rating,address, picslist):
                image_response = requests.post(wpAPI + "/media", headers=headers, files=image)
            except Exception as error:
                print("    An error uploading picture ' + picname+ ' occurred:", type(error).__name__) # An error occurred:
-            if ( image_response.status_code != 201 ):
+            if image_response.status_code != 201 :
                print ('    Error- Image ',picname,' was not successfully uploaded.  response: ',image_response)
            else:
                PicDict=image_response.json()
@@ -808,7 +818,7 @@ def post_to_wp(title, content,  headers,date, rating,address, picslist):
                print ('    Error- Image ',picname,' was not attached to post.  response: ',image_response)
            try:
                post_response = requests.post(wpAPI + "/posts/" + str(post_id), headers=headers)
-                if (link in str(post_response.text)):
+                if link in str(post_response.text):
                    print ('    Image link for ', picname, 'already in content of post: ',post_id, post_response.text, link)
                else:
                    linkslist.append({'file_id' : file_id , 'link' : link})
@@ -821,9 +831,9 @@ def post_to_wp(title, content,  headers,date, rating,address, picslist):
        #for loop in linkslist:
        print ('    Adding ', piclink['link'], ' to posting')
        try:
-            ext = (piclink['link'].split( '.')[-1] )
-            if (ext == 'mp4'):
-                if (firstMP4):
+            ext = piclink['link'].split( '.')[-1] 
+            if ext == 'mp4':
+                if firstMP4:
                    contentpics += '\n' +r'[evp_embed_video url="' + piclink['link'] + r'" autoplay="true"]'
                    firstMP4 = False
                else:
@@ -850,13 +860,13 @@ def make_video(inphotos):
        dir = inphotos[0].rsplit(r'/', 1)
        folder = dir[0]
        output = folder+"/montage.mp4"
-        if ((not os.path.exists(output)) and (len(inphotos) >1)):
+        if (not os.path.exists(output) and (len(inphotos) >1)):
            video = VideoFileClip(inphotos[0])
            for photo in inphotos :
                #clip = VideoFileClip("myHolidays.mp4").subclip(50,60)
                clip = VideoFileClip(photo)
                # Concatenate the photos into a clip
-                if (".jpg" in photo):
+                if ".jpg" in photo:
                    clip.duration = 2
                try:
                    video = concatenate_videoclips([video, clip], method="compose")
@@ -875,9 +885,6 @@ def make_video(inphotos):

 ##################################################################################################

-
-##################################################################################################
-    
 def post_to_instagram2 (title, content, date, rating, address, picslist, instasession):
    #post_to_instagram2(processrow[1].value, processrow[2].value ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['instagram'])
    #montageexists = "montage.mp4" in picslist
@@ -948,7 +955,7 @@ def hastags (address, name):
    fulltag = defaulttags+" "+citytag+" "+statetag+" "+ziptag
    # 153 Sugar Belle Dr, Winter Garden, FL 34787
    # inphotos[0].rsplit(r'/', 1)
-    return fulltag
+    return (fulltag)

 ##################################################################################################
    
@@ -956,7 +963,7 @@ def process_reviews(outputs):
    # Process
    webcount = xtwittercount = instagramcount = yelpcount = threadscount = facebookcount= tiktokcount = 0
    rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)))
-    if (google):
+    if google:
        print('Configuration says to update google Reviews prior to processing them')
        options = webdriver.ChromeOptions()
        options.add_argument("--log-level=3")
@@ -980,7 +987,6 @@ def process_reviews(outputs):
        driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})") 
        driver.get(URL)
        time.sleep(5)
-    #                    counter = counter()
        scrolling(counter(driver), driver)
        webdata = get_data(driver,outputs)
        write_to_xlsx2(webdata, outputs)
@@ -999,8 +1005,8 @@ def process_reviews(outputs):
            # Check to see if the website has already been written to according to the xls sheet, if it has not... then process
            if ((writtento["web"]) == 0 or writtento["instagram"]==0 or writtento["facebook"]==0 or writtento["xtwitter"]==0 or writtento["yelp"]==0 or writtento["tiktok"]==0 or writtento["threads"]==0 ) and (is_port_open(wpAPI, 443)) and (web or instagram or yelp or xtwitter or tiktok or facebook or threads or google)and (processrow[2].value != None) :
                if web :
-                    if (writtento["web"] == 0) :
-                        if (webcount <= postsperrun):
+                    if writtento["web"] == 0 :
+                        if webcount <= postsperrun:
                            try: 
                                #NewWebPost = post_to_wp(processrow[1].value, processrow[2].value, processrow[2].value ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value)
                                NewWebPost = post_to_wp(processrow[1].value, processrow[2].value, outputs['web'] ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value)
@@ -1025,8 +1031,8 @@ def process_reviews(outputs):
                    else:
                        print ('  Website: Skipping posting for ',processrow[1].value,' previously written')
                if instagram:
-                    if (writtento["instagram"] == 0 ):
-                        if (instagramcount <= postsperrun):
+                    if writtento["instagram"] == 0:
+                        if instagramcount <= postsperrun:
                            try: 
                                print('  Starting to generate Instagram post')
                                NewInstagramPost = post_to_instagram2(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['instagram'] )
@@ -1054,8 +1060,8 @@ def process_reviews(outputs):
                    else:
                        print ('  Instagram: Skipping posting for ',processrow[1].value,' previously written')
                if facebook:
-                    if (writtento["facebook"] == 0 ):
-                        if (facebookcount <= postsperrun):
+                    if writtento["facebook"] == 0:
+                        if facebookcount <= postsperrun:
                            try: 
                                print('  Starting to generate Facebook post')
                                NewFacebookPost = post_facebook3(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['facebook'] )
@@ -1083,8 +1089,8 @@ def process_reviews(outputs):
                    else:
                        print ('  Facebook: Skipping posting for ',processrow[1].value,' previously written')
                if False:
-                    if (writtento["xtwitter"] == 0 ):
-                        if (xtwittercount <= postsperrun):
+                    if writtento["xtwitter"] == 0:
+                        if xtwittercount <= postsperrun:
                            try: 
                                print('  Starting to generate xtwitter post')
                                NewxtwitterPost = post_x(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['xtwitter'] )
@@ -1138,9 +1144,9 @@ def socials(name, namedict,outputs, writtento, processrow,funct):# namecount, na
    #namedict{'name':name, 'namecount':namecount, 'namepost':namepost, 'subroutine':subroutine}
    #namedict{'name':xtwitter, 'namecount':xtwiteercount, 'namepost':NewxtwitterPost, 'subroutine':postx}
    if name:
-        if (writtento[name] == 0 ):
+        if writtento[name] == 0 :
            print (namedict['namecount'])
-            if (int(namedict['namecount']) <= postsperrun):
+            if int(namedict['namecount']) <= postsperrun:
                try: 
                    print('  Starting to generate xtwitter post : ',namedict['subroutine'])
                    postoutput = funct(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[4].value,outputs )