admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

more linting

Browse Source
This commit is contained in:
Joe Steele
2024-06-17 17:27:39 -04:00
parent 058477d87f
commit 55dfec90df
1 changed files with 31 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+31 -59
View File
@@ -6,21 +6,21 @@ import pathlib
#from selenium.webdriver.chrome.service import Service #from selenium.webdriver.chrome.service import Service
import re import re
#from openpyxl import Workbook, load_workbook #from openpyxl import Workbook, load_workbook
import pandas as pd
from datetime import datetime from datetime import datetime
from dateutil.relativedelta import relativedelta
import ast import ast
import base64 import base64
import requests
import datetime as dt import datetime as dt
from urllib.request import urlretrieve
import requests
#import json #import json
import jsonpickle import jsonpickle
from urllib.request import urlretrieve
import urllib3 import urllib3
#Instgram #Instgram
#from instapy import InstaPy #from instapy import InstaPy
#import instapy #import instapy
#from instabot import Bot #from instabot import Bot
import pandas as pd
from dateutil.relativedelta import relativedelta
from openpyxl import load_workbook from openpyxl import load_workbook
import instagrapi import instagrapi
#from instagrapi.types import StoryMention, StoryMedia, StoryLink, StoryHashtag #from instagrapi.types import StoryMention, StoryMedia, StoryLink, StoryHashtag
@@ -34,11 +34,8 @@ from selenium.common.exceptions import NoSuchElementException
#from selenium.webdriver.chrome.service import Service #from selenium.webdriver.chrome.service import Service
#twitter #twitter
import tweepy import tweepy
#import asyncio #import asyncio
#import aiohttp #import aiohttp
import sqlalchemy import sqlalchemy
#from sqlalchemy import create_engine #from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
@@ -605,7 +602,7 @@ def write_to_xlsx2(data, local_outputs):
print("Encode Object into JSON formatted Data using jsonpickle") print("Encode Object into JSON formatted Data using jsonpickle")
jsonposts = jsonpickle.encode(local_outputs['posts'], unpicklable=False) jsonposts = jsonpickle.encode(local_outputs['posts'], unpicklable=False)
for processrow in df2.values: for processrow in df2.values:
if (processrow[1] in df.values): if processrow[1] in df.values:
print (' Row ',processrow.id,' ', processrow.name ,' already in XLS sheet') print (' Row ',processrow.id,' ', processrow.name ,' already in XLS sheet')
d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=\ d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=\
processrow.rating,picsURL=processrow.picsURL,pics_local_path=processrow.\ processrow.rating,picsURL=processrow.picsURL,pics_local_path=processrow.\
@@ -875,7 +872,7 @@ def get_wordpress_featured_photo_id(post_id):
################################################################################################## ##################################################################################################
def post_to_x2(title, content, headers,date, rating, address, picslist,local_outputs): def post_to_x2(title, content, headers,date, rating, address, picslist,local_outputs):
""" """
Post to x2. Post to x2.
@@ -918,12 +915,13 @@ def post_to_x2(title, content, headers,date, rating, address, picslist,local_out
try: try:
video_path = imgs_vid[0] video_path = imgs_vid[0]
# Message to post along with the video # Message to post along with the video
attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all() attrib_list = local_outputs['postssession'].query(Posts).filter(Posts.name == title)\
.all()
business_url = attrib_list[0].businessurl business_url = attrib_list[0].businessurl
wpurl = attrib_list[0].wpurl wpurl = attrib_list[0].wpurl
if wpurl: # Don't post of website URL does not exist yet if wpurl: # Don't post of website URL does not exist yet
if business_url: #Sometimes Business URL does not exist, so account for it if business_url: #Sometimes Business URL does not exist, so account for it
status_message = str(title) + ': My Review - '+ wpurl + '\n Business website: '+ \ status_message=str(title)+': My Review - '+wpurl+'\n Business website: '+\
business_url + '\n' business_url + '\n'
else: else:
status_message = str(title) + ': My Review - '+ wpurl + '\n' status_message = str(title) + ': My Review - '+ wpurl + '\n'
@@ -1001,7 +999,7 @@ def post_facebook3(title, content,headers, date, rating, address, picslist, loca
return False return False
time.sleep(env.facebooksleep) time.sleep(env.facebooksleep)
print(' Facebook response: ',post_id) print(' Facebook response: ',post_id)
return (True) return True
################################################################################################## ##################################################################################################
@@ -1043,7 +1041,7 @@ def post_to_threads2(title, content, headers, date, rating, address, picslist, l
return False return False
####################################################################################################### ###################################################################################################
def post_to_tiktok(title, content, headers, date, rating, address, picslist, local_outputs): def post_to_tiktok(title, content, headers, date, rating, address, picslist, local_outputs):
""" """
@@ -1126,41 +1124,11 @@ def post_to_instagram2(title, content, headers,date, rating, address, picslist,l
if picslist != '[]' and "montage.mp4" in picslist: if picslist != '[]' and "montage.mp4" in picslist:
#content = content + get_hastags(address, title) #content = content + get_hastags(address, title)
pics = ((picslist[1:-1].replace(",","")).replace("'","")).split(" ") pics = ((picslist[1:-1].replace(",","")).replace("'","")).split(" ")
video, outputmontage = make_montage_video_from_google(pics)
try: try:
instasession.video_upload(outputmontage, data) instasession.video_upload(outputmontage, data)
# video2 = instasession.video_upload(outputmontage, data)
except AttributeError as error: except AttributeError as error:
print(" An error occurred uploading video to Instagram:", type(error).__name__) print(" An error occurred uploading video to Instagram:", type(error).__name__)
return False return False
#media_pk = instasession.media_pk_from_url('https://www.instagram.com/p/CGgDsi7JQdS/')
#media_path = instasession.video_download(media_pk)
# joeeatswhat = instasession.user_info_by_username('timberjoe')
# try: buildout = instagrapi.story.StoryBuilder(outputmontage,'Credits @timberjoe',[StoryMention(user=joeeatswhat)]).video(40) # seconds
# except AttributeError as error:
# print(" An error occurred uploading video to Instagram:", type(error).__name__) # An error occurred:
# try: instasession.video_upload_to_story(buildout.path,"Credits @example",mentions=buildout.mentions,links=[StoryLink(webUri='https://www.joeeatswhat.com')],medias=[StoryMedia(media_pk=outputmontage)])
# try:
# instasession.video_upload_to_story(
# outputmontage,
# "Credits @joeeatswhat",
# # mentions=[StoryMention(user='timberjoe', x=0.49892962, y=0.703125, width=0.8333333333333334, height=0.125)],
# links=[StoryLink(webUri='https://www.joeeatswhat.com')],
# # hashtags=[StoryHashtag(hashtag=get_hastags(address,title), x=0.23, y=0.32, width=0.5, height=0.22)],
# #medias=[StoryMedia(media_pk=media_pk, x=0.5, y=0.5, width=0.6, height=0.8)],
# )
# story = instasession.story_photo("path/to/photo.jpg")
# instasession.video_elements.add_link("https://www.joeeatswhat.com")
# #story.add_link("https://www.joeeatswhat.com")
# instasession.video_elements.add_hashtags(get_hastags)
# story = instasession.video_upload_to_story(outputmontage)
# story.upload()
# #instasession.video_upload_to_story(path:outputmontage,caption:content, mentions:r'@timberjoe',links:'https://www.joeeatswhat.com',hashtags: hastag) ( path: outputmontage, caption: content, mentions:['@timberjoe'], links: ['https://www.joeeatswhat.com'], hashtags: get_hastags )
# # temp = dict()
# # temp = instasession.video_upload_to_story(path=outputmontage,caption=content,mentions=r'@timberjoe',links='https://www.joeeatswhat.com',hashtags=get_hastags)
# except AttributeError as error:
# print(" An error occurred uploading video to Instagram:", type(error).__name__) # An error occurred:
# return False
return True return True
else: else:
print (' Missing wordpress post for instagram : ',title) print (' Missing wordpress post for instagram : ',title)
@@ -1188,7 +1156,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
None None
""" """
# post # post
newPost = False new_post = False
#countreview = False #countreview = False
addresshtml = re.sub(" ", ".",address) addresshtml = re.sub(" ", ".",address)
googleadress = r"<a href=https://www.google.com/maps/dir/?api=1&destination="+\ googleadress = r"<a href=https://www.google.com/maps/dir/?api=1&destination="+\
@@ -1202,7 +1170,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
linkslist=[] linkslist=[]
print (' Figuring out date of Post : ',title) print (' Figuring out date of Post : ',title)
#specifify the formatting of the date_string. #specifify the formatting of the date_string.
# formatting = '%b/%Y/%d' # formatting = '%b/%Y/%d'
date_string = date date_string = date
if "a day" in date_string: if "a day" in date_string:
date = dt.timedelta(days=-1) date = dt.timedelta(days=-1)
@@ -1256,7 +1224,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
print(" An error getting date occurred:",error) print(" An error getting date occurred:",error)
else: else:
#specifify the formatting of the date_string. #specifify the formatting of the date_string.
formatting = '%Y-%b-%d' formatting = '%Y-%b-%d'
month = date[:3] month = date[:3]
year = date[3:] year = date[3:]
day = '01' day = '01'
@@ -1305,7 +1273,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
if response.status_code != 201: if response.status_code != 201:
print ('Error: ',response, response.text) print ('Error: ',response, response.text)
else: else:
newPost = True new_post = True
post_id_json = response.json() post_id_json = response.json()
post_id = post_id_json.get('id') post_id = post_id_json.get('id')
print (' New post is has post_id = ',post_id) print (' New post is has post_id = ',post_id)
@@ -1339,7 +1307,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
print(" An error uploading picture ' + picname+ ' occurred:", \ print(" An error uploading picture ' + picname+ ' occurred:", \
type(error).__name__) type(error).__name__)
if image_response.status_code != 201 : if image_response.status_code != 201 :
print (' Error- Image ',picname,' was not successfully uploaded. response: ',\ print (' Error:Image ',picname,' was not successfully uploaded. response: ',\
image_response) image_response)
else: else:
pic_dic=image_response.json() pic_dic=image_response.json()
@@ -1432,15 +1400,15 @@ def process_reviews2(outputs,headers):
webcount = xtwittercount = instagramcount = facebookcount = 0 webcount = xtwittercount = instagramcount = facebookcount = 0
# webcount=xtwittercount=instagramcount=yelpcount=threadscount=facebookcount=tiktokcount = 0 # webcount=xtwittercount=instagramcount=yelpcount=threadscount=facebookcount=tiktokcount = 0
if env.datasource == 'xls': if env.datasource == 'xls':
rows_orig = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row))) rows_orig = list(outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row))
rows_orig2 = (outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)) rows_orig2 = outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)
# rows = [({0:p.id},{1:p.name}, { 2:p.comment}, {3: p.rating}, {4:p.picsURL},\ # rows = [({0:p.id},{1:p.name}, { 2:p.comment}, {3: p.rating}, {4:p.picsURL},\
# {5:p.pics_local_path},{6:p.source},{7:p.date},{8:p.address},{9:p.dict_post_complete})\ # {5:p.pics_local_path},{6:p.source},{7:p.date},{8:p.address},{9:p.dict_post_complete})\
# for p in rows_orig] # for p in rows_orig]
# rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row))) # rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)))
rows = outputs['data'] rows = outputs['data']
else: else:
rows_orig = (outputs['posts']) rows_orig = outputs['posts']
rows = rows_orig rows = rows_orig
if env.google: if env.google:
print('Configuration says to update google Reviews prior to processing them') print('Configuration says to update google Reviews prior to processing them')
@@ -1488,7 +1456,8 @@ def process_reviews2(outputs,headers):
print('Processing Reviews') print('Processing Reviews')
for processrow in rows: for processrow in rows:
if processrow.name != "name": # Skip header line of xls sheet if processrow.name != "name": # Skip header line of xls sheet
outputs['postssession'].query(Posts).filter(Posts.name == processrow.name).update({"google" : 1}) outputs['postssession'].query(Posts).filter(Posts.name == processrow.name).\
update({"google" : 1})
print ("Processing : ",processrow.name) print ("Processing : ",processrow.name)
writtento = (ast.literal_eval(processrow.dict_post_complete)) writtento = (ast.literal_eval(processrow.dict_post_complete))
# Check to see if the website has already been written to according to the xls sheet,\ # Check to see if the website has already been written to according to the xls sheet,\
@@ -1501,10 +1470,12 @@ def process_reviews2(outputs,headers):
if env.web : if env.web :
#if writtento["web"] == 0 : #if writtento["web"] == 0 :
try: try:
post_id, post_link = get_wordpress_post_id_and_link(processrow.name,outputs['web'] ) post_id, post_link = get_wordpress_post_id_and_link(processrow.name,\
outputs['web'] )
if env.forcegoogleupdate is True and env.block_google_maps is not True: if env.forcegoogleupdate is True and env.block_google_maps is not True:
if post_link: if post_link:
database_update_row(processrow.name,"wpurl",post_link,"forceall",outputs) database_update_row(processrow.name,"wpurl",post_link,"forceall"\
,outputs)
else: else:
print (' Error getting wordpress links to update databse') print (' Error getting wordpress links to update databse')
except AttributeError as error : except AttributeError as error :
@@ -1542,10 +1513,10 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
Count of the social that was selected Count of the social that was selected
""" """
writtento = (ast.literal_eval(social_post.dict_post_complete)) writtento = (ast.literal_eval(social_post.dict_post_complete))
if (len(local_outputs['postssession'].query(Posts).filter(Posts.name == social_post.name,getattr\ if (len(local_outputs['postssession'].query(Posts).filter(Posts.name == social_post.name,\
(Posts, social_name) is True).all())==0) and ((local_outputs['postssession'].query(Posts).\ getattr(Posts, social_name) is True).all())==0) and ((local_outputs['postssession'].\
filter(Posts.name == social_post.name).all()[0].wpurl != None)or social_name \ query(Posts).filter(Posts.name == social_post.name).all()[0].wpurl != \
== 'web'): None)or social_name == 'web'):
if social_count < env.postsperrun: if social_count < env.postsperrun:
try: try:
print(' Starting to generate ',social_name,' post') print(' Starting to generate ',social_name,' post')
@@ -1566,7 +1537,8 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
print(' write to xls for :',social_name) print(' write to xls for :',social_name)
local_outputs['datawb'].save(env.xls) local_outputs['datawb'].save(env.xls)
print(' Successfully wrote to xls for social - ',social_name) print(' Successfully wrote to xls for social - ',social_name)
# local_outputs['postssession'].update('dict_post_complete = '+str(writtento)+\ # local_outputs['postssession'].update('dict_post_complete = '+\
# str(writtento)+\
# ' where name == '+social_post.name) # ' where name == '+social_post.name)
# local_outputs['postssession'].commit() # local_outputs['postssession'].commit()
except AttributeError as error: except AttributeError as error: