admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixing linting problems

Browse Source
This commit is contained in:
Joe Steele
2024-06-05 08:00:09 -04:00
parent 62d70040c8
commit edf0678a49
1 changed files with 62 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+58 -45
View File
@@ -97,6 +97,11 @@ class Posts(Base):
date = sqlalchemy.Column(sqlalchemy.String(length=64, collation="utf8"))
address = sqlalchemy.Column(sqlalchemy.String(length=256, collation="utf8"))
dictPostComplete = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8"))
googleurl = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8"))
weburl = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8"))
businessurl = sqlalchemy.Column(sqlalchemy.String(length=128, collation="utf8"))
longitude = sqlalchemy.Column(sqlalchemy.Float())
latitude = sqlalchemy.Column(sqlalchemy.Float())
#active = sqlalchemy.Column(sqlalchemy.Boolean, default=True)
##################################################################################################
@@ -144,7 +149,7 @@ def authconnect():
print(' Connecting to facebook ...')
# page_id_1 = facebookpageID
# facebook_access_token = 'paste-your-page-access-token-here'
# image_url = 'https://graph.facebook.com/{}/photos'.format(page_id_1)
# image_url = 'https://graph.facebook.com/{}/photos'.formatting(page_id_1)
# image_location = 'http://image.careers-portal.co.za/f_output.jpg'
# img_payload = {
# 'message': msg,
@@ -196,8 +201,8 @@ def post_facebook(title, content, date, rating, address, picslist, instasession)
page_id_1 = env.facebookpageID
facebook_access_token = env.facebookpass
#facebook_access_token = 'paste-your-page-access-token-here'
# image_url = 'https://graph.facebook.com/{}/feed'.format(page_id_1)
image_url = 'https://graph.facebook.com/{}/photos'.format(page_id_1)
# image_url = 'https://graph.facebook.com/{}/feed'.formatting(page_id_1)
image_url = 'https://graph.facebook.com/{}/photos'.formatting(page_id_1)
image_location = pics[0]
img_payload = {
'message': content,
@@ -338,7 +343,7 @@ def post_facebook3(title, content, date, rating, address, picslist, instasession
def get_data(driver,outputs ):
# curl -X GET -H 'Content-Type: application/json' -H "X-Goog-Api-Key: API_KEY" -H
# "X-Goog-FieldMask: id,displayName,formattedAddress,plusCode"
# "X-Goog-FieldMask: id,displayName,formattingtedAddress,plusCode"
# https://places.googleapis.com/v1/places/ChIJj61dQgK6j4AR4GeTYWZsKWw #placeId #websiteUri
"""
this function get main text, score, name
@@ -911,7 +916,7 @@ def post_x(title, content, date, rating, address, picslist, instasession):
def post_to_wp(title, content, headers,date, rating,address, picslist):
# post
NewPost = False
newPost = False
#countreview = False
addresshtml = re.sub(" ", ".",address)
googleadress = r"<a href=https://www.google.com/maps/dir/?api=1&destination="+\
@@ -923,47 +928,47 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
pidchop = pic3.split(" ")
linkslist=[]
print (' Figuring out date of Post : ',title)
format = '%b/%Y/%d' #specifify the format of the date_string.
formatting = '%b/%Y/%d' #specifify the formatting of the date_string.
date_string = date
if "a day" in date_string:
date = dt.timedelta(days=-1)
# newdate = dt.datetime.strptime(date_string, format).date()
# newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() - date
else:
if "day" in date:
tempdate = -(int(re.sub( r'[^0-9]','',date_string)))
print ('Stuff - > ',tempdate)
# date = dt.timedelta(days=tempdate)
# newdate = dt.datetime.strptime(date_string, format).date()
# newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() + relativedelta(days=tempdate)
else:
if "a week" in date:
# date = dt.timedelta(weeks= -1)
# newdate = dt.datetime.strptime(date_string, format).date()
# newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() - relativedelta(weeks= -1)
else:
if "week" in date:
tempdate = -(int(re.sub( r'[^0-9]','',date_string)))
print ('Stuff - > ',tempdate)
# date = dt.timedelta(weeks= tempdate)
# newdate = dt.datetime.strptime(date_string, format).date()
# newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() + relativedelta(weeks= tempdate)
else:
if "a month" in date:
# date = dt.timedelta(months= -1)
# newdate = dt.datetime.strptime(date_string, format).date()
# newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() - relativedelta(months = -1)
else:
if "month" in date:
tempdate = -int(re.sub( r'[^0-9]','',date_string))
print ('Stuff - > ',tempdate)
# date = dt.timedelta(months= tempdate)
# newdate = dt.datetime.strptime(date_string, format).date()
# newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() + relativedelta(months = tempdate)
else:
if "a year" in date:
# date = dt.timedelta(years= -1)
# newdate = dt.datetime.strptime(date_string, format).date()
# newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() - relativedelta(years= -1)
else:
if "year" in date:
@@ -976,22 +981,22 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
except Exception as error:
print(" An error getting date occurred:",error)
else:
format = '%Y-%b-%d' #specifify the format of the date_string.
formatting = '%Y-%b-%d' #specifify the formatting of the date_string.
month = date[:3]
year = date[3:]
day = '01'
date_string = year+'-'+ month+'-'+day
try:
newdate = dt.datetime.strptime(date_string, format).date()
newdate = dt.datetime.strptime(date_string, formatting).date()
except Exception as error:
print(" An error getting date occurred:",error)
# try:
# newdate = dt.datetime.strptime(date_string, format).date()
# newdate = dt.datetime.strptime(date_string, formatting).date()
# except Exception as error:
# print(" An error getting date occurred:", type(error).c)
# print(" An error getting date occurred:", error)
newdate = str(newdate)
#format = '%b/%Y/%d' #specifify the format of the date_string.
#newdate2 = dt.datetime.strptime(str(newdate), format).date()
#formatting = '%b/%Y/%d' #specifify the formatting of the date_string.
#newdate2 = dt.datetime.strptime(str(newdate), formatting).date()
dateparts = (str(newdate)).split("-")
dateparts2 = dateparts[2].split(" ")
#dateparts = dateparts2[0]
@@ -1017,11 +1022,11 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
}
try:
headers2 = headers
response = requests.post(env.wpAPOurl, json = post_data, headers=headers2)
response = requests.post(env.wpAPOurl, json = post_data, headers=headers2,timeout=30)
if response.status_code != 201:
print ('Error: ',response, response.text)
else:
NewPost = True
newPost = True
post_id_json = response.json()
post_id = post_id_json.get('id')
print (' New post is has post_id = ',post_id)
@@ -1049,38 +1054,43 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
"description": description
}
try:
image_response = requests.post(env.wpAPI + "/media", headers=headers, files=image)
image_response = requests.post(env.wpAPI + "/media", headers=headers, files=image,timeout=30)
except Exception as error:
print(" An error uploading picture ' + picname+ ' occurred:", type(error).__name__) # An error occurred:
print(" An error uploading picture ' + picname+ ' occurred:", type(error).__name__)
if image_response.status_code != 201 :
print (' Error- Image ',picname,' was not successfully uploaded. response: ',image_response)
print (' Error- Image ',picname,' was not successfully uploaded. response: ', \
image_response)
else:
PicDict=image_response.json()
file_id= PicDict.get('id')
link = PicDict.get('guid').get("rendered")
pic_dic=image_response.json()
file_id= pic_dic.get('id')
link = pic_dic.get('guid').get("rendered")
print (' ',picname,' was successfully uploaded to website with ID: ',file_id, link)
try:
linksDict = {'file_id' : file_id , 'link' : link}
linkslist.append(linksDict)
except Exception as error:
print(" An error adding to dictionary " , file_id , link , " occurred:", type(error).__name__) # An error occurred:
print(" An error adding to dictionary " , file_id , link , " occurred:", \
type(error).__name__) # An error occurred:
else:
print (' Photo ',picname,' was already in library and added to post with ID: ',file_id,' : ',link)
print (' Photo ',picname,' was already in library and added to post with ID: ', \
file_id,' : ',link)
try:
image_response = requests.post(env.wpAPI + "/media/" + str(file_id), headers=headers, data={"post" : post_id})
image_response = requests.post(env.wpAPI + "/media/" + str(file_id), headers=headers, data={"post" : post_id},timeout=30)
except Exception as error:
print (' Error- Image ',picname,' was not attached to post. response: ',image_response+' '+error)
try:
post_response = requests.post(env.wpAPI + "/posts/" + str(post_id), headers=headers)
post_response = requests.post(env.wpAPI + "/posts/" + str(post_id), headers=headers,timeout=30)
if link in str(post_response.text):
print (' Image link for ', picname, 'already in content of post: ',post_id, post_response.text, link)
print (' Image link for ', picname, 'already in content of post: ' \
,post_id, post_response.text, link)
else:
linkslist.append({'file_id' : file_id , 'link' : link})
# countreview = True
except Exception as error:
print(" An error loading the metadata from the post " + post_response.title + ' occurred: '+type(error).__name__)
except BaseException as error:
print(" An error loading the metadata from the post " + post_response.title + \
' occurred: '+type(error).__name__)
#ratinghtml = post_response.text
firstMP4 = True
first_MP4 = True
fmedia = {}
for piclink in linkslist:
#for loop in linkslist:
@@ -1088,14 +1098,16 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
try:
ext = piclink['link'].split( '.')[-1]
if ext == 'mp4':
if firstMP4:
contentpics += '\n' +r'[evp_embed_video url="' + piclink['link'] + r'" autoplay="true"]'
firstMP4 = False
if first_MP4:
contentpics += '\n' +r'[evp_embed_video url="' + piclink['link'] + \
r'" autoplay="true"]'
first_MP4 = False
else:
contentpics += '\n' +r'[evp_embed_video url="' + piclink['link'] + r'"]'
#[evp_embed_video url="http://example.com/wp-content/uploads/videos/vid1.mp4" autoplay="true"]
#[evp_embed_video url="http://example.com/wp-content/uploads/videos/vid1.mp4" autoplay="true"]
else:
contentpics += '\n '+r'<div class="col-xs-4"><img id="'+str(file_id)+r'"' + r'src="' + piclink['link'] + r'"></div>'
contentpics += '\n '+r'<div class="col-xs-4"><img id="'+str(file_id)+r'"' + r'src="' + \
piclink['link'] + r'"></div>'
# fmedia.append = piclink{'file_id' }
# contentpics += '\n '+r'<img src="'+ piclink['link'] + '> \n'
#contentpics += r'<img src="'+ piclink['link'] + r' alt="' + title +r'">' +'\n\n'
@@ -1108,11 +1120,12 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
else:
fmedia = file_id
print ('featured_media2 = ',file_id)
response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), data={"content" : title+' = '+content+'\n'+googleadress+'\n'+rating + contentpics, "featured_media" : fmedia,"rank_math_focus_keyword" : title }, headers=headers)
response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), data={"content" : title+' = '+content+'\n'+googleadress+'\n'+rating + contentpics, "featured_media" : fmedia,"rank_math_focus_keyword" : title }, headers=headers,timeout=30)
print (response_piclinks)
except Exception as error:
print(" An error writing images to the post " + post_response.title + ' occurred:', type(error).__name__) # An error occurred')
return NewPost
print(" An error writing images to the post " + post_response.title + ' occurred:', \
type(error).__name__) # An error occurred')
return newPost
##################################################################################################
@@ -1120,8 +1133,8 @@ def make_video(inphotos):
# Load the photos from the folder
# Set the duration of each photo to 2 seconds
if inphotos:
dir = inphotos[0].rsplit(r'/', 1)
folder = dir[0]
directory = inphotos[0].rsplit(r'/', 1)
folder = directory[0]
output = folder+"/montage.mp4"
if not os.path.exists(output) and len(inphotos) >1:
video = VideoFileClip(inphotos[0])