mirror of
https://github.com/timberjoegithub/GoogleScrape.git
synced 2026-07-22 00:19:48 +00:00
continue to lint
This commit is contained in:
@@ -52,7 +52,7 @@ def preload():
|
|||||||
today = datetime.today().strftime('%Y-%m-%d')
|
today = datetime.today().strftime('%Y-%m-%d')
|
||||||
return
|
return
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
class Users(Base):
|
class Users(Base):
|
||||||
__tablename__ = 'userConfig'
|
__tablename__ = 'userConfig'
|
||||||
@@ -79,7 +79,7 @@ class Users(Base):
|
|||||||
instagrampass = sqlalchemy.Column(sqlalchemy.String(length=32, collation="utf8"))
|
instagrampass = sqlalchemy.Column(sqlalchemy.String(length=32, collation="utf8"))
|
||||||
#active = sqlalchemy.Column(sqlalchemy.Boolean, default=True)
|
#active = sqlalchemy.Column(sqlalchemy.Boolean, default=True)
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
class Posts(Base):
|
class Posts(Base):
|
||||||
__tablename__ = 'posts'
|
__tablename__ = 'posts'
|
||||||
@@ -102,7 +102,8 @@ def authconnect():
|
|||||||
if mariadb:
|
if mariadb:
|
||||||
print('Connecting to MariaDB for configuration and storage')
|
print('Connecting to MariaDB for configuration and storage')
|
||||||
from sqlalchemy import create_engine
|
from sqlalchemy import create_engine
|
||||||
engine = sqlalchemy.create_engine("mysql+mysqldb://"+mariadbuser+":"+mariadbpass+"@"+mariadbserver+"/"+mariadbdb+"?charset=utf8mb4", echo=False)
|
engine = sqlalchemy.create_engine("mysql+mysqldb://"+mariadbuser+":"+mariadbpass+
|
||||||
|
"@"+mariadbserver+"/"+mariadbdb+"?charset=utf8mb4", echo=False)
|
||||||
Session = sqlalchemy.orm.sessionmaker()
|
Session = sqlalchemy.orm.sessionmaker()
|
||||||
Session.configure(bind=engine)
|
Session.configure(bind=engine)
|
||||||
session = Session()
|
session = Session()
|
||||||
@@ -116,15 +117,15 @@ def authconnect():
|
|||||||
connections.update({'postssession':session})
|
connections.update({'postssession':session})
|
||||||
if data:
|
if data:
|
||||||
print(' loading XLS content data source ...')
|
print(' loading XLS content data source ...')
|
||||||
if (os.path.exists(xls)):
|
if os.path.exists(xls):
|
||||||
wb = load_workbook(filename = xls)
|
wb = load_workbook(filename = xls)
|
||||||
xlswbDF = pd.read_excel(xls)
|
xlswbDF = pd.read_excel(xls)
|
||||||
else:
|
else:
|
||||||
if (os.path.exists('./GoogleScrape/'+ xls)):
|
if os.path.exists('./GoogleScrape/'+ xls):
|
||||||
wb = load_workbook(filename = './GoogleScrape/'+ xls)
|
wb = load_workbook(filename = './GoogleScrape/'+ xls)
|
||||||
xlswbDF = pd.read_excel('./GoogleScrape/'+ xls)
|
xlswbDF = pd.read_excel('./GoogleScrape/'+ xls)
|
||||||
else:
|
else:
|
||||||
input("Not able to find xls file Press any key to continue...")
|
input("Not able to find xls file Press any key to continue...")
|
||||||
ws = wb['Sheet1']
|
ws = wb['Sheet1']
|
||||||
#xlswbDF = pd.read_excel(xls)
|
#xlswbDF = pd.read_excel(xls)
|
||||||
connections.update({'xlsdf':xlswbDF})
|
connections.update({'xlsdf':xlswbDF})
|
||||||
@@ -137,7 +138,6 @@ def authconnect():
|
|||||||
connections.update({'instagram':instasessionclient})
|
connections.update({'instagram':instasessionclient})
|
||||||
if facebook :
|
if facebook :
|
||||||
print(' Connecting to facebook ...')
|
print(' Connecting to facebook ...')
|
||||||
# msg = 'Purple Ombre Bob Lace Wig Natural Human Hair now available on https://lace-wigs.co.za/'
|
|
||||||
# page_id_1 = facebookpageID
|
# page_id_1 = facebookpageID
|
||||||
# facebook_access_token = 'paste-your-page-access-token-here'
|
# facebook_access_token = 'paste-your-page-access-token-here'
|
||||||
# image_url = 'https://graph.facebook.com/{}/photos'.format(page_id_1)
|
# image_url = 'https://graph.facebook.com/{}/photos'.format(page_id_1)
|
||||||
@@ -165,7 +165,7 @@ def authconnect():
|
|||||||
connections.update({'web' : headers})
|
connections.update({'web' : headers})
|
||||||
if tiktok :
|
if tiktok :
|
||||||
print(' Connecting to Instagram ...')
|
print(' Connecting to Instagram ...')
|
||||||
return connections
|
return (connections)
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
@@ -208,13 +208,13 @@ def postImage(group_id, img,auth_token):
|
|||||||
data = {
|
data = {
|
||||||
"published" : False
|
"published" : False
|
||||||
}
|
}
|
||||||
try:
|
try:
|
||||||
r = requests.post(url, files=files, data=data).json()
|
r = requests.post(url, files=files, data=data).json()
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
print(" An error getting date occurred:", type(error).c) # An error occurred:
|
print(" An error getting date occurred:", type(error).c) # An error occurred:
|
||||||
r = False
|
r = False
|
||||||
time.sleep(facebooksleep)
|
time.sleep(facebooksleep)
|
||||||
return r
|
return (r)
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
@@ -228,16 +228,15 @@ def postVideo(group_id, video_path,auth_token,title, content, date, rating, addr
|
|||||||
# my_dict['key'].append(1)
|
# my_dict['key'].append(1)
|
||||||
files.update({eachfile: open(eachfile, 'rb')})
|
files.update({eachfile: open(eachfile, 'rb')})
|
||||||
data = { "title":title,"description" : title + "\n"+ address+"\nGoogle map to destination: "
|
data = { "title":title,"description" : title + "\n"+ address+"\nGoogle map to destination: "
|
||||||
r"https://www.google.com/maps/dir/?api=1&destination="+addresshtml +"\n\n"+ content + "\n"+rating+"\n"+date+"\n\n"+ hastags(address, title)+"\n\nhttps://www.joeeatswhat.com"+"\n\n","published" : True
|
r"https://www.google.com/maps/dir/?api=1&destination="+addresshtml +"\n\n"+ content +
|
||||||
|
"\n"+rating+"\n"+date+"\n\n"+ hastags(address, title)+
|
||||||
|
"\n\nhttps://www.joeeatswhat.com"+"\n\n","published" : True
|
||||||
}
|
}
|
||||||
#try: r = requests.post(url, files=files, data=data)
|
|
||||||
try: r = requests.post(url, files=files, data=data).json()
|
try: r = requests.post(url, files=files, data=data).json()
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
print(" An error getting date occurred:", type(error).c) # An error occurred:
|
print(" An error getting date occurred:", type(error).c) # An error occurred:
|
||||||
r = False
|
r = False
|
||||||
time.sleep(facebooksleep)
|
time.sleep(facebooksleep)
|
||||||
print (r)
|
|
||||||
print ('r id = ',r['id'])
|
|
||||||
return r
|
return r
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
@@ -260,19 +259,18 @@ def post_facebook2(title, content, date, rating, address, picslist, instasession
|
|||||||
imgs_pic.append(img)
|
imgs_pic.append(img)
|
||||||
# if (imgs_vid ):
|
# if (imgs_vid ):
|
||||||
# print ("loop")
|
# print ("loop")
|
||||||
# try:
|
# try:
|
||||||
# post_id = postVideo(group_id, imgs_vid,auth_token)
|
# post_id = postVideo(group_id, imgs_vid,auth_token)
|
||||||
# imgs_id.append(post_id['id'])
|
# imgs_id.append(post_id['id'])
|
||||||
# except Exception as error:
|
# except Exception as error:
|
||||||
# print(" An error occurred:", type(error).c) # An error occurred:
|
# print(" An error occurred:", type(error).c) # An error occurred:
|
||||||
if (imgs_pic):
|
if (imgs_pic):
|
||||||
try:
|
try:
|
||||||
post_id = postImage(group_id ,imgs_pic,auth_token)
|
post_id = postImage(group_id ,imgs_pic,auth_token)
|
||||||
imgs_id.append(post_id['id'])
|
imgs_id.append(post_id['id'])
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
print(" An error occurred:", type(error)) # An error occurred:
|
print(" An error occurred:", type(error)) # An error occurred:
|
||||||
|
# try:
|
||||||
# try:
|
|
||||||
# imgs_id.append(post_id['id'])
|
# imgs_id.append(post_id['id'])
|
||||||
# except Exception as error:
|
# except Exception as error:
|
||||||
# print(" An error occurred:", type(error).c) # An error occurred:
|
# print(" An error occurred:", type(error).c) # An error occurred:
|
||||||
@@ -307,14 +305,15 @@ def post_facebook3(title, content, date, rating, address, picslist, instasession
|
|||||||
imgs_pic = []
|
imgs_pic = []
|
||||||
img_list = pics
|
img_list = pics
|
||||||
for img in img_list:
|
for img in img_list:
|
||||||
if ('montage.mp4' in img ):
|
if 'montage.mp4' in img:
|
||||||
imgs_vid.append(img.strip())
|
imgs_vid.append(img.strip())
|
||||||
else:
|
else:
|
||||||
imgs_pic.append(img.strip())
|
imgs_pic.append(img.strip())
|
||||||
if (imgs_vid ):
|
if imgs_vid:
|
||||||
# print ("loop")
|
# print ("loop")
|
||||||
try:
|
try:
|
||||||
post_id = postVideo(group_id, imgs_vid,auth_token,title, content, date, rating, address)
|
post_id = postVideo(group_id, imgs_vid,auth_token,title, content,
|
||||||
|
date, rating, address)
|
||||||
imgs_id.append(post_id['id'])
|
imgs_id.append(post_id['id'])
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
print(" An error occurred:", type(error).c) # An error occurred:
|
print(" An error occurred:", type(error).c) # An error occurred:
|
||||||
@@ -323,11 +322,13 @@ def post_facebook3(title, content, date, rating, address, picslist, instasession
|
|||||||
return (True)
|
return (True)
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
|
|
||||||
def get_data(driver,outputs ):
|
def get_data(driver,outputs ):
|
||||||
|
|
||||||
# curl -X GET -H 'Content-Type: application/json' -H "X-Goog-Api-Key: API_KEY" -H "X-Goog-FieldMask: id,displayName,formattedAddress,plusCode" https://places.googleapis.com/v1/places/ChIJj61dQgK6j4AR4GeTYWZsKWw #placeId #websiteUri AIzaSyB_hglzM7N8HzjXwi1dF1E8WYTql3akS7Q
|
# curl -X GET -H 'Content-Type: application/json' -H "X-Goog-Api-Key: API_KEY" -H
|
||||||
|
# "X-Goog-FieldMask: id,displayName,formattedAddress,plusCode"
|
||||||
|
# https://places.googleapis.com/v1/places/ChIJj61dQgK6j4AR4GeTYWZsKWw #placeId #websiteUri
|
||||||
"""
|
"""
|
||||||
this function get main text, score, name
|
this function get main text, score, name
|
||||||
"""
|
"""
|
||||||
@@ -336,8 +337,9 @@ def get_data(driver,outputs ):
|
|||||||
more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq')
|
more_elemets = driver.find_elements(By.CSS_SELECTOR, '.w8nwRe.kyuRq')
|
||||||
for list_more_element in more_elemets:
|
for list_more_element in more_elemets:
|
||||||
list_more_element.click()
|
list_more_element.click()
|
||||||
# Find Pictures that have the expansion indicator to see the rest of the pictures under them and click it to expose them all
|
# Find Pictures that have the expansion indicator to see the rest of the pictures under
|
||||||
more_pics = driver.find_elements(By.CLASS_NAME, 'Tya61d')
|
# them and click it to expose them all
|
||||||
|
more_pics = driver.find_elements(By.CLASS_NAME, 'Tya61d')
|
||||||
for list_more_pics in more_pics:
|
for list_more_pics in more_pics:
|
||||||
if 'showMorePhotos' in list_more_pics.get_attribute("jsaction") :
|
if 'showMorePhotos' in list_more_pics.get_attribute("jsaction") :
|
||||||
print('Found extra pics')
|
print('Found extra pics')
|
||||||
@@ -345,33 +347,38 @@ def get_data(driver,outputs ):
|
|||||||
elements = driver.find_elements(By.CLASS_NAME, 'jftiEf')
|
elements = driver.find_elements(By.CLASS_NAME, 'jftiEf')
|
||||||
lst_data = []
|
lst_data = []
|
||||||
for data in elements:
|
for data in elements:
|
||||||
name = data.find_element(By.CSS_SELECTOR, 'div.d4r55.YJxk2d').text
|
name = data.find_element(By.CSS_SELECTOR, 'div.d4r55.YJxk2d').text
|
||||||
try: address = data.find_element(By.CSS_SELECTOR, 'div.RfnDt.xJVozb').text
|
try: address = data.find_element(By.CSS_SELECTOR, 'div.RfnDt.xJVozb').text
|
||||||
except: address = 'Unknonwn'
|
except: address = 'Unknonwn'
|
||||||
print ('Name of location: ',name, ' Address:',address)
|
print ('Name of location: ',name, ' Address:',address)
|
||||||
try: visitdate = data.find_element(By.CSS_SELECTOR, 'span.rsqaWe').text
|
try: visitdate = data.find_element(By.CSS_SELECTOR, 'span.rsqaWe').text
|
||||||
except: visitdate = "Unknown"
|
except: visitdate = "Unknown"
|
||||||
print('Visited: ',visitdate)
|
print('Visited: ',visitdate)
|
||||||
try: text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text
|
try: text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text
|
||||||
except: text = ''
|
except: text = ''
|
||||||
try: score = data.find_element(By.CSS_SELECTOR, 'span.kvMYJc').get_attribute("aria-label") #find_element(By.CSS_SELECTOR,'aria-label').text #) ##QA0Szd > div > div > div.w6VYqd > div:nth-child(2) > div > div.e07Vkf.kA9KIf > div > div > div.m6QErb.DxyBCb.kA9KIf.dS8AEf > div.m6QErb > div:nth-child(3) > div:nth-child(2) > div > div:nth-child(4) > div.DU9Pgb > span.kvMYJc
|
try: score = data.find_element(By.CSS_SELECTOR, 'span.kvMYJc').get_attribute("aria-label")
|
||||||
|
#find_element(By.CSS_SELECTOR,'aria-label').text #) ##QA0Szd > div > div > div.w6VYqd >
|
||||||
|
# div:nth-child(2) > div > div.e07Vkf.kA9KIf > div > div > div.m6QErb.DxyBCb.kA9KIf.dS8AEf
|
||||||
|
# > div.m6QErb > div:nth-child(3) > div:nth-child(2) > div > div:nth-child(4) > div.DU9Pgb
|
||||||
|
# > span.kvMYJc
|
||||||
except: score = "Unknown"
|
except: score = "Unknown"
|
||||||
more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d')
|
more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d')
|
||||||
pics= []
|
pics= []
|
||||||
pics2 = []
|
pics2 = []
|
||||||
# check to see if folder for pictures and videos already exists, if not, create it
|
# check to see if folder for pictures and videos already exists, if not, create it
|
||||||
cleanname = re.sub( r'[^a-zA-Z0-9]','', name)
|
cleanname = re.sub( r'[^a-zA-Z0-9]','', name)
|
||||||
if not os.path.exists('./Output/Pics/'+cleanname):
|
if not os.path.exists('./Output/Pics/'+cleanname):
|
||||||
os.makedirs('./Output/Pics/'+cleanname)
|
os.makedirs('./Output/Pics/'+cleanname)
|
||||||
# Walk through all the pictures and videos for a given review
|
# Walk through all the pictures and videos for a given review
|
||||||
for lmpics in more_specific_pics:
|
for lmpics in more_specific_pics:
|
||||||
# if pics2 and not pics:
|
# Grab URL from style definiton (long multivalue string), and remove the -p-k so that
|
||||||
# pics2 = []
|
# it is full size
|
||||||
# Grab URL from style definiton (long multivalue string), and remove the -p-k so that it is full size
|
urlmedia = re.sub('=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]",
|
||||||
urlmedia = re.sub('=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]", lmpics.get_attribute("style")))[0])
|
lmpics.get_attribute("style")))[0])
|
||||||
print ('URL : ',urlmedia)
|
print ('URL : ',urlmedia)
|
||||||
pics.append(urlmedia)
|
pics.append(urlmedia)
|
||||||
# Grab the name of the file and remove all spaces and special charecters to name the folder
|
# Grab the name of the file and remove all spaces and special charecters to name the
|
||||||
|
# folder
|
||||||
filename = re.sub( r'[^a-zA-Z0-9]','', str(lmpics.get_attribute("aria-label")))
|
filename = re.sub( r'[^a-zA-Z0-9]','', str(lmpics.get_attribute("aria-label")))
|
||||||
if lmpics == more_specific_pics[0]:
|
if lmpics == more_specific_pics[0]:
|
||||||
lmpics.click()
|
lmpics.click()
|
||||||
@@ -380,8 +387,10 @@ def get_data(driver,outputs ):
|
|||||||
tempdate = str((driver.find_element(By.CLASS_NAME,'mqX5ad')).text).rsplit("-",1)
|
tempdate = str((driver.find_element(By.CLASS_NAME,'mqX5ad')).text).rsplit("-",1)
|
||||||
visitdate = re.sub( r'[^a-zA-Z0-9]','',tempdate[1])
|
visitdate = re.sub( r'[^a-zA-Z0-9]','',tempdate[1])
|
||||||
print ('Visited: ',visitdate)
|
print ('Visited: ',visitdate)
|
||||||
# Check to see if it has a sub div, which represents the label with the video length displayed, this will be done
|
# Check to see if it has a sub div, which represents the label with the video length
|
||||||
# because videos are represented by pictures in the main dialogue, so we need to click through and grab the video URL
|
# displayed, this will be done
|
||||||
|
# because videos are represented by pictures in the main dialogue, so we need to click
|
||||||
|
# through and grab the video URL
|
||||||
if (lmpics.find_elements(By.CSS_SELECTOR,'div.fontLabelMedium.e5A3N')) :
|
if (lmpics.find_elements(By.CSS_SELECTOR,'div.fontLabelMedium.e5A3N')) :
|
||||||
ext='.mp4'
|
ext='.mp4'
|
||||||
lmpics.click()
|
lmpics.click()
|
||||||
@@ -397,10 +406,11 @@ def get_data(driver,outputs ):
|
|||||||
driver.switch_to.default_content()
|
driver.switch_to.default_content()
|
||||||
else:
|
else:
|
||||||
# The default path if it is not a video link
|
# The default path if it is not a video link
|
||||||
ext='.jpg'
|
ext='.jpg'
|
||||||
# Add the correct extension to the file name
|
# Add the correct extension to the file name
|
||||||
filename = filename+ext
|
filename = filename+ext
|
||||||
# Test to see if file already exists, and if it does not grab the media and store it in location folder
|
# Test to see if file already exists, and if it does not grab the media and store it
|
||||||
|
# in location folder
|
||||||
if not os.path.exists('./Output/Pics/'+cleanname+'/'+visitdate):
|
if not os.path.exists('./Output/Pics/'+cleanname+'/'+visitdate):
|
||||||
os.makedirs('./Output/Pics/'+cleanname+'/'+visitdate)
|
os.makedirs('./Output/Pics/'+cleanname+'/'+visitdate)
|
||||||
if not os.path.isfile('./Output/Pics/'+cleanname+'/'+visitdate+'/'+filename):
|
if not os.path.isfile('./Output/Pics/'+cleanname+'/'+visitdate+'/'+filename):
|
||||||
@@ -410,10 +420,11 @@ def get_data(driver,outputs ):
|
|||||||
pics2.append(picsLocalpath)
|
pics2.append(picsLocalpath)
|
||||||
if pics2:
|
if pics2:
|
||||||
make_video(pics2)
|
make_video(pics2)
|
||||||
pics2.append("./Output/Pics/"+cleanname+"/"+visitdate+'/'+'montage.mp4')
|
pics2.append("./Output/Pics/"+cleanname+"/"+visitdate+'/'+'montage.mp4')
|
||||||
dictPostComplete= {'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,'instagram':0,'tiktok':0}
|
dictPostComplete= {'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,
|
||||||
lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address,dictPostComplete])
|
'instagram':0,'tiktok':0}
|
||||||
|
lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address,
|
||||||
|
dictPostComplete])
|
||||||
return lst_data
|
return lst_data
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
@@ -428,7 +439,8 @@ def scrolling(counter,driver):
|
|||||||
for _i in range(counter):
|
for _i in range(counter):
|
||||||
try:
|
try:
|
||||||
scrolling = driver.execute_script(
|
scrolling = driver.execute_script(
|
||||||
'document.getElementsByClassName("dS8AEf")[0].scrollTop = document.getElementsByClassName("dS8AEf")[0].scrollHeight',
|
'document.getElementsByClassName("dS8AEf")[0].scrollTop = /
|
||||||
|
document.getElementsByClassName("dS8AEf")[0].scrollHeight',
|
||||||
scrollable_div
|
scrollable_div
|
||||||
)
|
)
|
||||||
time.sleep(3)
|
time.sleep(3)
|
||||||
@@ -440,7 +452,8 @@ def scrolling(counter,driver):
|
|||||||
|
|
||||||
def write_to_xlsx(webdata, outputs):
|
def write_to_xlsx(webdata, outputs):
|
||||||
print('Start to write to excel...')
|
print('Start to write to excel...')
|
||||||
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address','dictPostComplete']
|
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
|
||||||
|
'dictPostComplete']
|
||||||
# rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)))
|
# rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)))
|
||||||
rows = list(webdata)
|
rows = list(webdata)
|
||||||
if needreversed:
|
if needreversed:
|
||||||
@@ -452,10 +465,13 @@ def write_to_xlsx(webdata, outputs):
|
|||||||
else:
|
else:
|
||||||
if (processrow[0] != None):
|
if (processrow[0] != None):
|
||||||
# processrow.append({'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,'Instagram':0,'tiktok':0})
|
# processrow.append({'google':1,'web':0,'yelp':0,'facebook':0,'xtwitter':0,'Instagram':0,'tiktok':0})
|
||||||
outputs['data'].append([processrow[0],processrow[1],processrow[2],str(processrow[3]),str(processrow[4]),str(processrow[5]),str(processrow[6]),str(processrow[7]),str(processrow[8])]) # sheet_obj.append([col1, col2])
|
outputs['data'].append([processrow[0],processrow[1],processrow[2],str(
|
||||||
|
processrow[3]),str(processrow[4]),str(processrow[5]),str(processrow[6]),
|
||||||
|
str(processrow[7]),str(processrow[8])]) # sheet_obj.append([col1, col2])
|
||||||
#outputs['data'].parent.save(xls)
|
#outputs['data'].parent.save(xls)
|
||||||
#outputs['datawb'].save(xls)
|
#outputs['datawb'].save(xls)
|
||||||
cols = ['num',"name", "comment", 'rating','picsURL','picsLocalpath','source','date','address','dictPostComplete', 'test']
|
cols = ['num',"name", "comment", 'rating','picsURL','picsLocalpath','source','date',
|
||||||
|
'address','dictPostComplete', 'test']
|
||||||
df = pd.DataFrame(outputs['data'], columns=cols)
|
df = pd.DataFrame(outputs['data'], columns=cols)
|
||||||
df.to_excel(xls)
|
df.to_excel(xls)
|
||||||
return True
|
return True
|
||||||
@@ -483,9 +499,6 @@ def write_to_xlsx2(data, outputs):
|
|||||||
if (processrow[1] in df.values):
|
if (processrow[1] in df.values):
|
||||||
print (' Row ',processrow[0],' ', processrow[1],' already in XLS sheet')
|
print (' Row ',processrow[0],' ', processrow[1],' already in XLS sheet')
|
||||||
d2_row = Posts(name=processrow[1],comment=processrow[2],rating=processrow[3],picsURL=processrow[4],picsLocalpath=processrow[5],source=processrow[6],date=processrow[7],address=processrow[8],dictPostComplete=processrow[9])
|
d2_row = Posts(name=processrow[1],comment=processrow[2],rating=processrow[3],picsURL=processrow[4],picsLocalpath=processrow[5],source=processrow[6],date=processrow[7],address=processrow[8],dictPostComplete=processrow[9])
|
||||||
# if not (processrow[1] in outputs['posts']) :
|
|
||||||
# outputs['postssession'].add(d2_row)
|
|
||||||
# outputs['postssession'].commit()
|
|
||||||
else:
|
else:
|
||||||
if (processrow[1] != None):
|
if (processrow[1] != None):
|
||||||
# Create a Python dictionary object with all the column values
|
# Create a Python dictionary object with all the column values
|
||||||
@@ -516,7 +529,7 @@ def database_read(data):
|
|||||||
db_connection_str = 'mysql+pymysql://mysql_user:mysql_password@mysql_host/mysql_db'
|
db_connection_str = 'mysql+pymysql://mysql_user:mysql_password@mysql_host/mysql_db'
|
||||||
db_connection = create_engine(db_connection_str)
|
db_connection = create_engine(db_connection_str)
|
||||||
df = pd.read_sql('SELECT * FROM table_name', con=db_connection)
|
df = pd.read_sql('SELECT * FROM table_name', con=db_connection)
|
||||||
return df
|
return (df)
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
@@ -531,7 +544,7 @@ def check_web_media(filename,headers):
|
|||||||
return file_id, link
|
return file_id, link
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
print(' No existing media with same name in Wordpress media folder: ' + filename)
|
print(' No existing media with same name in Wordpress media folder: ' + filename)
|
||||||
return False, False
|
return (False, False)
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
@@ -543,7 +556,7 @@ def check_web_post(postname,postdate,headers):
|
|||||||
post_date = result[0]['date']
|
post_date = result[0]['date']
|
||||||
if postdate == post_date:
|
if postdate == post_date:
|
||||||
return post_id
|
return post_id
|
||||||
except:
|
except:
|
||||||
print('No existing post with same name: ' + postname)
|
print('No existing post with same name: ' + postname)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -644,7 +657,6 @@ def post_x(title, content, date, rating, address, picslist, instasession):
|
|||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
|
|
||||||
def post_to_wp(title, content, headers,date, rating,address, picslist):
|
def post_to_wp(title, content, headers,date, rating,address, picslist):
|
||||||
# post
|
# post
|
||||||
NewPost = False
|
NewPost = False
|
||||||
@@ -655,10 +667,8 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
|
|||||||
picl = picslist[1:-1]
|
picl = picslist[1:-1]
|
||||||
pic2 = picl.replace(",","")#re.sub(r',','',picl) #re.sub( r'[^a-zA-Z0-9]','',tempdate[1])
|
pic2 = picl.replace(",","")#re.sub(r',','',picl) #re.sub( r'[^a-zA-Z0-9]','',tempdate[1])
|
||||||
pic3= pic2.replace("'","")
|
pic3= pic2.replace("'","")
|
||||||
# print (pic3)
|
|
||||||
pidchop = pic3.split(" ")
|
pidchop = pic3.split(" ")
|
||||||
linkslist=[]
|
linkslist=[]
|
||||||
# linkslist.clear
|
|
||||||
print (' Figuring out date of Post : ',title)
|
print (' Figuring out date of Post : ',title)
|
||||||
format = '%b/%Y/%d' #specifify the format of the date_string.
|
format = '%b/%Y/%d' #specifify the format of the date_string.
|
||||||
date_string = date
|
date_string = date
|
||||||
@@ -740,7 +750,7 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
|
|||||||
post_id = check_post(title,newdate2,headers)
|
post_id = check_post(title,newdate2,headers)
|
||||||
except Exception as error :
|
except Exception as error :
|
||||||
print ('Could not check to see post already exists', type(error).c)
|
print ('Could not check to see post already exists', type(error).c)
|
||||||
if ( post_id == False):
|
if post_id == False:
|
||||||
googleadress = r"<a href=https://www.google.com/maps/dir/?api=1&destination="+addresshtml + r">"+address+r"</a>"
|
googleadress = r"<a href=https://www.google.com/maps/dir/?api=1&destination="+addresshtml + r">"+address+r"</a>"
|
||||||
post_data = {
|
post_data = {
|
||||||
"title": title,
|
"title": title,
|
||||||
@@ -775,7 +785,7 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
|
|||||||
print (' Found Picture: ',picname)
|
print (' Found Picture: ',picname)
|
||||||
file_id, link = check_media(picname, headers)
|
file_id, link = check_media(picname, headers)
|
||||||
# link = linknew['rendered']
|
# link = linknew['rendered']
|
||||||
if (file_id) is False:
|
if file_id is False:
|
||||||
print (' '+picname+' was not already found in library, adding it')
|
print (' '+picname+' was not already found in library, adding it')
|
||||||
countreview = True
|
countreview = True
|
||||||
image = {
|
image = {
|
||||||
@@ -788,7 +798,7 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
|
|||||||
image_response = requests.post(wpAPI + "/media", headers=headers, files=image)
|
image_response = requests.post(wpAPI + "/media", headers=headers, files=image)
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
print(" An error uploading picture ' + picname+ ' occurred:", type(error).__name__) # An error occurred:
|
print(" An error uploading picture ' + picname+ ' occurred:", type(error).__name__) # An error occurred:
|
||||||
if ( image_response.status_code != 201 ):
|
if image_response.status_code != 201 :
|
||||||
print (' Error- Image ',picname,' was not successfully uploaded. response: ',image_response)
|
print (' Error- Image ',picname,' was not successfully uploaded. response: ',image_response)
|
||||||
else:
|
else:
|
||||||
PicDict=image_response.json()
|
PicDict=image_response.json()
|
||||||
@@ -808,7 +818,7 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
|
|||||||
print (' Error- Image ',picname,' was not attached to post. response: ',image_response)
|
print (' Error- Image ',picname,' was not attached to post. response: ',image_response)
|
||||||
try:
|
try:
|
||||||
post_response = requests.post(wpAPI + "/posts/" + str(post_id), headers=headers)
|
post_response = requests.post(wpAPI + "/posts/" + str(post_id), headers=headers)
|
||||||
if (link in str(post_response.text)):
|
if link in str(post_response.text):
|
||||||
print (' Image link for ', picname, 'already in content of post: ',post_id, post_response.text, link)
|
print (' Image link for ', picname, 'already in content of post: ',post_id, post_response.text, link)
|
||||||
else:
|
else:
|
||||||
linkslist.append({'file_id' : file_id , 'link' : link})
|
linkslist.append({'file_id' : file_id , 'link' : link})
|
||||||
@@ -821,9 +831,9 @@ def post_to_wp(title, content, headers,date, rating,address, picslist):
|
|||||||
#for loop in linkslist:
|
#for loop in linkslist:
|
||||||
print (' Adding ', piclink['link'], ' to posting')
|
print (' Adding ', piclink['link'], ' to posting')
|
||||||
try:
|
try:
|
||||||
ext = (piclink['link'].split( '.')[-1] )
|
ext = piclink['link'].split( '.')[-1]
|
||||||
if (ext == 'mp4'):
|
if ext == 'mp4':
|
||||||
if (firstMP4):
|
if firstMP4:
|
||||||
contentpics += '\n' +r'[evp_embed_video url="' + piclink['link'] + r'" autoplay="true"]'
|
contentpics += '\n' +r'[evp_embed_video url="' + piclink['link'] + r'" autoplay="true"]'
|
||||||
firstMP4 = False
|
firstMP4 = False
|
||||||
else:
|
else:
|
||||||
@@ -850,13 +860,13 @@ def make_video(inphotos):
|
|||||||
dir = inphotos[0].rsplit(r'/', 1)
|
dir = inphotos[0].rsplit(r'/', 1)
|
||||||
folder = dir[0]
|
folder = dir[0]
|
||||||
output = folder+"/montage.mp4"
|
output = folder+"/montage.mp4"
|
||||||
if ((not os.path.exists(output)) and (len(inphotos) >1)):
|
if (not os.path.exists(output) and (len(inphotos) >1)):
|
||||||
video = VideoFileClip(inphotos[0])
|
video = VideoFileClip(inphotos[0])
|
||||||
for photo in inphotos :
|
for photo in inphotos :
|
||||||
#clip = VideoFileClip("myHolidays.mp4").subclip(50,60)
|
#clip = VideoFileClip("myHolidays.mp4").subclip(50,60)
|
||||||
clip = VideoFileClip(photo)
|
clip = VideoFileClip(photo)
|
||||||
# Concatenate the photos into a clip
|
# Concatenate the photos into a clip
|
||||||
if (".jpg" in photo):
|
if ".jpg" in photo:
|
||||||
clip.duration = 2
|
clip.duration = 2
|
||||||
try:
|
try:
|
||||||
video = concatenate_videoclips([video, clip], method="compose")
|
video = concatenate_videoclips([video, clip], method="compose")
|
||||||
@@ -874,10 +884,7 @@ def make_video(inphotos):
|
|||||||
return False, False
|
return False, False
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
|
|
||||||
##################################################################################################
|
|
||||||
|
|
||||||
def post_to_instagram2 (title, content, date, rating, address, picslist, instasession):
|
def post_to_instagram2 (title, content, date, rating, address, picslist, instasession):
|
||||||
#post_to_instagram2(processrow[1].value, processrow[2].value ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['instagram'])
|
#post_to_instagram2(processrow[1].value, processrow[2].value ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['instagram'])
|
||||||
#montageexists = "montage.mp4" in picslist
|
#montageexists = "montage.mp4" in picslist
|
||||||
@@ -948,7 +955,7 @@ def hastags (address, name):
|
|||||||
fulltag = defaulttags+" "+citytag+" "+statetag+" "+ziptag
|
fulltag = defaulttags+" "+citytag+" "+statetag+" "+ziptag
|
||||||
# 153 Sugar Belle Dr, Winter Garden, FL 34787
|
# 153 Sugar Belle Dr, Winter Garden, FL 34787
|
||||||
# inphotos[0].rsplit(r'/', 1)
|
# inphotos[0].rsplit(r'/', 1)
|
||||||
return fulltag
|
return (fulltag)
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
@@ -956,7 +963,7 @@ def process_reviews(outputs):
|
|||||||
# Process
|
# Process
|
||||||
webcount = xtwittercount = instagramcount = yelpcount = threadscount = facebookcount= tiktokcount = 0
|
webcount = xtwittercount = instagramcount = yelpcount = threadscount = facebookcount= tiktokcount = 0
|
||||||
rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)))
|
rows = list((outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row)))
|
||||||
if (google):
|
if google:
|
||||||
print('Configuration says to update google Reviews prior to processing them')
|
print('Configuration says to update google Reviews prior to processing them')
|
||||||
options = webdriver.ChromeOptions()
|
options = webdriver.ChromeOptions()
|
||||||
options.add_argument("--log-level=3")
|
options.add_argument("--log-level=3")
|
||||||
@@ -980,7 +987,6 @@ def process_reviews(outputs):
|
|||||||
driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
|
driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
|
||||||
driver.get(URL)
|
driver.get(URL)
|
||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
# counter = counter()
|
|
||||||
scrolling(counter(driver), driver)
|
scrolling(counter(driver), driver)
|
||||||
webdata = get_data(driver,outputs)
|
webdata = get_data(driver,outputs)
|
||||||
write_to_xlsx2(webdata, outputs)
|
write_to_xlsx2(webdata, outputs)
|
||||||
@@ -999,8 +1005,8 @@ def process_reviews(outputs):
|
|||||||
# Check to see if the website has already been written to according to the xls sheet, if it has not... then process
|
# Check to see if the website has already been written to according to the xls sheet, if it has not... then process
|
||||||
if ((writtento["web"]) == 0 or writtento["instagram"]==0 or writtento["facebook"]==0 or writtento["xtwitter"]==0 or writtento["yelp"]==0 or writtento["tiktok"]==0 or writtento["threads"]==0 ) and (is_port_open(wpAPI, 443)) and (web or instagram or yelp or xtwitter or tiktok or facebook or threads or google)and (processrow[2].value != None) :
|
if ((writtento["web"]) == 0 or writtento["instagram"]==0 or writtento["facebook"]==0 or writtento["xtwitter"]==0 or writtento["yelp"]==0 or writtento["tiktok"]==0 or writtento["threads"]==0 ) and (is_port_open(wpAPI, 443)) and (web or instagram or yelp or xtwitter or tiktok or facebook or threads or google)and (processrow[2].value != None) :
|
||||||
if web :
|
if web :
|
||||||
if (writtento["web"] == 0) :
|
if writtento["web"] == 0 :
|
||||||
if (webcount <= postsperrun):
|
if webcount <= postsperrun:
|
||||||
try:
|
try:
|
||||||
#NewWebPost = post_to_wp(processrow[1].value, processrow[2].value, processrow[2].value ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value)
|
#NewWebPost = post_to_wp(processrow[1].value, processrow[2].value, processrow[2].value ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value)
|
||||||
NewWebPost = post_to_wp(processrow[1].value, processrow[2].value, outputs['web'] ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value)
|
NewWebPost = post_to_wp(processrow[1].value, processrow[2].value, outputs['web'] ,processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value)
|
||||||
@@ -1025,8 +1031,8 @@ def process_reviews(outputs):
|
|||||||
else:
|
else:
|
||||||
print (' Website: Skipping posting for ',processrow[1].value,' previously written')
|
print (' Website: Skipping posting for ',processrow[1].value,' previously written')
|
||||||
if instagram:
|
if instagram:
|
||||||
if (writtento["instagram"] == 0 ):
|
if writtento["instagram"] == 0:
|
||||||
if (instagramcount <= postsperrun):
|
if instagramcount <= postsperrun:
|
||||||
try:
|
try:
|
||||||
print(' Starting to generate Instagram post')
|
print(' Starting to generate Instagram post')
|
||||||
NewInstagramPost = post_to_instagram2(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['instagram'] )
|
NewInstagramPost = post_to_instagram2(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['instagram'] )
|
||||||
@@ -1054,8 +1060,8 @@ def process_reviews(outputs):
|
|||||||
else:
|
else:
|
||||||
print (' Instagram: Skipping posting for ',processrow[1].value,' previously written')
|
print (' Instagram: Skipping posting for ',processrow[1].value,' previously written')
|
||||||
if facebook:
|
if facebook:
|
||||||
if (writtento["facebook"] == 0 ):
|
if writtento["facebook"] == 0:
|
||||||
if (facebookcount <= postsperrun):
|
if facebookcount <= postsperrun:
|
||||||
try:
|
try:
|
||||||
print(' Starting to generate Facebook post')
|
print(' Starting to generate Facebook post')
|
||||||
NewFacebookPost = post_facebook3(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['facebook'] )
|
NewFacebookPost = post_facebook3(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['facebook'] )
|
||||||
@@ -1083,8 +1089,8 @@ def process_reviews(outputs):
|
|||||||
else:
|
else:
|
||||||
print (' Facebook: Skipping posting for ',processrow[1].value,' previously written')
|
print (' Facebook: Skipping posting for ',processrow[1].value,' previously written')
|
||||||
if False:
|
if False:
|
||||||
if (writtento["xtwitter"] == 0 ):
|
if writtento["xtwitter"] == 0:
|
||||||
if (xtwittercount <= postsperrun):
|
if xtwittercount <= postsperrun:
|
||||||
try:
|
try:
|
||||||
print(' Starting to generate xtwitter post')
|
print(' Starting to generate xtwitter post')
|
||||||
NewxtwitterPost = post_x(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['xtwitter'] )
|
NewxtwitterPost = post_x(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['xtwitter'] )
|
||||||
@@ -1138,9 +1144,9 @@ def socials(name, namedict,outputs, writtento, processrow,funct):# namecount, na
|
|||||||
#namedict{'name':name, 'namecount':namecount, 'namepost':namepost, 'subroutine':subroutine}
|
#namedict{'name':name, 'namecount':namecount, 'namepost':namepost, 'subroutine':subroutine}
|
||||||
#namedict{'name':xtwitter, 'namecount':xtwiteercount, 'namepost':NewxtwitterPost, 'subroutine':postx}
|
#namedict{'name':xtwitter, 'namecount':xtwiteercount, 'namepost':NewxtwitterPost, 'subroutine':postx}
|
||||||
if name:
|
if name:
|
||||||
if (writtento[name] == 0 ):
|
if writtento[name] == 0 :
|
||||||
print (namedict['namecount'])
|
print (namedict['namecount'])
|
||||||
if (int(namedict['namecount']) <= postsperrun):
|
if int(namedict['namecount']) <= postsperrun:
|
||||||
try:
|
try:
|
||||||
print(' Starting to generate xtwitter post : ',namedict['subroutine'])
|
print(' Starting to generate xtwitter post : ',namedict['subroutine'])
|
||||||
postoutput = funct(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[4].value,outputs )
|
postoutput = funct(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[4].value,outputs )
|
||||||
|
|||||||
Reference in New Issue
Block a user