mirror of
https://github.com/timberjoegithub/GoogleScrape.git
synced 2026-07-22 00:19:48 +00:00
Compare commits
9
Commits
c8296da736
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
778ce37c93 | ||
|
|
e5b8f829c7 | ||
|
|
26785e73b6 | ||
|
|
45a9ca04da | ||
|
|
875d2f5c2b | ||
|
|
f0f1eacfbd | ||
|
|
8005f47517 | ||
|
|
e5618abbe1 | ||
|
|
0d8f2dd565 |
@@ -164,6 +164,7 @@ def get_auth_connect():
|
||||
xls_wb_df = pd.read_excel('./GoogleScrape/'+ env.xls)
|
||||
else:
|
||||
input("Not able to find xls file Press any key to continue...")
|
||||
wb = ""
|
||||
ws = wb['Sheet1']
|
||||
#xls_wb_df = pd.read_excel(xls)
|
||||
connections |= {'xlsdf':xls_wb_df,'data':ws,'datawb':wb}
|
||||
@@ -443,7 +444,9 @@ def post_threads_video(group_id, video_path, auth_token, title, content, date, r
|
||||
"""
|
||||
# Get url from wordpress - right now threads can only get video and pics from and external URL
|
||||
|
||||
connect_url = f"https://graph.threads.net/v1.0/{env.threads_page_id}/threads?media_type=VIDEO&video_url=https://www.example.com/images/bronz-fonz.jpg&text=#BronzFonz&access_token=" + auth_token
|
||||
connect_url = f"https://graph.threads.net/v1.0/{env.threads_page_id}/threads?media_type="+\
|
||||
"VIDEO&video_url=https://www.example.com/images/bronz-fonz.jpg&text=#BronzFonz&"+\
|
||||
"access_token=" + auth_token
|
||||
addresshtml = re.sub(" ", ".",address)
|
||||
files = {eachfile: open(eachfile, 'rb') for eachfile in video_path}
|
||||
data = { "title":title,"description" : title + "\n"+ address+"\nGoogle map to destination: "
|
||||
@@ -503,7 +506,8 @@ def get_google_data(driver, local_outputs):
|
||||
except NoSuchElementException :
|
||||
visitdate = "Unknown"
|
||||
print(' Visited: ',visitdate)
|
||||
if re.match('^[a-zA-Z]+', visitdate) is None and re.match('^[a-zA-Z]+', visitdate) is not None:
|
||||
if re.match('^[a-zA-Z]+', visitdate) is None and re.match('^[a-zA-Z]+', visitdate) \
|
||||
is not None:
|
||||
#if visitdate[:1].isalpha:
|
||||
newdate,newdate2,visitdate = get_wordpress_post_date_string(visitdate, str(datetime.now()))
|
||||
print(' Visited altered to : ',visitdate)
|
||||
@@ -596,7 +600,7 @@ def get_google_data(driver, local_outputs):
|
||||
# switch to selected iframe
|
||||
driver.switch_to.frame(iframe)
|
||||
# Now find button and click on button
|
||||
video_elements = driver.find_elements(By.XPATH ,'//video') #.get_attribute('src')
|
||||
video_elements = driver.find_elements(By.XPATH ,'//video')#.get_attribute('src')
|
||||
urlmedia = str((video_elements[0]).get_attribute("src"))
|
||||
current_url = video_elements[0]._parent.current_url
|
||||
# return back away from iframe
|
||||
@@ -795,8 +799,8 @@ def write_to_database(data, local_outputs):
|
||||
# 'dictPostComplete':str(processrow[8])}
|
||||
# if env.forcegoogleupdate:
|
||||
# print (' Row ',processrow[0],' updated in database due to forcegoogleupdate')
|
||||
# local_outputs['postssession'].query(Posts).filter(Posts.name == processrow[0]).update\
|
||||
# (d2_dict)
|
||||
# local_outputs['postssession'].query(Posts).filter(Posts.name == processrow[0]).\
|
||||
# update (d2_dict)
|
||||
# local_outputs['postssession'].commit()
|
||||
# elif processrow[0] is not None:
|
||||
# # Create a Python dictionary object with all the column values
|
||||
@@ -989,8 +993,10 @@ def check_wordpress_post(postname, postdate, headers2,local_outputs):
|
||||
print ('Could not query for post on wordpress: ', postname,postdate, type(error))
|
||||
return False, False
|
||||
# try:
|
||||
# newdate,newdate2,visitdate = get_wordpress_post_date_string(result[0]['date'],result[0]['date'])
|
||||
# newdate3,newdate4,visitdate2 = get_wordpress_post_date_string(postdate,str(datetime.now()))
|
||||
# newdate,newdate2,visitdate = get_wordpress_post_date_string(result[0]['date'],\
|
||||
# result[0]['date'])
|
||||
# newdate3,newdate4,visitdate2 = get_wordpress_post_date_string(postdate,\
|
||||
# str(datetime.now()))
|
||||
# if visitdate and visitdate2:
|
||||
# print ('Post exists, checking visit date')
|
||||
# except KeyError as error:
|
||||
@@ -1027,8 +1033,10 @@ def check_wordpress_post2(postname, postdate, post_id, headers2,local_outputs):
|
||||
print ('Could not query for post on wordpress: ', postname,postdate, type(error))
|
||||
return False, False
|
||||
# try:
|
||||
# newdate,newdate2,visitdate = get_wordpress_post_date_string(result[0]['date'],result[0]['date'])
|
||||
# newdate3,newdate4,visitdate2 = get_wordpress_post_date_string(postdate,str(datetime.now()))
|
||||
# newdate,newdate2,visitdate = get_wordpress_post_date_string(result[0]['date'],\
|
||||
# result[0]['date'])
|
||||
# newdate3,newdate4,visitdate2 = get_wordpress_post_date_string(postdate,\
|
||||
# str(datetime.now()))
|
||||
# if visitdate and visitdate2:
|
||||
# print ('Post exists, checking visit date')
|
||||
# except KeyError as error:
|
||||
@@ -1447,8 +1455,8 @@ def post_to_tiktok(title, content, headers2, date, rating, address, picslist, lo
|
||||
'X-SS-TOKEN': env.tiktok_app_id, #[Your App ID],
|
||||
'X-TikTok-Signature': env.tiktok_client_secret, #[Your App Secret]
|
||||
}
|
||||
connect_response = requests.post('https://open.tiktokapis.com/v2/post/publish/video/init/',headers=connect_json_header,\
|
||||
data=connect_json_body,timeout=env.request_timeout)
|
||||
connect_response = requests.post('https://open.tiktokapis.com/v2/post/publish/video/init/',\
|
||||
headers=connect_json_header,data=connect_json_body,timeout=env.request_timeout)
|
||||
|
||||
#request https://api.tiktok.com/open-api/v2/aweme/post/?aweme-type=video
|
||||
# https://api.tiktok.com/open-api/v2/aweme/post/
|
||||
@@ -1473,20 +1481,21 @@ def post_to_tiktok(title, content, headers2, date, rating, address, picslist, lo
|
||||
'desc': content+tags
|
||||
}
|
||||
|
||||
post_response = requests.post('https://api.tiktok.com/open-api/v2/aweme/post/',headers=post_json_header,\
|
||||
data=post_json_body,timeout=env.request_timeout)
|
||||
post_response = requests.post('https://api.tiktok.com/open-api/v2/aweme/post/',\
|
||||
headers=post_json_header,data=post_json_body,timeout=env.request_timeout)
|
||||
|
||||
|
||||
|
||||
# headers = ['Content-Type: application/x-www-form-urlencoded','Cache-Control: no-cache']
|
||||
# data = ['client_key='+env.tiktok_client_key,'client_secret='+env.tiktok_client_secret,\
|
||||
# 'code='+env.tiktok_app_id,'grant_type=authorization_code','redirect_uri=http://www.joeeatswhat.com']
|
||||
# 'code='+env.tiktok_app_id,'grant_type=authorization_code',\
|
||||
# 'redirect_uri=http://www.joeeatswhat.com']
|
||||
# allheaders = headers + data
|
||||
|
||||
# try:
|
||||
# data2 = {'source_info':{'source':"FILE_UPLOAD",file_path}}
|
||||
# response = requests.post('https://open.tiktokapis.com/v2/oauth/token/,headers:',allheaders,\
|
||||
# 'data:',data2,'timeout:',env.request_timeout)
|
||||
# response = requests.post('https://open.tiktokapis.com/v2/oauth/token/,headers:',\
|
||||
# allheaders,'data:',data2,'timeout:',env.request_timeout)
|
||||
# except BaseException as error:
|
||||
# print(" An error occurred uploading video to TikTok:", type(error).__name__)
|
||||
# return False
|
||||
@@ -1539,8 +1548,10 @@ def post_to_instagram2(title, content, headers, date, rating, address, picslist,
|
||||
if "montage.mp4" in loop:
|
||||
outputmontage = loop
|
||||
break
|
||||
pathoutputmontage = outputmontage[1:-1]
|
||||
if pathlib.Path.exists(pathoutputmontage):
|
||||
pathoutputmontage = outputmontage
|
||||
# pathoutputmontage = outputmontage[1:-1]
|
||||
# if pathlib.Path.exists(pathoutputmontage):
|
||||
if Path(pathoutputmontage).exists():
|
||||
try:
|
||||
duration, fps, (width, height)=with_moviepy(outputmontage)
|
||||
if duration < 60:
|
||||
@@ -2215,9 +2226,7 @@ def post_to_wordpress2(title,content,headers,date,rating,address,picslist,local_
|
||||
database_update_row(title,"wpurl",post_link,"forceall",local_outputs)
|
||||
except AttributeError as error :
|
||||
print (f' Error: {error}')
|
||||
else:
|
||||
|
||||
if post_id:
|
||||
elif post_id:
|
||||
print (' Post already existed: Post ID : ',post_id)
|
||||
print (' Found post for : '+title)
|
||||
if env.force_web_create:
|
||||
@@ -2249,16 +2258,16 @@ def process_reviews2(outputs):
|
||||
webcount = xtwittercount = instagramcount = facebookcount = tiktokcount = 0
|
||||
# webcount=xtwittercount=instagramcount=yelpcount=threadscount=facebookcount=tiktokcount = 0
|
||||
if env.datasource == 'db':
|
||||
cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
|
||||
'address','dictPostComplete']
|
||||
#cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
|
||||
#'address','dictPostComplete']
|
||||
rows_orig = list(outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row))
|
||||
ttt = list(outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row))
|
||||
#ttt = list(outputs['data'].iter_rows(min_row=1, max_row=outputs['data'].max_row))
|
||||
ows_orig = list(outputs['data'])
|
||||
rows_orig2 = list(outputs['xlsdf'].values)
|
||||
rows3 = outputs['data'].iter_rows(min_row=1, max_row=outputs['data'])
|
||||
rows4 = pd.DataFrame(outputs['xlsdf'])
|
||||
rows5 = pd.DataFrame(outputs['posts'])
|
||||
rows6 = pd.DataFrame(outputs['data'])
|
||||
#rows3 = outputs['data'].iter_rows(min_row=1, max_row=outputs['data'])
|
||||
#rows4 = pd.DataFrame(outputs['xlsdf'])
|
||||
#rows5 = pd.DataFrame(outputs['posts'])
|
||||
#rows6 = pd.DataFrame(outputs['data'])
|
||||
# df2 = pd.DataFrame(outputs['xlsdf'].values, columns=cols2).iter_rows(min_row=1, max_row=outputs\
|
||||
# ['data'].max_row)
|
||||
# df3 = pd.DataFrame(outputs['xlsdf'].iter_rows(min_row=1, max_row=outputs['data'].max_row))
|
||||
@@ -2335,7 +2344,8 @@ def process_reviews2(outputs):
|
||||
#newdate,newdate2,visitdate = get_wordpress_post_date_string(date_string,date)
|
||||
post_id, post_link = get_wordpress_post_id_and_link(processrow.name,\
|
||||
outputs['web'] )
|
||||
# post_id, post_link = check_wordpress_post(processrow.name,processrow.date,outputs['web'],outputs)
|
||||
# post_id, post_link = check_wordpress_post(processrow.name,processrow.date,\
|
||||
# outputs['web'],outputs)
|
||||
if env.forcegoogleupdate is True and env.block_google_maps is not True:
|
||||
if post_link:
|
||||
database_update_row(processrow.name,"wpurl",post_link,"forceall"\
|
||||
@@ -2418,9 +2428,14 @@ def process_socials(social_name,social_post,headers,sub_process,social_count, lo
|
||||
print(" An error occurred writing database", type(error).__name__)
|
||||
except AttributeError as error:
|
||||
print(' Error writing social - ',social_name,' post : ',error,social_post.name,\
|
||||
social_post.comment, local_outputs,social_post.date, social_post.rating,\
|
||||
social_post.address, social_post.picsLocalpath, writtento[social_name],\
|
||||
# social_post.comment, local_outputs,social_post.date, social_post.rating,\
|
||||
# social_post.address, social_post.picsLocalpath, writtento[social_name],\
|
||||
type(error).__name__ )
|
||||
# except AttributeError as error:
|
||||
# print(' Error writing social - ',social_name,' post : ',error,social_post.name,\
|
||||
# social_post.comment, local_outputs,social_post.date, social_post.rating,\
|
||||
# social_post.address, social_post.picsLocalpath, writtento[social_name],\
|
||||
# type(error).__name__ )
|
||||
else:
|
||||
print (' Exceeded the number of social - ',social_name,' posts per run, skipping',\
|
||||
social_post.name)
|
||||
|
||||
Reference in New Issue
Block a user