admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/timberjoegithub/GoogleScrape into 3.0

Browse Source
This commit is contained in:
Joe Steele
2024-06-25 09:15:25 -04:00
parent c6af16b186 f7067ade72
commit 357bad0e1d
1 changed files with 31 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+31 -7
View File
@@ -96,6 +96,7 @@ class Posts(Base):
#businessurl = sqlalchemy.Column(sqlalchemy.String(length=512, collation="utf8")) #businessurl = sqlalchemy.Column(sqlalchemy.String(length=512, collation="utf8"))
pluscode = sqlalchemy.Column(sqlalchemy.String(length=64, collation="utf8")) pluscode = sqlalchemy.Column(sqlalchemy.String(length=64, collation="utf8"))
googleurl = sqlalchemy.Column(sqlalchemy.String(length=512, collation="utf8")) googleurl = sqlalchemy.Column(sqlalchemy.String(length=512, collation="utf8"))
visitdate = sqlalchemy.Column(sqlalchemy.String(length=64, collation="utf8"))
################################################################################################## ##################################################################################################
@@ -552,6 +553,7 @@ def get_google_data(driver, local_outputs):
'instagram':0,'tiktok':0} 'instagram':0,'tiktok':0}
lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address, lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address,
dict_post_complete]) dict_post_complete])
database_update_row(name,"visitdate",visitdate,"forceall",local_outputs)
return lst_data return lst_data
################################################################################################## ##################################################################################################
@@ -571,7 +573,7 @@ def google_scroll(counter_google_scroll,driver):
Raises: Raises:
AttributeError: If an error occurs during scrolling. AttributeError: If an error occurs during scrolling.
""" """
print('google_scroll...') print('google_scroll...', end ="")
time.sleep(3) time.sleep(3)
scrollable_div = driver.find_element(By.XPATH, scrollable_div = driver.find_element(By.XPATH,
'//*[@id="QA0Szd"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[5]/div[2]') '//*[@id="QA0Szd"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[5]/div[2]')
@@ -584,9 +586,11 @@ def google_scroll(counter_google_scroll,driver):
scrollable_div scrollable_div
) )
time.sleep(3) time.sleep(3)
print('.')
except AttributeError as e: except AttributeError as e:
print(f"Error while google_scroll: {e}") print(f"Error while google_scroll: {e}")
break break
print ('')
return google_scroller return google_scroller
################################################################################################## ##################################################################################################
@@ -673,9 +677,9 @@ def write_to_database(data, local_outputs):
if not (x[0].startswith('_') or 'metadata' in x[0]): if not (x[0].startswith('_') or 'metadata' in x[0]):
column_list.append(x[0]) column_list.append(x[0])
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address', cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
'dictPostComplete'] 'dictPostComplete','visitdate']
cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date', cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
'address','dictPostComplete'] 'address','dictPostComplete','visitdate']
#df = pd.DataFrame(local_outputs["xls"], columns=cols) #df = pd.DataFrame(local_outputs["xls"], columns=cols)
# df = pd.DataFrame(local_outputs['xlsdf']) # df = pd.DataFrame(local_outputs['xlsdf'])
df = pd.DataFrame(local_outputs['xlsdf'].values, columns=column_list) df = pd.DataFrame(local_outputs['xlsdf'].values, columns=column_list)
@@ -1255,6 +1259,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
date = dt.timedelta(days=-1) date = dt.timedelta(days=-1)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() - date newdate = datetime.today() - date
visitdate = newdate.strftime("%b%Y")
else: else:
if "day" in date: if "day" in date:
tempdate = -(int(re.sub( r'[^0-9]','',date_string))) tempdate = -(int(re.sub( r'[^0-9]','',date_string)))
@@ -1262,11 +1267,13 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
# date = dt.timedelta(days=tempdate) # date = dt.timedelta(days=tempdate)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() + relativedelta(days=tempdate) newdate = datetime.today() + relativedelta(days=tempdate)
visitdate = newdate.strftime("%b%Y")
else: else:
if "a week" in date: if "a week" in date:
# date = dt.timedelta(weeks= -1) # date = dt.timedelta(weeks= -1)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() - relativedelta(weeks= 1) newdate = datetime.today() - relativedelta(weeks= 1)
visitdate = newdate.strftime("%b%Y")
else: else:
if "week" in date: if "week" in date:
tempdate = -(int(re.sub( r'[^0-9]','',date_string))) tempdate = -(int(re.sub( r'[^0-9]','',date_string)))
@@ -1274,11 +1281,13 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
# date = dt.timedelta(weeks= tempdate) # date = dt.timedelta(weeks= tempdate)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() + relativedelta(weeks= tempdate) newdate = datetime.today() + relativedelta(weeks= tempdate)
visitdate = newdate.strftime("%b%Y")
else: else:
if "a month" in date: if "a month" in date:
# date = dt.timedelta(months= -1) # date = dt.timedelta(months= -1)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() - relativedelta(months = 1) newdate = datetime.today() - relativedelta(months = 1)
visitdate = newdate.strftime("%b%Y")
else: else:
if "month" in date: if "month" in date:
tempdate = -int(re.sub( r'[^0-9]','',date_string)) tempdate = -int(re.sub( r'[^0-9]','',date_string))
@@ -1286,11 +1295,13 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
# date = dt.timedelta(months= tempdate) # date = dt.timedelta(months= tempdate)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() + relativedelta(months = tempdate) newdate = datetime.today() + relativedelta(months = tempdate)
visitdate = newdate.strftime("%b%Y")
else: else:
if "a year" in date: if "a year" in date:
# date = dt.timedelta(years= -1) # date = dt.timedelta(years= -1)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() - relativedelta(years= 1) newdate = datetime.today() - relativedelta(years= 1)
visitdate = newdate.strftime("%b%Y")
else: else:
if "year" in date: if "year" in date:
try: try:
@@ -1299,6 +1310,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
# date = dt.timedelta( years= tempdate) # date = dt.timedelta( years= tempdate)
# newdate = dt.datetime.strptime(date_string).date() # newdate = dt.datetime.strptime(date_string).date()
newdate = datetime.today() + relativedelta(years= tempdate) newdate = datetime.today() + relativedelta(years= tempdate)
visitdate = newdate.strftime("%b%Y")
except AttributeError as error: except AttributeError as error:
print(" An error getting date occurred:",error) print(" An error getting date occurred:",error)
else: else:
@@ -1323,12 +1335,16 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
#newdate2 = dt.datetime.strptime(str(newdate), formatting).date() #newdate2 = dt.datetime.strptime(str(newdate), formatting).date()
dateparts = (str(newdate)).split("-") dateparts = (str(newdate)).split("-")
dateparts2 = dateparts[2].split(" ") dateparts2 = dateparts[2].split(" ")
visitdate2 = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()[0].visitdate
if visitdate != visitdate2:
database_update_row(name,"visitdate",visitdate,"forceall",local_outputs)
print (f'UPDATED: {visitdate2} to {visitdate} for {title}')
#dateparts = dateparts2[0] #dateparts = dateparts2[0]
# print ('dateparts',dateparts) # print ('dateparts',dateparts)
newdate2 = dateparts[0]+'-'+dateparts[1]+'-'+dateparts2[0]+'T22:00:00' newdate2 = dateparts[0]+'-'+dateparts[1]+'-'+dateparts2[0]+'T22:00:00'
#newdate2 = str(re.sub(r'-','/',str(newdate.date())))+'T22:00:00' #newdate2 = str(re.sub(r'-','/',str(newdate.date())))+'T22:00:00'
print (' Got Date: ', newdate2, newdate) print (' Got Date: ', newdate2, newdate)
post_id, post_link = check_wordpress_post(title,newdate2,headers) post_id, post_link = check_wordpress_post(title,visitdate,headers)
featured_photo_id = get_wordpress_featured_photo_id(post_id) featured_photo_id = get_wordpress_featured_photo_id(post_id)
print(f" Featured photo ID: for {title} post {post_id} is: {featured_photo_id}") print(f" Featured photo ID: for {title} post {post_id} is: {featured_photo_id}")
if env.block_google_maps is not True: if env.block_google_maps is not True:
@@ -1558,36 +1574,44 @@ def get_wordpress_post_date_string(date_string, date):
if "a day" in date_string: if "a day" in date_string:
date = dt.timedelta(days=-1) date = dt.timedelta(days=-1)
newdate = datetime.today() - date newdate = datetime.today() - date
visitdate = newdate.strftime("%b%Y")
else: else:
if "day" in date: if "day" in date:
tempdate = -(int(re.sub( r'[^0-9]','',date_string))) tempdate = -(int(re.sub( r'[^0-9]','',date_string)))
print ('Stuff - > ',tempdate) print ('Stuff - > ',tempdate)
newdate = datetime.today() + relativedelta(days=tempdate) newdate = datetime.today() + relativedelta(days=tempdate)
visitdate = newdate.strftime("%b%Y")
else: else:
if "a week" in date: if "a week" in date:
newdate = datetime.today() - relativedelta(weeks= 1) newdate = datetime.today() - relativedelta(weeks= 1)
visitdate = newdate.strftime("%b%Y")
else: else:
if "week" in date: if "week" in date:
tempdate = -(int(re.sub( r'[^0-9]','',date_string))) tempdate = -(int(re.sub( r'[^0-9]','',date_string)))
print ('Stuff - > ',tempdate) print ('Stuff - > ',tempdate)
newdate = datetime.today() + relativedelta(weeks= tempdate) newdate = datetime.today() + relativedelta(weeks= tempdate)
visitdate = newdate.strftime("%b%Y")
else: else:
if "a month" in date: if "a month" in date:
newdate = datetime.today() - relativedelta(months = 1) newdate = datetime.today() - relativedelta(months = 1)
visitdate = newdate.strftime("%b%Y")
else: else:
if "month" in date: if "month" in date:
tempdate = -int(re.sub( r'[^0-9]','',date_string)) tempdate = -int(re.sub( r'[^0-9]','',date_string))
print ('Stuff - > ',tempdate) print ('Stuff - > ',tempdate)
newdate = datetime.today() + relativedelta(months = tempdate) newdate = datetime.today() + relativedelta(months = tempdate)
visitdate = newdate.strftime("%b%Y")
else: else:
if "a year" in date: if "a year" in date:
newdate = datetime.today() - relativedelta(years= 1) newdate = datetime.today() - relativedelta(years= 1)
visitdate = newdate.strftime("%b%Y")
else: else:
if "year" in date: if "year" in date:
try: try:
tempdate = -int(re.sub( r'[^0-9]','',date_string)) tempdate = -int(re.sub( r'[^0-9]','',date_string))
print ('Stuff - > ',tempdate) print ('Stuff - > ',tempdate)
newdate = datetime.today() + relativedelta(years= tempdate) newdate = datetime.today() + relativedelta(years= tempdate)
visitdate = newdate.strftime("%b%Y")
except AttributeError as error: except AttributeError as error:
print(" An error getting date occurred:",error) print(" An error getting date occurred:",error)
else: else:
@@ -1609,7 +1633,7 @@ def get_wordpress_post_date_string(date_string, date):
dateparts2 = dateparts[2].split(" ") dateparts2 = dateparts[2].split(" ")
newdate2 = dateparts[0]+'-'+dateparts[1]+'-'+dateparts2[0]+'T22:00:00' newdate2 = dateparts[0]+'-'+dateparts[1]+'-'+dateparts2[0]+'T22:00:00'
print (' Got Date: ', newdate2, newdate) print (' Got Date: ', newdate2, newdate)
return (newdate,newdate2) return (newdate,newdate2,visitdate)
################################################################################################## ##################################################################################################
@@ -1809,7 +1833,7 @@ def post_to_wordpress2(title,content,headers,date,rating,address,picslist,local_
#specifify the formatting of the date_string. #specifify the formatting of the date_string.
# formatting = '%b/%Y/%d' # formatting = '%b/%Y/%d'
date_string = date date_string = date
newdate,newdate2 = get_wordpress_post_date_string(date_string,date) newdate,newdate2,visitdate = get_wordpress_post_date_string(date_string,date)
if picslist and picslist != '': if picslist and picslist != '':
content_pics = build_picslist(picchop,piclink) content_pics = build_picslist(picchop,piclink)
featured_photo_id = get_wordpress_featured_photo_id(post_id) featured_photo_id = get_wordpress_featured_photo_id(post_id)
@@ -1820,7 +1844,7 @@ def post_to_wordpress2(title,content,headers,date,rating,address,picslist,local_
except AttributeError as error : except AttributeError as error :
print (f' Error: {error}') print (f' Error: {error}')
else: else:
post_id, post_link = check_wordpress_post(title,newdate2,headers) post_id, post_link = check_wordpress_post(title,visitdate,headers)
if post_id: if post_id:
print (' Post already existed: Post ID : ',post_id) print (' Post already existed: Post ID : ',post_id)
print (' Found post for : '+title) print (' Found post for : '+title)