admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

working on date information for posts with no pics

Browse Source
This commit is contained in:
Joe Steele
2024-06-25 08:18:55 -04:00
parent 7992b28036
commit f7067ade72
1 changed files with 28 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+28 -7
View File
@@ -514,7 +514,6 @@ def get_google_data(driver, local_outputs):
#iframe = driver.find_element(By.TAG_NAME, "iframe") #iframe = driver.find_element(By.TAG_NAME, "iframe")
tempdate = str((driver.find_element(By.CLASS_NAME,'mqX5ad')).text).rsplit("-",1) tempdate = str((driver.find_element(By.CLASS_NAME,'mqX5ad')).text).rsplit("-",1)
visitdate = re.sub( r'[^a-zA-Z0-9]','',tempdate[1]) visitdate = re.sub( r'[^a-zA-Z0-9]','',tempdate[1])
database_update_row(name,"visitdate",True,"onlyempty",local_outputs)
#print (' Visited: ',visitdate) #print (' Visited: ',visitdate)
# Check to see if it has a sub div, which represents the label with the video length # Check to see if it has a sub div, which represents the label with the video length
# displayed, this will be done # displayed, this will be done
@@ -554,6 +553,7 @@ def get_google_data(driver, local_outputs):
'instagram':0,'tiktok':0} 'instagram':0,'tiktok':0}
lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address, lst_data.append([name , text, score,pics,pics2,"GoogleMaps",visitdate,address,
dict_post_complete]) dict_post_complete])
database_update_row(name,"visitdate",visitdate,"forceall",local_outputs)
return lst_data return lst_data
################################################################################################## ##################################################################################################
@@ -573,7 +573,7 @@ def google_scroll(counter_google_scroll,driver):
Raises: Raises:
AttributeError: If an error occurs during scrolling. AttributeError: If an error occurs during scrolling.
""" """
print('google_scroll...') print('google_scroll...', end ="")
time.sleep(3) time.sleep(3)
scrollable_div = driver.find_element(By.XPATH, scrollable_div = driver.find_element(By.XPATH,
'//*[@id="QA0Szd"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[5]/div[2]') '//*[@id="QA0Szd"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[5]/div[2]')
@@ -586,9 +586,11 @@ def google_scroll(counter_google_scroll,driver):
scrollable_div scrollable_div
) )
time.sleep(3) time.sleep(3)
print('.')
except AttributeError as e: except AttributeError as e:
print(f"Error while google_scroll: {e}") print(f"Error while google_scroll: {e}")
break break
print ('')
return google_scroller return google_scroller
################################################################################################## ##################################################################################################
@@ -675,9 +677,9 @@ def write_to_database(data, local_outputs):
if not (x[0].startswith('_') or 'metadata' in x[0]): if not (x[0].startswith('_') or 'metadata' in x[0]):
column_list.append(x[0]) column_list.append(x[0])
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address', cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
'dictPostComplete'] 'dictPostComplete','visitdate']
cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date', cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
'address','dictPostComplete'] 'address','dictPostComplete','visitdate']
#df = pd.DataFrame(local_outputs["xls"], columns=cols) #df = pd.DataFrame(local_outputs["xls"], columns=cols)
# df = pd.DataFrame(local_outputs['xlsdf']) # df = pd.DataFrame(local_outputs['xlsdf'])
df = pd.DataFrame(local_outputs['xlsdf'].values, columns=column_list) df = pd.DataFrame(local_outputs['xlsdf'].values, columns=column_list)
@@ -1257,6 +1259,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
date = dt.timedelta(days=-1) date = dt.timedelta(days=-1)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() - date newdate = datetime.today() - date
visitdate = newdate.strftime("%b%Y")
else: else:
if "day" in date: if "day" in date:
tempdate = -(int(re.sub( r'[^0-9]','',date_string))) tempdate = -(int(re.sub( r'[^0-9]','',date_string)))
@@ -1264,11 +1267,13 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
# date = dt.timedelta(days=tempdate) # date = dt.timedelta(days=tempdate)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() + relativedelta(days=tempdate) newdate = datetime.today() + relativedelta(days=tempdate)
visitdate = newdate.strftime("%b%Y")
else: else:
if "a week" in date: if "a week" in date:
# date = dt.timedelta(weeks= -1) # date = dt.timedelta(weeks= -1)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() - relativedelta(weeks= 1) newdate = datetime.today() - relativedelta(weeks= 1)
visitdate = newdate.strftime("%b%Y")
else: else:
if "week" in date: if "week" in date:
tempdate = -(int(re.sub( r'[^0-9]','',date_string))) tempdate = -(int(re.sub( r'[^0-9]','',date_string)))
@@ -1276,11 +1281,13 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
# date = dt.timedelta(weeks= tempdate) # date = dt.timedelta(weeks= tempdate)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() + relativedelta(weeks= tempdate) newdate = datetime.today() + relativedelta(weeks= tempdate)
visitdate = newdate.strftime("%b%Y")
else: else:
if "a month" in date: if "a month" in date:
# date = dt.timedelta(months= -1) # date = dt.timedelta(months= -1)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() - relativedelta(months = 1) newdate = datetime.today() - relativedelta(months = 1)
visitdate = newdate.strftime("%b%Y")
else: else:
if "month" in date: if "month" in date:
tempdate = -int(re.sub( r'[^0-9]','',date_string)) tempdate = -int(re.sub( r'[^0-9]','',date_string))
@@ -1288,11 +1295,13 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
# date = dt.timedelta(months= tempdate) # date = dt.timedelta(months= tempdate)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() + relativedelta(months = tempdate) newdate = datetime.today() + relativedelta(months = tempdate)
visitdate = newdate.strftime("%b%Y")
else: else:
if "a year" in date: if "a year" in date:
# date = dt.timedelta(years= -1) # date = dt.timedelta(years= -1)
# newdate = dt.datetime.strptime(date_string, formatting).date() # newdate = dt.datetime.strptime(date_string, formatting).date()
newdate = datetime.today() - relativedelta(years= 1) newdate = datetime.today() - relativedelta(years= 1)
visitdate = newdate.strftime("%b%Y")
else: else:
if "year" in date: if "year" in date:
try: try:
@@ -1301,6 +1310,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
# date = dt.timedelta( years= tempdate) # date = dt.timedelta( years= tempdate)
# newdate = dt.datetime.strptime(date_string).date() # newdate = dt.datetime.strptime(date_string).date()
newdate = datetime.today() + relativedelta(years= tempdate) newdate = datetime.today() + relativedelta(years= tempdate)
visitdate = newdate.strftime("%b%Y")
except AttributeError as error: except AttributeError as error:
print(" An error getting date occurred:",error) print(" An error getting date occurred:",error)
else: else:
@@ -1325,7 +1335,10 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
#newdate2 = dt.datetime.strptime(str(newdate), formatting).date() #newdate2 = dt.datetime.strptime(str(newdate), formatting).date()
dateparts = (str(newdate)).split("-") dateparts = (str(newdate)).split("-")
dateparts2 = dateparts[2].split(" ") dateparts2 = dateparts[2].split(" ")
visitdate = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()[0].visitdate visitdate2 = local_outputs['postssession'].query(Posts).filter(Posts.name == title).all()[0].visitdate
if visitdate != visitdate2:
database_update_row(name,"visitdate",visitdate,"forceall",local_outputs)
print (f'UPDATED: {visitdate2} to {visitdate} for {title}')
#dateparts = dateparts2[0] #dateparts = dateparts2[0]
# print ('dateparts',dateparts) # print ('dateparts',dateparts)
newdate2 = dateparts[0]+'-'+dateparts[1]+'-'+dateparts2[0]+'T22:00:00' newdate2 = dateparts[0]+'-'+dateparts[1]+'-'+dateparts2[0]+'T22:00:00'
@@ -1536,36 +1549,44 @@ def get_wordpress_post_date_string(date_string,date):
if "a day" in date_string: if "a day" in date_string:
date = dt.timedelta(days=-1) date = dt.timedelta(days=-1)
newdate = datetime.today() - date newdate = datetime.today() - date
visitdate = newdate.strftime("%b%Y")
else: else:
if "day" in date: if "day" in date:
tempdate = -(int(re.sub( r'[^0-9]','',date_string))) tempdate = -(int(re.sub( r'[^0-9]','',date_string)))
print ('Stuff - > ',tempdate) print ('Stuff - > ',tempdate)
newdate = datetime.today() + relativedelta(days=tempdate) newdate = datetime.today() + relativedelta(days=tempdate)
visitdate = newdate.strftime("%b%Y")
else: else:
if "a week" in date: if "a week" in date:
newdate = datetime.today() - relativedelta(weeks= 1) newdate = datetime.today() - relativedelta(weeks= 1)
visitdate = newdate.strftime("%b%Y")
else: else:
if "week" in date: if "week" in date:
tempdate = -(int(re.sub( r'[^0-9]','',date_string))) tempdate = -(int(re.sub( r'[^0-9]','',date_string)))
print ('Stuff - > ',tempdate) print ('Stuff - > ',tempdate)
newdate = datetime.today() + relativedelta(weeks= tempdate) newdate = datetime.today() + relativedelta(weeks= tempdate)
visitdate = newdate.strftime("%b%Y")
else: else:
if "a month" in date: if "a month" in date:
newdate = datetime.today() - relativedelta(months = 1) newdate = datetime.today() - relativedelta(months = 1)
visitdate = newdate.strftime("%b%Y")
else: else:
if "month" in date: if "month" in date:
tempdate = -int(re.sub( r'[^0-9]','',date_string)) tempdate = -int(re.sub( r'[^0-9]','',date_string))
print ('Stuff - > ',tempdate) print ('Stuff - > ',tempdate)
newdate = datetime.today() + relativedelta(months = tempdate) newdate = datetime.today() + relativedelta(months = tempdate)
visitdate = newdate.strftime("%b%Y")
else: else:
if "a year" in date: if "a year" in date:
newdate = datetime.today() - relativedelta(years= 1) newdate = datetime.today() - relativedelta(years= 1)
visitdate = newdate.strftime("%b%Y")
else: else:
if "year" in date: if "year" in date:
try: try:
tempdate = -int(re.sub( r'[^0-9]','',date_string)) tempdate = -int(re.sub( r'[^0-9]','',date_string))
print ('Stuff - > ',tempdate) print ('Stuff - > ',tempdate)
newdate = datetime.today() + relativedelta(years= tempdate) newdate = datetime.today() + relativedelta(years= tempdate)
visitdate = newdate.strftime("%b%Y")
except AttributeError as error: except AttributeError as error:
print(" An error getting date occurred:",error) print(" An error getting date occurred:",error)
else: else:
@@ -1587,7 +1608,7 @@ def get_wordpress_post_date_string(date_string,date):
dateparts2 = dateparts[2].split(" ") dateparts2 = dateparts[2].split(" ")
newdate2 = dateparts[0]+'-'+dateparts[1]+'-'+dateparts2[0]+'T22:00:00' newdate2 = dateparts[0]+'-'+dateparts[1]+'-'+dateparts2[0]+'T22:00:00'
print (' Got Date: ', newdate2, newdate) print (' Got Date: ', newdate2, newdate)
return (newdate,newdate2) return (newdate,newdate2,visitdate)
################################################################################################## ##################################################################################################
@@ -1764,7 +1785,7 @@ def post_to_wordpress2(title,content,headers,date,rating,address,picslist,local_
#specifify the formatting of the date_string. #specifify the formatting of the date_string.
# formatting = '%b/%Y/%d' # formatting = '%b/%Y/%d'
date_string = date date_string = date
newdate,newdate2 = get_wordpress_post_date_string(date_string,date) newdate,newdate2,visitdate = get_wordpress_post_date_string(date_string,date)
if picslist and picslist != '': if picslist and picslist != '':
content_pics = build_picslist(picchop,piclink) content_pics = build_picslist(picchop,piclink)
featured_photo_id = get_wordpress_featured_photo_id(post_id) featured_photo_id = get_wordpress_featured_photo_id(post_id)