admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/timberjoegithub/GoogleScrape into 3.0

Browse Source
This commit is contained in:
Joe Steele
2024-06-26 16:11:34 -04:00
parent 8ffb9f46a8 2b33ff35c8
commit 5dd64d5d5e
1 changed files with 19 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+19 -15
View File
@@ -448,7 +448,11 @@ def get_google_data(driver, local_outputs):
visitdate = data.find_element(By.CSS_SELECTOR, 'span.rsqaWe').text visitdate = data.find_element(By.CSS_SELECTOR, 'span.rsqaWe').text
except NoSuchElementException : except NoSuchElementException :
visitdate = "Unknown" visitdate = "Unknown"
print(' Visited: ',visitdate) print(' Visited: ',visitdate, visitdate[:1])
if re.match('^[a-zA-Z]+', visitdate) is None:
#if visitdate[:1].isalpha:
newdate,newdate2,visitdate = get_wordpress_post_date_string(str(visitdate), str(datetime.now()))
print(' Visited altered to : ',visitdate)
try: try:
text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text
except NoSuchElementException : except NoSuchElementException :
@@ -677,14 +681,14 @@ def write_to_database(data, local_outputs):
for x in inspect.getmembers(Posts): for x in inspect.getmembers(Posts):
if not (x[0].startswith('_') or 'metadata' in x[0] or 'registry' in x[0]): if not (x[0].startswith('_') or 'metadata' in x[0] or 'registry' in x[0]):
column_list.append(x[0]) column_list.append(x[0])
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address', #cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
'dictPostComplete','visitdate'] # 'dictPostComplete','visitdate']
cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date', #cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
'address','dictPostComplete','visitdate'] # 'address','dictPostComplete','visitdate']
#df = pd.DataFrame(local_outputs["xls"], columns=cols) #df = pd.DataFrame(local_outputs["xls"], columns=cols)
# df = pd.DataFrame(local_outputs['xlsdf']) # df = pd.DataFrame(local_outputs['xlsdf'])
df = pd.DataFrame(local_outputs['xlsdf'].values, columns=column_list) df = pd.DataFrame(local_outputs['xlsdf'].values, columns=column_list)
df2 = pd.DataFrame(local_outputs['posts']) #df2 = pd.DataFrame(local_outputs['posts'])
# print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True)) # print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True))
# rows = list(data) # rows = list(data)
# if env.needreversed: # if env.needreversed:
@@ -804,7 +808,7 @@ def check_wordpress_media(filename, headers):
################################################################################################## ##################################################################################################
def check_is_port_open(host, port): def check_is_port_open(host):
""" """
Checks if a port on a host is open. Checks if a port on a host is open.
@@ -1593,37 +1597,37 @@ def get_wordpress_post_date_string(date_string, date):
newdate = datetime.today() - date newdate = datetime.today() - date
visitdate = newdate.strftime("%b%Y") visitdate = newdate.strftime("%b%Y")
else: else:
if "day" in date: if "day" in date_string:
tempdate = -(int(re.sub( r'[^0-9]','',date_string))) tempdate = -(int(re.sub( r'[^0-9]','',date_string)))
print ('Stuff - > ',tempdate) print ('Stuff - > ',tempdate)
newdate = datetime.today() + relativedelta(days=tempdate) newdate = datetime.today() + relativedelta(days=tempdate)
visitdate = newdate.strftime("%b%Y") visitdate = newdate.strftime("%b%Y")
else: else:
if "a week" in date: if "a week" in date_string:
newdate = datetime.today() - relativedelta(weeks= 1) newdate = datetime.today() - relativedelta(weeks= 1)
visitdate = newdate.strftime("%b%Y") visitdate = newdate.strftime("%b%Y")
else: else:
if "week" in date: if "week" in date_string:
tempdate = -(int(re.sub( r'[^0-9]','',date_string))) tempdate = -(int(re.sub( r'[^0-9]','',date_string)))
print ('Stuff - > ',tempdate) print ('Stuff - > ',tempdate)
newdate = datetime.today() + relativedelta(weeks= tempdate) newdate = datetime.today() + relativedelta(weeks= tempdate)
visitdate = newdate.strftime("%b%Y") visitdate = newdate.strftime("%b%Y")
else: else:
if "a month" in date: if "a month" in date_string:
newdate = datetime.today() - relativedelta(months = 1) newdate = datetime.today() - relativedelta(months = 1)
visitdate = newdate.strftime("%b%Y") visitdate = newdate.strftime("%b%Y")
else: else:
if "month" in date: if "month" in date_string:
tempdate = -int(re.sub( r'[^0-9]','',date_string)) tempdate = -int(re.sub( r'[^0-9]','',date_string))
print ('Stuff - > ',tempdate) print ('Stuff - > ',tempdate)
newdate = datetime.today() + relativedelta(months = tempdate) newdate = datetime.today() + relativedelta(months = tempdate)
visitdate = newdate.strftime("%b%Y") visitdate = newdate.strftime("%b%Y")
else: else:
if "a year" in date: if "a year" in date_string:
newdate = datetime.today() - relativedelta(years= 1) newdate = datetime.today() - relativedelta(years= 1)
visitdate = newdate.strftime("%b%Y") visitdate = newdate.strftime("%b%Y")
else: else:
if "year" in date: if "year" in date_string:
try: try:
tempdate = -int(re.sub( r'[^0-9]','',date_string)) tempdate = -int(re.sub( r'[^0-9]','',date_string))
print ('Stuff - > ',tempdate) print ('Stuff - > ',tempdate)
@@ -1975,7 +1979,7 @@ def process_reviews2(outputs):
# if it has not... then process # if it has not... then process
if (writtento["web"] == 0 or writtento["instagram"]==0 or writtento["facebook"]==0 or\ if (writtento["web"] == 0 or writtento["instagram"]==0 or writtento["facebook"]==0 or\
writtento["xtwitter"]==0 or writtento["yelp"]==0 or writtento["tiktok"]==0 or \ writtento["xtwitter"]==0 or writtento["yelp"]==0 or writtento["tiktok"]==0 or \
writtento["threads"]==0 ) and (check_is_port_open(env.wpAPI, 443)) and (env.web \ writtento["threads"]==0 ) and (check_is_port_open(env.wpAPI)) and (env.web \
or env.instagram or env.yelp or env.xtwitter or env.tiktok or env.facebook or \ or env.instagram or env.yelp or env.xtwitter or env.tiktok or env.facebook or \
env.threads or env.google)and (processrow.comment is not None) : env.threads or env.google)and (processrow.comment is not None) :
if env.web and processrow.web is False or env.force_web_create is True: if env.web and processrow.web is False or env.force_web_create is True: