admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

cleanup

Browse Source
This commit is contained in:
Joe Steele
2024-06-26 12:24:21 -04:00
parent 70b7b4ae80
commit a26b334571
1 changed files with 9 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+9 -7
View File
@@ -449,6 +449,7 @@ def get_google_data(driver, local_outputs):
except NoSuchElementException : except NoSuchElementException :
visitdate = "Unknown" visitdate = "Unknown"
print(' Visited: ',visitdate) print(' Visited: ',visitdate)
newdate,newdate2,visitdate = get_wordpress_post_date_string(visitdate, datetime.now())
try: try:
text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text
except NoSuchElementException : except NoSuchElementException :
@@ -677,14 +678,14 @@ def write_to_database(data, local_outputs):
for x in inspect.getmembers(Posts): for x in inspect.getmembers(Posts):
if not (x[0].startswith('_') or 'metadata' in x[0] or 'registry' in x[0]): if not (x[0].startswith('_') or 'metadata' in x[0] or 'registry' in x[0]):
column_list.append(x[0]) column_list.append(x[0])
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address', #cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
'dictPostComplete','visitdate'] # 'dictPostComplete','visitdate']
cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date', #cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
'address','dictPostComplete','visitdate'] # 'address','dictPostComplete','visitdate']
#df = pd.DataFrame(local_outputs["xls"], columns=cols) #df = pd.DataFrame(local_outputs["xls"], columns=cols)
# df = pd.DataFrame(local_outputs['xlsdf']) # df = pd.DataFrame(local_outputs['xlsdf'])
df = pd.DataFrame(local_outputs['xlsdf'].values, columns=column_list) df = pd.DataFrame(local_outputs['xlsdf'].values, columns=column_list)
df2 = pd.DataFrame(local_outputs['posts']) #df2 = pd.DataFrame(local_outputs['posts'])
# print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True)) # print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True))
# rows = list(data) # rows = list(data)
# if env.needreversed: # if env.needreversed:
@@ -804,7 +805,7 @@ def check_wordpress_media(filename, headers):
################################################################################################## ##################################################################################################
def check_is_port_open(host, port): def check_is_port_open(host):
""" """
Checks if a port on a host is open. Checks if a port on a host is open.
@@ -1341,6 +1342,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,local_o
# .date() # .date()
# except AttributeError as error: # except AttributeError as error:
# print(" An error getting date occurred:", error) # print(" An error getting date occurred:", error)
visitdate = newdate.strftime("%b%Y")
newdate = str(newdate) newdate = str(newdate)
#formatting = '%b/%Y/%d' #specifify the formatting of the date_string. #formatting = '%b/%Y/%d' #specifify the formatting of the date_string.
#newdate2 = dt.datetime.strptime(str(newdate), formatting).date() #newdate2 = dt.datetime.strptime(str(newdate), formatting).date()
@@ -1974,7 +1976,7 @@ def process_reviews2(outputs):
# if it has not... then process # if it has not... then process
if (writtento["web"] == 0 or writtento["instagram"]==0 or writtento["facebook"]==0 or\ if (writtento["web"] == 0 or writtento["instagram"]==0 or writtento["facebook"]==0 or\
writtento["xtwitter"]==0 or writtento["yelp"]==0 or writtento["tiktok"]==0 or \ writtento["xtwitter"]==0 or writtento["yelp"]==0 or writtento["tiktok"]==0 or \
writtento["threads"]==0 ) and (check_is_port_open(env.wpAPI, 443)) and (env.web \ writtento["threads"]==0 ) and (check_is_port_open(env.wpAPI)) and (env.web \
or env.instagram or env.yelp or env.xtwitter or env.tiktok or env.facebook or \ or env.instagram or env.yelp or env.xtwitter or env.tiktok or env.facebook or \
env.threads or env.google)and (processrow.comment is not None) : env.threads or env.google)and (processrow.comment is not None) :
if env.web and processrow.web is False or env.force_web_create is True: if env.web and processrow.web is False or env.force_web_create is True: