admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

working on wpurl, need to clean DB, got dup records

Browse Source
This commit is contained in:
Joe Steele
2024-06-11 17:33:52 -04:00
parent 09d694d937
commit ee0110eae9
1 changed files with 9 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+9 -12
View File
@@ -147,7 +147,7 @@ def get_auth_connect():
usersession = session.query(Users).filter(Users.user=='joesteele') usersession = session.query(Users).filter(Users.user=='joesteele')
dbuser = usersession[0] dbuser = usersession[0]
#for userloop in usersession: #for userloop in usersession:
print(" - " + dbuser.user + ' ' + dbuser.googleurl) print("- " + dbuser.user + ' ' + dbuser.googleurl)
connections.update({'user':dbuser}) connections.update({'user':dbuser})
posts = session.query(Posts).all() posts = session.query(Posts).all()
connections.update({'posts':posts}) connections.update({'posts':posts})
@@ -345,7 +345,7 @@ def get_google_data(driver,outputs ):
more_pics = driver.find_elements(By.CLASS_NAME, 'Tya61d') more_pics = driver.find_elements(By.CLASS_NAME, 'Tya61d')
for list_more_pics in more_pics: for list_more_pics in more_pics:
if 'showMorePhotos' in list_more_pics.get_attribute("jsaction") : if 'showMorePhotos' in list_more_pics.get_attribute("jsaction") :
print('Found extra pics') print(' Found extra pics')
list_more_pics.click() list_more_pics.click()
elements = driver.find_elements(By.CLASS_NAME, 'jftiEf') elements = driver.find_elements(By.CLASS_NAME, 'jftiEf')
@@ -361,7 +361,7 @@ def get_google_data(driver,outputs ):
visitdate = data.find_element(By.CSS_SELECTOR, 'span.rsqaWe').text visitdate = data.find_element(By.CSS_SELECTOR, 'span.rsqaWe').text
except Exception: except Exception:
visitdate = "Unknown" visitdate = "Unknown"
print('Visited: ',visitdate) print(' Visited: ',visitdate)
try: try:
text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text text = data.find_element(By.CSS_SELECTOR, 'div.MyEned').text
except Exception: except Exception:
@@ -416,7 +416,7 @@ def get_google_data(driver,outputs ):
# it is full size # it is full size
urlmedia = re.sub('=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]", urlmedia = re.sub('=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]",
lmpics.get_attribute("style")))[0]) lmpics.get_attribute("style")))[0])
print ('Pic URL : ',urlmedia) print (' Pic URL : ',urlmedia)
pics.append(urlmedia) pics.append(urlmedia)
# Grab the name of the file and remove all spaces and special charecters to name the # Grab the name of the file and remove all spaces and special charecters to name the
# folder # folder
@@ -660,7 +660,7 @@ def get_wordpress_post_id_and_link(postname,headers2):
result = response.json() result = response.json()
if len(result) > 0 : if len(result) > 0 :
post_id = int(result[0]['id']) post_id = int(result[0]['id'])
post_date = result[0]['date'] # post_date = result[0]['date']
post_link = result[0]['link'] post_link = result[0]['link']
return post_id, post_link return post_id, post_link
else: else:
@@ -1249,7 +1249,7 @@ def process_reviews(outputs):
new_web_post=post_to_wordpress(processrow[1].value,processrow[2].\ new_web_post=post_to_wordpress(processrow[1].value,processrow[2].\
value,outputs['web'] ,processrow[7].value, processrow[3].value\ value,outputs['web'] ,processrow[7].value, processrow[3].value\
, processrow[8].value, processrow[5].value,outputs) , processrow[8].value, processrow[5].value,outputs)
print (' Success Posting to Wordpress: '+processrow[1].value)# ',processrow[1].value, processrow[2].value, headers,processrow[7].value, processrow[3].value,processrow[8].value, processrow[5].value, temp3["web"] ) print (' Success Posting to Wordpress: '+processrow[1].value)
if new_web_post: if new_web_post:
webcount +=1 webcount +=1
try: try:
@@ -1257,20 +1257,17 @@ def process_reviews(outputs):
outputs['datawb'].save(env.xls) outputs['datawb'].save(env.xls)
print(' Successfully updated spreadsheet') print(' Successfully updated spreadsheet')
except Exception as error: except Exception as error:
print(" An error occurred writing Excel file:", type(error).__name__) # An error occurred: try: print(" An error occurred writing Excel file:", type(error).__name__)
try: try:
print(' write to DB for web') print(' write to DB for web')
outputs['postssession'].query(Posts).filter(Posts.name == processrow[1].value).update({"web" : 1}) outputs['postssession'].query(Posts).filter(Posts.name == processrow[1].value).update({"web" : 1})
outputs['postssession'].commit() outputs['postssession'].commit()
print(' Successfully wrote to database') print(' Successfully wrote to database')
except Exception as error: except Exception as error:
print(" An error occurred writing database", type(error).__name__) # An error occurred: print(" An error occurred writing database", type(error).__name__)
except Exception as error: except Exception as error:
print (' Error writing web post : ',processrow[1].value, processrow[2].value,processrow[7].value, processrow[3].value,processrow[8].value, processrow[5].value, writtento["web"])
print (error) #print (' Error writing web post : ',processrow[1].value, processrow[2].value, outputs['web'],processrow[7].value, processrow[3].value,processrow[8].value, processrow[5].value, writtento["web"] )
print (' Error writing web post : ',processrow[1].value, processrow[2].value,processrow[7].value, processrow[3].value,processrow[8].value, processrow[5].value, writtento["web"],' ',error) print (' Error writing web post : ',processrow[1].value, processrow[2].value,processrow[7].value, processrow[3].value,processrow[8].value, processrow[5].value, writtento["web"],' ',error)
print (error) print (error)
#print (' Error writing web post : ',processrow[1].value, processrow[2].value, outputs['web'],processrow[7].value, processrow[3].value,processrow[8].value, processrow[5].value, writtento["web"] )
else: else:
print (' Exceeded the number of web posts per run, skipping', processrow[1].value) print (' Exceeded the number of web posts per run, skipping', processrow[1].value)
else: else:
@@ -1287,7 +1284,7 @@ def process_reviews(outputs):
processrow[9].value = str(writtento) processrow[9].value = str(writtento)
except Exception as error: except Exception as error:
print(" An error occurred setting value to go into Excel file:", type(error).__name__) print(" An error occurred setting value to go into Excel file:", type(error).__name__)
print (' Success Posting to Instagram: '+processrow[1].value)# ',processrow[1].value, processrow[2].value, headers,processrow[7].value, processrow[3].value,processrow[8].value, processrow[5].value, temp3["web"] ) print (' Success Posting to Instagram: '+processrow[1].value)
if NewInstagramPost: if NewInstagramPost:
instagramcount +=1 instagramcount +=1
try: try: