admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updated trues to 1's

Browse Source
This commit is contained in:
Joe Steele
2024-06-16 17:19:33 -04:00
parent 9cd8c248dd
commit df0dce5b50
1 changed files with 35 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
social.py
+22 -13
View File
@@ -444,7 +444,7 @@ def get_google_data(driver,outputs ):
print ('Error: ',error)
more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d')
# Grab more info from google maps entry on this particular review
if len(outputs['postssession'].query(Posts).filter(Posts.name == name,Posts.google is not True).all()) == 0 or env.forcegoogleupdate:
if len(outputs['postssession'].query(Posts).filter(Posts.name == name,Posts.google is not True).all()) == 0 or env.forcegoogleupdate or env.block_google_maps is not True:
gmaps = googlemaps.Client(env.googleapipass)
place_ids = gmaps.find_place(name+address, input_type = 'textquery', fields='')
if len(place_ids['candidates']) == 1 :
@@ -469,6 +469,7 @@ def get_google_data(driver,outputs ):
print('Error writing business details from google maps : ',error)
else:
print (' Post was already in database, skipping update unless you activate override')
database_update_row(name,"google",True,"forceall",outputs)
pics= []
pics2 = []
# check to see if folder for pictures and videos already exists, if not, create it
@@ -623,7 +624,6 @@ def write_to_xlsx2(data, outputs):
def write_to_database(data, outputs):
print('write to database ...')
sqlalchemy.null()
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
'dictPostComplete']
# cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
@@ -632,12 +632,12 @@ def write_to_database(data, outputs):
# df2 = pd.DataFrame(outputs['xlsdf'].values, columns=cols2)
# print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True))
rows = list(data)
if env.needreversed:
rows = reversed(rows)
# if env.needreversed:
# rows = reversed(rows)
#jsonposts = json.dumps(outputs['posts'], default=Posts)
print("Encode Object into JSON formatted Data using jsonpickle")
jsonposts = jsonpickle.encode(outputs['posts'], unpicklable=False)
for processrow in outputs['postssession']:
for processrow in data:
if (processrow.name in df.values):
print (' Row ',processrow.id,' ', processrow.name ,' already in database')
d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=processrow.rating,
@@ -1200,8 +1200,9 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs
newdate2 = dateparts[0]+'-'+dateparts[1]+'-'+dateparts2[0]+'T22:00:00'
#newdate2 = str(re.sub(r'-','/',str(newdate.date())))+'T22:00:00'
print (' Got Date: ', newdate2, newdate)
try:
post_id, post_link = check_wordpress_post(title,newdate2,headers)
if env.block_google_maps is not True:
try:
database_update_row(title,"wpurl",post_link,"forceall",outputs)
except AttributeError as error :
print ('Could not check to see post already exists',error)
@@ -1295,6 +1296,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs
#ratinghtml = post_response.text
first_mp4 = True
fmedia = {}
contentpics = ""
for piclink in linkslist:
#for loop in linkslist:
print (' Adding ', piclink['link'], ' to posting')
@@ -1325,10 +1327,10 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs
fmedia = file_id
# print ('featured_media2 = ',file_id)
business_url_list = outputs['postssession'].query(Posts).filter(Posts.name == title).all()
business_url = business_url_list[0].businessurl
business_url = str(business_url_list[0].businessurl)
# wpurllist = outputs['postssession'].query(Posts).filter(Posts.name == title).all()
# wpurl = wpurllist[0].wpurl
if business_url:
# if business_url or business_url is False:
status_message = str(title) + ': Business website: '+ business_url
response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), \
data={"content" : title+' = '+status_message+'\n\n'+content+'\n'+googleadress+'\n'+rating + contentpics,\
@@ -1339,7 +1341,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs
print(" An error writing images to the post " + post_response.title + ' occurred:', \
type(error).__name__) # An error occurred')
return False
return newPost
return True
##################################################################################################
@@ -1396,6 +1398,9 @@ def process_reviews(outputs):
time.sleep(5)
google_scroll(counter_google(driver), driver)
webdata = get_google_data(driver,outputs)
if env.datasource == 'db':
write_to_database(webdata, outputs)
else:
write_to_xlsx2(webdata, outputs)
driver.close()
# outputs['data'].save(xls)
@@ -1419,7 +1424,7 @@ def process_reviews(outputs):
if env.web :
#if writtento["web"] == 0 :
try:
if env.forcegoogleupdate:
if env.forcegoogleupdate and env.block_google_maps is not True :
post_id, post_link = get_wordpress_post_id_and_link(processrow.name,outputs['web'] )
if post_link:
database_update_row(processrow.name,"wpurl",post_link,"forceall",outputs)
@@ -1630,17 +1635,21 @@ def process_reviews2(outputs):
time.sleep(5)
google_scroll(counter_google(driver), driver)
webdata = get_google_data(driver,outputs)
if env.datasource == 'db':
write_to_database(webdata, outputs)
else:
write_to_xlsx2(webdata, outputs)
driver.close()
# outputs['data'].save(xls)
print('Done getting google reviews and writing them to xls file !')
else:
print ('Configuration says to skip creation of new reviews from google for this run')
if env.needreversed:
rows = reversed(rows)
# if env.needreversed:
# rows = reversed(rows)
print('Processing Reviews')
for processrow in rows:
if processrow.name != "name": # Skip header line of xls sheet
outputs['postssession'].query(Posts).filter(Posts.name == processrow.name).update({"google" : 1})
print ("Processing : ",processrow.name)
writtento = (ast.literal_eval(processrow.dictPostComplete))
# Check to see if the website has already been written to according to the xls sheet,\
@@ -1653,8 +1662,8 @@ def process_reviews2(outputs):
if env.web :
#if writtento["web"] == 0 :
try:
if env.forcegoogleupdate:
post_id, post_link = get_wordpress_post_id_and_link(processrow.name,outputs['web'] )
if env.forcegoogleupdate is True and env.block_google_maps is not True:
if post_link:
database_update_row(processrow.name,"wpurl",post_link,"forceall",outputs)
else: