mirror of
https://github.com/timberjoegithub/GoogleScrape.git
synced 2026-07-22 00:19:48 +00:00
Updated trues to 1's
This commit is contained in:
@@ -444,7 +444,7 @@ def get_google_data(driver,outputs ):
|
|||||||
print ('Error: ',error)
|
print ('Error: ',error)
|
||||||
more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d')
|
more_specific_pics = data.find_elements(By.CLASS_NAME, 'Tya61d')
|
||||||
# Grab more info from google maps entry on this particular review
|
# Grab more info from google maps entry on this particular review
|
||||||
if len(outputs['postssession'].query(Posts).filter(Posts.name == name,Posts.google is not True).all()) == 0 or env.forcegoogleupdate:
|
if len(outputs['postssession'].query(Posts).filter(Posts.name == name,Posts.google is not True).all()) == 0 or env.forcegoogleupdate or env.block_google_maps is not True:
|
||||||
gmaps = googlemaps.Client(env.googleapipass)
|
gmaps = googlemaps.Client(env.googleapipass)
|
||||||
place_ids = gmaps.find_place(name+address, input_type = 'textquery', fields='')
|
place_ids = gmaps.find_place(name+address, input_type = 'textquery', fields='')
|
||||||
if len(place_ids['candidates']) == 1 :
|
if len(place_ids['candidates']) == 1 :
|
||||||
@@ -469,6 +469,7 @@ def get_google_data(driver,outputs ):
|
|||||||
print('Error writing business details from google maps : ',error)
|
print('Error writing business details from google maps : ',error)
|
||||||
else:
|
else:
|
||||||
print (' Post was already in database, skipping update unless you activate override')
|
print (' Post was already in database, skipping update unless you activate override')
|
||||||
|
database_update_row(name,"google",True,"forceall",outputs)
|
||||||
pics= []
|
pics= []
|
||||||
pics2 = []
|
pics2 = []
|
||||||
# check to see if folder for pictures and videos already exists, if not, create it
|
# check to see if folder for pictures and videos already exists, if not, create it
|
||||||
@@ -623,7 +624,6 @@ def write_to_xlsx2(data, outputs):
|
|||||||
|
|
||||||
def write_to_database(data, outputs):
|
def write_to_database(data, outputs):
|
||||||
print('write to database ...')
|
print('write to database ...')
|
||||||
sqlalchemy.null()
|
|
||||||
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
|
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
|
||||||
'dictPostComplete']
|
'dictPostComplete']
|
||||||
# cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
|
# cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
|
||||||
@@ -632,12 +632,12 @@ def write_to_database(data, outputs):
|
|||||||
# df2 = pd.DataFrame(outputs['xlsdf'].values, columns=cols2)
|
# df2 = pd.DataFrame(outputs['xlsdf'].values, columns=cols2)
|
||||||
# print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True))
|
# print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True))
|
||||||
rows = list(data)
|
rows = list(data)
|
||||||
if env.needreversed:
|
# if env.needreversed:
|
||||||
rows = reversed(rows)
|
# rows = reversed(rows)
|
||||||
#jsonposts = json.dumps(outputs['posts'], default=Posts)
|
#jsonposts = json.dumps(outputs['posts'], default=Posts)
|
||||||
print("Encode Object into JSON formatted Data using jsonpickle")
|
print("Encode Object into JSON formatted Data using jsonpickle")
|
||||||
jsonposts = jsonpickle.encode(outputs['posts'], unpicklable=False)
|
jsonposts = jsonpickle.encode(outputs['posts'], unpicklable=False)
|
||||||
for processrow in outputs['postssession']:
|
for processrow in data:
|
||||||
if (processrow.name in df.values):
|
if (processrow.name in df.values):
|
||||||
print (' Row ',processrow.id,' ', processrow.name ,' already in database')
|
print (' Row ',processrow.id,' ', processrow.name ,' already in database')
|
||||||
d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=processrow.rating,
|
d2_row = Posts(name=processrow.name ,comment=processrow.comment,rating=processrow.rating,
|
||||||
@@ -1200,8 +1200,9 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs
|
|||||||
newdate2 = dateparts[0]+'-'+dateparts[1]+'-'+dateparts2[0]+'T22:00:00'
|
newdate2 = dateparts[0]+'-'+dateparts[1]+'-'+dateparts2[0]+'T22:00:00'
|
||||||
#newdate2 = str(re.sub(r'-','/',str(newdate.date())))+'T22:00:00'
|
#newdate2 = str(re.sub(r'-','/',str(newdate.date())))+'T22:00:00'
|
||||||
print (' Got Date: ', newdate2, newdate)
|
print (' Got Date: ', newdate2, newdate)
|
||||||
try:
|
|
||||||
post_id, post_link = check_wordpress_post(title,newdate2,headers)
|
post_id, post_link = check_wordpress_post(title,newdate2,headers)
|
||||||
|
if env.block_google_maps is not True:
|
||||||
|
try:
|
||||||
database_update_row(title,"wpurl",post_link,"forceall",outputs)
|
database_update_row(title,"wpurl",post_link,"forceall",outputs)
|
||||||
except AttributeError as error :
|
except AttributeError as error :
|
||||||
print ('Could not check to see post already exists',error)
|
print ('Could not check to see post already exists',error)
|
||||||
@@ -1295,6 +1296,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs
|
|||||||
#ratinghtml = post_response.text
|
#ratinghtml = post_response.text
|
||||||
first_mp4 = True
|
first_mp4 = True
|
||||||
fmedia = {}
|
fmedia = {}
|
||||||
|
contentpics = ""
|
||||||
for piclink in linkslist:
|
for piclink in linkslist:
|
||||||
#for loop in linkslist:
|
#for loop in linkslist:
|
||||||
print (' Adding ', piclink['link'], ' to posting')
|
print (' Adding ', piclink['link'], ' to posting')
|
||||||
@@ -1325,10 +1327,10 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs
|
|||||||
fmedia = file_id
|
fmedia = file_id
|
||||||
# print ('featured_media2 = ',file_id)
|
# print ('featured_media2 = ',file_id)
|
||||||
business_url_list = outputs['postssession'].query(Posts).filter(Posts.name == title).all()
|
business_url_list = outputs['postssession'].query(Posts).filter(Posts.name == title).all()
|
||||||
business_url = business_url_list[0].businessurl
|
business_url = str(business_url_list[0].businessurl)
|
||||||
# wpurllist = outputs['postssession'].query(Posts).filter(Posts.name == title).all()
|
# wpurllist = outputs['postssession'].query(Posts).filter(Posts.name == title).all()
|
||||||
# wpurl = wpurllist[0].wpurl
|
# wpurl = wpurllist[0].wpurl
|
||||||
if business_url:
|
# if business_url or business_url is False:
|
||||||
status_message = str(title) + ': Business website: '+ business_url
|
status_message = str(title) + ': Business website: '+ business_url
|
||||||
response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), \
|
response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), \
|
||||||
data={"content" : title+' = '+status_message+'\n\n'+content+'\n'+googleadress+'\n'+rating + contentpics,\
|
data={"content" : title+' = '+status_message+'\n\n'+content+'\n'+googleadress+'\n'+rating + contentpics,\
|
||||||
@@ -1339,7 +1341,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs
|
|||||||
print(" An error writing images to the post " + post_response.title + ' occurred:', \
|
print(" An error writing images to the post " + post_response.title + ' occurred:', \
|
||||||
type(error).__name__) # An error occurred')
|
type(error).__name__) # An error occurred')
|
||||||
return False
|
return False
|
||||||
return newPost
|
return True
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
@@ -1396,6 +1398,9 @@ def process_reviews(outputs):
|
|||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
google_scroll(counter_google(driver), driver)
|
google_scroll(counter_google(driver), driver)
|
||||||
webdata = get_google_data(driver,outputs)
|
webdata = get_google_data(driver,outputs)
|
||||||
|
if env.datasource == 'db':
|
||||||
|
write_to_database(webdata, outputs)
|
||||||
|
else:
|
||||||
write_to_xlsx2(webdata, outputs)
|
write_to_xlsx2(webdata, outputs)
|
||||||
driver.close()
|
driver.close()
|
||||||
# outputs['data'].save(xls)
|
# outputs['data'].save(xls)
|
||||||
@@ -1419,7 +1424,7 @@ def process_reviews(outputs):
|
|||||||
if env.web :
|
if env.web :
|
||||||
#if writtento["web"] == 0 :
|
#if writtento["web"] == 0 :
|
||||||
try:
|
try:
|
||||||
if env.forcegoogleupdate:
|
if env.forcegoogleupdate and env.block_google_maps is not True :
|
||||||
post_id, post_link = get_wordpress_post_id_and_link(processrow.name,outputs['web'] )
|
post_id, post_link = get_wordpress_post_id_and_link(processrow.name,outputs['web'] )
|
||||||
if post_link:
|
if post_link:
|
||||||
database_update_row(processrow.name,"wpurl",post_link,"forceall",outputs)
|
database_update_row(processrow.name,"wpurl",post_link,"forceall",outputs)
|
||||||
@@ -1630,17 +1635,21 @@ def process_reviews2(outputs):
|
|||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
google_scroll(counter_google(driver), driver)
|
google_scroll(counter_google(driver), driver)
|
||||||
webdata = get_google_data(driver,outputs)
|
webdata = get_google_data(driver,outputs)
|
||||||
|
if env.datasource == 'db':
|
||||||
|
write_to_database(webdata, outputs)
|
||||||
|
else:
|
||||||
write_to_xlsx2(webdata, outputs)
|
write_to_xlsx2(webdata, outputs)
|
||||||
driver.close()
|
driver.close()
|
||||||
# outputs['data'].save(xls)
|
# outputs['data'].save(xls)
|
||||||
print('Done getting google reviews and writing them to xls file !')
|
print('Done getting google reviews and writing them to xls file !')
|
||||||
else:
|
else:
|
||||||
print ('Configuration says to skip creation of new reviews from google for this run')
|
print ('Configuration says to skip creation of new reviews from google for this run')
|
||||||
if env.needreversed:
|
# if env.needreversed:
|
||||||
rows = reversed(rows)
|
# rows = reversed(rows)
|
||||||
print('Processing Reviews')
|
print('Processing Reviews')
|
||||||
for processrow in rows:
|
for processrow in rows:
|
||||||
if processrow.name != "name": # Skip header line of xls sheet
|
if processrow.name != "name": # Skip header line of xls sheet
|
||||||
|
outputs['postssession'].query(Posts).filter(Posts.name == processrow.name).update({"google" : 1})
|
||||||
print ("Processing : ",processrow.name)
|
print ("Processing : ",processrow.name)
|
||||||
writtento = (ast.literal_eval(processrow.dictPostComplete))
|
writtento = (ast.literal_eval(processrow.dictPostComplete))
|
||||||
# Check to see if the website has already been written to according to the xls sheet,\
|
# Check to see if the website has already been written to according to the xls sheet,\
|
||||||
@@ -1653,8 +1662,8 @@ def process_reviews2(outputs):
|
|||||||
if env.web :
|
if env.web :
|
||||||
#if writtento["web"] == 0 :
|
#if writtento["web"] == 0 :
|
||||||
try:
|
try:
|
||||||
if env.forcegoogleupdate:
|
|
||||||
post_id, post_link = get_wordpress_post_id_and_link(processrow.name,outputs['web'] )
|
post_id, post_link = get_wordpress_post_id_and_link(processrow.name,outputs['web'] )
|
||||||
|
if env.forcegoogleupdate is True and env.block_google_maps is not True:
|
||||||
if post_link:
|
if post_link:
|
||||||
database_update_row(processrow.name,"wpurl",post_link,"forceall",outputs)
|
database_update_row(processrow.name,"wpurl",post_link,"forceall",outputs)
|
||||||
else:
|
else:
|
||||||
|
|||||||
Reference in New Issue
Block a user