admin/GoogleScrape
1
0
Fork 0
mirror of https://github.com/timberjoegithub/GoogleScrape.git synced 2026-07-22 00:19:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/timberjoegithub/GoogleScrape

Browse Source
This commit is contained in:
Joe Steele
2024-06-13 21:26:04 -04:00
parent 24975dbc38 2bc34775c0
commit ad5ab9ce48
2 changed files with 75 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
requirements.txt
+3
View File
@@ -17,3 +17,6 @@ tweepy
sqlalchemy sqlalchemy
googlemaps googlemaps
moviepy moviepy
mysqlclient
mysql-connector-python
pymysql
social.py
+71 -34
View File
@@ -1,5 +1,9 @@
import time import time
import os import os
from pathlib import Path
#from selenium.webdriver.chrome.webdriver import WebDriver
#from selenium.webdriver.chrome.service import Service
import re import re
import urllib3 import urllib3
from urllib.request import urlretrieve from urllib.request import urlretrieve
@@ -36,11 +40,13 @@ import tweepy
#import asyncio #import asyncio
#import aiohttp #import aiohttp
from pathlib import Path
#from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
import sqlalchemy import sqlalchemy
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import null from sqlalchemy import null
from selenium import webdriver
from selenium.webdriver.common.by import By
import googlemaps import googlemaps
#import mysqlclient #import mysqlclient
#import mysql-connector-python #import mysql-connector-python
@@ -479,7 +485,7 @@ def get_google_data(driver,outputs ):
for lmpics in more_specific_pics: for lmpics in more_specific_pics:
# Grab URL from style definiton (long multivalue string), and remove the -p-k so that # Grab URL from style definiton (long multivalue string), and remove the -p-k so that
# it is full size # it is full size
urlmedia = re.sub('=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]", urlmedia = re.sub(r'=\S*-p-k-no', '=-no', (re.findall(r"['\"](.*?)['\"]",
lmpics.get_attribute("style")))[0]) lmpics.get_attribute("style")))[0])
print (' Pic URL : ',urlmedia) print (' Pic URL : ',urlmedia)
pics.append(urlmedia) pics.append(urlmedia)
@@ -713,8 +719,7 @@ def check_wordpress_media(filename,headers):
link = result[0]['guid']['rendered'] link = result[0]['guid']['rendered']
return file_id, link return file_id, link
except Exception as error: except Exception as error:
print(' No existing media with same name in Wordpress media folder: '+filename+' '\ print(' No existing media with same name in Wordpress media folder: '+filename)
,error)
return (False, False) return (False, False)
################################################################################################## ##################################################################################################
@@ -803,24 +808,24 @@ def get_wordpress_featured_photo_id(post_id):
################################################################################################## ##################################################################################################
def post_to_x2(title, content, date, rating, address, picslist, instasession): def post_to_x2(title, content, date, rating, address, picslist, instasession,outputs):
""" """
Post to x2. Post to x2.
This function posts content to a social media platform using the provided data. This function posts content to a social media platform using the provided data.
Args: Args:
title (str): The title of the post. title (str): The title of the post.
content (str): The content of the post. content (str): The content of the post.
date (str): The date of the post. date (str): The date of the post.
rating (int): The rating of the post. rating (int): The rating of the post.
address (str): The address associated with the post. address (str): The address associated with the post.
picslist (list): A list of pictures for the post. picslist (list): A list of pictures for the post.
instasession: The Instagram session for posting. instasession: The Instagram session for posting.
Returns: Returns:
str: The media ID of the posted content. str: The media ID of the posted content.
""" """
pics = ((picslist[1:-1]).replace("'","")).split(",") pics = ((picslist[1:-1]).replace("'","")).split(",")
# Replace the following strings with your own keys and secrets # Replace the following strings with your own keys and secrets
@@ -855,7 +860,11 @@ Returns:
# Path to the video you want to upload # Path to the video you want to upload
#video_path = 'path_to_video.mp4' #video_path = 'path_to_video.mp4'
# Message to post along with the video # Message to post along with the video
status_message = str(title) + ': Review https://www.joeeatswhat.com' business_url_list = outputs['postssession'].query(Posts).filter(Posts.name == title).all()
business_url = business_url_list[0].business_url_list
wpurllist = outputs['postssession'].query(Posts).filter(Posts.name == title).all()
wpurl = wpurllist[0].wpurl
status_message = str(title) + ': My Review - '+ wpurl + '\n Business website: '+ business_url
status_message2 = status_message +' '+str(get_hastags(address, title, 'short'))+' ' status_message2 = status_message +' '+str(get_hastags(address, title, 'short'))+' '
status_message_short = status_message2[:279] status_message_short = status_message2[:279]
# if len(status_message) > 279: # if len(status_message) > 279:
@@ -876,6 +885,23 @@ Returns:
################################################################################################## ##################################################################################################
def post_facebook3(title, content, date, rating, address, picslist, instasession): def post_facebook3(title, content, date, rating, address, picslist, instasession):
"""
Post to Facebook3.
This function posts content to Facebook using the provided data.
Args:
title (str): The title of the post.
content (str): The content of the post.
date (str): The date of the post.
rating (int): The rating of the post.
address (str): The address associated with the post.
picslist (list): A list of pictures for the post.
instasession: The Instagram session for posting.
Returns:
bool: Indicates if the post was successfully made.
"""
pics = ((picslist[1:-1]).replace("'","")).split(",") pics = ((picslist[1:-1]).replace("'","")).split(",")
group_id = env.facebookpageID group_id = env.facebookpageID
auth_token = env.facebookpass auth_token = env.facebookpass
@@ -1063,18 +1089,24 @@ def post_to_instagram2 (title, content, date, rating, address, picslist, instase
################################################################################################## ##################################################################################################
def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs): def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs):
""" Post to wordpress site """
Post to WordPress.
This function posts content to a WordPress site using the provided data.
Args: Args:
title (_type_): _description_ title (str): The title of the post.
content (_type_): _description_ content (str): The content of the post.
headers (_type_): _description_ headers: Headers for the request.
date (_type_): _description_ date (str): The date of the post.
rating (_type_): _description_ rating (str): The rating of the post.
address (_type_): _description_ address (str): The address associated with the post.
picslist (_type_): _description_ picslist (list): A list of pictures for the post.
outputs (_type_): _description_ outputs: Outputs for the post.
"""
Returns:
None
"""
# post # post
newPost = False newPost = False
#countreview = False #countreview = False
@@ -1247,7 +1279,7 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs
try: try:
post_response = requests.post(env.wpAPI + "/posts/" + str(post_id),\ post_response = requests.post(env.wpAPI + "/posts/" + str(post_id),\
headers=headers,timeout=30) headers=headers,timeout=30)
if link in str(post_response.text): if link in post_response.text:
print (' Image link for ', picname, 'already in content of post: ' \ print (' Image link for ', picname, 'already in content of post: ' \
,post_id, post_response.text, link) ,post_id, post_response.text, link)
else: else:
@@ -1287,8 +1319,13 @@ def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs
else: else:
fmedia = file_id fmedia = file_id
# print ('featured_media2 = ',file_id) # print ('featured_media2 = ',file_id)
business_url_list = outputs['postssession'].query(Posts).filter(Posts.name == title).all()
business_url = business_url_list[0].business_url_list
# wpurllist = outputs['postssession'].query(Posts).filter(Posts.name == title).all()
# wpurl = wpurllist[0].wpurl
status_message = str(title) + ': Business website: '+ business_url
response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), \ response_piclinks = requests.post(env.wpAPI+"/posts/"+ str(post_id), \
data={"content" : title+' = '+content+'\n'+googleadress+'\n'+rating + contentpics,\ data={"content" : title+' = '+status_message+'\n\n'+content+'\n'+googleadress+'\n'+rating + contentpics,\
"featured_media" : fmedia,"rank_math_focus_keyword" : title }, headers=headers,\ "featured_media" : fmedia,"rank_math_focus_keyword" : title }, headers=headers,\
timeout=30) timeout=30)
print (' ',response_piclinks) print (' ',response_piclinks)
@@ -1470,7 +1507,7 @@ def process_reviews(outputs):
if xtwittercount < env.postsperrun: if xtwittercount < env.postsperrun:
try: try:
print(' Starting to generate xtwitter post') print(' Starting to generate xtwitter post')
NewxtwitterPost = post_to_x2(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['posts'] ) NewxtwitterPost = post_to_x2(processrow[1].value, processrow[2].value, processrow[7].value, processrow[3].value, processrow[8].value, processrow[5].value,outputs['posts'],outputs )
try: try:
print (' Start generating content to post to xtwitter') print (' Start generating content to post to xtwitter')
writtento["xtwitter"] = 1 writtento["xtwitter"] = 1