mirror of
https://github.com/timberjoegithub/GoogleScrape.git
synced 2026-07-22 00:19:48 +00:00
added comments
This commit is contained in:
@@ -1,10 +1,6 @@
|
|||||||
#data
|
#data
|
||||||
import time
|
import time
|
||||||
import os
|
import os
|
||||||
from selenium import webdriver
|
|
||||||
#from selenium.webdriver.chrome.webdriver import WebDriver
|
|
||||||
#from selenium.webdriver.chrome.service import Service
|
|
||||||
from selenium.webdriver.common.by import By
|
|
||||||
import re
|
import re
|
||||||
import urllib3
|
import urllib3
|
||||||
from urllib.request import urlretrieve
|
from urllib.request import urlretrieve
|
||||||
@@ -32,7 +28,10 @@ import instagrapi
|
|||||||
#from instagrapi.story import StoryBuilder
|
#from instagrapi.story import StoryBuilder
|
||||||
from moviepy.editor import VideoFileClip, concatenate_videoclips
|
from moviepy.editor import VideoFileClip, concatenate_videoclips
|
||||||
#import moviepy
|
#import moviepy
|
||||||
|
from selenium import webdriver
|
||||||
|
#from selenium.webdriver.chrome.webdriver import WebDriver
|
||||||
|
#from selenium.webdriver.chrome.service import Service
|
||||||
|
from selenium.webdriver.common.by import By
|
||||||
#twitter
|
#twitter
|
||||||
import tweepy
|
import tweepy
|
||||||
|
|
||||||
@@ -126,10 +125,10 @@ def preload():
|
|||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
def clearlist (list):
|
def clearlist (my_list):
|
||||||
for listelement in list:
|
for listelement in my_list:
|
||||||
listelement.clear
|
listelement.clear
|
||||||
return list
|
return my_list
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
@@ -549,22 +548,20 @@ def write_to_database(data, outputs):
|
|||||||
sqlalchemy.null()
|
sqlalchemy.null()
|
||||||
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
|
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
|
||||||
'dictPostComplete']
|
'dictPostComplete']
|
||||||
cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
|
# cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
|
||||||
'address','dictPostComplete']
|
# 'address','dictPostComplete']
|
||||||
df = pd.DataFrame(data, columns=cols)
|
df = pd.DataFrame(data, columns=cols)
|
||||||
df2 = pd.DataFrame(outputs['xlsdf'].values, columns=cols2)
|
# df2 = pd.DataFrame(outputs['xlsdf'].values, columns=cols2)
|
||||||
#df2 = df1.where((pd.notnull(df)), None) # take out NAN problems
|
# print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True))
|
||||||
#df3.astype(object).where(pd.notnull(df2), None)
|
|
||||||
print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True))
|
|
||||||
rows = list(data)
|
rows = list(data)
|
||||||
if env.needreversed:
|
if env.needreversed:
|
||||||
rows = reversed(rows)
|
rows = reversed(rows)
|
||||||
#jsonposts = json.dumps(outputs['posts'], default=Posts)
|
#jsonposts = json.dumps(outputs['posts'], default=Posts)
|
||||||
print("Encode Object into JSON formatted Data using jsonpickle")
|
print("Encode Object into JSON formatted Data using jsonpickle")
|
||||||
jsonposts = jsonpickle.encode(outputs['posts'], unpicklable=False)
|
jsonposts = jsonpickle.encode(outputs['posts'], unpicklable=False)
|
||||||
for processrow in df2.values:
|
for processrow in outputs['postssession']:
|
||||||
if (processrow[1] in df.values):
|
if (processrow[1] in df.values):
|
||||||
print (' Row ',processrow[0],' ', processrow[1],' already in XLS sheet')
|
print (' Row ',processrow[0],' ', processrow[1],' already in database')
|
||||||
d2_row = Posts(name=processrow[1],comment=processrow[2],rating=processrow[3],
|
d2_row = Posts(name=processrow[1],comment=processrow[2],rating=processrow[3],
|
||||||
picsURL=processrow[4],picsLocalpath=processrow[5],source=processrow[6],
|
picsURL=processrow[4],picsLocalpath=processrow[5],source=processrow[6],
|
||||||
date=processrow[7],address=processrow[8],dictPostComplete=processrow[9])
|
date=processrow[7],address=processrow[8],dictPostComplete=processrow[9])
|
||||||
@@ -670,6 +667,16 @@ def get_wordpress_post_id_and_link(postname,headers2):
|
|||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
def check_wordpress_post(postname,postdate,headers2):
|
def check_wordpress_post(postname,postdate,headers2):
|
||||||
|
""" Checks workpress website to see if a post already exists
|
||||||
|
|
||||||
|
Args:
|
||||||
|
postname (_type_): _description_
|
||||||
|
postdate (_type_): _description_
|
||||||
|
headers2 (_type_): _description_
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
_type_: _description_
|
||||||
|
"""
|
||||||
response = requests.get(env.wpAPI+"/posts?search="+postname, headers=headers2,timeout=40)
|
response = requests.get(env.wpAPI+"/posts?search="+postname, headers=headers2,timeout=40)
|
||||||
result = response.json()
|
result = response.json()
|
||||||
if len(result) > 0 :
|
if len(result) > 0 :
|
||||||
@@ -689,6 +696,14 @@ def check_wordpress_post(postname,postdate,headers2):
|
|||||||
|
|
||||||
# Function to get the featured photo ID of a WordPress post
|
# Function to get the featured photo ID of a WordPress post
|
||||||
def get_wordpress_featured_photo_id(post_id):
|
def get_wordpress_featured_photo_id(post_id):
|
||||||
|
""" Get photoid from featured photo
|
||||||
|
|
||||||
|
Args:
|
||||||
|
post_id (_type_): _description_
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
_type_: _description_
|
||||||
|
"""
|
||||||
# Make a GET request to the WordPress REST API to retrieve media details
|
# Make a GET request to the WordPress REST API to retrieve media details
|
||||||
response = requests.get(f"{env.wpAPI}?parent={post_id}",timeout=50)
|
response = requests.get(f"{env.wpAPI}?parent={post_id}",timeout=50)
|
||||||
# Check if the request was successful
|
# Check if the request was successful
|
||||||
@@ -711,6 +726,17 @@ def get_wordpress_featured_photo_id(post_id):
|
|||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
def post_to_x2(title, content, date, rating, address, picslist, instasession):
|
def post_to_x2(title, content, date, rating, address, picslist, instasession):
|
||||||
|
"""Post to twitter
|
||||||
|
|
||||||
|
Args:
|
||||||
|
title (_type_): _description_
|
||||||
|
content (_type_): _description_
|
||||||
|
date (_type_): _description_
|
||||||
|
rating (_type_): _description_
|
||||||
|
address (_type_): _description_
|
||||||
|
picslist (_type_): _description_
|
||||||
|
instasession (_type_): _description_
|
||||||
|
"""
|
||||||
pics = ((picslist[1:-1]).replace("'","")).split(",")
|
pics = ((picslist[1:-1]).replace("'","")).split(",")
|
||||||
# Replace the following strings with your own keys and secrets
|
# Replace the following strings with your own keys and secrets
|
||||||
CONSUMER_KEY = env.x_consumer_key
|
CONSUMER_KEY = env.x_consumer_key
|
||||||
@@ -952,6 +978,18 @@ def post_to_instagram2 (title, content, date, rating, address, picslist, instase
|
|||||||
##################################################################################################
|
##################################################################################################
|
||||||
|
|
||||||
def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs):
|
def post_to_wordpress(title,content,headers,date,rating,address,picslist,outputs):
|
||||||
|
""" Post to wordpress site
|
||||||
|
|
||||||
|
Args:
|
||||||
|
title (_type_): _description_
|
||||||
|
content (_type_): _description_
|
||||||
|
headers (_type_): _description_
|
||||||
|
date (_type_): _description_
|
||||||
|
rating (_type_): _description_
|
||||||
|
address (_type_): _description_
|
||||||
|
picslist (_type_): _description_
|
||||||
|
outputs (_type_): _description_
|
||||||
|
"""
|
||||||
# post
|
# post
|
||||||
newPost = False
|
newPost = False
|
||||||
#countreview = False
|
#countreview = False
|
||||||
|
|||||||
Reference in New Issue
Block a user