mirror of
https://github.com/timberjoegithub/GoogleScrape.git
synced 2026-07-22 00:19:48 +00:00
working on writing to DB and excel
This commit is contained in:
@@ -1,52 +1,34 @@
|
|||||||
|
""" Script to download google maps reviews (that you create) and store them
|
||||||
|
and then on a cron'd basis post them to social media connectors """
|
||||||
import time
|
import time
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import pathlib
|
import pathlib
|
||||||
#from selenium.webdriver.chrome.webdriver import WebDriver
|
|
||||||
#from selenium.webdriver.chrome.service import Service
|
|
||||||
import re
|
import re
|
||||||
#from openpyxl import Workbook, load_workbook
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import ast
|
import ast
|
||||||
import base64
|
import base64
|
||||||
import datetime as dt
|
import datetime as dt
|
||||||
from urllib.request import urlretrieve
|
from urllib.request import urlretrieve
|
||||||
import requests
|
import requests
|
||||||
#import json
|
|
||||||
import jsonpickle
|
import jsonpickle
|
||||||
import urllib3
|
import urllib3
|
||||||
#Instgram
|
|
||||||
#from instapy import InstaPy
|
|
||||||
#import instapy
|
|
||||||
#from instabot import Bot
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from dateutil.relativedelta import relativedelta
|
from dateutil.relativedelta import relativedelta
|
||||||
from openpyxl import load_workbook
|
from openpyxl import load_workbook
|
||||||
import instagrapi
|
import instagrapi
|
||||||
#from instagrapi.types import StoryMention, StoryMedia, StoryLink, StoryHashtag
|
|
||||||
#from instagrapi.story import StoryBuilder
|
|
||||||
from moviepy.editor import VideoFileClip, concatenate_videoclips
|
from moviepy.editor import VideoFileClip, concatenate_videoclips
|
||||||
#import moviepy
|
|
||||||
from selenium import webdriver
|
from selenium import webdriver
|
||||||
from selenium.webdriver.common.by import By
|
from selenium.webdriver.common.by import By
|
||||||
from selenium.common.exceptions import NoSuchElementException
|
from selenium.common.exceptions import NoSuchElementException
|
||||||
#from selenium.webdriver.chrome.webdriver import WebDriver
|
|
||||||
#from selenium.webdriver.chrome.service import Service
|
|
||||||
#twitter
|
|
||||||
import tweepy
|
import tweepy
|
||||||
#import asyncio
|
#import asyncio
|
||||||
#import aiohttp
|
#import aiohttp
|
||||||
import sqlalchemy
|
import sqlalchemy
|
||||||
#from sqlalchemy import create_engine
|
|
||||||
from sqlalchemy.ext.declarative import declarative_base
|
from sqlalchemy.ext.declarative import declarative_base
|
||||||
from sqlalchemy import null
|
from sqlalchemy import null
|
||||||
#from selenium import webdriver
|
|
||||||
#from selenium.webdriver.common.by import By
|
|
||||||
import googlemaps
|
import googlemaps
|
||||||
#import mysqlclient
|
|
||||||
#import mysql-connector-python
|
|
||||||
import env
|
import env
|
||||||
#from social import Posts
|
|
||||||
Base = declarative_base()
|
Base = declarative_base()
|
||||||
|
|
||||||
##################################################################################################
|
##################################################################################################
|
||||||
@@ -652,16 +634,17 @@ def write_to_database(data, local_outputs):
|
|||||||
print('write to database ...')
|
print('write to database ...')
|
||||||
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
|
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
|
||||||
'dictPostComplete']
|
'dictPostComplete']
|
||||||
# cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
|
cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
|
||||||
# 'address','dictPostComplete']
|
'address','dictPostComplete']
|
||||||
df = pd.DataFrame(data, columns=cols)
|
#df = pd.DataFrame(local_outputs["xls"], columns=cols)
|
||||||
# df2 = pd.DataFrame(local_outputs['xlsdf'].values, columns=cols2)
|
df = pd.DataFrame(local_outputs['xlsdf'].values, columns=cols)
|
||||||
|
df2 = pd.DataFrame(local_outputs['posts'].values, columns=cols2)
|
||||||
# print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True))
|
# print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True))
|
||||||
rows = list(data)
|
# rows = list(data)
|
||||||
# if env.needreversed:
|
# if env.needreversed:
|
||||||
# rows = reversed(rows)
|
# rows = reversed(rows)
|
||||||
#jsonposts = json.dumps(local_outputs['posts'], default=Posts)
|
#jsonposts = json.dumps(local_outputs['posts'], default=Posts)
|
||||||
print("Encode Object into JSON formatted Data using jsonpickle")
|
#print("Encode Object into JSON formatted Data using jsonpickle")
|
||||||
jsonposts = jsonpickle.encode(local_outputs['posts'], unpicklable=False)
|
jsonposts = jsonpickle.encode(local_outputs['posts'], unpicklable=False)
|
||||||
for processrow in data:
|
for processrow in data:
|
||||||
if processrow.name in df.values:
|
if processrow.name in df.values:
|
||||||
|
|||||||
@@ -1,3 +1,14 @@
|
|||||||
|
"""
|
||||||
|
Sends a text message to a phone number through the specified carrier.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
phone_number_inside (str): The phone number to send the message to.
|
||||||
|
carrier_inside (str): The carrier of the phone number.
|
||||||
|
message_inside (str): The message content to be sent.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None
|
||||||
|
"""
|
||||||
import smtplib
|
import smtplib
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
@@ -11,13 +22,24 @@ CARRIERS = {
|
|||||||
EMAIL = "EMAIL"
|
EMAIL = "EMAIL"
|
||||||
PASSWORD = "PASSWORD"
|
PASSWORD = "PASSWORD"
|
||||||
|
|
||||||
def send_message(phone_number, carrier, message):
|
def send_message(phone_number_inside, carrier_inside, message_inside):
|
||||||
recipient = phone_number + CARRIERS[carrier]
|
"""
|
||||||
|
Sends a text message to a phone number through the specified carrier.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
phone_number_inside (str): The phone number to send the message to.
|
||||||
|
carrier_inside (str): The carrier of the phone number.
|
||||||
|
message_inside (str): The message content to be sent.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None
|
||||||
|
"""
|
||||||
|
recipient = phone_number_inside + CARRIERS[carrier_inside]
|
||||||
auth = (EMAIL, PASSWORD)
|
auth = (EMAIL, PASSWORD)
|
||||||
server = smtplib.SMTP("smtp.gmail.com", 587)
|
server = smtplib.SMTP("smtp.gmail.com", 587)
|
||||||
server.starttls()
|
server.starttls()
|
||||||
server.login(auth[0], auth[1])
|
server.login(auth[0], auth[1])
|
||||||
server.sendmail(auth[0], recipient, message)
|
server.sendmail(auth[0], recipient, message_inside)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
if len(sys.argv) < 4:
|
if len(sys.argv) < 4:
|
||||||
|
|||||||
Reference in New Issue
Block a user