mirror of
https://github.com/timberjoegithub/GoogleScrape.git
synced 2026-07-22 00:19:48 +00:00
working on writing to DB and excel
This commit is contained in:
@@ -1,52 +1,34 @@
|
||||
""" Script to download google maps reviews (that you create) and store them
|
||||
and then on a cron'd basis post them to social media connectors """
|
||||
import time
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pathlib
|
||||
#from selenium.webdriver.chrome.webdriver import WebDriver
|
||||
#from selenium.webdriver.chrome.service import Service
|
||||
import re
|
||||
#from openpyxl import Workbook, load_workbook
|
||||
from datetime import datetime
|
||||
import ast
|
||||
import base64
|
||||
import datetime as dt
|
||||
from urllib.request import urlretrieve
|
||||
import requests
|
||||
#import json
|
||||
import jsonpickle
|
||||
import urllib3
|
||||
#Instgram
|
||||
#from instapy import InstaPy
|
||||
#import instapy
|
||||
#from instabot import Bot
|
||||
import pandas as pd
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from openpyxl import load_workbook
|
||||
import instagrapi
|
||||
#from instagrapi.types import StoryMention, StoryMedia, StoryLink, StoryHashtag
|
||||
#from instagrapi.story import StoryBuilder
|
||||
from moviepy.editor import VideoFileClip, concatenate_videoclips
|
||||
#import moviepy
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.common.exceptions import NoSuchElementException
|
||||
#from selenium.webdriver.chrome.webdriver import WebDriver
|
||||
#from selenium.webdriver.chrome.service import Service
|
||||
#twitter
|
||||
import tweepy
|
||||
#import asyncio
|
||||
#import aiohttp
|
||||
import sqlalchemy
|
||||
#from sqlalchemy import create_engine
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy import null
|
||||
#from selenium import webdriver
|
||||
#from selenium.webdriver.common.by import By
|
||||
import googlemaps
|
||||
#import mysqlclient
|
||||
#import mysql-connector-python
|
||||
import env
|
||||
#from social import Posts
|
||||
Base = declarative_base()
|
||||
|
||||
##################################################################################################
|
||||
@@ -652,16 +634,17 @@ def write_to_database(data, local_outputs):
|
||||
print('write to database ...')
|
||||
cols = ["name", "comment", 'rating','picsURL','picsLocalpath','source','date','address',
|
||||
'dictPostComplete']
|
||||
# cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
|
||||
# 'address','dictPostComplete']
|
||||
df = pd.DataFrame(data, columns=cols)
|
||||
# df2 = pd.DataFrame(local_outputs['xlsdf'].values, columns=cols2)
|
||||
cols2 = ["num","name", "comment", 'rating','picsURL','picsLocalpath','source','date',
|
||||
'address','dictPostComplete']
|
||||
#df = pd.DataFrame(local_outputs["xls"], columns=cols)
|
||||
df = pd.DataFrame(local_outputs['xlsdf'].values, columns=cols)
|
||||
df2 = pd.DataFrame(local_outputs['posts'].values, columns=cols2)
|
||||
# print ('Dropped items not included in sync to database: ',df2.dropna(inplace=True))
|
||||
rows = list(data)
|
||||
# rows = list(data)
|
||||
# if env.needreversed:
|
||||
# rows = reversed(rows)
|
||||
#jsonposts = json.dumps(local_outputs['posts'], default=Posts)
|
||||
print("Encode Object into JSON formatted Data using jsonpickle")
|
||||
#print("Encode Object into JSON formatted Data using jsonpickle")
|
||||
jsonposts = jsonpickle.encode(local_outputs['posts'], unpicklable=False)
|
||||
for processrow in data:
|
||||
if processrow.name in df.values:
|
||||
|
||||
@@ -1,23 +1,45 @@
|
||||
"""
|
||||
Sends a text message to a phone number through the specified carrier.
|
||||
|
||||
Args:
|
||||
phone_number_inside (str): The phone number to send the message to.
|
||||
carrier_inside (str): The carrier of the phone number.
|
||||
message_inside (str): The message content to be sent.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
import smtplib
|
||||
import sys
|
||||
|
||||
|
||||
CARRIERS = {
|
||||
"att": "@mms.att.net",
|
||||
"tmobile": "@tmomail.net",
|
||||
"verizon": "@vtext.com",
|
||||
"sprint": "@messaging.sprintpcs.com"
|
||||
}
|
||||
|
||||
|
||||
EMAIL = "EMAIL"
|
||||
PASSWORD = "PASSWORD"
|
||||
|
||||
def send_message(phone_number, carrier, message):
|
||||
recipient = phone_number + CARRIERS[carrier]
|
||||
|
||||
def send_message(phone_number_inside, carrier_inside, message_inside):
|
||||
"""
|
||||
Sends a text message to a phone number through the specified carrier.
|
||||
|
||||
Args:
|
||||
phone_number_inside (str): The phone number to send the message to.
|
||||
carrier_inside (str): The carrier of the phone number.
|
||||
message_inside (str): The message content to be sent.
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
recipient = phone_number_inside + CARRIERS[carrier_inside]
|
||||
auth = (EMAIL, PASSWORD)
|
||||
server = smtplib.SMTP("smtp.gmail.com", 587)
|
||||
server.starttls()
|
||||
server.login(auth[0], auth[1])
|
||||
server.sendmail(auth[0], recipient, message)
|
||||
server.sendmail(auth[0], recipient, message_inside)
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 4:
|
||||
@@ -26,4 +48,4 @@ if __name__ == "__main__":
|
||||
phone_number = sys.argv[1]
|
||||
carrier = sys.argv[2]
|
||||
message = sys.argv[3]
|
||||
send_message(phone_number, carrier, message)
|
||||
send_message(phone_number, carrier, message)
|
||||
|
||||
Reference in New Issue
Block a user