diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cfd7802 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +Driver/chromedriver* +__pycache__/ \ No newline at end of file diff --git a/Driver/driver_links.txt b/Driver/driver_links.txt new file mode 100644 index 0000000..c0fbe22 --- /dev/null +++ b/Driver/driver_links.txt @@ -0,0 +1,4 @@ +chrome: https://chromedriver.chromium.org/downloads +edge : https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/ +firefox: https://github.com/mozilla/geckodriver/releases +safari: https://webkit.org/blog/6900/webdriver-support-in-safari-10/ diff --git a/README.md b/README.md deleted file mode 100644 index af0691c..0000000 --- a/README.md +++ /dev/null @@ -1 +0,0 @@ -# GoogleScrape diff --git a/app.py b/app.py new file mode 100644 index 0000000..482cd2e --- /dev/null +++ b/app.py @@ -0,0 +1,83 @@ +import time + +from selenium import webdriver +from selenium.webdriver.chrome.webdriver import WebDriver + +from openpyxl import Workbook +import pandas as pd + +from env import URL, DriverLocation + +def get_data(driver): + """ + this function get main text, score, name + """ + print('get data...') + more_elemets = driver.find_elements_by_class_name('w8nwRe kyuRq') + for list_more_element in more_elemets: + list_more_element.click() + + elements = driver.find_elements_by_class_name( + 'jftiEf') + lst_data = [] + for data in elements: + name = data.find_element_by_xpath( + './/a/div[@class="d4r55"]/span').text + text = data.find_element_by_xpath( + './/div[@class="MyEned"]/span[2]').text + score = data.find_element_by_xpath( + './/span[@class="kvMYJc"]').get_attribute("aria-label") + + lst_data.append([name + " from GoogleMaps", text, score[1]]) + + return lst_data + + +def counter(): + result = driver.find_element_by_class_name('jANrlb').find_element_by_class_name('fontBodySmall').text + result = result.replace(',', '') + result = result.split(' ') + result = result[0].split('\n') + return int(int(result[0])/10)+1 + + +def scrolling(counter): + print('scrolling...') + scrollable_div = driver.find_element_by_xpath( + '//div[@class="lXJj5c Hk4XGb"]') + for _i in range(counter): + scrolling = driver.execute_script( + 'document.getElementsByClassName("dS8AEf")[0].scrollTop = document.getElementsByClassName("dS8AEf")[0].scrollHeight', + scrollable_div + ) + time.sleep(3) + + +def write_to_xlsx(data): + print('write to excel...') + cols = ["name", "comment", 'rating'] + df = pd.DataFrame(data, columns=cols) + df.to_excel('out.xlsx') + + +if __name__ == "__main__": + + print('starting...') + options = webdriver.ChromeOptions() + options.add_argument("--headless") # show browser or not + options.add_argument("--lang=en-US") + options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'}) + DriverPath = DriverLocation + driver = webdriver.Chrome(DriverPath, options=options) + + driver.get(URL) + time.sleep(5) + + counter = counter() + scrolling(counter) + + data = get_data(driver) + driver.close() + + write_to_xlsx(data) + print('Done!') diff --git a/env.py b/env.py new file mode 100644 index 0000000..312d489 --- /dev/null +++ b/env.py @@ -0,0 +1,2 @@ +URL = "https://www.google.com/maps/place/Google+UK/@51.5332608,-0.1304879,17z/data=!4m7!3m6!1s0x48761b3c54efa6e1:0xc7053ab04745950d!8m2!3d51.5332609!4d-0.1260032!9m1!1b1" +DriverLocation = "./Driver/chromedriver.exe" \ No newline at end of file diff --git a/geckodriver.log b/geckodriver.log new file mode 100644 index 0000000..f08ea6f --- /dev/null +++ b/geckodriver.log @@ -0,0 +1,65 @@ +1707759920203 geckodriver INFO Listening on 127.0.0.1:50118 +1707759923905 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "50119" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jsadmin\\AppData\\Local\\Temp\\rust_mozprofileuKrhCR" +1707759989020 geckodriver INFO Listening on 127.0.0.1:50774 +1707759991998 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "50775" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jsadmin\\AppData\\Local\\Temp\\rust_mozprofilew3w9uT" +console.warn: services.settings: Ignoring preference override of remote settings server +console.warn: services.settings: Allow by setting MOZ_REMOTE_SETTINGS_DEVTOOLS=1 in the environment +1707759994161 Marionette INFO Marionette enabled +Dynamically enable window occlusion 0 +1707759994529 Marionette INFO Listening on port 50842 +WebDriver BiDi listening on ws://127.0.0.1:50775 +Read port: 50842 +1707759995346 RemoteAgent WARN TLS certificate errors will be ignored for this session +DevTools listening on ws://127.0.0.1:50775/devtools/browser/f4f97816-72e9-4390-a75b-6daae26b6adf +JavaScript warning: https://www.google.com/maps/_/js/k=maps.m.en.d-cA6F9zu-8.2019.O/m=sc2,per,mo,lp,ep,ti,ds,stx,dwi,enr,pwd,dw,pcb,log,b/am=BgAjAAI/rt=j/d=1/rs=ACT90oEz0eIzV6O4M5RYhpfrQJX3B8BiXg?wli=m.v_7ldjm66Dg.loadSv.O%3A%3Bm.0lcgipW5GjU.mapcore.O%3A%3B, line 2596: WEBGL_debug_renderer_info is deprecated in Firefox and will be removed. Please use RENDERER. +JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow. +JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization. +JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow. +JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization. +JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow. +JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization. +JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow. +JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization. +1707763079140 geckodriver INFO Listening on 127.0.0.1:64062 +1707763082520 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "64063" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jsadmin\\AppData\\Local\\Temp\\rust_mozprofile53onUu" +console.warn: services.settings: Ignoring preference override of remote settings server +console.warn: services.settings: Allow by setting MOZ_REMOTE_SETTINGS_DEVTOOLS=1 in the environment +1707763084943 Marionette INFO Marionette enabled +Dynamically enable window occlusion 0 +1707763085546 Marionette INFO Listening on port 64084 +WebDriver BiDi listening on ws://127.0.0.1:64063 +Read port: 64084 +1707763086345 RemoteAgent WARN TLS certificate errors will be ignored for this session +DevTools listening on ws://127.0.0.1:64063/devtools/browser/23b549b7-d008-49ee-a23c-85bde674b8aa +JavaScript warning: https://www.google.com/maps/_/js/k=maps.m.en.d-cA6F9zu-8.2019.O/m=sc2,per,mo,lp,ep,ti,ds,stx,dwi,enr,pwd,dw,pcb,log,b/am=BgAjAAI/rt=j/d=1/rs=ACT90oEz0eIzV6O4M5RYhpfrQJX3B8BiXg?wli=m.v_7ldjm66Dg.loadSv.O%3A%3Bm.0lcgipW5GjU.mapcore.O%3A%3B, line 2596: WEBGL_debug_renderer_info is deprecated in Firefox and will be removed. Please use RENDERER. +JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow. +JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization. +JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow. +JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization. +JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow. +JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization. +JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow. +JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization. +1707763115955 addons.xpi ERROR System addon update list error SyntaxError: XMLHttpRequest.open: 'http://%(server)s/dummy-system-addons.xml' is not a valid URL. +JavaScript error: , line 0: NS_ERROR_NOT_AVAILABLE +1707769513078 geckodriver INFO Listening on 127.0.0.1:52732 +1707769516172 mozrunner::runner INFO Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "52733" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jsadmin\\AppData\\Local\\Temp\\rust_mozprofiles5JV3J" +console.warn: services.settings: Ignoring preference override of remote settings server +console.warn: services.settings: Allow by setting MOZ_REMOTE_SETTINGS_DEVTOOLS=1 in the environment +1707769518456 Marionette INFO Marionette enabled +Dynamically enable window occlusion 0 +1707769518837 Marionette INFO Listening on port 52755 +WebDriver BiDi listening on ws://127.0.0.1:52733 +Read port: 52755 +1707769519785 RemoteAgent WARN TLS certificate errors will be ignored for this session +DevTools listening on ws://127.0.0.1:52733/devtools/browser/0169f414-06b0-4328-93a9-357e17d60486 +JavaScript warning: https://www.google.com/maps/_/js/k=maps.m.en.d-cA6F9zu-8.2019.O/m=sc2,per,mo,lp,ep,ti,ds,stx,dwi,enr,pwd,dw,pcb,log,b/am=BgAjAAI/rt=j/d=1/rs=ACT90oEz0eIzV6O4M5RYhpfrQJX3B8BiXg?wli=m.v_7ldjm66Dg.loadSv.O%3A%3Bm.0lcgipW5GjU.mapcore.O%3A%3B, line 2596: WEBGL_debug_renderer_info is deprecated in Firefox and will be removed. Please use RENDERER. +JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow. +JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization. +1707769548713 addons.xpi ERROR System addon update list error SyntaxError: XMLHttpRequest.open: 'http://%(server)s/dummy-system-addons.xml' is not a valid URL. +JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow. +JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization. +JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow. +JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization. +JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow. +JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization. diff --git a/google2.py b/google2.py index 97be592..bc4cec8 100644 --- a/google2.py +++ b/google2.py @@ -19,7 +19,7 @@ request_result=requests.get( url ) soup = bs4.BeautifulSoup(request_result.text, "html.parser") #print(soup) -heading_object=soup.find_all( 'h3' ) +heading_object=soup #.find_all( 'h1' ) # Iterate through the object # and print it as a string. diff --git a/google3.py b/google3.py new file mode 100644 index 0000000..1b03afc --- /dev/null +++ b/google3.py @@ -0,0 +1,45 @@ + +# Import the beautifulsoup +# and request libraries of python. +import requests +#import bs4 + +#from here +from bs4 import BeautifulSoup +from selenium import webdriver +driver = webdriver.Firefox() +driver.get('https://www.google.com/maps/contrib/109274792898041753066/reviews') + +html = driver.page_source +soup = BeautifulSoup(html) + +innerHTML = driver.execute_script("return document.body.innerHTML") +print("\nJS PAGE SOURCE:", "\n", driver.page_source) + +for info in soup: + print (info) + +# to here + +# Make two strings with default google search URL +# 'https://google.com/search?q=' and +# our customized search keyword. +# Concatenate them +# text= "joe" +# url = "https://www.google.com/maps/contrib/109274792898041753066/reviews" + +# # Fetch the URL data using requests.get(url), +# # store it in a variable, request_result. +# request_result=requests.get( url ) + +# # Creating soup from the fetched request +# soup = bs4.BeautifulSoup(request_result.text, +# "html.parser") +# #print(soup) +# heading_object=soup #.find_all( 'h1' ) + +# # Iterate through the object +# # and print it as a string. +# for info in heading_object: +# print(info.getText()) +# print("------") \ No newline at end of file diff --git a/google4.py b/google4.py new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..abebd85 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +autopep8==1.5.7 +et-xmlfile==1.1.0 +numpy==1.21.2 +openpyxl==3.0.9 +pandas==1.3.3 +pkg_resources==0.0.0 +pycodestyle==2.7.0 +python-dateutil==2.8.2 +pytz==2021.1 +selenium==3.141.0 +six==1.16.0 +toml==0.10.2 +urllib3==1.26.6