new stuff

.gitignore

@@ -0,0 +1,2 @@
+Driver/chromedriver*
+__pycache__/

Driver/driver_links.txt

@@ -0,0 +1,4 @@
+chrome: https://chromedriver.chromium.org/downloads
+edge : https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/
+firefox: https://github.com/mozilla/geckodriver/releases
+safari: https://webkit.org/blog/6900/webdriver-support-in-safari-10/

README.md

@@ -1 +0,0 @@
-# GoogleScrape

app.py

@@ -0,0 +1,83 @@
+import time
+
+from selenium import webdriver
+from selenium.webdriver.chrome.webdriver import WebDriver
+
+from openpyxl import Workbook
+import pandas as pd
+
+from env import URL, DriverLocation
+
+def get_data(driver):
+    """
+    this function get main text, score, name
+    """
+    print('get data...')
+    more_elemets = driver.find_elements_by_class_name('w8nwRe kyuRq')
+    for list_more_element in more_elemets:
+        list_more_element.click()
+    
+    elements = driver.find_elements_by_class_name(
+        'jftiEf')
+    lst_data = []
+    for data in elements:
+        name = data.find_element_by_xpath(
+            './/a/div[@class="d4r55"]/span').text
+        text = data.find_element_by_xpath(
+            './/div[@class="MyEned"]/span[2]').text
+        score = data.find_element_by_xpath(
+            './/span[@class="kvMYJc"]').get_attribute("aria-label")
+
+        lst_data.append([name + " from GoogleMaps", text, score[1]])
+
+    return lst_data
+
+
+def counter():
+    result = driver.find_element_by_class_name('jANrlb').find_element_by_class_name('fontBodySmall').text
+    result = result.replace(',', '')
+    result = result.split(' ')
+    result = result[0].split('\n')
+    return int(int(result[0])/10)+1
+
+
+def scrolling(counter):
+    print('scrolling...')
+    scrollable_div = driver.find_element_by_xpath(
+        '//div[@class="lXJj5c Hk4XGb"]')
+    for _i in range(counter):
+        scrolling = driver.execute_script(
+            'document.getElementsByClassName("dS8AEf")[0].scrollTop = document.getElementsByClassName("dS8AEf")[0].scrollHeight',
+            scrollable_div
+        )
+        time.sleep(3)
+
+
+def write_to_xlsx(data):
+    print('write to excel...')
+    cols = ["name", "comment", 'rating']
+    df = pd.DataFrame(data, columns=cols)
+    df.to_excel('out.xlsx')
+
+
+if __name__ == "__main__":
+
+    print('starting...')
+    options = webdriver.ChromeOptions()
+    options.add_argument("--headless")  # show browser or not
+    options.add_argument("--lang=en-US")
+    options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'})
+    DriverPath = DriverLocation
+    driver = webdriver.Chrome(DriverPath, options=options)
+
+    driver.get(URL)
+    time.sleep(5)
+
+    counter = counter()
+    scrolling(counter)
+
+    data = get_data(driver)
+    driver.close()
+
+    write_to_xlsx(data)
+    print('Done!')

env.py

@@ -0,0 +1,2 @@
+URL = "https://www.google.com/maps/place/Google+UK/@51.5332608,-0.1304879,17z/data=!4m7!3m6!1s0x48761b3c54efa6e1:0xc7053ab04745950d!8m2!3d51.5332609!4d-0.1260032!9m1!1b1"
+DriverLocation = "./Driver/chromedriver.exe"

geckodriver.log

@@ -0,0 +1,65 @@
+1707759920203	geckodriver	INFO	Listening on 127.0.0.1:50118
+1707759923905	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "50119" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jsadmin\\AppData\\Local\\Temp\\rust_mozprofileuKrhCR"
+1707759989020	geckodriver	INFO	Listening on 127.0.0.1:50774
+1707759991998	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "50775" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jsadmin\\AppData\\Local\\Temp\\rust_mozprofilew3w9uT"
+console.warn: services.settings: Ignoring preference override of remote settings server
+console.warn: services.settings: Allow by setting MOZ_REMOTE_SETTINGS_DEVTOOLS=1 in the environment
+1707759994161	Marionette	INFO	Marionette enabled
+Dynamically enable window occlusion 0
+1707759994529	Marionette	INFO	Listening on port 50842
+WebDriver BiDi listening on ws://127.0.0.1:50775
+Read port: 50842
+1707759995346	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+DevTools listening on ws://127.0.0.1:50775/devtools/browser/f4f97816-72e9-4390-a75b-6daae26b6adf
+JavaScript warning: https://www.google.com/maps/_/js/k=maps.m.en.d-cA6F9zu-8.2019.O/m=sc2,per,mo,lp,ep,ti,ds,stx,dwi,enr,pwd,dw,pcb,log,b/am=BgAjAAI/rt=j/d=1/rs=ACT90oEz0eIzV6O4M5RYhpfrQJX3B8BiXg?wli=m.v_7ldjm66Dg.loadSv.O%3A%3Bm.0lcgipW5GjU.mapcore.O%3A%3B, line 2596: WEBGL_debug_renderer_info is deprecated in Firefox and will be removed. Please use RENDERER.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization.
+1707763079140	geckodriver	INFO	Listening on 127.0.0.1:64062
+1707763082520	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "64063" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jsadmin\\AppData\\Local\\Temp\\rust_mozprofile53onUu"
+console.warn: services.settings: Ignoring preference override of remote settings server
+console.warn: services.settings: Allow by setting MOZ_REMOTE_SETTINGS_DEVTOOLS=1 in the environment
+1707763084943	Marionette	INFO	Marionette enabled
+Dynamically enable window occlusion 0
+1707763085546	Marionette	INFO	Listening on port 64084
+WebDriver BiDi listening on ws://127.0.0.1:64063
+Read port: 64084
+1707763086345	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+DevTools listening on ws://127.0.0.1:64063/devtools/browser/23b549b7-d008-49ee-a23c-85bde674b8aa
+JavaScript warning: https://www.google.com/maps/_/js/k=maps.m.en.d-cA6F9zu-8.2019.O/m=sc2,per,mo,lp,ep,ti,ds,stx,dwi,enr,pwd,dw,pcb,log,b/am=BgAjAAI/rt=j/d=1/rs=ACT90oEz0eIzV6O4M5RYhpfrQJX3B8BiXg?wli=m.v_7ldjm66Dg.loadSv.O%3A%3Bm.0lcgipW5GjU.mapcore.O%3A%3B, line 2596: WEBGL_debug_renderer_info is deprecated in Firefox and will be removed. Please use RENDERER.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization.
+1707763115955	addons.xpi	ERROR	System addon update list error SyntaxError: XMLHttpRequest.open: 'http://%(server)s/dummy-system-addons.xml' is not a valid URL.
+JavaScript error: , line 0: NS_ERROR_NOT_AVAILABLE
+1707769513078	geckodriver	INFO	Listening on 127.0.0.1:52732
+1707769516172	mozrunner::runner	INFO	Running command: "C:\\Program Files\\Mozilla Firefox\\firefox.exe" "--marionette" "--remote-debugging-port" "52733" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\jsadmin\\AppData\\Local\\Temp\\rust_mozprofiles5JV3J"
+console.warn: services.settings: Ignoring preference override of remote settings server
+console.warn: services.settings: Allow by setting MOZ_REMOTE_SETTINGS_DEVTOOLS=1 in the environment
+1707769518456	Marionette	INFO	Marionette enabled
+Dynamically enable window occlusion 0
+1707769518837	Marionette	INFO	Listening on port 52755
+WebDriver BiDi listening on ws://127.0.0.1:52733
+Read port: 52755
+1707769519785	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+DevTools listening on ws://127.0.0.1:52733/devtools/browser/0169f414-06b0-4328-93a9-357e17d60486
+JavaScript warning: https://www.google.com/maps/_/js/k=maps.m.en.d-cA6F9zu-8.2019.O/m=sc2,per,mo,lp,ep,ti,ds,stx,dwi,enr,pwd,dw,pcb,log,b/am=BgAjAAI/rt=j/d=1/rs=ACT90oEz0eIzV6O4M5RYhpfrQJX3B8BiXg?wli=m.v_7ldjm66Dg.loadSv.O%3A%3Bm.0lcgipW5GjU.mapcore.O%3A%3B, line 2596: WEBGL_debug_renderer_info is deprecated in Firefox and will be removed. Please use RENDERER.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization.
+1707769548713	addons.xpi	ERROR	System addon update list error SyntaxError: XMLHttpRequest.open: 'http://%(server)s/dummy-system-addons.xml' is not a valid URL.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Texture has not been initialized prior to a partial upload, forcing the browser to clear it. This may be slow.
+JavaScript warning: , line 0: WebGL warning: texSubImage: Tex image TEXTURE_2D level 0 is incurring lazy initialization.

google2.py

@@ -19,7 +19,7 @@ request_result=requests.get( url )
 soup = bs4.BeautifulSoup(request_result.text, 
                          "html.parser") 
 #print(soup) 
-heading_object=soup.find_all( 'h3' ) 
+heading_object=soup #.find_all( 'h1' ) 
   
 # Iterate through the object  
 # and print it as a string. 

google3.py

@@ -0,0 +1,45 @@
+
+# Import the beautifulsoup  
+# and request libraries of python. 
+import requests 
+#import bs4 
+
+#from here
+from bs4 import BeautifulSoup
+from selenium import webdriver
+driver = webdriver.Firefox()
+driver.get('https://www.google.com/maps/contrib/109274792898041753066/reviews')
+
+html = driver.page_source
+soup = BeautifulSoup(html)
+
+innerHTML = driver.execute_script("return document.body.innerHTML")
+print("\nJS PAGE SOURCE:", "\n", driver.page_source)
+
+for info in soup:
+    print (info)
+
+# to here
+
+# Make two strings with default google search URL 
+# 'https://google.com/search?q=' and 
+# our customized search keyword. 
+# Concatenate them 
+# text= "joe"
+# url = "https://www.google.com/maps/contrib/109274792898041753066/reviews"
+  
+# # Fetch the URL data using requests.get(url), 
+# # store it in a variable, request_result. 
+# request_result=requests.get( url ) 
+  
+# # Creating soup from the fetched request 
+# soup = bs4.BeautifulSoup(request_result.text, 
+#                          "html.parser") 
+# #print(soup) 
+# heading_object=soup #.find_all( 'h1' ) 
+  
+# # Iterate through the object  
+# # and print it as a string. 
+# for info in heading_object: 
+#     print(info.getText()) 
+#     print("------") 

requirements.txt

@@ -0,0 +1,13 @@
+autopep8==1.5.7
+et-xmlfile==1.1.0
+numpy==1.21.2
+openpyxl==3.0.9
+pandas==1.3.3
+pkg_resources==0.0.0
+pycodestyle==2.7.0
+python-dateutil==2.8.2
+pytz==2021.1
+selenium==3.141.0
+six==1.16.0
+toml==0.10.2
+urllib3==1.26.6