Scraping historical merchant review data from Google

import sys
sys.path.insert(0,'/usr/bin/chromedriver')
from selenium import webdriver
import time, os, psutil, random
from selenium.webdriver.common.keys import Keys
import pandas as pd
from datetime import datetime

process = psutil.Process(os.getpid())
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome('chromedriver', options=chrome_options)

name='catch.com.au'
url = f'https://www.google.com/shopping/customerreviews/merchantreviews?q={name}'
probability = 0.1
month = 'August'
year = 2018
out_file = 'review_2018.csv'

driver.get(url)
all_match_found = False
previous = list()

def filter_elements(element):
  if len(previous) == 0:
    return True
  if element in previous:
    return False
  return True


def extract_review_for_month_year(revs, month:str, year:int):
  # extract review data
  global all_match_found
  month_index = months.index(month)
  for rev in revs:
    review_date = rev.find_element_by_class_name("CfOeR").text
    if (month in review_date or any(n in review_date for n in months[month_index:12])) and (str(year) in review_date):
      if random.random() < probability:
          print(f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")}::date in process: {review_date}')
      score.append(rev.find_element_by_class_name("ujLN8e").get_attribute("style"))
      date.append(rev.find_element_by_class_name("CfOeR").text)
      text.append(rev.find_element_by_css_selector("div:nth-of-type(2)").text)
    elif str(year - 1) in review_date:
      all_match_found = True
      print(f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")}::We crossed our review month, nothing else to look for here!!')
    else:
      #all_match_found = True
      if random.random() < probability:
        #all_match_found = True
        print(f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")}::date in process: {review_date}, is before our start date!!!')

previous_count = 0
while not all_match_found:
  time.sleep(3)
  driver.find_element_by_class_name("VfPpkd-vQzf8d").click()
  current = driver.find_elements_by_class_name("SzAWKe")

  if len(current) - previous_count < 10:
      print(f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")}:: >>>>>> It only fetched {len(current) - previous_count} elements!!! Probably it did not load 10 reviews!!!! There will be duplicate data!!')
  
  previous_count = len(current)
  to_process = current[-10:]
  
  if len(list(filter(filter_elements, to_process))) == 0:
      print('{datetime.now().strftime("%d/%m/%Y %H:%M:%S")} >>>>>>>>!!!! ALERT !!!!either there are no more reviews to process, or selenium could not load any more reviews!!')
      break

  if random.random() < probability:
      print(f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")}::Reviews scanned so far: {len(current)} && review to be processed {len(to_process)}')
      print(f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")}:: memory used in MB : {process.memory_info().rss/1048576}')
  extract_review_for_month_year(to_process, month, year)
  previous = to_process

print(f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")}::Processing done. Total reviews processed: {previous_count}')

df=pd.DataFrame()
score, text, date = [], [], []
months = ['December', 'November', 'October', 'September', 'August', 'July', 'June', 'May', 'April', 'March', 'February', 'January']

text = ['' if i[:11]=='Written on ' else i for i in text]
df['score'], df['text'], df['date'] = score, text, date
df.score = df.score.astype(str).str.replace('width: ', '').str.replace('%;','').astype(int)
df.date = pd.to_datetime(df.date.astype(str).str.replace('Written on ', ''))
df.to_csv('reviews_2021.csv')
df.head()