from time import sleep from selenium.webdriver import Firefox from selenium.webdriver.common.by import By from selenium.webdriver.firefox.options import Options from selenium.webdriver.support.select import Select class Scraper: def __init__(self, base_url, headless=True): options = Options() if headless: options.add_argument("--headless") self.driver = Firefox(options=options) self.driver.get(base_url) def close_modal(self): sleep(3) self.driver.execute_script('closeModel({modal_id:"validateError"})') sleep(1) def select(self, i_d, value): while True: try: element = self.driver.find_element(By.ID, i_d) break except: sleep(0.2) pass select = Select(element) select.select_by_visible_text(value) sleep(1) def scrape_states(self): element = self.driver.find_element(By.ID, 'sess_state_code') options = Select(element).options states = [ option.text for option in options[1:] ] print(f'STATES: {states}') sleep(0.2) return states def scrape_districts(self): element = self.driver.find_element(By.ID, 'sess_dist_code') options = Select(element).options districts = [ option.text for option in options[1:] ] print(f'DISTRICTS: {districts}') return districts def scrape_complexes(self): element = self.driver.find_element(By.ID, 'court_complex_code') options = Select(element).options complexes = [ option.text for option in options[1:] ] print(f'COMPLEXES: {complexes}') return complexes def establishments_visible(self): return self.driver.find_element(By.ID, 'court_est_code').is_displayed() def scrape_establishments(self): element = self.driver.find_element(By.ID, 'court_est_code') options = Select(element).options establishments = [ option.text for option in options[1:] if option.text != '' ] print(f'ESTABLISHMENTS: {establishments}') return establishments