import csv
from time import sleep
from modules.scraper_orders import ScraperOrders
from selenium.webdriver.common.by import By
from concurrent.futures import ThreadPoolExecutor, as_completed
import threading
class ThreadSafeCSVWriter:
def __init__(self, filename):
self.file = open(filename, 'w', newline='')
self.writer = csv.writer(self.file)
self.lock = threading.Lock()
def writerow(self, row):
with self.lock:
self.writer.writerow(row)
print(f'Wrote: {row}')
def close(self):
self.file.close()
def scrape_district(state, district, csv_writer):
try:
config = {}
scraper = ScraperOrders(None, config)
scraper.close_modal()
scraper.select('sess_state_code', state)
scraper.select('sess_dist_code', district)
complexes = scraper.scrape_complexes()
scraper.select('court_complex_code', complexes[0])
sleep(2)
scraper.goto_courtnumber()
for cmplx in complexes:
while True:
sleep(0.5)
try:
modal_is_open = scraper.driver.find_element(By.CLASS_NAME, 'modal').is_displayed()
if modal_is_open:
scraper.close_modal()
continue
break
except:
break
scraper.select('court_complex_code', cmplx)
sleep(0.5)
court_numbers = scraper.get_court_numbers()
for court_number in court_numbers:
row = [state, district, cmplx, court_number]
csv_writer.writerow(row)
scraper.driver.quit()
except Exception as e:
print(f"Error scraping district {district}: {e}")
def scrape_courts():
state = 'Uttar Pradesh'
config = {}
scraper = ScraperOrders(None, config)
scraper.close_modal()
scraper.select('sess_state_code', state)
districts = scraper.scrape_districts()
scraper.driver.quit()
csv_writer = ThreadSafeCSVWriter('csv/court_numbers.csv')
csv_writer.writerow(['State', 'District', 'Cmplx', 'Court number'])
with ThreadPoolExecutor(max_workers=5) as executor:
futures = [
executor.submit(scrape_district, state, district, csv_writer)
for district in districts
]
for future in as_completed(futures):
try:
future.result()
except Exception as e:
print(f"A thread encountered an error: {e}")
csv_writer.close()
if __name__ == '__main__':
scrape_courts()