<jupyter_start><jupyter_text>Import Packages<jupyter_code>import pandas as pd 
import os 
import numpy as np 
import re
import string
from urllib.request import urlopen
from bs4 import BeautifulSoup
import requests
from selenium import webdriver
import math
from selenium.common.exceptions import NoSuchElementException
import googlemaps
import json<jupyter_output><empty_output><jupyter_text>Load Webdriver<jupyter_code>EXE_PATH = r"C:\Users\nikhi\Downloads\chromedriver_win32\chromedriver.exe"
browser = webdriver.Chrome(executable_path=EXE_PATH)<jupyter_output><empty_output><jupyter_text>Open Delhi Metro Website<jupyter_code>base = "https://www.delhimetrorail.com/"
browser.get(base)
browser.implicitly_wait(10)
browser.find_element_by_id("FromStation").click()
html = browser.page_source
soup = BeautifulSoup(html)<jupyter_output><empty_output><jupyter_text>Find all the different lines (routes)<jupyter_code>line_results = browser.find_elements_by_class_name('popup-result-line')
lines = soup.find_all('div', class_ = 'popup-result-line')<jupyter_output><empty_output><jupyter_text>Scrape Data for All Stations on all the lines<jupyter_code>stations_prev = ''
dic = {}
count = 0
for i in range(len(line_results)-1):
    
    line_name = lines[i].text
    
    line_results[i].click()

    while True:
        html = browser.page_source
        soup = BeautifulSoup(html)
        if str(soup.find('div', class_ = 'sub-popup-result').find('div', class_ = 'layout')) != stations_prev:
            break
            
    print(line_name)
    stations = soup.find('div', class_ = 'sub-popup-result').find('div', class_ = 'layout').find_all('a', class_ = 'clearfix')
    
    for station in stations:
        dic[count] = {"Line": line_name}
        dic[count]['n'] = station.find('div', class_ = 'sub-result-left').text
        dic[count]['name'] = station.find('div', class_ = 'sub-result-name').text
        dic[count]['disabled friendly'] = 1 if "Divyang" in str(station.find('div', class_ = 'sub-result-list')) else 0
        dic[count]['parking available'] = 1 if "Parking" in str(station.find('div', class_ = 'sub-result-list')) else 0
        dic[count]['elevator available'] = 1 if "Lift" in str(station.find('div', class_ = 'sub-result-list')) else 0
        count+=1
        
    stations_prev = str(soup.find('div', class_ = 'sub-popup-result').find('div', class_ = 'layout'))
pd.DataFrame.from_dict(dic).T<jupyter_output><empty_output><jupyter_text>Geocode Stations using Google Maps API<jupyter_code>g_API = 'api_key'
gmaps_key = googlemaps.Client(key=g_API)

for i in range(len(dic)):
    search_term = "Delhi Metro " + dic[i]['Line'][:-2] + " " + dic[i]['name']
    geocode_obj = gmaps_key.geocode(search_term)
    dic[i]['map_info'] = geocode_obj
for i in range(len(dic)):
    if len(dic[i]['map_info']) > 0:
        lat = dic[i]['map_info'][0]['geometry']['location']['lat']
        lon = dic[i]['map_info'][0]['geometry']['location']['lng']
        dic[i]['lat'] = lat
        dic[i]['long'] = lon
        
    else:
        print(dic[i])
df = pd.DataFrame.from_dict(dic).T
df.drop(columns=['map_info']).to_excel(r"C:\Users\nikhi\Downloads\Delhi_metro.xlsx")  
for i in range(len(dic)):
    dic[i].pop('map_info')
    
with open(r"C:\Users\nikhi\Downloads\Delhi_metro.json", "w") as outfile:
    json.dump(dic, outfile,indent = 4)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1559171597633136, 'fraction_numerical': 0.005621301775147929, 'mean_word_length': 4.64440734557596, 'pattern_counts': {'":': 1, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '493900', 'license': 'CC0-1.0', 'n_tokens_mistral': 1220, 'n_tokens_neox': 1206, 'n_words': 215, 'path': 'Delhi Metro.ipynb', 'repo_name': 'nikhilklath/delhi-metro-stations', 'size': '3380'}