<jupyter_start><jupyter_text>Table of Contents<jupyter_code>import requests
import json
import pandas as pd
routes = json.load(open('../data/raw/moovit_routes.json', 'r'))
routes = pd.DataFrame(routes['routes'])
agencias = pd.DataFrame(json.load(open('../data/raw/moovit_agencias.json', 'r')))
empresas = pd.read_csv('../data/raw/empresas_moovit.csv', encoding='latin1')
sel_agencias = pd.merge(agencias, empresas, left_on='agencyName', right_on='empresas_moovit')
sel_routes = routes[routes['agencyId'].isin(sel_agencias['agencyId'])]
def get_trips(route_id):
    
    url = f"https://editor.moovit.com/web/routes/{route_id}/trips"

    headers = {
        'host': "editor.moovit.com",
        'connection': "keep-alive",
        'accept': "application/json, text/plain, */*",
        'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.86 Safari/537.36",
        'sec-gpc': "1",
        'sec-fetch-site': "same-origin",
        'sec-fetch-mode': "cors",
        'sec-fetch-dest': "empty",
        'referer': "https://editor.moovit.com/web/routes/19692",
        'accept-encoding': "gzip, deflate, br",
        'accept-language': "en-GB,en-US;q=0.9,en;q=0.8",
        'cookie': "cookieconsent_status=dismiss; EDITORSESSIONID=Bp5CKqRGp5ZxrOrER0DRKJmD.undefined; JSESSIONIDSSO=sgKs7crBsVSLH+lknszk7Gme",
        'cache-control': "no-cache",
        'postman-token': "<PASSWORD>-77f6-1d65-b5d5-f0939ff0bca9"
        }
    response = requests.request("GET", url, headers=headers)
    
    if response.ok:
        json.dump(response.json(), open(f'../data/raw/moovit_trips/{route_id}.json', 'w'))
    else:
        print(f'FAILED: {route_id}')
import requests


def fecth_and_save_stops(trip_id):
    
    url = f"https://editor.moovit.com/web/trips/mobile-mappings/{trip_id}"

    headers = {
        'host': "editor.moovit.com",
        'connection': "keep-alive",
        'accept': "application/json, text/plain, */*",
        'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.86 Safari/537.36",
        'sec-gpc': "1",
        'sec-fetch-site': "same-origin",
        'sec-fetch-mode': "cors",
        'sec-fetch-dest': "empty",
        'referer': "https://editor.moovit.com/web/trips/31699",
        'accept-encoding': "gzip, deflate, br",
        'accept-language': "en-GB,en-US;q=0.9,en;q=0.8",
        'cookie': "cookieconsent_status=dismiss; EDITORSESSIONID=Bp5CKqRGp5ZxrOrER0DRKJmD.undefined; JSESSIONIDSSO=sgKs7crBsVSLH+lknszk7Gme",
        'cache-control': "no-cache",
        'postman-token': "c19ef460-5f4d-0936-1225-18569ee1f312"
        }

    response = requests.request("GET", url, headers=headers)
    
    if response.ok:
        json.dump(response.json(), open(f'../data/raw/moovit_stops/{trip_id}.json', 'w'))
    else:
        print(f'FAILED: {trip_id}')
sel_routes.head()
import time
import tqdm
for route_id in tqdm.tqdm(sel_routes['routeId'].unique()):
    get_trips(route_id)
    time.sleep(1)
from pathlib import Path
trips = [json.load(open(p)) for p in Path('../data/raw/moovit_trips/').glob('*')]
flatten = lambda t: [item for sublist in t for item in sublist]
trips = flatten(trips)
trips = pd.DataFrame(trips)
len(trips)
for trip_id in tqdm.tqdm(trips['tripId'].unique()):
    fecth_and_save_stops(trip_id)
    time.sleep(0.5)
!ls ../data/raw/moovit_stops/
stops = [json.load(open(p)) for p in Path('../data/raw/moovit_stops/').glob('*')]
stops = flatten(stops)
fstops = flatten([s['content']['stops'] for s in stops])
fstops = pd.DataFrame(fstops)[['lat', 'lng', 'name']]
fstops['lat'] = fstops['lat'].apply(lambda x: round(x, 5))
fstops['lng'] = fstops['lng'].apply(lambda x: round(x, 5))
fstops = fstops.drop_duplicates()
fstops.to_csv('../data/output/moovit_paradas.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18501291989664082, 'fraction_numerical': 0.03255813953488372, 'mean_word_length': 3.773119605425401, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '593757', 'license': 'MIT', 'n_tokens_mistral': 1612, 'n_tokens_neox': 1547, 'n_words': 231, 'path': 'notebooks/Untitled2.ipynb', 'repo_name': 'RJ-SMTR/rGTFS', 'size': '3868'}