<commit_before>import rp from 'request-promise';
import {toLatLon} from 'utm';
import LRU from 'lru-cache';

const cache = LRU({
    maxAge: 1000 * 20,
    max: 200
});
const baseUrl = 'https://reisapi.ruter.no';

// mapper between how transport type is represented in
// realtime api and line api.
// http://reisapi.ruter.no/Help/ResourceModel?modelName=VehicleModeEnum
// http://reisapi.ruter.no/Help/ResourceModel?modelName=TransportationType
const vehicleModeToTransportType = {
    0: 2,
    1: 5,
    2: 6,
    3: 7,
    4: 8
};

function parseStopInfo(info) {
    return {
        id: info.ID,
        name: info.Name.trim(),
        shortName: info.ShortName,
        zone: info.Zone,
        geoLocation: toLatLon(info.X, info.Y, 32, null, true), // should be "V"?
        utmLocation: {x: info.X, y: info.Y},
        isHub: info.IsHub,
        district: info.District,
        placeType: info.PlaceType || "Stop",
        walkingTimeMins: info.WalkingMinutes || null
    };
}

function parsePoiInfo(info) {
    return {
        id: info.ID,
        name: info.Name.trim(),
        geoLocation: toLatLon(info.X, info.Y, 32, null, true), // should be "V"?
        utmLocation: {x: info.X, y: info.Y},
        district: info.District,
        placeType: "POI",
        nearbyStops: info.Stops
            .filter(e => e.ID !== 0)
            .map(parseStopInfo)
    };
}

function parseAreaInfo(info) {
    return {
        id: info.ID,
        name: info.Name.trim(),
        geoLocation: toLatLon(info.Center.X, info.Center.Y, 32, null, true), // should be "V"?
        utmLocation: {x: info.Center.X, y: info.Center.Y},
        district: info.District,
        placeType: "Area",
        stops: info.Stops.map(parseStopInfo)
    };
}


function parseLineInfo(info) {
    return {
        id: info.ID,
        name: info.Name.trim(),
        transportationType: info.Transportation,
        color: info.LineColour
    };
}

function parseDeviations(e) {
    return e ? e.map(parseDeviation) : [];
}

function parseDeviation(e) {
    return { 
        id: e.ID,
        header: e.Header
    };
}

function parseVisit(e) {
    return {
        stopId: parseInt(e.MonitoringRef, 10),
        lineId: parseInt(e.MonitoredVehicleJourney.LineRef),
        destinationName: e.MonitoredVehicleJourney.DestinationName,
        name: e.MonitoredVehicleJourney.PublishedLineName,
        direction: e.MonitoredVehicleJourney.DirectionRef,
        recordedAtTime: new Date(e.RecordedAtTime).toString(),
        expectedArrival: new Date(e.MonitoredVehicleJourney.MonitoredCall.ExpectedArrivalTime).toString(),
        deviations: parseDeviations(e.Extensions.Deviations),
        lineColour: e.Extensions.LineColour,
        platform: e.MonitoredVehicleJourney.MonitoredCall.DeparturePlatformName,
        inCongestion: e.MonitoredVehicleJourney.InCongestion,
        monitored: e.MonitoredVehicleJourney.Monitored,
        transportationType: vehicleModeToTransportType[e.MonitoredVehicleJourney.VehicleMode],
        lowFloor: e.MonitoredVehicleJourney.VehicleFeatureRef 
                    ? e.MonitoredVehicleJourney.VehicleFeatureRef == 'lowFloor'
                    : false
    };
}

function parsePlace(e) {
    // todo: place types: Stop, POI, Area, Street
    if (e.PlaceType == "Stop") {
        return parseStopInfo(e);
    }
    else if (e.PlaceType == "POI") {
        return parsePoiInfo(e);
    }
    else if (e.PlaceType == "Street") {
        return {
            id: e.ID,
            name: e.Name,
            district: e.District,
            placeType: e.PlaceType
        };
    }
    else if (e.PlaceType == "Area") {
        return parseAreaInfo(e);
    }
    else {
        return {
            id: e.ID,
            name: e.Name,
            district: e.District,
            placeType: e.PlaceType
        }; 
    }
}

function getJson(url, query = null) {
    // todo: escape URI
    const key = url + ":" + JSON.stringify(query || {});
    const data = cache.get(key);
    if (data) { 
        console.log("cache hit:  ", key);
        return data;
    } 
    else {
        console.log("cache miss: ", key);
    }

    const response = rp({ uri: url, json: true, qs: query });
    cache.set(key, response);
    return response;
}

export function linesForStop(id) {
    console.log("fetching lines for stop", id);
    return getJson(`${baseUrl}/Line/GetLinesByStopID/${id}`)
               .then(e => e.map(parseLineInfo));
}

export function stopsForLine(id) {
    console.log("fetching stops for line", id);
    return getJson(`${baseUrl}/Line/GetStopsByLineID/${id}`)
        .then(e => e.map(parseStopInfo));
}

export function lineInfo(id) {
    console.log("fetching lines info for", id);
    return getJson(`${baseUrl}/Line/GetDataByLineID/${id}`)
        .then(parseLineInfo);<commit_msg>Escape URLs properly<commit_after>import {format as urlFormat, parse as urlParse} from 'url';
import rp from 'request-promise';
import {toLatLon} from 'utm';
import LRU from 'lru-cache';

const cache = LRU({
    maxAge: 1000 * 20,
    max: 200
});
const baseUrl = 'https://reisapi.ruter.no';

// mapper between how transport type is represented in
// realtime api and line api.
// http://reisapi.ruter.no/Help/ResourceModel?modelName=VehicleModeEnum
// http://reisapi.ruter.no/Help/ResourceModel?modelName=TransportationType
const vehicleModeToTransportType = {
    0: 2,
    1: 5,
    2: 6,
    3: 7,
    4: 8
};

function parseStopInfo(info) {
    return {
        id: info.ID,
        name: info.Name.trim(),
        shortName: info.ShortName,
        zone: info.Zone,
        geoLocation: toLatLon(info.X, info.Y, 32, null, true), // should be "V"?
        utmLocation: {x: info.X, y: info.Y},
        isHub: info.IsHub,
        district: info.District,
        placeType: info.PlaceType || "Stop",
        walkingTimeMins: info.WalkingMinutes || null
    };
}

function parsePoiInfo(info) {
    return {
        id: info.ID,
        name: info.Name.trim(),
        geoLocation: toLatLon(info.X, info.Y, 32, null, true), // should be "V"?
        utmLocation: {x: info.X, y: info.Y},
        district: info.District,
        placeType: "POI",
        nearbyStops: info.Stops
            .filter(e => e.ID !== 0)
            .map(parseStopInfo)
    };
}

function parseAreaInfo(info) {
    return {
        id: info.ID,
        name: info.Name.trim(),
        geoLocation: toLatLon(info.Center.X, info.Center.Y, 32, null, true), // should be "V"?
        utmLocation: {x: info.Center.X, y: info.Center.Y},
        district: info.District,
        placeType: "Area",
        stops: info.Stops.map(parseStopInfo)
    };
}


function parseLineInfo(info) {
    return {
        id: info.ID,
        name: info.Name.trim(),
        transportationType: info.Transportation,
        color: info.LineColour
    };
}

function parseDeviations(e) {
    return e ? e.map(parseDeviation) : [];
}

function parseDeviation(e) {
    return { 
        id: e.ID,
        header: e.Header
    };
}

function parseVisit(e) {
    return {
        stopId: parseInt(e.MonitoringRef, 10),
        lineId: parseInt(e.MonitoredVehicleJourney.LineRef),
        destinationName: e.MonitoredVehicleJourney.DestinationName,
        name: e.MonitoredVehicleJourney.PublishedLineName,
        direction: e.MonitoredVehicleJourney.DirectionRef,
        recordedAtTime: new Date(e.RecordedAtTime).toString(),
        expectedArrival: new Date(e.MonitoredVehicleJourney.MonitoredCall.ExpectedArrivalTime).toString(),
        deviations: parseDeviations(e.Extensions.Deviations),
        lineColour: e.Extensions.LineColour,
        platform: e.MonitoredVehicleJourney.MonitoredCall.DeparturePlatformName,
        inCongestion: e.MonitoredVehicleJourney.InCongestion,
        monitored: e.MonitoredVehicleJourney.Monitored,
        transportationType: vehicleModeToTransportType[e.MonitoredVehicleJourney.VehicleMode],
        lowFloor: e.MonitoredVehicleJourney.VehicleFeatureRef 
                    ? e.MonitoredVehicleJourney.VehicleFeatureRef == 'lowFloor'
                    : false
    };
}

function parsePlace(e) {
    if (e.PlaceType == "Stop") {
        return parseStopInfo(e);
    }
    else if (e.PlaceType == "POI") {
        return parsePoiInfo(e);
    }
    else if (e.PlaceType == "Street") {
        return {
            id: e.ID,
            name: e.Name,
            district: e.District,
            placeType: e.PlaceType
        };
    }
    else if (e.PlaceType == "Area") {
        return parseAreaInfo(e);
    }
    else {
        return {
            id: e.ID,
            name: e.Name,
            district: e.District,
            placeType: e.PlaceType
        }; 
    }
}

function sanitizeUrl(url) {
    return urlFormat(urlParse(url));
}

function getJson(url, query = null) {
    url = sanitizeUrl(url);
    const key = url + ":" + JSON.stringify(query || {});
    const data = cache.get(key);
    if (data) { 
        console.log("cache hit:  ", key);
        return data;
    } 
    else {
        console.log("cache miss: ", key);
    }

    const response = rp({ uri: url, json: true, qs: query });
    cache.set(key, response);
    return response;
}

export function linesForStop(id) {
    console.log("fetching lines for stop", id);
    return getJson(`${baseUrl}/Line/GetLinesByStopID/${id}`)
               .then(e => e.map(parseLineInfo));
}

export function stopsForLine(id) {
    console.log("fetching stops for line", id);
    return getJson(`${baseUrl}/Line/GetStopsByLineID/${id}`)
        .then(e => e.map(parseStopInfo));
}

export function lineInfo(id) {
    console.log("fetching lines info for", id);
    return getJson(`${baseUrl}/Line/GetDataByLineID/${id}`)
        .then(parseLineInfo);
{'fraction_non_alphanumeric': 0.13153301160283395, 'fraction_numerical': 0.005750077009959955, 'mean_word_length': 2.4612651030561477, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2308297', 'n_tokens_mistral': 3304, 'n_tokens_neox': 3210, 'n_words': 656}