<commit_before>from rdflib import Graph, Namespace, RDF, Literal, XSD, URIRef
from csv import DictReader
from rdflib.namespace import FOAF, DC
from rdflib.resource import Resource

__author__ = 'Diarmuid'

class gtfsCsvToRdf:

    def __init__(self, uri, output_file, format='n3'):
        self.output_file = output_file
        self.graph = Graph(identifier=uri)
        self.GTFS = Namespace("http://vocab.gtfs.org/terms#")
        self.graph.bind("gtfs", self.GTFS)
        self.GEO = Namespace("http://www.w3.org/2003/01/geo/wgs84_pos#")
        self.graph.bind("geo", self.GEO)
        self.uri = uri
        self.format = format
        self.agency_ids = {}
        self.stop_ids = {}
        self.route_ids = {}

    def convert_agency(self, csv_filename):
        read_agency = DictReader(open(csv_filename))
        print(read_agency.fieldnames)
        for row in read_agency:
            url = row['agency_url']
            agency = Resource(self.graph, URIRef(url))
            agency.add(RDF.type, self.GTFS.Agency)
            name = Literal(row['agency_name'], datatype=XSD.string)
            agency.add(FOAF.name, name)
            timezone = Literal(row['agency_timezone'], datatype=XSD.string)
            agency.add(self.GTFS.timeZone, timezone)
            if 'agency_lang' in row:
                agency.add(DC.language, Literal(row['agency_lang'], datatype=XSD.string))
            if 'agency_phone' in row:
                agency.add(FOAF.phone, Literal(row['agency_phone'], datatype=XSD.string))
            if 'agency_fare_url' in row:
                agency.add(self.GTFS.fareUrl, URIRef(row['agency_fare_url']))
            if 'agency_id' in row:
                self.agency_ids[row['agency_id']] = agency
                agency.add(DC.identifier, Literal(row['agency_id'], datatype=XSD.string))

    def convert_stops(self, csv_filename):
        read_stops = DictReader(open(csv_filename))
        print(read_stops.fieldnames)
        for row in read_stops:
            stop = self.get_stop(row['stop_id'])
            stop.add(FOAF.name, Literal(row['stop_name'], datatype=XSD.string))
            stop.add(self.GEO.long, Literal(row['stop_lon'], datatype=XSD.string))
            stop.add(self.GEO.lat, Literal(row['stop_lat'], datatype=XSD.string))
            if 'stop_code' in row:
                stop.add(self.GTFS.code, Literal(row['stop_code'], datatype=XSD.string))
            if 'stop_desc' in row:
                stop.add(DC.description, Literal(row['stop_desc'], datatype=XSD.string))
            if 'zone_id' in row:
                stop.add(self.GTFS.zone, self.get_zone(row['zone_id']))
            if 'stop_url' in row:
                stop.add(FOAF.page ,Literal(row['stop_url'], datatype=XSD.string))
            if 'location_type' in row and row['location_type'] is "1":
                stop.set(RDF.type, self.GTFS.Station)
            else:
                stop.set(RDF.type, self.GTFS.Stop)
            if 'parent_station' in row:
                parent_id = row['parent_station']
                if parent_id in self.stop_ids:
                    stop.add(self.GTFS.parentStation, self.stop_ids[parent_id])
                elif parent_id is not "":
                    stop.add(self.GTFS.parentStation, self.get_stop(parent_id))
            if 'stop_timezone' in row:
                stop.add(self.GTFS.timeZone, Literal(row['stop_timezone'], datatype=XSD.string))
            if 'wheelchair_boarding' in row:
                wheelchair = row['wheelchair_boarding']
                if wheelchair is "1":
                    accessibility = self.GTFS.WheelchairAccessible
                elif wheelchair is "2":
                    accessibility = self.GTFS.NotWheelchairAccessible
                else:
                    accessibility = self.GTFS.CheckParentStation
                stop.add(self.GTFS.wheelchairAccesssible, accessibility)

    def convert_routes(self, csv_filename):
        route_types = [self.GTFS.LightRail, self.GTFS.Subway, self.GTFS.Rail, self.GTFS.Bus, self.GTFS.Ferry, self.GTFS.CableCar, self.GTFS.Gondola, self.GTFS.Funicular]
        read_routes = DictReader(open(csv_filename))
        print(read_routes.fieldnames)
        for row in read_routes:
            route = self.get_route(row["route_id"])
            route.add(self.GTFS.shortName, Literal(row["route_short_name"], datatype=XSD.string))
            route.add(self.GTFS.longName, Literal(row["route_long_name"], datatype=XSD.string))
            route.add(self.GTFS.routeType, route_types[int(row["route_type"])])
            if "agency_id" in row:
                route.add(self.GTFS.agency, self.agency_ids[row["agency_id"]])
            if "route_desc" in row:
                route.add(DC.description, Literal(row["route_desc"], datatype=XSD.string))
            # not implemented yet
            # if "route_url" in row:
            #     route.add(self.GTFS.routeUrl, row["route_url"])
            if "route_color" in row:
                route.add(self.GTFS.color, Literal(row["route_color"], datatype=XSD.string))
            if "route_text_color" in row:
                route.add(self.GTFS.textColor, Literal(row["route_text_color"], datatype=XSD.string))

    def output(self):
        self.graph.serialize(destination=self.output_file,format=self.format)

    def get_stop(self, id):
        stop = Resource(self.graph, URIRef(self.uri + id))
        self.stop_ids[id] = stop
        stop.set(DC.identifier, Literal(id, datatype=XSD.string))
        return stop

    def get_zone(self, id):
        the_zone = Resource(self.graph, URIRef(self.uri + id))
        the_zone.set(RDF.type, self.GTFS.Zone)
        the_zone.set(DC.identifier, Literal(id, datatype=XSD.string))
        return the_zone

    def get_route(self, id):
        route = Resource(self.graph, URIRef(self.uri + id))
        self.route_ids[id] = route
        route.set(RDF.type, self.GTFS.Route)
        route.set(DC.identifier, Literal(id, datatype=XSD.string))
        return route<commit_msg>Added Trips conversion<commit_after>from rdflib import Graph, Namespace, RDF, Literal, XSD, URIRef
from csv import DictReader
from rdflib.namespace import FOAF, DC
from rdflib.resource import Resource

__author__ = 'Diarmuid'

class gtfsCsvToRdf:

    def __init__(self, uri, output_file, format='n3'):
        self.output_file = output_file
        self.graph = Graph(identifier=uri)
        self.GTFS = Namespace("http://vocab.gtfs.org/terms#")
        self.graph.bind("gtfs", self.GTFS)
        self.GEO = Namespace("http://www.w3.org/2003/01/geo/wgs84_pos#")
        self.graph.bind("geo", self.GEO)
        self.uri = uri
        self.format = format
        self.agency_ids = {}

    def convert_agency(self, csv_filename):
        read_agency = DictReader(open(csv_filename))
        print(read_agency.fieldnames)
        for row in read_agency:
            url = row['agency_url']
            agency = Resource(self.graph, URIRef(url))
            agency.add(RDF.type, self.GTFS.Agency)
            name = Literal(row['agency_name'], datatype=XSD.string)
            agency.add(FOAF.name, name)
            timezone = Literal(row['agency_timezone'], datatype=XSD.string)
            agency.add(self.GTFS.timeZone, timezone)
            if 'agency_lang' in row:
                agency.add(DC.language, Literal(row['agency_lang'], datatype=XSD.string))
            if 'agency_phone' in row:
                agency.add(FOAF.phone, Literal(row['agency_phone'], datatype=XSD.string))
            if 'agency_fare_url' in row:
                agency.add(self.GTFS.fareUrl, URIRef(row['agency_fare_url']))
            if 'agency_id' in row:
                self.agency_ids[row['agency_id']] = agency
                agency.add(DC.identifier, Literal(row['agency_id'], datatype=XSD.string))

    def convert_stops(self, csv_filename):
        read_stops = DictReader(open(csv_filename))
        print(read_stops.fieldnames)
        for row in read_stops:
            stop = self.get_stop(row['stop_id'])
            stop.add(FOAF.name, Literal(row['stop_name'], datatype=XSD.string))
            stop.add(self.GEO.long, Literal(row['stop_lon'], datatype=XSD.string))
            stop.add(self.GEO.lat, Literal(row['stop_lat'], datatype=XSD.string))
            if 'stop_code' in row:
                stop.add(self.GTFS.code, Literal(row['stop_code'], datatype=XSD.string))
            if 'stop_desc' in row:
                stop.add(DC.description, Literal(row['stop_desc'], datatype=XSD.string))
            if 'zone_id' in row:
                stop.add(self.GTFS.zone, self.get_zone(row['zone_id']))
            if 'stop_url' in row:
                stop.add(FOAF.page ,Literal(row['stop_url'], datatype=XSD.string))
            if 'location_type' in row and row['location_type'] is "1":
                stop.set(RDF.type, self.GTFS.Station)
            else:
                stop.set(RDF.type, self.GTFS.Stop)
            if 'parent_station' in row:
                if row['parent_station'] is not "":
                    stop.add(self.GTFS.parentStation, self.get_stop(row['parent_station']))
            if 'stop_timezone' in row:
                stop.add(self.GTFS.timeZone, Literal(row['stop_timezone'], datatype=XSD.string))
            if 'wheelchair_boarding' in row:
                accessibility = self.get_wheelchair_accessible(row['wheelchair_boarding'])
                stop.add(self.GTFS.wheelchairAccesssible, accessibility)

    def convert_routes(self, csv_filename):
        route_types = [self.GTFS.LightRail, self.GTFS.Subway, self.GTFS.Rail, self.GTFS.Bus, self.GTFS.Ferry, self.GTFS.CableCar, self.GTFS.Gondola, self.GTFS.Funicular]
        read_routes = DictReader(open(csv_filename))
        print(read_routes.fieldnames)
        for row in read_routes:
            route = self.get_route(row["route_id"])
            route.add(self.GTFS.shortName, Literal(row["route_short_name"], datatype=XSD.string))
            route.add(self.GTFS.longName, Literal(row["route_long_name"], datatype=XSD.string))
            route.add(self.GTFS.routeType, route_types[int(row["route_type"])])
            if "agency_id" in row:
                route.add(self.GTFS.agency, self.agency_ids[row["agency_id"]])
            if "route_desc" in row:
                route.add(DC.description, Literal(row["route_desc"], datatype=XSD.string))
            # not implemented yet
            # if "route_url" in row:
            #     route.add(self.GTFS.routeUrl, row["route_url"])
            if "route_color" in row:
                route.add(self.GTFS.color, Literal(row["route_color"], datatype=XSD.string))
            if "route_text_color" in row:
                route.add(self.GTFS.textColor, Literal(row["route_text_color"], datatype=XSD.string))

    def convert_trips(self, csv_filename):
        read_trips = DictReader(open(csv_filename))
        print(read_trips.fieldnames)
        for row in read_trips:
            trip = self.get_trip(row["trip_id"])
            trip.add(self.GTFS.route, self.get_route(row["route_id"]))
            trip.add(self.GTFS.service, self.get_service(row["service_id"]))
            if "trip_headsign" in row:
                trip.add(self.GTFS.headsign, Literal(row["trip_headsign"], datatype=XSD.string))
            if "trip_short_name" in row:
                trip.add(self.GTFS.shortName, Literal(row["trip_short_name"], datatype=XSD.string))
            if "direction_id" in row:
                trip.add(self.GTFS.direction, Literal(row["direction_id"], datatype=XSD.boolean))
            if "block_id" in row:
                trip.add(self.GTFS.block, Literal(row["block_id"], datatype=XSD.nonNegativeInteger))
            if "shape_id" in row:
                trip.add(self.GTFS.shape, self.get_shape(row["shape_id"]))
            if "wheelchair_accessible" in row:
                accessibility = self.get_wheelchair_accessible(row['wheelchair_accessible'])
                trip.add(self.GTFS.wheelchairAccessible, accessibility)
            if "bikes_allowed" in row:
                bikes = self.get_bikes_allowed(row["bikes_allowed"])
                trip.add(self.GTFS.bikesAllowed, bikes)


    def output(self):
        self.graph.serialize(destination=self.output_file, format=self.format)

    def get_stop(self, id):
        stop = Resource(self.graph, URIRef(self.uri + id))
        stop.set(DC.identifier, Literal(id, datatype=XSD.string))
        return stop

    def get_zone(self, id):
        the_zone = Resource(self.graph, URIRef(self.uri + id))
        the_zone.set(RDF.type, self.GTFS.Zone)
        the_zone.set(DC.identifier, Literal(id, datatype=XSD.string))
        return the_zone

    def get_route(self, id):
        route = Resource(self.graph, URIRef(self.uri + id))
        route.set(RDF.type, self.GTFS.Route)
        route.set(DC.identifier, Literal(id, datatype=XSD.string))
        return route

    def get_trip(self, id):
        trip = Resource(self.graph, URIRef(self.uri + id))
        trip.set(RDF.type, self.GTFS.Trip)
        trip.set(DC.identifier, Literal(id, datatype=XSD.string))
        return trip

    def get_service(self, id):
        service = Resource(self.graph, URIRef(self.uri + id))
        service.set(RDF.type, self.GTFS.Service)
        service.set(DC.identifier, Literal(id, datatype=XSD.string))
        return service

    def get_shape(self, id):
        shape = Resource(self.graph, URIRef(self.uri + id))
        shape.set(RDF.type, self.GTFS.Shape)
        shape.set(DC.identifier, Literal(id, datatype=XSD.string))
        return shape

    def get_wheelchair_accessible(self, wheelchair):
        if wheelchair is "1":
            accessibility = self.GTFS.WheelchairAccessible
        elif wheelchair is "2":
            accessibility = self.GTFS.NotWheelchairAccessible
        else:
            accessibility = self.GTFS.CheckParentStation
        return accessibility

    def get_bikes_allowed(self, code):
        bikes = 0
        if code is "0" or "2":
            bikes = 0
        elif code is "1":
            bikes = 1
        return Literal(bikes, datatype=XSD.boolean)
{'fraction_non_alphanumeric': 0.12385873027107368, 'fraction_numerical': 0.0022648453535282042, 'mean_word_length': 2.4396299902629015, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3462311', 'n_tokens_mistral': 4663, 'n_tokens_neox': 4498, 'n_words': 777}