<commit_before>namespace :stops do
  desc "Fetch the tram stops from the HSL data"
  task fetch: :environment do
    query = %(
    {
      routes(modes: "TRAM") {
        id
        agency {
          id
        }
        shortName
        longName
        desc
        patterns {
          code
          trips {
            serviceId
          }
          stops {
            id
            gtfsId
            name
            code
            lat
            lon
          }
        }
      }
    }
    )

    endpoint_url = "https://api.digitransit.fi/routing/v1/routers/finland/index/graphql"

    resp = HTTP.headers("Content-Type": "application/graphql").
      post(endpoint_url, body: query)

    data = JSON.load(resp.body.to_s)

    # Delete the #5 tram since it doesn't exist yet
    # FIXME: reinstate it in the future
    data["data"]["routes"].delete_if { |r| r["shortName"] == "5" }

    # Get the route with the most trips, that's likely to be the “main” route
    # for that tram line.
    data["data"]["routes"].each do |route|
      route["patterns"].sort_by! {|p| p["trips"].length }.reverse!
      if route["shortName"] == "6T"
        route["stops"] = route["patterns"][1]["stops"]
      else
        route["stops"] = route["patterns"][0]["stops"]
      end
    end

    stops = {}

    data.dig("data", "routes").each do |route|
      route["stops"].each_with_index do |s, index|
        stops[s["name"]] ||= {
          coordinates: [],
          hsl_ids: [],
          stop_numbers: [],
          routes: [],
          stop_positions: {},
          active: true
        }
        stops[s["name"]][:coordinates] << [s["lat"], s["lon"]]
        stops[s["name"]][:hsl_ids] << Base64.decode64(s["id"])
        stops[s["name"]][:stop_numbers] << s["code"]
        stops[s["name"]][:routes] << route["shortName"]
        stops[s["name"]][:stop_positions][route["shortName"]] = index

        stops[s["name"]].each do |k, v|
          v.uniq! if v.respond_to?(:uniq!)
        end
      end
    end

    Stop.transaction do
      records = stops.map do |name, data|
        print "Creating #{name}"

        stop = Stop.find_or_initialize_by(name: name)
        print "."

        centre = Geocoder::Calculations.geographic_center data.delete(:coordinates)
        stop.latitude, stop.longitude = centre
        print "."

        stop.assign_attributes(data)

        stop.save!
        puts ". done"

        stop
      end

      print "Removing old stops"
      Stop.where.not(id: records.map(&:id)).update_all(active: false)
      puts "... done"<commit_msg>Add line 5, fix stops on lines 5 and 6.<commit_after>namespace :stops do
  desc "Fetch the tram stops from the HSL data"
  task fetch: :environment do
    query = %(
    {
      routes(modes: "TRAM") {
        id
        agency {
          id
        }
        shortName
        longName
        desc
        patterns {
          code
          trips {
            serviceId
          }
          stops {
            id
            gtfsId
            name
            code
            lat
            lon
          }
        }
      }
    }
    )

    endpoint_url = "https://api.digitransit.fi/routing/v1/routers/finland/index/graphql"

    resp = HTTP.headers("Content-Type": "application/graphql").
      post(endpoint_url, body: query)

    data = JSON.load(resp.body.to_s)

    # Delete the old 4T if it's present
    data["data"]["routes"].delete_if { |r| r["shortName"] == "4T" }

    # Get the route with the most trips, that's likely to be the “main” route
    # for that tram line.
    data["data"]["routes"].each do |route|
      route["patterns"].sort_by! {|p| p["trips"].length }.reverse!
      route["stops"] = route["patterns"][0]["stops"]
      if route["shortName"] == "6T" || route["shortName"] == "6"
        extra_stops = route["patterns"][1]["stops"].select {|stop| stop["name"] == "Hietalahdenkatu" || stop["name"] == "Kalevankatu"}
        insert_index = route["stops"].index {|stop| stop["name"] == "Hietalahdentori"}
        if extra_stops.size == 2 && insert_index
          route["stops"].insert(insert_index, extra_stops[1])
          route["stops"].insert(insert_index, extra_stops[0])
        end
      end
      if route["shortName"] == "6T"
        extra_stops = route["patterns"][1]["stops"].select {|stop| stop["name"] == "Bunkkeri"}
        insert_index = route["stops"].index {|stop| stop["name"] == "Huutokonttori"}
        if extra_stops.size == 1 && insert_index
          route["stops"].insert(insert_index, extra_stops[0])
        end
      end
      if route["shortName"] == "5"
        extra_stops = route["patterns"][1]["stops"].select {|stop| stop["name"] == "Ylioppilastalo"}
        insert_index = route["stops"].index {|stop| stop["name"] == "Mikonkatu"}
        if extra_stops.size == 1 && insert_index
          route["stops"].insert(insert_index, extra_stops[0])
        end
      end
    end

    stops = {}

    data.dig("data", "routes").each do |route|
      route["stops"].each_with_index do |s, index|
        stops[s["name"]] ||= {
          coordinates: [],
          hsl_ids: [],
          stop_numbers: [],
          routes: [],
          stop_positions: {},
          active: true
        }
        stops[s["name"]][:coordinates] << [s["lat"], s["lon"]]
        stops[s["name"]][:hsl_ids] << Base64.decode64(s["id"])
        stops[s["name"]][:stop_numbers] << s["code"]
        stops[s["name"]][:routes] << route["shortName"]
        stops[s["name"]][:stop_positions][route["shortName"]] = index

        stops[s["name"]].each do |k, v|
          v.uniq! if v.respond_to?(:uniq!)
        end
      end
    end

    Stop.transaction do
      records = stops.map do |name, data|
        print "Creating #{name}"
 
        stop = Stop.find_or_initialize_by(name: name)
        print "."
 
        centre = Geocoder::Calculations.geographic_center data.delete(:coordinates)
        stop.latitude, stop.longitude = centre
        print "."
 
        stop.assign_attributes(data)
 
        stop.save!
        puts ". done"
 
        stop
      end
 
      print "Removing old stops"
      Stop.where.not(id: records.map(&:id)).update_all(active: false)
      puts "... done"
{'fraction_non_alphanumeric': 0.15631036703155182, 'fraction_numerical': 0.005634256278171281, 'mean_word_length': 2.0545722713864305, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5680989', 'n_tokens_mistral': 2023, 'n_tokens_neox': 1998, 'n_words': 410}