<commit_before>class GTFSGraph2
  class Error < StandardError
  end

  attr_accessor :feed, :feed_version

  CHANGE_PAYLOAD_MAX_ENTITIES = Figaro.env.feed_eater_change_payload_max_entities.try(:to_i) || 1_000
  STOP_TIMES_MAX_LOAD = Figaro.env.feed_eater_stop_times_max_load.try(:to_i) || 100_000
  ENTITY_CLASS_MAP = {
    GTFS::Stop => Stop,
    GTFS::Route => Route,
    GTFS::Trip => RouteStopPattern
  }

  def initialize(feed, feed_version)
    # GTFS Graph / TransitLand wrapper
    @feed = feed
    @feed_version = feed_version
    @gtfs = nil
    # Log
    @log = []
    # Lookup
    @entity_tl = {}
    @onestop_tl = {}
  end


  def load_graph
    @gtfs = @feed_version.open_gtfs
    @gtfs.load_graph
    @gtfs.load_shapes
    @gtfs.load_service_periods

    # gtfs_agency_id => operator
    oifs = Hash[@feed.operators_in_feed.map { |oif| [oif.gtfs_agency_id, oif.operator] }]

    # Operators
    @gtfs.agencies.each do |gtfs_agency|
      log "GTFS_AGENCY: #{gtfs_agency.agency_id}"
      tl_operator = oifs[gtfs_agency.agency_id]
      next unless tl_operator
      @entity_tl[gtfs_agency] = tl_operator
      add_eiff(tl_operator, gtfs_agency)

      # Routes
      gtfs_agency.routes.each do |gtfs_route|
        log "\tGTFS_ROUTE: #{gtfs_route.route_id}"

        # Trips: Pass 1: Create Stops
        tl_route_serves = []
        tl_trip_stop_sequence = {}
        gtfs_route.trips.each do |gtfs_trip|
          log "\t\tGTFS_TRIP: #{gtfs_trip.trip_id}"
          tl_trip_stop_sequence[gtfs_trip] = []
          # Stops
          gtfs_trip.stop_sequence.each do |gtfs_stop|
            log "\t\t\tGTFS_STOP: #{gtfs_stop.stop_id}"
            tl_stop = find_or_initialize_stop(gtfs_stop, operated_by: tl_operator)
            tl_route_serves << tl_stop
            tl_trip_stop_sequence[gtfs_trip] << tl_stop
          end
        end

        # Create Route
        tl_route = find_or_initialize_route(gtfs_route, serves: tl_route_serves.to_set, operated_by: tl_operator)

        # Trips: Pass 2: Create RSPs
        tl_route_rsps = Set.new
        gtfs_route.trips.each do |gtfs_trip|
          next unless gtfs_trip.stop_sequence.size > 1
          tl_rsp = find_or_initialize_rsp(gtfs_trip, serves: tl_trip_stop_sequence[gtfs_trip], traversed_by: tl_route)
          calculate_rsp_distance(tl_rsp) # TODO
          tl_route_rsps << tl_rsp
        end

        # Update Route geometry
        representative_rsps = Route.representative_geometry(tl_route, tl_route_rsps)
        Route.geometry_from_rsps(tl_route, representative_rsps)
      end
    end
  end

  def create_changeset
    # Convert associations
    entities = @entity_tl.values.to_set
    entities.each do |tl_entity|
      if tl_entity.instance_of?(StopPlatform)
        tl_entity.parent_stop_onestop_id = tl_entity.parent_stop.onestop_id
      elsif tl_entity.instance_of?(Route)
        tl_entity.serves = tl_entity.serves.map(&:onestop_id).uniq
        tl_entity.operated_by = tl_entity.operated_by.onestop_id
      elsif tl_entity.instance_of?(RouteStopPattern)
        tl_entity.traversed_by = tl_entity.traversed_by.onestop_id
      end
    end

    # Changeset
    changeset = Changeset.create!(
      imported_from_feed: @feed,
      imported_from_feed_version: @feed_version
    )
    changeset.create_change_payloads(entities)
    changeset
  end

  # Compatibility
  def import_log
    @log.join("\n")
  end

  def cleanup
    @feed_version.delete_schedule_stop_pairs!
  end

  def create_change_osr
    load_graph
    before_create_changeset
    changeset = create_changeset
    changeset.apply!
  end

  def ssp_schedule_async
    # pass
  end

  def ssp_perform_async(gtfs_trip_ids, agency_map, route_map, stop_map, rsp_map)
    # pass
  end

  private

  def log(msg)
    puts msg
  end

  def before_create_changeset
    ##### Backwards compat #####
    # Create FeedVersion issue and fail if no matching operators found.
    Issue.where(issue_type: 'feed_import_no_operators_found').issues_of_entity(feed_version).each(&:deprecate)
    # ... create new issue
    if @entity_tl.size == 0
      known_agency_ids = @feed.operators_in_feed.map(&:gtfs_agency_id).map{ |s| "\"#{s}\"" }.join(', ')
      feed_agency_ids = @gtfs.agencies.map(&:agency_id).map{ |s| "\"#{s}\"" }.join(', ')
      details = "No agencies found.\noperators_in_feed agency_ids: #{known_agency_ids}\nfeed agency_ids: #{feed_agency_ids}"
      issue = Issue.new(issue_type: 'feed_import_no_operators_found', details: details)
      issue.entities_with_issues.new(entity: @feed_version)
      issue.save!
      fail GTFSGraph::Error.new('No agencies found that match operators_in_feed')
    end

    # Update Feed Geometry
    @feed.set_bounding_box_from_stops(@entity_tl.values.select { |i| i.is_a?(Stop) })
    @feed.save!
  end

  def find_by_eiff(gtfs_entity)
    eiff = EntityImportedFromFeed.find_by(
      feed_version: @feed.active_feed_version,
      entity_type: ENTITY_CLASS_MAP[gtfs_entity.class],
      gtfs_id: gtfs_entity.id
    )
    return unless eiff
    return unless eiff.entity
    return eiff.entity
  end

  def add_eiff(tl_entity, gtfs_entity)
    tl_entity.add_imported_from_feeds ||= Set.new
    tl_entity.add_imported_from_feeds << {feedVersion: @feed_version.sha1, gtfsId: gtfs_entity.id}
  end

  def find_or_initialize_stop(gtfs_entity, operated_by: nil)
    # Check cache
    tl_entity = @entity_tl[gtfs_entity]
    if tl_entity
      # log "FOUND: #{tl_entity}"
      return tl_entity
    end<commit_msg>Private create_change_osr<commit_after>class GTFSGraph2
  class Error < StandardError
  end

  attr_accessor :feed, :feed_version

  CHANGE_PAYLOAD_MAX_ENTITIES = Figaro.env.feed_eater_change_payload_max_entities.try(:to_i) || 1_000
  STOP_TIMES_MAX_LOAD = Figaro.env.feed_eater_stop_times_max_load.try(:to_i) || 100_000
  ENTITY_CLASS_MAP = {
    GTFS::Stop => Stop,
    GTFS::Route => Route,
    GTFS::Trip => RouteStopPattern
  }

  def initialize(feed, feed_version)
    # GTFS Graph / TransitLand wrapper
    @feed = feed
    @feed_version = feed_version
    @gtfs = nil
    # Log
    @log = []
    # Lookup
    @entity_tl = {}
    @onestop_tl = {}
  end


  def load_graph
    @gtfs = @feed_version.open_gtfs
    @gtfs.load_graph
    @gtfs.load_shapes
    @gtfs.load_service_periods

    # gtfs_agency_id => operator
    oifs = Hash[@feed.operators_in_feed.map { |oif| [oif.gtfs_agency_id, oif.operator] }]

    # Operators
    @gtfs.agencies.each do |gtfs_agency|
      log "GTFS_AGENCY: #{gtfs_agency.agency_id}"
      tl_operator = oifs[gtfs_agency.agency_id]
      next unless tl_operator
      @entity_tl[gtfs_agency] = tl_operator
      add_eiff(tl_operator, gtfs_agency)

      # Routes
      gtfs_agency.routes.each do |gtfs_route|
        log "\tGTFS_ROUTE: #{gtfs_route.route_id}"

        # Trips: Pass 1: Create Stops
        tl_route_serves = []
        tl_trip_stop_sequence = {}
        gtfs_route.trips.each do |gtfs_trip|
          log "\t\tGTFS_TRIP: #{gtfs_trip.trip_id}"
          tl_trip_stop_sequence[gtfs_trip] = []
          gtfs_trip.stop_sequence.each do |gtfs_stop|
            log "\t\t\tGTFS_STOP: #{gtfs_stop.stop_id}"
            tl_stop = find_or_initialize_stop(gtfs_stop, operated_by: tl_operator)
            tl_route_serves << tl_stop
            tl_trip_stop_sequence[gtfs_trip] << tl_stop
          end
        end

        # Create Route
        tl_route = find_or_initialize_route(gtfs_route, serves: tl_route_serves.to_set, operated_by: tl_operator)

        # Trips: Pass 2: Create RSPs
        tl_route_rsps = Set.new
        gtfs_route.trips.each do |gtfs_trip|
          next unless gtfs_trip.stop_sequence.size > 1
          tl_rsp = find_or_initialize_rsp(gtfs_trip, serves: tl_trip_stop_sequence[gtfs_trip], traversed_by: tl_route)
          calculate_rsp_distance(tl_rsp) # TODO
          tl_route_rsps << tl_rsp
        end

        # Update Route geometry
        representative_rsps = Route.representative_geometry(tl_route, tl_route_rsps)
        Route.geometry_from_rsps(tl_route, representative_rsps)
      end
    end
  end

  def create_changeset
    # Convert associations
    entities = @entity_tl.values.to_set
    entities.each do |tl_entity|
      if tl_entity.instance_of?(StopPlatform)
        tl_entity.parent_stop_onestop_id = tl_entity.parent_stop.onestop_id
      elsif tl_entity.instance_of?(Route)
        tl_entity.serves = tl_entity.serves.map(&:onestop_id).uniq
        tl_entity.operated_by = tl_entity.operated_by.onestop_id
      elsif tl_entity.instance_of?(RouteStopPattern)
        tl_entity.traversed_by = tl_entity.traversed_by.onestop_id
      end
    end

    # Changeset
    changeset = Changeset.create!(
      imported_from_feed: @feed,
      imported_from_feed_version: @feed_version
    )
    changeset.create_change_payloads(entities)
    changeset
  end

  # Compatibility
  def import_log
    @log.join("\n")
  end

  def cleanup
    @feed_version.delete_schedule_stop_pairs!
  end

  def create_change_osr
    private_create_change_osr
  end

  def ssp_schedule_async
    # pass
  end

  def ssp_perform_async(gtfs_trip_ids, agency_map, route_map, stop_map, rsp_map)
    # pass
  end

  private

  def log(msg)
    puts msg
  end

  def private_create_change_osr
    ##### Backwards compat #####
    load_graph
    entities = @entity_tl.values.to_set

    # Create FeedVersion issue and fail if no matching operators found.
    Issue.where(issue_type: 'feed_import_no_operators_found').issues_of_entity(feed_version).each(&:deprecate)
    # ... create new issue
    if @entity_tl.size == 0
      known_agency_ids = @feed.operators_in_feed.map(&:gtfs_agency_id).map{ |s| "\"#{s}\"" }.join(', ')
      feed_agency_ids = @gtfs.agencies.map(&:agency_id).map{ |s| "\"#{s}\"" }.join(', ')
      details = "No agencies found.\noperators_in_feed agency_ids: #{known_agency_ids}\nfeed agency_ids: #{feed_agency_ids}"
      issue = Issue.new(issue_type: 'feed_import_no_operators_found', details: details)
      issue.entities_with_issues.new(entity: @feed_version)
      issue.save!
      fail GTFSGraph::Error.new('No agencies found that match operators_in_feed')
    end

    # Update Feed Geometry
    @feed.set_bounding_box_from_stops(entities.select { |i| i.is_a?(Stop) })
    @feed.save!

    # Create changeset
    changeset = create_changeset

    # Apply changeset
    changeset.apply!
  end

  def find_by_eiff(gtfs_entity)
    eiff = EntityImportedFromFeed.find_by(
      feed_version: @feed.active_feed_version,
      entity_type: ENTITY_CLASS_MAP[gtfs_entity.class],
      gtfs_id: gtfs_entity.id
    )
    return unless eiff
    return unless eiff.entity
    return eiff.entity
  end

  def add_eiff(tl_entity, gtfs_entity)
    tl_entity.add_imported_from_feeds ||= Set.new
    tl_entity.add_imported_from_feeds << {feedVersion: @feed_version.sha1, gtfsId: gtfs_entity.id}
  end

  def find_or_initialize_stop(gtfs_entity, operated_by: nil)
    # Check cache
    tl_entity = @entity_tl[gtfs_entity]
    if tl_entity
      # log "FOUND: #{tl_entity}"
      return tl_entity
    end
{'fraction_non_alphanumeric': 0.09225358463342051, 'fraction_numerical': 0.0028857426278293807, 'mean_word_length': 3.211925560197493, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '488362', 'n_tokens_mistral': 4175, 'n_tokens_neox': 4011, 'n_words': 671}