<commit_before>package org.mtransit.parser.mt;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;

import org.apache.commons.lang3.StringUtils;
import org.mtransit.parser.Constants;
import org.mtransit.parser.Pair;
import org.mtransit.parser.gtfs.GAgencyTools;
import org.mtransit.parser.gtfs.data.GAgency;
import org.mtransit.parser.gtfs.data.GCalendar;
import org.mtransit.parser.gtfs.data.GCalendarDate;
import org.mtransit.parser.gtfs.data.GFrequency;
import org.mtransit.parser.gtfs.data.GRoute;
import org.mtransit.parser.gtfs.data.GSpec;
import org.mtransit.parser.gtfs.data.GStop;
import org.mtransit.parser.gtfs.data.GStopTime;
import org.mtransit.parser.gtfs.data.GTrip;
import org.mtransit.parser.gtfs.data.GTripStop;
import org.mtransit.parser.mt.data.MAgency;
import org.mtransit.parser.mt.data.MFrequency;
import org.mtransit.parser.mt.data.MRoute;
import org.mtransit.parser.mt.data.MSchedule;
import org.mtransit.parser.mt.data.MServiceDate;
import org.mtransit.parser.mt.data.MSpec;
import org.mtransit.parser.mt.data.MStop;
import org.mtransit.parser.mt.data.MTrip;
import org.mtransit.parser.mt.data.MTripStop;

public class GenerateMObjectsTask implements Callable<MSpec> {

	private GAgencyTools agencyTools;
	private long routeId;
	private GSpec gtfs;
	private Map<String, GStop> gstops;

	public GenerateMObjectsTask(GAgencyTools agencyTools, long routeId, GSpec gtfs, Map<String, GStop> gstops) {
		this.agencyTools = agencyTools;
		this.routeId = routeId;
		this.gtfs = gtfs;
		this.gstops = gstops;
	}

	@Override
	public MSpec call() {
		try {
			return calll();
		} catch (Exception e) {
			System.out.printf("\n%s: FATAL ERROR!\n", this.routeId);
			e.printStackTrace();
			System.exit(-1);
			return null;
		}
	}

	public MSpec calll() {
		final long startAt = System.currentTimeMillis();
		System.out.printf("\n%s: processing... ", this.routeId);
		HashMap<String, MAgency> mAgencies = new HashMap<String, MAgency>();
		Set<MServiceDate> mServiceDates = new HashSet<MServiceDate>();
		HashMap<String, MSchedule> mSchedules = new HashMap<String, MSchedule>();
		HashMap<String, MFrequency> mFrequencies = new HashMap<String, MFrequency>();
		Map<Long, MRoute> mRoutes = new HashMap<Long, MRoute>();
		Map<Long, MTrip> mTrips = new HashMap<Long, MTrip>();
		Map<String, MTripStop> allMTripStops = new HashMap<String, MTripStop>();
		Map<Integer, MStop> mStops = new HashMap<Integer, MStop>();
		Set<Integer> tripStopIds = new HashSet<Integer>(); // the list of stop IDs used by trips
		Set<String> serviceIds = new HashSet<String>();
		MAgency mAgency;
		for (GAgency gAgency : this.gtfs.agencies) {
			mAgency = new MAgency(gAgency.agency_id, gAgency.agency_timezone, this.agencyTools.getAgencyColor(), this.agencyTools.getAgencyRouteType());
			if (mAgencies.containsKey(mAgency.getId()) && !mAgencies.get(mAgency.getId()).equals(mAgency)) {
				System.out.printf("\n%s: Agency %s already in list!", this.routeId, mAgency.getId());
				System.out.printf("\n%s: %s", this.routeId, mAgency.toString());
				System.out.printf("\n%s: %s", this.routeId, mAgencies.get(mAgency.getId()).toString());
				System.exit(-1);
			}
			mAgencies.put(mAgency.getId(), mAgency);
		}
		parseRTS(mSchedules, mFrequencies, mRoutes, mTrips, mStops, allMTripStops, tripStopIds, serviceIds);
		Set<String> gCalendarDateServiceRemoved = new HashSet<String>();
		for (GCalendarDate gCalendarDate : this.gtfs.calendarDates) {
			switch (gCalendarDate.exception_type) {
			case SERVICE_REMOVED: // keep list of removed service for calendars processing
				gCalendarDateServiceRemoved.add(gCalendarDate.getUID());
				break;
			case SERVICE_ADDED:
				mServiceDates.add(new MServiceDate(this.agencyTools.cleanServiceId(gCalendarDate.service_id), gCalendarDate.date));
				break;
			default:
				System.out.printf("\n%s: Unexpected calendar date exeception type '%s'!", this.routeId, gCalendarDate.exception_type);
			}
		}
		for (GCalendar gCalendar : this.gtfs.calendars) {
			for (GCalendarDate gCalendarDate : gCalendar.getDates()) {
				if (gCalendarDateServiceRemoved.contains(gCalendarDate.getUID())) {
					continue; // service REMOVED at this date
				}
				mServiceDates.add(new MServiceDate(this.agencyTools.cleanServiceId(gCalendarDate.service_id), gCalendarDate.date));
			}
		}
		MTrip mTrip;
		for (MSchedule mSchedule : mSchedules.values()) {
			mTrip = mTrips.get(mSchedule.getTripId());
			if (mTrip.getHeadsignType() == mSchedule.getHeadsignType() && StringUtils.equals(mTrip.getHeadsignValue(), mSchedule.getHeadsignValue())) {
				mSchedule.clearHeadsign();
			}
		}
		List<MAgency> mAgenciesList = new ArrayList<MAgency>(mAgencies.values());
		Collections.sort(mAgenciesList);
		List<MStop> mStopsList = new ArrayList<MStop>(mStops.values());
		Collections.sort(mStopsList);
		List<MRoute> mRoutesList = new ArrayList<MRoute>(mRoutes.values());
		Collections.sort(mRoutesList);
		List<MTrip> mTripsList = new ArrayList<MTrip>(mTrips.values());
		Collections.sort(mTripsList);
		List<MTripStop> mTripStopsList = new ArrayList<MTripStop>(allMTripStops.values());
		Collections.sort(mTripStopsList);
		setTripStopDecentOnly(mTripStopsList);
		List<MServiceDate> mServiceDatesList = new ArrayList<MServiceDate>(mServiceDates);
		Collections.sort(mServiceDatesList);
		List<MSchedule> mSchedulesList = new ArrayList<MSchedule>(mSchedules.values());
		Collections.sort(mSchedulesList);
		List<MFrequency> mFrequenciesList = new ArrayList<MFrequency>(mFrequencies.values());
		Collections.sort(mFrequenciesList);
		Map<Integer, List<MSchedule>> mStopScheduleMap = new HashMap<Integer, List<MSchedule>>();
		for (MSchedule schedule : mSchedulesList) {
			if (!mStopScheduleMap.containsKey(schedule.getStopId())) {
				mStopScheduleMap.put(schedule.getStopId(), new ArrayList<MSchedule>());
			}
			mStopScheduleMap.get(schedule.getStopId()).add(schedule);
		}
		Map<Long, List<MFrequency>> mRouteFrequencies = new HashMap<Long, List<MFrequency>>();
		if (mFrequenciesList != null && mFrequenciesList.size() > 0) {
			mRouteFrequencies.put(this.routeId, mFrequenciesList);
		}
		MSpec myrouteSpec = new MSpec(mAgenciesList, mStopsList, mRoutesList, mTripsList, mTripStopsList, mServiceDatesList, null, mStopScheduleMap,
				mRouteFrequencies);
		System.out.printf("\n%s: processing... DONE in %s." + this.routeId, org.mtransit.parser.Utils.getPrettyDuration(System.currentTimeMillis() - startAt));
		return myrouteSpec;
	}

	private void parseRTS(HashMap<String, MSchedule> mSchedules, HashMap<String, MFrequency> mFrequencies, Map<Long, MRoute> mRoutes, Map<Long, MTrip> mTrips,
			Map<Integer, MStop> mStops, Map<String, MTripStop> allMTripStops, Set<Integer> tripStopIds, Set<String> serviceIds) {
		MRoute mRoute;
		boolean mergeSuccessful;
		HashMap<Long, String> mTripStopTimesHeadsign;
		HashMap<Long, List<MTripStop>> tripIdToMTripStops;
		Set<String> mTripHeasignStrings;
		boolean headsignTypeString;
		boolean tripKeptNonDescriptiveHeadsign;
		for (GRoute gRoute : this.gtfs.routes.values()) {
			mRoute = new MRoute(this.agencyTools.getRouteId(gRoute), this.agencyTools.getRouteShortName(gRoute), this.agencyTools.getRouteLongName(gRoute),
					this.agencyTools.getRouteColor(gRoute));
			if (mRoutes.containsKey(mRoute.id) && !mRoute.equals(mRoutes.get(mRoute.id))) {
				mergeSuccessful = false;
				if (mRoute.equalsExceptLongName(mRoutes.get(mRoute.id))) {
					mergeSuccessful = this.agencyTools.mergeRouteLongName(mRoute, mRoutes.get(mRoute.id));
				}
				if (!mergeSuccessful) {
					System.out.println(this.routeId + ": Route " + mRoute.id + " already in list!");
					System.out.println(this.routeId + ": " + mRoute.toString());
					System.out.println(this.routeId + ": " + mRoutes.get(mRoute.id).toString());
					System.exit(-1);
				}
			}
			mTripStopTimesHeadsign = new HashMap<Long, String>();
			// find route trips
			tripIdToMTripStops = new HashMap<Long, List<MTripStop>>();
			parseTrips(mSchedules, mFrequencies, mTrips, mStops, serviceIds, mRoute, mTripStopTimesHeadsign, tripIdToMTripStops, gRoute);
			mTripHeasignStrings = new HashSet<String>();<commit_msg>Minor.<commit_after>package org.mtransit.parser.mt;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;

import org.apache.commons.lang3.StringUtils;
import org.mtransit.parser.Constants;
import org.mtransit.parser.Pair;
import org.mtransit.parser.gtfs.GAgencyTools;
import org.mtransit.parser.gtfs.data.GAgency;
import org.mtransit.parser.gtfs.data.GCalendar;
import org.mtransit.parser.gtfs.data.GCalendarDate;
import org.mtransit.parser.gtfs.data.GFrequency;
import org.mtransit.parser.gtfs.data.GRoute;
import org.mtransit.parser.gtfs.data.GSpec;
import org.mtransit.parser.gtfs.data.GStop;
import org.mtransit.parser.gtfs.data.GStopTime;
import org.mtransit.parser.gtfs.data.GTrip;
import org.mtransit.parser.gtfs.data.GTripStop;
import org.mtransit.parser.mt.data.MAgency;
import org.mtransit.parser.mt.data.MFrequency;
import org.mtransit.parser.mt.data.MRoute;
import org.mtransit.parser.mt.data.MSchedule;
import org.mtransit.parser.mt.data.MServiceDate;
import org.mtransit.parser.mt.data.MSpec;
import org.mtransit.parser.mt.data.MStop;
import org.mtransit.parser.mt.data.MTrip;
import org.mtransit.parser.mt.data.MTripStop;

public class GenerateMObjectsTask implements Callable<MSpec> {

	private GAgencyTools agencyTools;
	private long routeId;
	private GSpec gtfs;
	private Map<String, GStop> gstops;

	public GenerateMObjectsTask(GAgencyTools agencyTools, long routeId, GSpec gtfs, Map<String, GStop> gstops) {
		this.agencyTools = agencyTools;
		this.routeId = routeId;
		this.gtfs = gtfs;
		this.gstops = gstops;
	}

	@Override
	public MSpec call() {
		try {
			return calll();
		} catch (Exception e) {
			System.out.printf("\n%s: FATAL ERROR!\n", this.routeId);
			e.printStackTrace();
			System.exit(-1);
			return null;
		}
	}

	public MSpec calll() {
		final long startAt = System.currentTimeMillis();
		System.out.printf("\n%s: processing... ", this.routeId);
		HashMap<String, MAgency> mAgencies = new HashMap<String, MAgency>();
		Set<MServiceDate> mServiceDates = new HashSet<MServiceDate>();
		HashMap<String, MSchedule> mSchedules = new HashMap<String, MSchedule>();
		HashMap<String, MFrequency> mFrequencies = new HashMap<String, MFrequency>();
		Map<Long, MRoute> mRoutes = new HashMap<Long, MRoute>();
		Map<Long, MTrip> mTrips = new HashMap<Long, MTrip>();
		Map<String, MTripStop> allMTripStops = new HashMap<String, MTripStop>();
		Map<Integer, MStop> mStops = new HashMap<Integer, MStop>();
		Set<Integer> tripStopIds = new HashSet<Integer>(); // the list of stop IDs used by trips
		Set<String> serviceIds = new HashSet<String>();
		MAgency mAgency;
		for (GAgency gAgency : this.gtfs.agencies) {
			mAgency = new MAgency(gAgency.agency_id, gAgency.agency_timezone, this.agencyTools.getAgencyColor(), this.agencyTools.getAgencyRouteType());
			if (mAgencies.containsKey(mAgency.getId()) && !mAgencies.get(mAgency.getId()).equals(mAgency)) {
				System.out.printf("\n%s: Agency %s already in list!", this.routeId, mAgency.getId());
				System.out.printf("\n%s: %s", this.routeId, mAgency.toString());
				System.out.printf("\n%s: %s", this.routeId, mAgencies.get(mAgency.getId()).toString());
				System.exit(-1);
			}
			mAgencies.put(mAgency.getId(), mAgency);
		}
		parseRTS(mSchedules, mFrequencies, mRoutes, mTrips, mStops, allMTripStops, tripStopIds, serviceIds);
		Set<String> gCalendarDateServiceRemoved = new HashSet<String>();
		for (GCalendarDate gCalendarDate : this.gtfs.calendarDates) {
			switch (gCalendarDate.exception_type) {
			case SERVICE_REMOVED: // keep list of removed service for calendars processing
				gCalendarDateServiceRemoved.add(gCalendarDate.getUID());
				break;
			case SERVICE_ADDED:
				mServiceDates.add(new MServiceDate(this.agencyTools.cleanServiceId(gCalendarDate.service_id), gCalendarDate.date));
				break;
			default:
				System.out.printf("\n%s: Unexpected calendar date exeception type '%s'!", this.routeId, gCalendarDate.exception_type);
			}
		}
		for (GCalendar gCalendar : this.gtfs.calendars) {
			for (GCalendarDate gCalendarDate : gCalendar.getDates()) {
				if (gCalendarDateServiceRemoved.contains(gCalendarDate.getUID())) {
					continue; // service REMOVED at this date
				}
				mServiceDates.add(new MServiceDate(this.agencyTools.cleanServiceId(gCalendarDate.service_id), gCalendarDate.date));
			}
		}
		MTrip mTrip;
		for (MSchedule mSchedule : mSchedules.values()) {
			mTrip = mTrips.get(mSchedule.getTripId());
			if (mTrip.getHeadsignType() == mSchedule.getHeadsignType() && StringUtils.equals(mTrip.getHeadsignValue(), mSchedule.getHeadsignValue())) {
				mSchedule.clearHeadsign();
			}
		}
		List<MAgency> mAgenciesList = new ArrayList<MAgency>(mAgencies.values());
		Collections.sort(mAgenciesList);
		List<MStop> mStopsList = new ArrayList<MStop>(mStops.values());
		Collections.sort(mStopsList);
		List<MRoute> mRoutesList = new ArrayList<MRoute>(mRoutes.values());
		Collections.sort(mRoutesList);
		List<MTrip> mTripsList = new ArrayList<MTrip>(mTrips.values());
		Collections.sort(mTripsList);
		List<MTripStop> mTripStopsList = new ArrayList<MTripStop>(allMTripStops.values());
		Collections.sort(mTripStopsList);
		setTripStopDecentOnly(mTripStopsList);
		List<MServiceDate> mServiceDatesList = new ArrayList<MServiceDate>(mServiceDates);
		Collections.sort(mServiceDatesList);
		List<MSchedule> mSchedulesList = new ArrayList<MSchedule>(mSchedules.values());
		Collections.sort(mSchedulesList);
		List<MFrequency> mFrequenciesList = new ArrayList<MFrequency>(mFrequencies.values());
		Collections.sort(mFrequenciesList);
		Map<Integer, List<MSchedule>> mStopScheduleMap = new HashMap<Integer, List<MSchedule>>();
		for (MSchedule schedule : mSchedulesList) {
			if (!mStopScheduleMap.containsKey(schedule.getStopId())) {
				mStopScheduleMap.put(schedule.getStopId(), new ArrayList<MSchedule>());
			}
			mStopScheduleMap.get(schedule.getStopId()).add(schedule);
		}
		Map<Long, List<MFrequency>> mRouteFrequencies = new HashMap<Long, List<MFrequency>>();
		if (mFrequenciesList != null && mFrequenciesList.size() > 0) {
			mRouteFrequencies.put(this.routeId, mFrequenciesList);
		}
		MSpec myrouteSpec = new MSpec(mAgenciesList, mStopsList, mRoutesList, mTripsList, mTripStopsList, mServiceDatesList, null, mStopScheduleMap,
				mRouteFrequencies);
		System.out.printf("\n%s: processing... DONE in %s.", this.routeId, org.mtransit.parser.Utils.getPrettyDuration(System.currentTimeMillis() - startAt));
		return myrouteSpec;
	}

	private void parseRTS(HashMap<String, MSchedule> mSchedules, HashMap<String, MFrequency> mFrequencies, Map<Long, MRoute> mRoutes, Map<Long, MTrip> mTrips,
			Map<Integer, MStop> mStops, Map<String, MTripStop> allMTripStops, Set<Integer> tripStopIds, Set<String> serviceIds) {
		MRoute mRoute;
		boolean mergeSuccessful;
		HashMap<Long, String> mTripStopTimesHeadsign;
		HashMap<Long, List<MTripStop>> tripIdToMTripStops;
		Set<String> mTripHeasignStrings;
		boolean headsignTypeString;
		boolean tripKeptNonDescriptiveHeadsign;
		for (GRoute gRoute : this.gtfs.routes.values()) {
			mRoute = new MRoute(this.agencyTools.getRouteId(gRoute), this.agencyTools.getRouteShortName(gRoute), this.agencyTools.getRouteLongName(gRoute),
					this.agencyTools.getRouteColor(gRoute));
			if (mRoutes.containsKey(mRoute.id) && !mRoute.equals(mRoutes.get(mRoute.id))) {
				mergeSuccessful = false;
				if (mRoute.equalsExceptLongName(mRoutes.get(mRoute.id))) {
					mergeSuccessful = this.agencyTools.mergeRouteLongName(mRoute, mRoutes.get(mRoute.id));
				}
				if (!mergeSuccessful) {
					System.out.printf("\n%s: Route %s already in list!", this.routeId, mRoute.id);
					System.out.printf("\n%s: %s", this.routeId, mRoute.toString());
					System.out.printf("\n%s: %s.\n", this.routeId, mRoutes.get(mRoute.id).toString());
					System.exit(-1);
				}
			}
			mTripStopTimesHeadsign = new HashMap<Long, String>();
			// find route trips
			tripIdToMTripStops = new HashMap<Long, List<MTripStop>>();
			parseTrips(mSchedules, mFrequencies, mTrips, mStops, serviceIds, mRoute, mTripStopTimesHeadsign, tripIdToMTripStops, gRoute);
			mTripHeasignStrings = new HashSet<String>();
{'fraction_non_alphanumeric': 0.13272518576693046, 'fraction_numerical': 0.0006041200990756962, 'mean_word_length': 5.537914691943128, 'pattern_counts': {'":': 3, '<': 135, '<?xml version=': 0, '>': 137, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3627351', 'n_tokens_mistral': 6198, 'n_tokens_neox': 5759, 'n_words': 987}