<commit_before>package org.mtransit.parser.gtfs;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang3.StringUtils;
import org.mtransit.parser.Utils;
import org.mtransit.parser.gtfs.data.GAgency;
import org.mtransit.parser.gtfs.data.GCalendar;
import org.mtransit.parser.gtfs.data.GCalendarDate;
import org.mtransit.parser.gtfs.data.GCalendarDatesExceptionType;
import org.mtransit.parser.gtfs.data.GFrequency;
import org.mtransit.parser.gtfs.data.GPickupType;
import org.mtransit.parser.gtfs.data.GRoute;
import org.mtransit.parser.gtfs.data.GSpec;
import org.mtransit.parser.gtfs.data.GStop;
import org.mtransit.parser.gtfs.data.GStopTime;
import org.mtransit.parser.gtfs.data.GTrip;

public class GReader {

	private static final String SLASH = "/";

	public static final Charset UTF8 = Charset.forName("UTF-8");

	private interface LineProcessor {
		void processLine(HashMap<String, String> line);
	}

	public static GSpec readGtfsZipFile(String gtfsFile, final GAgencyTools agencyTools, boolean calendarsOnly, boolean routeTripCalendarsOnly) {
		System.out.printf("\nReading GTFS file '%s'...", gtfsFile);
		long start = System.currentTimeMillis();
		final GSpec gSpec = new GSpec();
		ZipInputStream zip = null;
		InputStreamReader isr = null;
		BufferedReader reader = null;
		try {
			zip = new ZipInputStream(new FileInputStream(gtfsFile));
			isr = new InputStreamReader(zip, UTF8);
			reader = new BufferedReader(isr);
			for (ZipEntry entry = zip.getNextEntry(); entry != null; entry = zip.getNextEntry()) {
				if (entry.isDirectory()) {
					continue;
				}
				String filename = entry.getName();
				while (filename.contains(SLASH)) { // remove directory from file name
					filename = filename.substring(filename.indexOf(SLASH) + 1);
				}
				if (filename.equals(GAgency.FILENAME)) { // AGENCY
					if (calendarsOnly || routeTripCalendarsOnly) {
						continue;
					}
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processAgency(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equals(GCalendarDate.FILENAME)) { // CALENDAR DATES
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processCalendarDate(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equals(GCalendar.FILENAME)) { // CALENDAR
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processCalendar(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equals(GRoute.FILENAME)) { // ROUTE
					if (calendarsOnly) {
						continue;
					}
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processRoute(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equals(GStop.FILENAME)) { // STOP
					if (calendarsOnly || routeTripCalendarsOnly) {
						continue;
					}
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processStop(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equals(GTrip.FILENAME)) { // TRIP
					if (calendarsOnly) {
						continue;
					}
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processTrip(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equals(GStopTime.FILENAME)) { // STOP TIME
					if (calendarsOnly || routeTripCalendarsOnly) {
						continue;
					}
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processStopTime(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equals(GFrequency.FILENAME)) { // FREQUENCY
					if (calendarsOnly || routeTripCalendarsOnly) {
						continue;
					}
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processFrequency(agencyTools, gSpec, line);
						}
					});
				} else {
					System.out.printf("\nFile not used: %s", filename);
				}
			}
		} catch (IOException ioe) {
			System.out.printf("\nI/O Error while reading GTFS file!\n");
			ioe.printStackTrace();
			System.exit(-1);
		} finally {
			if (reader != null) {<commit_msg>Compatibility with wrong GTFS file name case.<commit_after>package org.mtransit.parser.gtfs;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang3.StringUtils;
import org.mtransit.parser.Utils;
import org.mtransit.parser.gtfs.data.GAgency;
import org.mtransit.parser.gtfs.data.GCalendar;
import org.mtransit.parser.gtfs.data.GCalendarDate;
import org.mtransit.parser.gtfs.data.GCalendarDatesExceptionType;
import org.mtransit.parser.gtfs.data.GFrequency;
import org.mtransit.parser.gtfs.data.GPickupType;
import org.mtransit.parser.gtfs.data.GRoute;
import org.mtransit.parser.gtfs.data.GSpec;
import org.mtransit.parser.gtfs.data.GStop;
import org.mtransit.parser.gtfs.data.GStopTime;
import org.mtransit.parser.gtfs.data.GTrip;

public class GReader {

	private static final String SLASH = "/";

	public static final Charset UTF8 = Charset.forName("UTF-8");

	private interface LineProcessor {
		void processLine(HashMap<String, String> line);
	}

	public static GSpec readGtfsZipFile(String gtfsFile, final GAgencyTools agencyTools, boolean calendarsOnly, boolean routeTripCalendarsOnly) {
		System.out.printf("\nReading GTFS file '%s'...", gtfsFile);
		long start = System.currentTimeMillis();
		final GSpec gSpec = new GSpec();
		ZipInputStream zip = null;
		InputStreamReader isr = null;
		BufferedReader reader = null;
		try {
			zip = new ZipInputStream(new FileInputStream(gtfsFile));
			isr = new InputStreamReader(zip, UTF8);
			reader = new BufferedReader(isr);
			for (ZipEntry entry = zip.getNextEntry(); entry != null; entry = zip.getNextEntry()) {
				if (entry.isDirectory()) {
					continue;
				}
				String filename = entry.getName();
				while (filename.contains(SLASH)) { // remove directory from file name
					filename = filename.substring(filename.indexOf(SLASH) + 1);
				}
				if (filename.equalsIgnoreCase(GAgency.FILENAME)) { // AGENCY
					if (calendarsOnly || routeTripCalendarsOnly) {
						continue;
					}
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processAgency(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equalsIgnoreCase(GCalendarDate.FILENAME)) { // CALENDAR DATES
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processCalendarDate(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equalsIgnoreCase(GCalendar.FILENAME)) { // CALENDAR
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processCalendar(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equalsIgnoreCase(GRoute.FILENAME)) { // ROUTE
					if (calendarsOnly) {
						continue;
					}
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processRoute(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equalsIgnoreCase(GStop.FILENAME)) { // STOP
					if (calendarsOnly || routeTripCalendarsOnly) {
						continue;
					}
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processStop(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equalsIgnoreCase(GTrip.FILENAME)) { // TRIP
					if (calendarsOnly) {
						continue;
					}
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processTrip(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equalsIgnoreCase(GStopTime.FILENAME)) { // STOP TIME
					if (calendarsOnly || routeTripCalendarsOnly) {
						continue;
					}
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processStopTime(agencyTools, gSpec, line);
						}
					});
				} else if (filename.equalsIgnoreCase(GFrequency.FILENAME)) { // FREQUENCY
					if (calendarsOnly || routeTripCalendarsOnly) {
						continue;
					}
					readCsv(filename, reader, null, null, new LineProcessor() {
						@Override
						public void processLine(HashMap<String, String> line) {
							processFrequency(agencyTools, gSpec, line);
						}
					});
				} else {
					System.out.printf("\nFile not used: %s", filename);
				}
			}
		} catch (IOException ioe) {
			System.out.printf("\nI/O Error while reading GTFS file!\n");
			ioe.printStackTrace();
			System.exit(-1);
		} finally {
			if (reader != null) {
{'fraction_non_alphanumeric': 0.11964038727524205, 'fraction_numerical': 0.0011855364552459987, 'mean_word_length': 3.499111111111111, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2545985', 'n_tokens_mistral': 4015, 'n_tokens_neox': 3182, 'n_words': 688}