<commit_before>// Package conf defines
package conf

// DB is our database config used by both busapi and busloader
type DB struct {

	// DBAddr is the "host:port" we use for connecting to postgres.
	// Default: "localhost:5432"
	// Environment variable: $BUS_DB_ADDR
	DBAddr string `envconfig:"db_addr" default:"localhost:5432"`

	// DBUser is the username we use for connecting to postgres.
	// Default: postgres
	// Environment variable: $BUS_DB_USER
	DBUser string `envconfig:"db_user" default:"postgres"`

	// DBName is the database name we use in postgres.
	// Default: postgres
	// Environment variable: $BUS_DB_NAME
	DBName string `envconfig:"db_name" default:"postgres"`
}

// API is our config spec used by busapi
type API struct {

	// APIAddr is the "host:port" we listen to for incoming HTTP connections.
	// The host can be blank.
	// Default: "0.0.0.0:8000"
	// Environment variable: $BUS_API_ADDR
	APIAddr string `envconfig:"api_addr" default:"0.0.0.0:8000"`

	// RedisAddr is the "host:port" we use for connecting to redis.
	// Default: "localhost:6379"
	// Environment variable: $BUS_REDIS_ADDR
	RedisAddr string `envconfig:"api_addr" default:"localhost:6379"`

	// BusAPIKey is the API key for accessing http://bustime.mta.info/
	// Default: None
	// Environment variable: $BUS_MTA_BUSTIME_API_KEY
	BustimeAPIKey string `envconfig:"mta_bustime_api_key" required:"true"`

	// DatamineAPIKey is the API key for accessing http://datamine.mta.info/
	// Default: None
	// Environment variable: $BUS_MTA_DATAMINE_API_KEY
	DatamineAPIKey string `envconfig:"mta_datamine_api_key" required:"true"`
}

// Loader is our config spec used by busloader
type Loader struct {
	// TmpDir is the root directory we use for creating temporary
	// files when loading new data.
	// Default: None (use system default)
	// Environment variable: $BUS_TMP_DIR
	TmpDir string `envconfig:"tmp_dir"`

	// RouteFilter is a list of route_ids we want
	// to specifically extract from the transit files
	// Default: None (no filter)
	// Environment variable: $BUS_ROUTE_FILTER (comma-delimited list)
	RouteFilter []string `envconfig:route_filter"`

	// GTFSURLs is a pipe-delimited list of URLs that have zipped GTFS
	// feeds. https://developers.google.com/transit/gtfs/
	// Default: None
	// Environment variable: $BUS_GTFS_URLS (comma-delimited list)
	GTFSURLs []string `envconfig:"gtfs_urls"`

	// LoadForever is a boolean that determines whether we just load
	// the GTFS URLs once and exit or whether we continually load them (every
	// 24 hours).
	// Default: true
	// Environment variable: $BUS_LOAD_FOREVER
	LoadForever bool `envconfig:"load_forever" default`
}
<commit_msg>still broken<commit_after>// Package conf defines config structs with struct tags as expected
// by https://github.com/kelseyhightower/envconfig
package conf

var (
	// DB is current database config
	DB DBSpec

	// API is the current API config
	API APISpec

	// Loader is the current loader config
	Loader LoaderSpec
)

// DBSpec is our database config used by both busapi and busloader
type DBSpec struct {

	// DBAddr is the "host:port" we use for connecting to postgres
	// Default: "localhost:5432"
	// Environment variable: $BUS_DB_ADDR
	DBAddr string `envconfig:"db_addr" default:"localhost:5432"`

	// DBUser is the username we use for connecting to postgres
	// Default: postgres
	// Environment variable: $BUS_DB_USER
	DBUser string `envconfig:"db_user" default:"postgres"`

	// DBName is the database name we use in postgres
	// Default: postgres
	// Environment variable: $BUS_DB_NAME
	DBName string `envconfig:"db_name" default:"postgres"`
}

// APISpec is our config spec used by busapi
type APISpec struct {

	// APIAddr is the "host:port" we listen to for incoming HTTP connections
	// Default: "0:8000"
	// Environment variable: $BUS_API_ADDR
	APIAddr string `envconfig:"api_addr" default:"0:8000"`

	// RedisAddr is the "host:port" we use for connecting to redis
	// Default: "localhost:6379"
	// Environment variable: $BUS_REDIS_ADDR
	RedisAddr string `envconfig:"api_addr" default:"localhost:6379"`

	// BusAPIKey is the API key for accessing http://bustime.mta.info/
	// Default: None
	// Environment variable: $BUS_MTA_BUSTIME_API_KEY
	BustimeAPIKey string `envconfig:"mta_bustime_api_key" required:"true"`

	// DatamineAPIKey is the API key for accessing http://datamine.mta.info/
	// Default: None
	// Environment variable: $BUS_MTA_DATAMINE_API_KEY
	DatamineAPIKey string `envconfig:"mta_datamine_api_key" required:"true"`
}

// LoaderSpec is our config spec used by busloader
type LoaderSpec struct {
	// TmpDir is the root directory we use for creating temporary
	// files when loading new data
	// Default: None (use system default)
	// Environment variable: $BUS_TMP_DIR
	TmpDir string `envconfig:"tmp_dir"`

	// RouteFilter is a comma-delimited list of route_ids we want to
	// specifically extract from the transit files
	// Default: None (no filter)
	// Environment variable: $BUS_ROUTE_FILTER (comma-delimited list)
	RouteFilter []string `envconfig:route_filter"`

	// GTFSURLs is a comma-delimited list of URLs that have zipped GTFS
	// feeds, see: https://developers.google.com/transit/gtfs/
	// Default: None
	// Environment variable: $BUS_GTFS_URLS (comma-delimited list)
	GTFSURLs []string `envconfig:"gtfs_urls"`

	// LoadForever is a boolean that determines whether we just load
	// the GTFS URLs once and exit or whether we continually load them (every
	// 24 hours)
	// Default: true
	// Environment variable: $BUS_LOAD_FOREVER
	LoadForever bool `envconfig:"load_forever" default:"false"`
}

{'fraction_non_alphanumeric': 0.09942877543734381, 'fraction_numerical': 0.011067475901463763, 'mean_word_length': 4.1975881261595545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3909763', 'n_tokens_mistral': 1811, 'n_tokens_neox': 1744, 'n_words': 603}