<commit_before>export interface IConfig {
  url: string;
  database: string;
  port: number;
  schedulerThreadsSource: number;
  schedulerThreadsAggregation: number;
  removeThreshold: number;
  source: ISourceConfig;
  aggregation: IAggregationConfig;
  homeDir: string;
  ssl?: {
    key: string;
    cert: string;
  };
}

export interface ISourceDriverMeta {
  latestDate: Date;
  ids?: string[];
}

export type SourceDriverCallback = (data: ISourceDriverEntry[]) => any;

export interface ISourceDriverConfig<DriverConfig = any> {
  getIds: (config: DriverConfig) => string[] | Set<string>;
  getData: (
    config: DriverConfig,
    callback: SourceDriverCallback,
    meta: ISourceDriverMeta
  ) => Promise<void>;
}

export interface ISourceConfigFile<DriverConfigType = any> {
  driver: string | ISourceDriverConfig<DriverConfigType>;
  driverConfig: DriverConfigType;
  disabled?: boolean;
  version?: number;
  removeThreshold?: number;
  copyMissingIdsCronRule?: string;
  copyCronRule?: string;
  removeCronRule?: string;
  migration?: () => Promise<void>;
}

export interface ISourceConfigElement extends ISourceConfigFile {
  collection: string;
}

export type ISourceConfig = IObject<ISourceConfigElement>;

export type IAggregationConfig = IObject<IAggregationConfigElement>;

export interface IAggregationConfigFile<
  SourceDataType = any,
  AggregationResultType = any
> {
  sources: {
    [key: string]: IAggregationCallback<SourceDataType, AggregationResultType>;
  };
  disabled?: boolean;
  chunkSize?: number;
  [key: string]: any;
}
export interface IAggregationConfigElement extends IAggregationConfigFile {
  collection: string;
}

export type IAggregationCallback<<commit_msg>disallow free keys in aggregation config object<commit_after>export interface IConfig {
  url: string;
  database: string;
  port: number;
  schedulerThreadsSource: number;
  schedulerThreadsAggregation: number;
  removeThreshold: number;
  source: ISourceConfig;
  aggregation: IAggregationConfig;
  homeDir: string;
  ssl?: {
    key: string;
    cert: string;
  };
}

export interface ISourceDriverMeta {
  latestDate: Date;
  ids?: string[];
}

export type SourceDriverCallback = (data: ISourceDriverEntry[]) => any;

export interface ISourceDriverConfig<DriverConfig = any> {
  getIds: (config: DriverConfig) => string[] | Set<string>;
  getData: (
    config: DriverConfig,
    callback: SourceDriverCallback,
    meta: ISourceDriverMeta
  ) => Promise<void>;
}

export interface ISourceConfigFile<DriverConfigType = any> {
  driver: string | ISourceDriverConfig<DriverConfigType>;
  driverConfig: DriverConfigType;
  disabled?: boolean;
  version?: number;
  removeThreshold?: number;
  copyMissingIdsCronRule?: string;
  copyCronRule?: string;
  removeCronRule?: string;
  migration?: () => Promise<void>;
}

export interface ISourceConfigElement extends ISourceConfigFile {
  collection: string;
}

export type ISourceConfig = IObject<ISourceConfigElement>;

export type IAggregationConfig = IObject<IAggregationConfigElement>;

export interface IAggregationConfigFile<
  SourceDataType = any,
  AggregationResultType = any
> {
  sources: {
    [key: string]: IAggregationCallback<SourceDataType, AggregationResultType>;
  };
  disabled?: boolean;
  chunkSize?: number;
}
export interface IAggregationConfigElement extends IAggregationConfigFile {
  collection: string;
}

export type IAggregationCallback<
{'fraction_non_alphanumeric': 0.09566998244587478, 'fraction_numerical': 0.0, 'mean_word_length': 5.600386100386101, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '51044', 'n_tokens_mistral': 985, 'n_tokens_neox': 985, 'n_words': 248}