<commit_before>import * as path from 'path';
import * as chokidar from 'chokidar';
import { Observer, Observable } from 'rxjs';
import { ensureUnixPath } from '../util/path';
import * as log from '../util/log';

export type FileWatchEvent = 'change' | 'unlink' | 'add' | 'unlinkDir' | 'addDir';
export interface FileChangedEvent {
  filePath: string;
  event: FileWatchEvent;
}

export function createFileWatch(
  projectPath: string,
  ignoredPaths: (RegExp | string)[] = []
): Observable<FileChangedEvent> {
  log.debug(`Watching for changes: projectPath: ${projectPath}, ignoredPaths: ${ignoredPaths}`);

  const watch = chokidar.watch(projectPath, {
    ignoreInitial: true,
    ignored: [...ignoredPaths, /((^[\/\\])\..)|(\.js$)|(\.map$)|(\.metadata\.json)|(node_modules)/],
    persistent: true
  });

  const handleFileChange = (event: FileWatchEvent, filePath: string, observer: Observer<FileChangedEvent>) => {
    log.debug(`Watch: Path changed. Event: ${event}, Path: ${filePath}`);

    observer.next({
      filePath: path.resolve(ensureUnixPath(filePath)),
      event
    });
  };

  return Observable.create(observer => {
    watch.on('all', (event: FileWatchEvent, filePath: string) => handleFileChange(event, filePath, observer));
  });
}<commit_msg>perf: add teardown logic for watch (#980)<commit_after>import * as path from 'path';
import * as chokidar from 'chokidar';
import { Observer, Observable } from 'rxjs';
import { ensureUnixPath } from '../util/path';
import * as log from '../util/log';

export type FileWatchEvent = 'change' | 'unlink' | 'add' | 'unlinkDir' | 'addDir';
export interface FileChangedEvent {
  filePath: string;
  event: FileWatchEvent;
}

export function createFileWatch(
  projectPath: string,
  ignoredPaths: (RegExp | string)[] = []
): Observable<FileChangedEvent> {
  log.debug(`Watching for changes: projectPath: ${projectPath}, ignoredPaths: ${ignoredPaths}`);

  const watch = chokidar.watch(projectPath, {
    ignoreInitial: true,
    ignored: [...ignoredPaths, /((^[\/\\])\..)|(\.js$)|(\.map$)|(\.metadata\.json)|(node_modules)/],
    persistent: true
  });

  const handleFileChange = (event: FileWatchEvent, filePath: string, observer: Observer<FileChangedEvent>) => {
    log.debug(`Watch: Path changed. Event: ${event}, Path: ${filePath}`);

    observer.next({
      filePath: path.resolve(ensureUnixPath(filePath)),
      event
    });
  };

  return Observable.create(observer => {
    watch.on('all', (event: FileWatchEvent, filePath: string) => handleFileChange(event, filePath, observer));
    return () => watch.close();
  });
}
{'fraction_non_alphanumeric': 0.17257318952234207, 'fraction_numerical': 0.0011556240369799693, 'mean_word_length': 4.4789029535864975, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3282223', 'n_tokens_mistral': 822, 'n_tokens_neox': 810, 'n_words': 196}