<commit_before>import * as glob from "glob-promise"
import * as isGlob from "is-glob"

import * as configuration from "../configuration/index"
import { UserError } from "../errors/user-error"
import * as files from "./"

/**
 * Returns the FullPaths of all files/directories relative to the given sourceDir
 * that match the given glob
 */
export async function getFileNames(config: configuration.Data): Promise<files.FullFile[]> {
  let filenames = await getFiles(config)
  filenames = removeExcludedFiles(filenames, config.exclude)
  filenames = removeExcludedFiles(filenames, "node_modules")
  if (!config.emptyWorkspace) {
    const relativeWorkspace = config.sourceDir.relativeStr(config.workspace.platformified())
    if (relativeWorkspace !== "") {
      filenames = removeExcludedFiles(filenames, relativeWorkspace)
    }
  }
  return filenames
}

/**
 * Returns files described by the given configuration.
 * Filenames are relative to config.sourceDir.
 */
async function getFiles(config: configuration.Data): Promise<files.FullFile[]> {
  const fullGlob = config.sourceDir.joinStr(config.files)
  if (config.files === "") {
    return markdownFilesInDir("", config.sourceDir)
  } else if (await files.hasDirectory(fullGlob)) {
    return markdownFilesInDir(fullGlob, config.sourceDir)
  } else if (await files.isMarkdownFile(fullGlob)) {
    return [new files.FullFile(config.files)]
  } else if (isGlob(config.files)) {
    return config.sourceDir.fullFilesMatchingGlob(fullGlob)
  } else {
    throw new UserError(`file or directory does not exist: ${config.files}`)
  }
}

/**
 * returns all the markdown files in this directory and its children,<commit_msg>Better configuration check<commit_after>import * as glob from "glob-promise"
import * as isGlob from "is-glob"

import * as configuration from "../configuration/index"
import { UserError } from "../errors/user-error"
import * as files from "./"

/**
 * Returns the FullPaths of all files/directories relative to the given sourceDir
 * that match the given glob
 */
export async function getFileNames(config: configuration.Data): Promise<files.FullFile[]> {
  let filenames = await getFiles(config)
  filenames = removeExcludedFiles(filenames, config.exclude)
  filenames = removeExcludedFiles(filenames, "node_modules")
  if (config.emptyWorkspace === false) {
    const relativeWorkspace = config.sourceDir.relativeStr(config.workspace.platformified())
    if (relativeWorkspace !== "") {
      filenames = removeExcludedFiles(filenames, relativeWorkspace)
    }
  }
  return filenames
}

/**
 * Returns files described by the given configuration.
 * Filenames are relative to config.sourceDir.
 */
async function getFiles(config: configuration.Data): Promise<files.FullFile[]> {
  const fullGlob = config.sourceDir.joinStr(config.files)
  if (config.files === "") {
    return markdownFilesInDir("", config.sourceDir)
  } else if (await files.hasDirectory(fullGlob)) {
    return markdownFilesInDir(fullGlob, config.sourceDir)
  } else if (await files.isMarkdownFile(fullGlob)) {
    return [new files.FullFile(config.files)]
  } else if (isGlob(config.files)) {
    return config.sourceDir.fullFilesMatchingGlob(fullGlob)
  } else {
    throw new UserError(`file or directory does not exist: ${config.files}`)
  }
}

/**
 * returns all the markdown files in this directory and its children,
{'fraction_non_alphanumeric': 0.11296076099881094, 'fraction_numerical': 0.0, 'mean_word_length': 4.617696160267112, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6980109', 'n_tokens_mistral': 982, 'n_tokens_neox': 990, 'n_words': 266}