<commit_before>import { DangerDSLType } from "../node_modules/danger/distribution/dsl/DangerDSL"
import { DangerResults } from "../node_modules/danger/distribution/dsl/DangerResults"

declare var danger: DangerDSLType
declare var results: DangerResults

declare function message(message: string): void
declare function warn(message: string): void
declare function fail(message: string): void
declare function markdown(message: string): void
declare function markdown(message: string): void

import * as cspell from "cspell-lib"
import mdspell from "markdown-spellcheck"
import * as minimatch from "minimatch"
import context from "./string-index-context"

const implicitSettingsFilename = "spellcheck.json"

/**
 * Optional ...options.
 *
 * Today it offers:
 *
 *  - `settings` a peril-like-GH-path to the JSON file of ignored words. e.g.
 *    "orta/words@ignore_words.json" which is the repo orta/words and the file
 *    "ignore_words.json". See the README for usage.
 *
 *  - `ignore` a list of words to ignore
 *  - `whitelistFiles` a list of files to ignore
 *  - `codeSpellCheck` a list of regexes to run cspell against
 */
export interface SpellCheckOptions {
  settings?: string
  ignore?: string[]
  whitelistFiles?: string[]
  codeSpellCheck?: string[]
}

/**
 * A de-null'd version of the spell settings
 */
export interface SpellCheckSettings {
  ignore: string[]
  "cSpell.words"?: string[]
  whitelistFiles: string[]
  hasLocalSettings?: boolean
}

/**
 * This is the _expected_ structure of the JSON file for settings.
 */
export type SpellCheckJSONSettings = Partial<SpellCheckSettings>

export interface SpellCheckWord {
  word: string
  index: number
}

interface SpellCheckContext {
  info: string
  lineNumber: number
}

const leftSquareBracket = "&#91;"

export enum SpellChecker {
  MDSpellCheck,
  CSpell,
}

export const spellCheck = async (
  file: string,
  sourceText: string,
  type: SpellChecker,
  ignoredWords: string[],
  ignoredRegexs: string[]
) => {
  const errorFunc = type === SpellChecker.MDSpellCheck ? mdSpellCheck : codeSpellCheck
  const errors = await errorFunc(sourceText, file)

  const presentableErrors = errors
    .filter(e => ignoredWords.indexOf(e.word.toLowerCase()) === -1)
    .filter(e => !ignoredRegexs.find(r => !!e.word.match(new RegExp(r.substring(1)))))

  const contextualErrors = presentableErrors.map(e =>
    context.getBlock(sourceText, e.index, e.word.length)
  ) as SpellCheckContext[]

  if (contextualErrors.length > 0) {
    markdown(`
### Typos for ${danger.github.utils.fileLinks([file])}

| Line | Typo |
| ---- | ---- |
${contextualErrors.map(contextualErrorToMarkdown).join("\n")}
      `)
  }
}

const contextualErrorToMarkdown = (error: SpellCheckContext) => {
  const sanitizedMarkdown = error.info.replace(/\[/, leftSquareBracket)
  return `${error.lineNumber} | ${sanitizedMarkdown}`
}

const getPRParams = path => ({ ...danger.github.thisPR, path, ref: danger.github.pr.head.ref })

export const mdSpellCheck = (sourceText: string): SpellCheckWord[] =>
  mdspell.spell(sourceText, { ignoreNumbers: true, ignoreAcronyms: true })

export const codeSpellCheck = (sourceText: string, path: string): Promise<SpellCheckWord[]> =>
  cspell.checkText(sourceText, { source: { name: path } }).then(info => {
    return info.items
      .filter(i => i.isError)
      .map(item => ({
        word: item.text,
        index: item.startPos,
      }))
  })

export const githubRepresentationForPath = (value: string) => {
  if (value.includes("@")) {
    return {
      path: value.split("@")[1] as string,
      owner: value.split("@")[0].split("/")[0] as string,
      repo: value.split("@")[0].split("/")[1] as string,
    }
  }
}

export const parseSettingsFromFile = async (path: string, repo: string): Promise<SpellCheckSettings> => {
  const data = await danger.github.utils.fileContents(path, repo)
  if (data) {
    const settings = JSON.parse(data) as SpellCheckJSONSettings
    return {
      ignore: (settings.ignore || settings["cSpell.words"] || []).map(w => w.toLowerCase()),
      whitelistFiles: settings.whitelistFiles || [],
    }<commit_msg>Wrap up the spell checker<commit_after>import { DangerDSLType } from "../node_modules/danger/distribution/dsl/DangerDSL"
import { DangerResults } from "../node_modules/danger/distribution/dsl/DangerResults"

declare var danger: DangerDSLType
declare var results: DangerResults

declare function message(message: string): void
declare function warn(message: string): void
declare function fail(message: string): void
declare function markdown(message: string): void
declare function markdown(message: string): void

import * as cspell from "cspell-lib"
import mdspell from "markdown-spellcheck"
import * as minimatch from "minimatch"
import { extname } from "path"
import context from "./string-index-context"

const implicitSettingsFilename = "spellcheck.json"

/**
 * Optional ...options.
 *
 * Today it offers:
 *
 *  - `settings` a peril-like-GH-path to the JSON file of ignored words. e.g.
 *    "orta/words@ignore_words.json" which is the repo orta/words and the file
 *    "ignore_words.json". See the README for usage.
 *
 *  - `ignore` a list of words to ignore
 *  - `whitelistFiles` a list of files to ignore
 *  - `codeSpellCheck` a list of regexes to run cspell against
 */
export interface SpellCheckOptions {
  settings?: string
  ignore?: string[]
  whitelistFiles?: string[]
  codeSpellCheck?: string[]
}

/**
 * A de-null'd version of the spell settings
 */
export interface SpellCheckSettings {
  ignore: string[]
  "cSpell.words"?: string[]
  whitelistFiles: string[]
  hasLocalSettings?: boolean
}

/**
 * This is the _expected_ structure of the JSON file for settings.
 */
export type SpellCheckJSONSettings = Partial<SpellCheckSettings>

export interface SpellCheckWord {
  word: string
  index: number
}

interface SpellCheckContext {
  info: string
  lineNumber: number
}

const leftSquareBracket = "&#91;"

export enum SpellChecker {
  MDSpellCheck,
  CSpell,
}

export const spellCheck = async (
  file: string,
  sourceText: string,
  type: SpellChecker,
  ignoredWords: string[],
  ignoredRegexs: string[]
) => {
  const errorFunc = type === SpellChecker.MDSpellCheck ? mdSpellCheck : codeSpellCheck
  const errors = await errorFunc(sourceText, file)

  const presentableErrors = errors
    .filter(e => ignoredWords.indexOf(e.word.toLowerCase()) === -1)
    .filter(e => !ignoredRegexs.find(r => !!e.word.match(new RegExp(r.substring(1)))))

  const contextualErrors = presentableErrors.map(e =>
    context.getBlock(sourceText, e.index, e.word.length)
  ) as SpellCheckContext[]

  if (contextualErrors.length > 0) {
    markdown(`
### Typos for ${danger.github.utils.fileLinks([file])}

| Line | Typo |
| ---- | ---- |
${contextualErrors.map(contextualErrorToMarkdown).join("\n")}
      `)
  }
}

const contextualErrorToMarkdown = (error: SpellCheckContext) => {
  const sanitizedMarkdown = error.info.replace(/\[/, leftSquareBracket)
  return `${error.lineNumber} | ${sanitizedMarkdown}`
}

const getPRParams = path => ({ ...danger.github.thisPR, path, ref: danger.github.pr.head.ref })

export const mdSpellCheck = (sourceText: string): SpellCheckWord[] =>
  mdspell.spell(sourceText, { ignoreNumbers: true, ignoreAcronyms: true })

export const codeSpellCheck = (sourceText: string, path: string): Promise<SpellCheckWord[]> => {
  const ext = extname(path)
  const languageIds = cspell.getLanguagesForExt(ext)
  const mergedSettings = cspell.mergeSettings(cspell.getDefaultSettings(), { source: { name: path, filename: path } })
  const config = cspell.constructSettingsForText(mergedSettings, sourceText, languageIds)

  return cspell.checkText(sourceText, config).then(info => {
    return info.items
      .filter(i => i.isError)
      .map(item => ({
        word: item.text,
        index: item.startPos,
      }))
  })
}

export const githubRepresentationForPath = (value: string) => {
  if (value.includes("@")) {
    return {
      path: value.split("@")[1] as string,
      owner: value.split("@")[0].split("/")[0] as string,
      repo: value.split("@")[0].split("/")[1] as string,
    }
  }
}

export const parseSettingsFromFile = async (path: string, repo: string): Promise<SpellCheckSettings> => {
  const data = await danger.github.utils.fileContents(path, repo)
  if (data) {
    const settings = JSON.parse(data) as SpellCheckJSONSettings
    return {
      ignore: (settings.ignore || settings["cSpell.words"] || []).map(w => w.toLowerCase()),
      whitelistFiles: settings.whitelistFiles || [],
    }
{'fraction_non_alphanumeric': 0.13459969700501107, 'fraction_numerical': 0.0023307306840694556, 'mean_word_length': 4.182367149758454, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3893394', 'n_tokens_mistral': 2703, 'n_tokens_neox': 2696, 'n_words': 693}