<commit_before>import * as React from "react"
import { observer } from "mobx-react"
import {
    observable,
    computed,
    action,
    runInAction,
    reaction,
    IReactionDisposer,
} from "mobx"
import parse from "csv-parse"

import unidecode from "unidecode"
import FuzzySet from "fuzzyset"

import { AdminLayout } from "./AdminLayout"
import { SelectField, SelectGroupsField, SelectGroup } from "./Forms"
import {
    CountryNameFormat,
    CountryNameFormatDefs,
    CountryDefByKey,
} from "../adminSiteClient/CountryNameFormat"
import { uniq, toString, csvEscape, sortBy } from "../clientUtils/Util"
import { AdminAppContext, AdminAppContextType } from "./AdminAppContext"
import { faDownload } from "@fortawesome/free-solid-svg-icons/faDownload"
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"

class CSV {
    @observable filename?: string
    @observable rows: string[][]
    @observable countryEntriesMap: Map<string, CountryEntry>
    @observable mapCountriesInputToOutput: Record<string, string>
    @observable autoMatchedCount: number = 0
    @observable parseError?: string
    @observable findSimilarCountries: boolean = true

    constructor() {
        this.countryEntriesMap = new Map<string, CountryEntry>()
        this.rows = []
        this.mapCountriesInputToOutput = {}
    }

    @computed get allCountries(): string[] {
        const standardNames = Object.values(
            this.mapCountriesInputToOutput
        ).filter((value: string | undefined) => value !== undefined) as string[]
        return uniq(sortBy(standardNames)) as string[]
    }

    @computed get countryColumnIndex() {
        const { rows } = this
        if (rows.length === 0) {
            return -1
        }
        return rows[0].findIndex(
            (columnName) => columnName.toLowerCase() === "country"
        )
    }

    @computed get showDownloadOption() {
        const { rows, validationError } = this
        if (rows.length > 0 && validationError === undefined) {
            return true
        }
        return false
    }

    @computed get numCountries() {
        return this.rows.length - 1
    }

    @computed get validationError(): string | undefined {
        const { parseError } = this
        if (parseError !== undefined) {
            return `Could not parse file (error: ${parseError}). Check if it is a valid CSV file.`
        }

        const { rows, countryColumnIndex } = this
        if (rows.length === 0) return undefined

        if (countryColumnIndex < 0) {
            return "Could not find a column name with the header 'Country'"
        }

        return undefined
    }

    @action.bound onFileUpload(
        filename: string,
        rows: string[][],
        err: { message: string } | undefined,
        similarityMatch: boolean
    ) {
        this.filename = filename
        if (err) {
            this.parseError = err.message
            this.rows = []
        } else {
            this.parseError = undefined
            this.rows = rows
        }
        this.findSimilarCountries = similarityMatch
        this.parseCSV()
    }

    @action.bound onFormatChange(
        countryMap: Record<string, string>,
        findSimilarCountries: boolean
    ) {
        this.mapCountriesInputToOutput = countryMap
        this.findSimilarCountries = findSimilarCountries
        this.parseCSV()
    }

    @action.bound parseCSV() {
        const {
            rows,
            countryColumnIndex,
            mapCountriesInputToOutput,
            findSimilarCountries,
        } = this

        if (countryColumnIndex < 0) {
            this.countryEntriesMap = new Map<string, CountryEntry>()
            this.autoMatchedCount = 0
            return
        }

        const entriesByCountry = new Map<string, CountryEntry>()
        const countries = rows
            .slice(1) // remove header row
            .map((row: string[]) =>
                unidecode(row[countryColumnIndex] as string)
            )
            .filter(
                (country?: string) => country !== "" && country !== undefined
            ) // exclude empty strings

        // for fuzzy-match, use the input and output values as target to improve matching potential
        const inputCountries = Object.keys(mapCountriesInputToOutput).filter(
            (key) => mapCountriesInputToOutput[key] !== undefined
        )
        const outputCountries = inputCountries.map(
            (key) => mapCountriesInputToOutput[key]
        ) as string[]
        const fuzz = FuzzySet(inputCountries.concat(outputCountries))

        let autoMatched = 0

        countries.map((country: string) => {
            const outputCountry =
                mapCountriesInputToOutput[country.toLowerCase()]
            let approximatedMatches: string[] = []

            if (outputCountry === undefined) {
                if (findSimilarCountries) {
                    approximatedMatches = fuzz
                        .get(country)
                        .map(
                            (fuzzyMatch: [number, string]) =>
                                mapCountriesInputToOutput[fuzzyMatch[1]] ||
                                fuzzyMatch[1]
                        )
                    approximatedMatches = approximatedMatches.filter(
                        (key) => key !== undefined
                    )
                }
            } else {
                autoMatched += 1
            }

            const entry: CountryEntry = {
                originalName: country,
                standardizedName: outputCountry || undefined,
                approximatedMatches: approximatedMatches,
                selectedMatch: "",
                customName: "",
            }
            entriesByCountry.set(country, entry)
        })

        this.countryEntriesMap = entriesByCountry
        this.autoMatchedCount = autoMatched
    }
}

interface CountryEntry extends React.HTMLAttributes<HTMLTableRowElement> {
    originalName: string
    standardizedName?: string
    approximatedMatches: string[]
    selectedMatch?: string
    customName?: string<commit_msg>fix(country standardizer): handle missing fuzz matches gracefully<commit_after>import * as React from "react"
import { observer } from "mobx-react"
import {
    observable,
    computed,
    action,
    runInAction,
    reaction,
    IReactionDisposer,
} from "mobx"
import parse from "csv-parse"

import unidecode from "unidecode"
import FuzzySet from "fuzzyset"

import { AdminLayout } from "./AdminLayout"
import { SelectField, SelectGroupsField, SelectGroup } from "./Forms"
import {
    CountryNameFormat,
    CountryNameFormatDefs,
    CountryDefByKey,
} from "../adminSiteClient/CountryNameFormat"
import { uniq, toString, csvEscape, sortBy } from "../clientUtils/Util"
import { AdminAppContext, AdminAppContextType } from "./AdminAppContext"
import { faDownload } from "@fortawesome/free-solid-svg-icons/faDownload"
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"

class CSV {
    @observable filename?: string
    @observable rows: string[][]
    @observable countryEntriesMap: Map<string, CountryEntry>
    @observable mapCountriesInputToOutput: Record<string, string>
    @observable autoMatchedCount: number = 0
    @observable parseError?: string
    @observable findSimilarCountries: boolean = true

    constructor() {
        this.countryEntriesMap = new Map<string, CountryEntry>()
        this.rows = []
        this.mapCountriesInputToOutput = {}
    }

    @computed get allCountries(): string[] {
        const standardNames = Object.values(
            this.mapCountriesInputToOutput
        ).filter((value: string | undefined) => value !== undefined) as string[]
        return uniq(sortBy(standardNames)) as string[]
    }

    @computed get countryColumnIndex() {
        const { rows } = this
        if (rows.length === 0) {
            return -1
        }
        return rows[0].findIndex(
            (columnName) => columnName.toLowerCase() === "country"
        )
    }

    @computed get showDownloadOption() {
        const { rows, validationError } = this
        if (rows.length > 0 && validationError === undefined) {
            return true
        }
        return false
    }

    @computed get numCountries() {
        return this.rows.length - 1
    }

    @computed get validationError(): string | undefined {
        const { parseError } = this
        if (parseError !== undefined) {
            return `Could not parse file (error: ${parseError}). Check if it is a valid CSV file.`
        }

        const { rows, countryColumnIndex } = this
        if (rows.length === 0) return undefined

        if (countryColumnIndex < 0) {
            return "Could not find a column name with the header 'Country'"
        }

        return undefined
    }

    @action.bound onFileUpload(
        filename: string,
        rows: string[][],
        err: { message: string } | undefined,
        similarityMatch: boolean
    ) {
        this.filename = filename
        if (err) {
            this.parseError = err.message
            this.rows = []
        } else {
            this.parseError = undefined
            this.rows = rows
        }
        this.findSimilarCountries = similarityMatch
        this.parseCSV()
    }

    @action.bound onFormatChange(
        countryMap: Record<string, string>,
        findSimilarCountries: boolean
    ) {
        this.mapCountriesInputToOutput = countryMap
        this.findSimilarCountries = findSimilarCountries
        this.parseCSV()
    }

    @action.bound parseCSV() {
        const {
            rows,
            countryColumnIndex,
            mapCountriesInputToOutput,
            findSimilarCountries,
        } = this

        if (countryColumnIndex < 0) {
            this.countryEntriesMap = new Map<string, CountryEntry>()
            this.autoMatchedCount = 0
            return
        }

        const entriesByCountry = new Map<string, CountryEntry>()
        const countries = rows
            .slice(1) // remove header row
            .map((row: string[]) =>
                unidecode(row[countryColumnIndex] as string)
            )
            .filter(
                (country?: string) => country !== "" && country !== undefined
            ) // exclude empty strings

        // for fuzzy-match, use the input and output values as target to improve matching potential
        const inputCountries = Object.keys(mapCountriesInputToOutput).filter(
            (key) => mapCountriesInputToOutput[key] !== undefined
        )
        const outputCountries = inputCountries.map(
            (key) => mapCountriesInputToOutput[key]
        ) as string[]
        const fuzz = FuzzySet(inputCountries.concat(outputCountries))

        let autoMatched = 0

        countries.map((country: string) => {
            const outputCountry =
                mapCountriesInputToOutput[country.toLowerCase()]
            let approximatedMatches: string[] = []

            if (outputCountry === undefined) {
                if (findSimilarCountries) {
                    const fuzzMatches = fuzz.get(country) ?? []
                    approximatedMatches = fuzzMatches
                        .map(
                            (fuzzyMatch: [number, string]) =>
                                mapCountriesInputToOutput[fuzzyMatch[1]] ||
                                fuzzyMatch[1]
                        )
                        .filter((key) => key !== undefined)
                    approximatedMatches = uniq(approximatedMatches)
                }
            } else {
                autoMatched += 1
            }

            const entry: CountryEntry = {
                originalName: country,
                standardizedName: outputCountry || undefined,
                approximatedMatches: approximatedMatches,
                selectedMatch: "",
                customName: "",
            }
            entriesByCountry.set(country, entry)
        })

        this.countryEntriesMap = entriesByCountry
        this.autoMatchedCount = autoMatched
    }
}

interface CountryEntry extends React.HTMLAttributes<HTMLTableRowElement> {
    originalName: string
    standardizedName?: string
    approximatedMatches: string[]
    selectedMatch?: string
    customName?: string
{'fraction_non_alphanumeric': 0.09295911745619727, 'fraction_numerical': 0.0024334847501622323, 'mean_word_length': 1.9888484848484849, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1448669', 'n_tokens_mistral': 3226, 'n_tokens_neox': 3107, 'n_words': 779}