<commit_before>import * as React from 'react'
import {observer} from 'mobx-react'
import {observable, computed, action, runInAction} from 'mobx'
import * as parse from 'csv-parse'

const fuzzysort = require("fuzzysort")
const unidecode = require("unidecode")

import Admin from './Admin'
import AdminLayout from './AdminLayout'
import { SelectField } from './Forms'
import CountryNameFormat, { CountryNameFormatDefs, CountryDefByKey } from '../standardizer/CountryNameFormat'

class CSV {
    @observable filename: string|undefined
    @observable rows: string[][]
    @observable countryEntriesMap: Map<string, CountryEntry>
    @observable mapCountriesInputToOutput: { [key: string]: string }
    @observable autoMatchedCount: number = 0
    @observable parseError: string|undefined
    @observable findSimilarCountries: boolean = true

    constructor() {
        this.countryEntriesMap = new Map<string, CountryEntry>()
        this.rows = []
        this.mapCountriesInputToOutput = {}
    }

    @computed get countryColumnIndex() {
        const { rows } = this
        if (rows.length === 0) {
            return -1
        }
        return rows[0].findIndex((columnName: string) => columnName.toLowerCase() === "country")
    }

    @computed get showDownloadOption() {
        const { rows, validationError } = this
        if (rows.length > 0 && validationError === null) {
            return true
        }
        return false
    }

    @computed get numCountries() {
        return this.rows.length - 1
    }

    @computed get validationError() {
        const { parseError } = this
        if (parseError !== undefined) {
            return "Could not parse file (error: " + parseError + "). Check if it is a valid CSV file."
        }

        const { rows, countryColumnIndex } = this
        if (rows.length === 0) return null

        if (countryColumnIndex < 0) {
            return "Could not find a column name with the header 'Country'"
        }

        return null
    }

    @action.bound onFileUpload(filename: string, rows: string[][], err: any, similarityMatch: boolean) {
        this.filename = filename
        if (err !== null) {
            this.parseError = err.message
            this.rows = []
        } else {
            this.parseError = undefined
            this.rows = rows
        }
        this.findSimilarCountries = similarityMatch
        this.parseCSV()
    }

    @action.bound onFormatChange(countryMap: any, findSimilarCountries: boolean) {
        this.mapCountriesInputToOutput = countryMap
        this.findSimilarCountries = findSimilarCountries
        this.parseCSV()
    }

    parseCSV() {
        console.log("parsing CSV")

        const { rows, countryColumnIndex, mapCountriesInputToOutput, findSimilarCountries } = this

        if (countryColumnIndex < 0) {
            this.countryEntriesMap = new Map<string, CountryEntry>()
            this.autoMatchedCount = 0
            return
        }

        const entriesByCountry = new Map<string, CountryEntry>()
        const countries = rows.slice(1).map((row: string[]) => unidecode(row[countryColumnIndex] as string))

        // for fuzzy-sort
        let targetValues = Object.keys(mapCountriesInputToOutput).filter(key => typeof(mapCountriesInputToOutput[key]) === 'string')
        targetValues = targetValues.map(key => mapCountriesInputToOutput[key])
        targetValues = targetValues.map(val => fuzzysort.prepare(val))

        let autoMatched = 0

        countries.map((country: string) => {
            const outputCountry = mapCountriesInputToOutput[country.toLowerCase()]
            let approximatedMatches: FuzzyMatch[] = []

            if (outputCountry === undefined || outputCountry === null) {
                if (findSimilarCountries) {
                    approximatedMatches = fuzzysort.go(country, targetValues)
                }
            } else {
                autoMatched += 1
            }

            const entry: CountryEntry = {
                originalName: country,
                standardizedName: outputCountry || undefined,
                approximatedMatches: approximatedMatches,
                selectedMatch: "",
                customName: ""<commit_msg>fix duplicate names appearing in the fuzzy matches<commit_after>import * as React from 'react'
import {observer} from 'mobx-react'
import {observable, computed, action, runInAction} from 'mobx'
import * as parse from 'csv-parse'

const fuzzysort = require("fuzzysort")
const unidecode = require("unidecode")

import Admin from './Admin'
import AdminLayout from './AdminLayout'
import { SelectField } from './Forms'
import CountryNameFormat, { CountryNameFormatDefs, CountryDefByKey } from '../standardizer/CountryNameFormat'
import { uniq } from '../charts/Util'

class CSV {
    @observable filename: string|undefined
    @observable rows: string[][]
    @observable countryEntriesMap: Map<string, CountryEntry>
    @observable mapCountriesInputToOutput: { [key: string]: string }
    @observable autoMatchedCount: number = 0
    @observable parseError: string|undefined
    @observable findSimilarCountries: boolean = true

    constructor() {
        this.countryEntriesMap = new Map<string, CountryEntry>()
        this.rows = []
        this.mapCountriesInputToOutput = {}
    }

    @computed get countryColumnIndex() {
        const { rows } = this
        if (rows.length === 0) {
            return -1
        }
        return rows[0].findIndex((columnName: string) => columnName.toLowerCase() === "country")
    }

    @computed get showDownloadOption() {
        const { rows, validationError } = this
        if (rows.length > 0 && validationError === null) {
            return true
        }
        return false
    }

    @computed get numCountries() {
        return this.rows.length - 1
    }

    @computed get validationError() {
        const { parseError } = this
        if (parseError !== undefined) {
            return "Could not parse file (error: " + parseError + "). Check if it is a valid CSV file."
        }

        const { rows, countryColumnIndex } = this
        if (rows.length === 0) return null

        if (countryColumnIndex < 0) {
            return "Could not find a column name with the header 'Country'"
        }

        return null
    }

    @action.bound onFileUpload(filename: string, rows: string[][], err: any, similarityMatch: boolean) {
        this.filename = filename
        if (err !== null) {
            this.parseError = err.message
            this.rows = []
        } else {
            this.parseError = undefined
            this.rows = rows
        }
        this.findSimilarCountries = similarityMatch
        this.parseCSV()
    }

    @action.bound onFormatChange(countryMap: any, findSimilarCountries: boolean) {
        this.mapCountriesInputToOutput = countryMap
        this.findSimilarCountries = findSimilarCountries
        this.parseCSV()
    }

    parseCSV() {
        console.log("parsing CSV")

        const { rows, countryColumnIndex, mapCountriesInputToOutput, findSimilarCountries } = this

        if (countryColumnIndex < 0) {
            this.countryEntriesMap = new Map<string, CountryEntry>()
            this.autoMatchedCount = 0
            return
        }

        const entriesByCountry = new Map<string, CountryEntry>()
        const countries = rows.slice(1).map((row: string[]) => unidecode(row[countryColumnIndex] as string))

        // for fuzzy-sort
        let targetValues = Object.keys(mapCountriesInputToOutput).filter(key => typeof(mapCountriesInputToOutput[key]) === 'string')
        targetValues = uniq(targetValues.map(key => mapCountriesInputToOutput[key]))
        targetValues = targetValues.map(val => fuzzysort.prepare(val))

        let autoMatched = 0

        countries.map((country: string) => {
            const outputCountry = mapCountriesInputToOutput[country.toLowerCase()]
            let approximatedMatches: FuzzyMatch[] = []

            if (outputCountry === undefined || outputCountry === null) {
                if (findSimilarCountries) {
                    approximatedMatches = fuzzysort.go(country, targetValues)
                }
            } else {
                autoMatched += 1
            }

            const entry: CountryEntry = {
                originalName: country,
                standardizedName: outputCountry || undefined,
                approximatedMatches: approximatedMatches,
                selectedMatch: "",
                customName: ""
{'fraction_non_alphanumeric': 0.1017943004573707, 'fraction_numerical': 0.003049138032133224, 'mean_word_length': 2.392203659506762, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5433008', 'n_tokens_mistral': 2310, 'n_tokens_neox': 2212, 'n_words': 575}