<commit_before>import { VariableDisplaySettings } from "./Variable"
import { uniq } from "./Util"

interface OwidRow {
    entity: string
    value: number
    year: number
}

export interface TabularDataset {
    rows: OwidRow[]
}

export function tabularDataToOwidDataset(dataset: TabularDataset): OwidDataset {
    const { rows } = dataset
    const entityKey: { [id: string]: EntityMeta } = {}
    const entities = lodash.uniq(rows.map(row => row.entity))
    const entitiesByName: { [name: string]: number } = {}
    entities.forEach((entityName, index) => {
        entitiesByName[entityName] = index
        entityKey[index.toString()] = {
            id: index,
            name: entityName,
            code: ""
        }
    })
    return {<commit_msg>Import fix<commit_after>import { VariableDisplaySettings } from "./Variable"
import { uniq } from "./Util"

interface OwidRow {
    entity: string
    value: number
    year: number
}

export interface TabularDataset {
    rows: OwidRow[]
}

export function tabularDataToOwidDataset(dataset: TabularDataset): OwidDataset {
    const { rows } = dataset
    const entityKey: { [id: string]: EntityMeta } = {}
    const entities = uniq(rows.map(row => row.entity))
    const entitiesByName: { [name: string]: number } = {}
    entities.forEach((entityName, index) => {
        entitiesByName[entityName] = index
        entityKey[index.toString()] = {
            id: index,
            name: entityName,
            code: ""
        }
    })
    return {
{'fraction_non_alphanumeric': 0.12087186261558784, 'fraction_numerical': 0.0, 'mean_word_length': 2.7780548628428927, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6572992', 'n_tokens_mistral': 433, 'n_tokens_neox': 448, 'n_words': 109}