<commit_before>import {
    formatYear,
    csvEscape,
    parseDelimited,
    min,
    max,
    orderBy,
    Grid,
    rowsFromGrid,
    range,
    difference,
    intersection,
    flatten,
    sum,
    differenceBy,
    uniqBy,
} from "grapher/utils/Util"
import { queryParamsToStr } from "utils/client/url"
import { CoreColumn, ColumnTypeMap } from "./CoreTableColumns"
import {
    ColumnSlug,
    CoreColumnStore,
    ColumnTypeNames,
    CoreColumnDef,
    CoreRow,
    CoreTableInputOption,
    PrimitiveType,
    SortOrder,
    Time,
    TransformType,
    ValueRange,
    CoreQuery,
    CoreValueType,
} from "./CoreTableConstants"
import {
    AlignedTextTableOptions,
    toAlignedTextTable,
    toDelimited,
    toMarkdownTable,
} from "./CoreTablePrinters"
import {
    makeAutoTypeFn,
    columnStoreToRows,
    imemo,
    makeKeyFn,
    makeRowFromColumnStore,
    standardizeSlugs,
    concatColumnStores,
    rowsToColumnStore,
    autodetectColumnDefs,
    reverseColumnStore,
    renameColumnStore,
    replaceNonPositives,
    replaceRandomCellsInColumnStore,
    getDropIndexes,
} from "./CoreTableUtils"
import { InvalidCellTypes } from "./InvalidCells"
import { OwidTableSlugs } from "./OwidTableConstants"

const TransformsRequiringCompute = new Set([
    TransformType.Load,
    TransformType.AppendRows,
    TransformType.AppendColumns,
    TransformType.UpdateRows,
])

interface AdvancedOptions {
    tableDescription?: string
    transformCategory?: TransformType
    parent?: CoreTable
    rowConversionFunction?: (row: any) => CoreRow
    filterMask?: FilterMask
}

// The complex generic with default here just enables you to optionally specify a more
// narrow interface for the input rows. This is helpful for OwidTable.
export class CoreTable<
    ROW_TYPE extends CoreRow = CoreRow,
    COL_DEF_TYPE extends CoreColumnDef = CoreColumnDef
> {
    private _columns: Map<ColumnSlug, CoreColumn> = new Map()
    protected parent?: this
    private tableDescription: string
    private transformCategory: TransformType
    private timeToLoad = 0
    private initTime = Date.now()

    private originalInput: CoreTableInputOption
    private advancedOptions: AdvancedOptions

    private inputColumnDefs: COL_DEF_TYPE[]
    constructor(
        rowsOrColumnsOrDsv: CoreTableInputOption = [],
        inputColumnDefs: COL_DEF_TYPE[] = [],
        advancedOptions: AdvancedOptions = {}
    ) {
        const start = Date.now() // Perf aid
        const {
            parent,
            tableDescription = "",
            transformCategory = TransformType.Load,
        } = advancedOptions

        this.originalInput = rowsOrColumnsOrDsv
        this.tableDescription = tableDescription
        this.transformCategory = transformCategory
        this.parent = parent as this
        this.inputColumnDefs = inputColumnDefs
        this.inputColumnDefs.forEach((def) => this.setColumn(def))
        this.advancedOptions = advancedOptions

        // If this has a parent table, than we expect all defs. This makes "deletes" and "renames" fast.
        // If this is the first input table, then we do a simple check to generate any missing column defs.
        if (!parent)
            autodetectColumnDefs(
                this.inputColumnStore,
                this._columns
            ).forEach((def) => this.setColumn(def as COL_DEF_TYPE))

        this.timeToLoad = Date.now() - start // Perf aid
    }

    // If the input is a column store, returns that. If it is DSV, parses that and turns it into a column store.
    // If it is a Rows[], turns it into a column store.
    @imemo private get inputColumnStore() {
        const { originalInput } = this

        if (typeof originalInput === "string")
            return this.delimitedAsColumnStore
        else if (Array.isArray(originalInput))
            return rowsToColumnStore(originalInput)
        return originalInput
    }

    @imemo get columnStore() {
        const {
            inputColumnStore,
            inputColumnsToComputedColumns,
            inputColumnsToParsedColumnStore,
        } = this

        // Set blank columns
        let columnStore = Object.assign({}, this.blankColumnStore)

        // Append input columns
        columnStore = Object.assign(columnStore, inputColumnStore)

        // Overwrite any non-parsed columns with parsed values
        if (Object.keys(inputColumnsToParsedColumnStore).length)
            columnStore = Object.assign(
                columnStore,
                inputColumnsToParsedColumnStore
            )

        // Adppend any computed columns
        if (Object.keys(inputColumnsToComputedColumns).length)
            columnStore = Object.assign(
                columnStore,
                inputColumnsToComputedColumns
            )

        return this.advancedOptions.filterMask
            ? this.advancedOptions.filterMask.apply(columnStore)
            : columnStore
    }

    getValuesFor(columnSlug: ColumnSlug) {
        return this.columnStore[columnSlug]
    }

    private get blankColumnStore() {
        const columnsObject: CoreColumnStore = {}
        this.columnSlugs.forEach((slug) => {
            columnsObject[slug] = []
        })
        return columnsObject
    }

    @imemo private get delimitedAsColumnStore() {
        const { originalInput, advancedOptions } = this
        const parsed = parseDelimited(
            originalInput as string,
            undefined,
            advancedOptions.rowConversionFunction ??
                makeAutoTypeFn(this._numericColumnSlugs)
        ) as any
        // dsv_parse adds a columns prop to the result we don't want since we handle our own column defs.
        // https://github.com/d3/d3-dsv#dsv_parse
        delete parsed.columns

        const renamedRows = standardizeSlugs(parsed) // todo: pass renamed defs back in.<commit_msg>fix: throw error if `getValuesFor` is called on a column that doesn't exist<commit_after>import {
    formatYear,
    csvEscape,
    parseDelimited,
    min,
    max,
    orderBy,
    Grid,
    rowsFromGrid,
    range,
    difference,
    intersection,
    flatten,
    sum,
    differenceBy,
    uniqBy,
} from "grapher/utils/Util"
import { queryParamsToStr } from "utils/client/url"
import { CoreColumn, ColumnTypeMap } from "./CoreTableColumns"
import {
    ColumnSlug,
    CoreColumnStore,
    ColumnTypeNames,
    CoreColumnDef,
    CoreRow,
    CoreTableInputOption,
    PrimitiveType,
    SortOrder,
    Time,
    TransformType,
    ValueRange,
    CoreQuery,
    CoreValueType,
} from "./CoreTableConstants"
import {
    AlignedTextTableOptions,
    toAlignedTextTable,
    toDelimited,
    toMarkdownTable,
} from "./CoreTablePrinters"
import {
    makeAutoTypeFn,
    columnStoreToRows,
    imemo,
    makeKeyFn,
    makeRowFromColumnStore,
    standardizeSlugs,
    concatColumnStores,
    rowsToColumnStore,
    autodetectColumnDefs,
    reverseColumnStore,
    renameColumnStore,
    replaceNonPositives,
    replaceRandomCellsInColumnStore,
    getDropIndexes,
} from "./CoreTableUtils"
import { InvalidCellTypes } from "./InvalidCells"
import { OwidTableSlugs } from "./OwidTableConstants"

const TransformsRequiringCompute = new Set([
    TransformType.Load,
    TransformType.AppendRows,
    TransformType.AppendColumns,
    TransformType.UpdateRows,
])

interface AdvancedOptions {
    tableDescription?: string
    transformCategory?: TransformType
    parent?: CoreTable
    rowConversionFunction?: (row: any) => CoreRow
    filterMask?: FilterMask
}

// The complex generic with default here just enables you to optionally specify a more
// narrow interface for the input rows. This is helpful for OwidTable.
export class CoreTable<
    ROW_TYPE extends CoreRow = CoreRow,
    COL_DEF_TYPE extends CoreColumnDef = CoreColumnDef
> {
    private _columns: Map<ColumnSlug, CoreColumn> = new Map()
    protected parent?: this
    private tableDescription: string
    private transformCategory: TransformType
    private timeToLoad = 0
    private initTime = Date.now()

    private originalInput: CoreTableInputOption
    private advancedOptions: AdvancedOptions

    private inputColumnDefs: COL_DEF_TYPE[]
    constructor(
        rowsOrColumnsOrDsv: CoreTableInputOption = [],
        inputColumnDefs: COL_DEF_TYPE[] = [],
        advancedOptions: AdvancedOptions = {}
    ) {
        const start = Date.now() // Perf aid
        const {
            parent,
            tableDescription = "",
            transformCategory = TransformType.Load,
        } = advancedOptions

        this.originalInput = rowsOrColumnsOrDsv
        this.tableDescription = tableDescription
        this.transformCategory = transformCategory
        this.parent = parent as this
        this.inputColumnDefs = inputColumnDefs
        this.inputColumnDefs.forEach((def) => this.setColumn(def))
        this.advancedOptions = advancedOptions

        // If this has a parent table, than we expect all defs. This makes "deletes" and "renames" fast.
        // If this is the first input table, then we do a simple check to generate any missing column defs.
        if (!parent)
            autodetectColumnDefs(
                this.inputColumnStore,
                this._columns
            ).forEach((def) => this.setColumn(def as COL_DEF_TYPE))

        this.timeToLoad = Date.now() - start // Perf aid
    }

    // If the input is a column store, returns that. If it is DSV, parses that and turns it into a column store.
    // If it is a Rows[], turns it into a column store.
    @imemo private get inputColumnStore() {
        const { originalInput } = this

        if (typeof originalInput === "string")
            return this.delimitedAsColumnStore
        else if (Array.isArray(originalInput))
            return rowsToColumnStore(originalInput)
        return originalInput
    }

    @imemo get columnStore() {
        const {
            inputColumnStore,
            inputColumnsToComputedColumns,
            inputColumnsToParsedColumnStore,
        } = this

        // Set blank columns
        let columnStore = Object.assign({}, this.blankColumnStore)

        // Append input columns
        columnStore = Object.assign(columnStore, inputColumnStore)

        // Overwrite any non-parsed columns with parsed values
        if (Object.keys(inputColumnsToParsedColumnStore).length)
            columnStore = Object.assign(
                columnStore,
                inputColumnsToParsedColumnStore
            )

        // Adppend any computed columns
        if (Object.keys(inputColumnsToComputedColumns).length)
            columnStore = Object.assign(
                columnStore,
                inputColumnsToComputedColumns
            )

        return this.advancedOptions.filterMask
            ? this.advancedOptions.filterMask.apply(columnStore)
            : columnStore
    }

    getValuesFor(columnSlug: ColumnSlug): CoreValueType[] {
        if (!this.columnStore[columnSlug])
            throw new Error(
                `getValuesFor() was called with a columnSlug that doesn't exist on the table: ${columnSlug}`
            )

        return this.columnStore[columnSlug]
    }

    private get blankColumnStore() {
        const columnsObject: CoreColumnStore = {}
        this.columnSlugs.forEach((slug) => {
            columnsObject[slug] = []
        })
        return columnsObject
    }

    @imemo private get delimitedAsColumnStore() {
        const { originalInput, advancedOptions } = this
        const parsed = parseDelimited(
            originalInput as string,
            undefined,
            advancedOptions.rowConversionFunction ??
                makeAutoTypeFn(this._numericColumnSlugs)
        ) as any
        // dsv_parse adds a columns prop to the result we don't want since we handle our own column defs.
        // https://github.com/d3/d3-dsv#dsv_parse
        delete parsed.columns

        const renamedRows = standardizeSlugs(parsed) // todo: pass renamed defs back in.
{'fraction_non_alphanumeric': 0.07376571476146865, 'fraction_numerical': 0.000499542086420781, 'mean_word_length': 2.5112540192926045, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5682973', 'n_tokens_mistral': 3247, 'n_tokens_neox': 3155, 'n_words': 852}