<commit_before>#! /usr/bin/env jest

import {
    SampleColumnSlugs,
    SynthesizeFruitTable,
    SynthesizeGDPTable,
} from "./OwidTableSynthesizers"
import { BlankOwidTable, OwidTable } from "./OwidTable"
import { ColumnTypeNames } from "./CoreColumnDef"
import { ErrorValueTypes } from "./ErrorValues"
import { OwidColumnDef } from "./OwidTableConstants"

const sampleRows = [
    {
        year: 2020,
        time: 2020,
        entityName: "United States",
        population: 3e8,
        entityId: 1,
        entityCode: "USA",
    },
]

it("can create a table and detect columns", () => {
    const table = new OwidTable(sampleRows)
    expect(table.numRows).toEqual(1)
    expect(table.numColumns).toEqual(6)
})

it("can create a new table by adding a column", () => {
    const table = new OwidTable(sampleRows, [
        {
            slug: "populationInMillions",
            values: sampleRows.map((row) => row.population / 1000000),
        },
    ])
    expect(table.rows[0].populationInMillions).toEqual(300)
})

it("can perform queries needed by discrete bar", () => {
    const table = SynthesizeGDPTable(
        {
            entityCount: 3,
            timeRange: [2000, 2004],
        },
        10
    )
    expect(table.rowIndicesByEntityName.size).toEqual(3)
    const entityNames = table.availableEntityNames
    expect(
        table.getClosestIndexForEachEntity(entityNames, 2003, 0).length
    ).toEqual(3)
    expect(
        table.getClosestIndexForEachEntity(entityNames, 2004, 1).length
    ).toEqual(3)
    expect(
        table.getClosestIndexForEachEntity(entityNames, 2005, 1).length
    ).toEqual(0)
})

it("can parse data to Javascript data structures", () => {
    const table = SynthesizeGDPTable({
        timeRange: [2000, 2010],
    })

    const parsed = table.get("Population").values
    expect(parsed.filter((item) => isNaN(item))).toEqual([])

    table.get("Population").owidRows.forEach((row) => {
        expect(typeof row.entityName).toBe("string")
        expect(row.value).toBeGreaterThan(100)
        expect(row.time).toBeGreaterThan(1999)
    })
})

it("can drop random cells", () => {
    const table = SynthesizeGDPTable({
        timeRange: [2000, 2010],
        entityCount: 1,
    })
    expect(table.get(SampleColumnSlugs.GDP).numValues).toBe(10)
    expect(
        table
            .replaceRandomCells(7, [SampleColumnSlugs.GDP])
            .get(SampleColumnSlugs.GDP).numValues
    ).toBe(3)
})

it("can group data by entity and time", () => {
    const table = SynthesizeGDPTable({
        timeRange: [2000, 2010],
        entityCount: 5,
    })

    const timeValues = Array.from(
        table.get("Population").valueByEntityNameAndTime.values()
    )
        .map((value) => Array.from(value.values()))
        .flat()

    expect(timeValues.length).toEqual(50)
    expect(timeValues.filter((value) => isNaN(value as number))).toEqual([])
})

it("prefers a day column when both year and day are in the chart", () => {
    const csv = `entityName,entityCode,entityId,pop,year,day
usa,usa,1,322,2000,2`

    const table = new OwidTable(csv)
    expect(table.timeColumn!.slug).toBe("day")
})

it("can get the latest values for an entity", () => {
    const csv = `entityName,entityCode,entityId,pop,year,coal
usa,usa,1,322,2000,10
usa,usa,1,322,2001,
usa,usa,1,4,2002,`

    const table = new OwidTable(csv)
    expect(table.getLatestValueForEntity("usa", "coal")).toBe(10)
    expect(table.getLatestValueForEntity("usa", "pop")).toBe(4)
    expect(table.getLatestValueForEntity("does not exit", "pop")).toBe(
        undefined
    )
})<commit_msg>test(table): use "time" as the canonical timeColumn<commit_after>#! /usr/bin/env jest

import {
    SampleColumnSlugs,
    SynthesizeFruitTable,
    SynthesizeGDPTable,
} from "./OwidTableSynthesizers"
import { BlankOwidTable, OwidTable } from "./OwidTable"
import { ColumnTypeNames } from "./CoreColumnDef"
import { ErrorValueTypes } from "./ErrorValues"
import { OwidColumnDef, OwidTableSlugs } from "./OwidTableConstants"

const sampleRows = [
    {
        year: 2020,
        time: 2020,
        entityName: "United States",
        population: 3e8,
        entityId: 1,
        entityCode: "USA",
    },
]

it("can create a table and detect columns", () => {
    const table = new OwidTable(sampleRows)
    expect(table.numRows).toEqual(1)
    expect(table.numColumns).toEqual(6)
})

it("can create a new table by adding a column", () => {
    const table = new OwidTable(sampleRows, [
        {
            slug: "populationInMillions",
            values: sampleRows.map((row) => row.population / 1000000),
        },
    ])
    expect(table.rows[0].populationInMillions).toEqual(300)
})

it("can perform queries needed by discrete bar", () => {
    const table = SynthesizeGDPTable(
        {
            entityCount: 3,
            timeRange: [2000, 2004],
        },
        10
    )
    expect(table.rowIndicesByEntityName.size).toEqual(3)
    const entityNames = table.availableEntityNames
    expect(
        table.getClosestIndexForEachEntity(entityNames, 2003, 0).length
    ).toEqual(3)
    expect(
        table.getClosestIndexForEachEntity(entityNames, 2004, 1).length
    ).toEqual(3)
    expect(
        table.getClosestIndexForEachEntity(entityNames, 2005, 1).length
    ).toEqual(0)
})

it("can parse data to Javascript data structures", () => {
    const table = SynthesizeGDPTable({
        timeRange: [2000, 2010],
    })

    const parsed = table.get("Population").values
    expect(parsed.filter((item) => isNaN(item))).toEqual([])

    table.get("Population").owidRows.forEach((row) => {
        expect(typeof row.entityName).toBe("string")
        expect(row.value).toBeGreaterThan(100)
        expect(row.time).toBeGreaterThan(1999)
    })
})

it("can drop random cells", () => {
    const table = SynthesizeGDPTable({
        timeRange: [2000, 2010],
        entityCount: 1,
    })
    expect(table.get(SampleColumnSlugs.GDP).numValues).toBe(10)
    expect(
        table
            .replaceRandomCells(7, [SampleColumnSlugs.GDP])
            .get(SampleColumnSlugs.GDP).numValues
    ).toBe(3)
})

it("can group data by entity and time", () => {
    const table = SynthesizeGDPTable({
        timeRange: [2000, 2010],
        entityCount: 5,
    })

    const timeValues = Array.from(
        table.get("Population").valueByEntityNameAndTime.values()
    )
        .map((value) => Array.from(value.values()))
        .flat()

    expect(timeValues.length).toEqual(50)
    expect(timeValues.filter((value) => isNaN(value as number))).toEqual([])
})

describe("timeColumn", () => {
    it("uses 'time' as the canonical timeColumn", () => {
        const columnStore = {
            [OwidTableSlugs.entityName]: ["usa"],
            [OwidTableSlugs.time]: [2000],
            year: [2000],
            day: ["2000-01-01"],
            x: [0],
        }
        const colDefs: OwidColumnDef[] = [
            {
                slug: "year",
                type: ColumnTypeNames.Year,
            },
            {
                slug: "day",
                type: ColumnTypeNames.Day,
            },
            {
                slug: OwidTableSlugs.time,
                type: ColumnTypeNames.Year,
            },
            {
                slug: "x",
                type: ColumnTypeNames.Numeric,
            },
        ]
        const table = new OwidTable(columnStore, colDefs)
        expect(table.timeColumn.slug).toEqual(OwidTableSlugs.time)
    })

    it("prefers a day column when both year and day are in the chart", () => {
        const csv = `entityName,entityCode,entityId,pop,year,day
    usa,usa,1,322,2000,2`

        const table = new OwidTable(csv)
        expect(table.timeColumn!.slug).toBe("day")
    })
})

it("can get the latest values for an entity", () => {
    const csv = `entityName,entityCode,entityId,pop,year,coal
usa,usa,1,322,2000,10
usa,usa,1,322,2001,
usa,usa,1,4,2002,`

    const table = new OwidTable(csv)
    expect(table.getLatestValueForEntity("usa", "coal")).toBe(10)
    expect(table.getLatestValueForEntity("usa", "pop")).toBe(4)
    expect(table.getLatestValueForEntity("does not exit", "pop")).toBe(
        undefined
    )
})
{'fraction_non_alphanumeric': 0.1402357516101592, 'fraction_numerical': 0.0334183983473083, 'mean_word_length': 2.725667722951562, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4009846', 'n_tokens_mistral': 2816, 'n_tokens_neox': 2575, 'n_words': 505}