<commit_before>import {
    dateDiffInDays,
    flatten,
    map,
    groupBy,
    parseFloatOrUndefined,
    difference,
    entries,
    minBy,
    sortBy,
    cloneDeep,
    retryPromise,
    memoize,
    fetchText,
    fetchJSON
} from "charts/Util"
import moment from "moment"
import {
    ParsedCovidCsvRow,
    MetricKind,
    CountryOption,
    CovidGrapherRow,
    SmoothingOption
} from "./CovidTypes"
import { csv } from "d3-fetch"
import { csvParse } from "d3-dsv"
import {
    OwidTable,
    ComputedColumnSpec,
    RowToValueMapper,
    ColumnSpec,
    entityName,
    columnSlug,
    columnTypes
} from "charts/owidData/OwidTable"
import { CovidConstrainedQueryParams, CovidQueryParams } from "./CovidChartUrl"
import { covidAnnotations } from "./CovidAnnotations"
import {
    covidDataPath,
    covidLastUpdatedPath,
    covidChartAndVariableMetaPath,
    sourceVariables
} from "./CovidConstants"

type MetricKey = {
    [K in MetricKind]: number
}

interface AnnotationsRow {
    location: entityName
    date: string
    cases_annotations: string
    deaths_annotations: string
}

const stringColumnSlugs = new Set(
    `iso_code location date tests_units continent`.split(" ")
)

export const buildColumnSlug = (
    name: MetricKind,
    perCapita: number,
    daily?: boolean,
    rollingAverage?: number
) =>
    [
        name,
        perCapita === 1e3
            ? "perThousand"
            : perCapita === 1e6
            ? "perMil"
            : undefined,
        daily ? "daily" : "cumulative",
        rollingAverage ? rollingAverage + "DayAvg" : undefined
    ]
        .filter(i => i)
        .join("-")

export class CovidExplorerTable {
    table: OwidTable
    columnSpecs: { [name: string]: ColumnSpec } = {}

    constructor(
        table: OwidTable,
        data: CovidGrapherRow[],
        owidVariableSpecs = {}
    ) {
        this.initColumnSpecs(owidVariableSpecs)
        this.table = table
        this.table.rows = data
        this.table.addSpecs(this.getBaseSpecs(data[0]))
        this.table.columnsBySlug.forEach(col => {
            // Ensure all columns have a OwidVarId for now. Todo: rely on just column slug in Grapher.
            if (!col.spec.owidVariableId)
                col.spec = CovidExplorerTable.makeSpec(col.spec)
        })
        this.table.addCategoricalColumnSpec(this.columnSpecs.continents)
        this.addAnnotationColumns()
    }

    private getBaseSpecs(row: CovidGrapherRow) {
        return Object.keys(row).map(slug => {
            return {
                slug,
                type: stringColumnSlugs.has(slug)
                    ? "String"
                    : ("Numeric" as columnTypes)
            }
        })
    }

    private static colOwidVarIdGuid = 90210
    private static makeSpec(spec: ColumnSpec): ColumnSpec {
        return {
            owidVariableId: CovidExplorerTable.colOwidVarIdGuid++,
            unit: "",
            description: "",
            coverage: "",
            display: { includeInTable: false },
            datasetName: "",
            source: {
                id: 1,
                name: "",
                dataPublishedBy: "",
                dataPublisherSource: "",
                link: "",
                retrievedDate: "",
                additionalInfo: ""
            },
            ...spec
        }
    }

    private initColumnSpecs(owidVariableSpecs: any) {
        this.columnSpecs = {
            positive_test_rate: {
                ...owidVariableSpecs[sourceVariables.positive_test_rate],
                isDailyMeasurement: true,
                description:
                    "The number of confirmed cases divided by the number of tests, expressed as a percentage. Tests may refer to the number of tests performed or the number of people tested – depending on which is reported by the particular country.",
                display: {
                    shortUnit: "%",
                    tableDisplay: { hideRelativeChange: true }
                }
            },
            tests_per_case: {
                ...owidVariableSpecs[sourceVariables.tests_per_case],
                isDailyMeasurement: true,
                description:
                    "The number of tests divided by the number of confirmed cases. Not all countries report testing data on a daily basis."
            },
            case_fatality_rate: {
                ...owidVariableSpecs[sourceVariables.case_fatality_rate],
                annotationsColumnSlug: "case_fatality_rate_annotations",
                isDailyMeasurement: true,
                description: `The Case Fatality Rate (CFR) is the ratio between confirmed deaths and confirmed cases. During an outbreak of a pandemic the CFR is a poor measure of the mortality risk of the disease. We explain this in detail at OurWorldInData.org/Coronavirus`,
                display: {
                    shortUnit: "%",
                    tableDisplay: { hideRelativeChange: true }
                }
            },
            cases: {
                ...owidVariableSpecs[sourceVariables.cases],
                isDailyMeasurement: true,
                annotationsColumnSlug: "cases_annotations",
                name: "Confirmed cases of COVID-19",
                description: `The number of confirmed cases is lower than the number of actual cases; the main reason for that is limited testing.`
            },
            deaths: {
                ...owidVariableSpecs[sourceVariables.deaths],
                isDailyMeasurement: true,
                annotationsColumnSlug: "deaths_annotations",
                name: "Confirmed deaths due to COVID-19",
                description: `Limited testing and challenges in the attribution of the cause of death means that the number of confirmed deaths may not be an accurate count of the true number of deaths from COVID-19.`
            },
            tests: {
                ...owidVariableSpecs[sourceVariables.tests],
                isDailyMeasurement: true,
                description: "",
                name: "tests",
                annotationsColumnSlug: "tests_units"
            },
            days_since: {
                ...owidVariableSpecs[sourceVariables.days_since],
                isDailyMeasurement: true,
                description: "",
                name: "days_since"
            },
            continents: {
                ...owidVariableSpecs[sourceVariables.continents],
                description: "",
                name: "continent",
                slug: "continent"
            }
        }
        Object.keys(this.columnSpecs).forEach(key => {
            this.columnSpecs[key].owidVariableId = (sourceVariables as any)[key]
        })
    }

    private addAnnotationColumns() {
        const caseSlug = "cases_annotations"
        const deathSlug = "deaths_annotations"
        const cfrSlug = "case_fatality_rate_annotations"
        this.table.addStringColumnSpec({ slug: caseSlug })
        this.table.addStringColumnSpec({
            slug: deathSlug
        })
        this.table.addStringColumnSpec({
            slug: cfrSlug
        })

        const index = this.table.entityIndex
        const annotationRows = csvParse(covidAnnotations) as AnnotationsRow[]
        annotationRows.forEach(annoRow => {
            const rows = index.get(annoRow.location)<commit_msg>Fix: do not overwrite existing variable display properties<commit_after>import {
    dateDiffInDays,
    flatten,
    map,
    groupBy,
    parseFloatOrUndefined,
    difference,
    entries,
    minBy,
    sortBy,
    cloneDeep,
    retryPromise,
    memoize,
    fetchText,
    fetchJSON
} from "charts/Util"
import moment from "moment"
import {
    ParsedCovidCsvRow,
    MetricKind,
    CountryOption,
    CovidGrapherRow,
    SmoothingOption
} from "./CovidTypes"
import { csv } from "d3-fetch"
import { csvParse } from "d3-dsv"
import {
    OwidTable,
    ComputedColumnSpec,
    RowToValueMapper,
    ColumnSpec,
    entityName,
    columnSlug,
    columnTypes
} from "charts/owidData/OwidTable"
import { CovidConstrainedQueryParams, CovidQueryParams } from "./CovidChartUrl"
import { covidAnnotations } from "./CovidAnnotations"
import {
    covidDataPath,
    covidLastUpdatedPath,
    covidChartAndVariableMetaPath,
    sourceVariables
} from "./CovidConstants"

type MetricKey = {
    [K in MetricKind]: number
}

interface AnnotationsRow {
    location: entityName
    date: string
    cases_annotations: string
    deaths_annotations: string
}

const stringColumnSlugs = new Set(
    `iso_code location date tests_units continent`.split(" ")
)

export const buildColumnSlug = (
    name: MetricKind,
    perCapita: number,
    daily?: boolean,
    rollingAverage?: number
) =>
    [
        name,
        perCapita === 1e3
            ? "perThousand"
            : perCapita === 1e6
            ? "perMil"
            : undefined,
        daily ? "daily" : "cumulative",
        rollingAverage ? rollingAverage + "DayAvg" : undefined
    ]
        .filter(i => i)
        .join("-")

export class CovidExplorerTable {
    table: OwidTable
    columnSpecs: { [name: string]: ColumnSpec } = {}

    constructor(
        table: OwidTable,
        data: CovidGrapherRow[],
        owidVariableSpecs = {}
    ) {
        this.initColumnSpecs(owidVariableSpecs)
        this.table = table
        this.table.rows = data
        this.table.addSpecs(this.getBaseSpecs(data[0]))
        this.table.columnsBySlug.forEach(col => {
            // Ensure all columns have a OwidVarId for now. Todo: rely on just column slug in Grapher.
            if (!col.spec.owidVariableId)
                col.spec = CovidExplorerTable.makeSpec(col.spec)
        })
        this.table.addCategoricalColumnSpec(this.columnSpecs.continents)
        this.addAnnotationColumns()
    }

    private getBaseSpecs(row: CovidGrapherRow) {
        return Object.keys(row).map(slug => {
            return {
                slug,
                type: stringColumnSlugs.has(slug)
                    ? "String"
                    : ("Numeric" as columnTypes)
            }
        })
    }

    private static colOwidVarIdGuid = 90210
    private static makeSpec(spec: ColumnSpec): ColumnSpec {
        return {
            owidVariableId: CovidExplorerTable.colOwidVarIdGuid++,
            unit: "",
            description: "",
            coverage: "",
            display: { includeInTable: false },
            datasetName: "",
            source: {
                id: 1,
                name: "",
                dataPublishedBy: "",
                dataPublisherSource: "",
                link: "",
                retrievedDate: "",
                additionalInfo: ""
            },
            ...spec
        }
    }

    private initColumnSpecs(owidVariableSpecs: any) {
        this.columnSpecs = {
            positive_test_rate: {
                ...owidVariableSpecs[sourceVariables.positive_test_rate],
                isDailyMeasurement: true,
                description:
                    "The number of confirmed cases divided by the number of tests, expressed as a percentage. Tests may refer to the number of tests performed or the number of people tested – depending on which is reported by the particular country."
            },
            tests_per_case: {
                ...owidVariableSpecs[sourceVariables.tests_per_case],
                isDailyMeasurement: true,
                description:
                    "The number of tests divided by the number of confirmed cases. Not all countries report testing data on a daily basis."
            },
            case_fatality_rate: {
                ...owidVariableSpecs[sourceVariables.case_fatality_rate],
                annotationsColumnSlug: "case_fatality_rate_annotations",
                isDailyMeasurement: true,
                description: `The Case Fatality Rate (CFR) is the ratio between confirmed deaths and confirmed cases. During an outbreak of a pandemic the CFR is a poor measure of the mortality risk of the disease. We explain this in detail at OurWorldInData.org/Coronavirus`
            },
            cases: {
                ...owidVariableSpecs[sourceVariables.cases],
                isDailyMeasurement: true,
                annotationsColumnSlug: "cases_annotations",
                name: "Confirmed cases of COVID-19",
                description: `The number of confirmed cases is lower than the number of actual cases; the main reason for that is limited testing.`
            },
            deaths: {
                ...owidVariableSpecs[sourceVariables.deaths],
                isDailyMeasurement: true,
                annotationsColumnSlug: "deaths_annotations",
                name: "Confirmed deaths due to COVID-19",
                description: `Limited testing and challenges in the attribution of the cause of death means that the number of confirmed deaths may not be an accurate count of the true number of deaths from COVID-19.`
            },
            tests: {
                ...owidVariableSpecs[sourceVariables.tests],
                isDailyMeasurement: true,
                description: "",
                name: "tests",
                annotationsColumnSlug: "tests_units"
            },
            days_since: {
                ...owidVariableSpecs[sourceVariables.days_since],
                isDailyMeasurement: true,
                description: "",
                name: "days_since"
            },
            continents: {
                ...owidVariableSpecs[sourceVariables.continents],
                description: "",
                name: "continent",
                slug: "continent"
            }
        }
        Object.keys(this.columnSpecs).forEach(key => {
            this.columnSpecs[key].owidVariableId = (sourceVariables as any)[key]
        })

        // Todo: move to the grapher specs?
        this.columnSpecs.positive_test_rate.display!.tableDisplay = {
            hideRelativeChange: true
        } as any
        this.columnSpecs.case_fatality_rate.display!.tableDisplay = {
            hideRelativeChange: true
        } as any
    }

    private addAnnotationColumns() {
        const caseSlug = "cases_annotations"
        const deathSlug = "deaths_annotations"
        const cfrSlug = "case_fatality_rate_annotations"
        this.table.addStringColumnSpec({ slug: caseSlug })
        this.table.addStringColumnSpec({
            slug: deathSlug
        })
        this.table.addStringColumnSpec({
            slug: cfrSlug
        })

        const index = this.table.entityIndex
        const annotationRows = csvParse(covidAnnotations) as AnnotationsRow[]
        annotationRows.forEach(annoRow => {
            const rows = index.get(annoRow.location)
{'fraction_non_alphanumeric': 0.08313041125541125, 'fraction_numerical': 0.0025703463203463205, 'mean_word_length': 1.8809431021044427, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3392166', 'n_tokens_mistral': 4036, 'n_tokens_neox': 3845, 'n_words': 960}