<commit_before>import {
    fromPairs,
    defaultTo,
    head,
    indexBy,
    keys,
    last,
    map,
    prop,
    range,
} from 'ramda'

export function accumulateData (data, column, series, extraYears = 0) {
    let currentYear
    let total = 0

    const dataYears = keys(data.data)
    const first = parseInt(head(dataYears))
    const latest = parseInt(last(dataYears))
    const years = range(first, latest + extraYears)
    const latestData = data.data[latest][column]

    console.debug('Range:', first, latest, latestData, column, total, data)

    // TODO Write separate function that uses world production data until the estimation continues!
    return indexBy(prop('Year'), map((year) => {
        if (year <= latest) {
            currentYear = defaultTo(0, parseFloat(data.data[year][column]))
        } else {
            currentYear = latestData
        }
        total += currentYear

        return fromPairs([
            ['Year', year],
            [series, total],
        ])
    }, years))
}<commit_msg>Fix accumulateData for production series where latest year is missing a value<commit_after>import {
    filter,
    fromPairs,
    defaultTo,
    head,
    indexBy,
    is,
    keys,
    last,
    map,
    prop,
    range,
} from 'ramda'
import { productionSeries } from './csv'

export function accumulateData (data, column, series, extraYears = 0) {
    let currentYear
    let total = 0

    const production = productionSeries(data.series)
    const dataYears = filter((row) => is(Number, row[production]), data.data)
    const first = parseInt(head(keys(dataYears)))
    const latest = parseInt(last(keys(dataYears)))
    const years = range(first, latest + extraYears)
    const latestData = data.data[latest][column]

    console.debug('Range:', first, latest, latestData, column, total, data)

    // TODO Write separate function that uses world production data until the estimation continues!
    return indexBy(prop('Year'), map((year) => {
        if (year <= latest) {
            currentYear = defaultTo(0, parseFloat(data.data[year][column]))
        } else {
            currentYear = latestData
        }
        total += currentYear

        return fromPairs([
            ['Year', year],
            [series, total],
        ])
    }, years))
}
{'fraction_non_alphanumeric': 0.11333914559721012, 'fraction_numerical': 0.0026155187445510027, 'mean_word_length': 2.637083993660856, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1661632', 'n_tokens_mistral': 659, 'n_tokens_neox': 646, 'n_words': 189}