<commit_before>import CSVReader from "readers/csv/csv";
import { isNumber } from "base/utils";

const CSVTimeInColumnsReader = CSVReader.extend({

  _name: "csv-time_in_columns",

  init(readerInfo) {
    this._super(readerInfo);
    this.timeKey = "time";
  },

  load() {
    return this._super()
      .then(({ rows, columns }) => {
        const indicatorKey = columns[this.keySize];

        const concepts = rows.reduce((result, row) => {
          Object.keys(row).forEach(concept => {
            concept = concept === indicatorKey ? row[indicatorKey] : concept;

            if (String(Number(concept)) !== String(concept) && !result.includes(concept)) {
              result.push(concept);
            }
          });

          return result;
        }, []);
        concepts.splice(1, 0, this.timeKey);

        const indicators = concepts.slice(2);
        const [entityDomain] = concepts;
        return {
          columns: concepts,
          rows: rows.reduce((result, row) => {
            const resultRows = result.filter(resultRow => resultRow[entityDomain] === row[entityDomain]);

            if (resultRows.length) {
              resultRows.forEach(resultRow => {
                resultRow[row[indicatorKey]] = row[resultRow[this.timeKey]];
              });<commit_msg>Fix identifying of concepts in csv-time_in_columns when columns in data are empty<commit_after>import CSVReader from "readers/csv/csv";
import { isNumber } from "base/utils";

const CSVTimeInColumnsReader = CSVReader.extend({

  _name: "csv-time_in_columns",

  init(readerInfo) {
    this._super(readerInfo);
    this.timeKey = "time";
  },

  load() {
    return this._super()
      .then(({ rows, columns }) => {
        const indicatorKey = columns[this.keySize];

        const concepts = rows.reduce((result, row) => {
          Object.keys(row).forEach(header => {
            const concept = header === indicatorKey ? row[indicatorKey] : header;

            if (String(Number(concept)) !== String(concept) && !result.includes(concept) && concept) {
              result.push(concept);
            }
          });

          return result;
        }, []);
        concepts.splice(1, 0, this.timeKey);

        const indicators = concepts.slice(2);
        const [entityDomain] = concepts;
        return {
          columns: concepts,
          rows: rows.reduce((result, row) => {
            const resultRows = result.filter(resultRow => resultRow[entityDomain] === row[entityDomain]);

            if (resultRows.length) {
              resultRows.forEach(resultRow => {
                resultRow[row[indicatorKey]] = row[resultRow[this.timeKey]];
              });
{'fraction_non_alphanumeric': 0.13745318352059926, 'fraction_numerical': 0.0022471910112359553, 'mean_word_length': 2.363979848866499, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5624847', 'n_tokens_mistral': 768, 'n_tokens_neox': 729, 'n_words': 149}