<commit_before>var google = require('googleapis');
var bigquery = google.bigquery('v2');
var webdriver = require('protractor/node_modules/selenium-webdriver');

var TABLE_FIELDS = [
  {
    "name": 'runId',
    "type": 'STRING',
    "description": 'git SHA and uuid for the benchmark run'
  },
  {
    "name": 'benchmarkId',
    "type": 'STRING',
    "description": 'id of the benchmark'
  },
  {
    "name": 'index',
    "type": 'INTEGER',
    "description": 'index within the sample'
  },
  {
    "name": 'creationTime',
    "type": 'TIMESTAMP'
  },
  {
    "name": 'browser',
    "type": 'STRING',
    "description": 'navigator.platform'
  },
  {
    "name": 'forceGc',
    "type": 'BOOLEAN',
    "description": 'whether gc was forced at end of action'
  },
  {
    "name": 'stable',
    "type": 'BOOLEAN',
    "description": 'whether this entry was part of the stable sample'
  },
  {
    "name": 'params',
    "type": 'RECORD',
    "description": 'parameters of the benchmark',
    "mode": 'REPEATED',
    "fields": [
      {
        "name": 'name',
        "type": 'STRING',
        "description": 'param name'
      },
      {
        "name": 'strvalue',
        "type": 'STRING',
        "description": 'param value for strings'
      },
      {
        "name": 'numvalue',
        "type": 'FLOAT',
        "description": 'param value for numbers'
      }
    ]
  },
  {
    "name": 'metrics',
    "type": 'RECORD',
    "description": 'metrics of the benchmark',
    "mode": 'REPEATED',
    "fields": [
      {
        "name": 'name',
        "type": 'STRING',
        "description": 'metric name'
      },
      {
        "name": 'value',
        "type": 'FLOAT',
        "description": 'metric value'
      }
    ]
  }
];

var RETRY_COUNT = 3;

class CloudReporter {
  constructor(benchmarkConfig) {
    this.tableConfig = createTableConfig(benchmarkConfig);
    this.authConfig = benchmarkConfig.cloudReporter.auth;
    this.benchmarkConfig = benchmarkConfig;
    this.allSample = [];
    var self = this;
    browser.executeScript('return navigator.userAgent').then(function(userAgent) {
      self.browserUserAgent = userAgent;
    });
  }
  begin() {
    var self = this;
    var flow = browser.driver.controlFlow();
    flow.execute(function() {
      return authenticate(self.authConfig, RETRY_COUNT).then(function(authClient) {
        self.authClient = authClient;
      });
    });
    flow.execute(function() {
      return getOrCreateTable(self.authClient, self.tableConfig, RETRY_COUNT);
    });
  }
  add(data) {
    this.allSample.push(data);
  }
  end(stableSample) {
    var self = this;
    var flow = browser.driver.controlFlow();
    var allRows = this.allSample.map(function(data) {
      return self._convertToTableRow(data, stableSample);
    });
    return insertRows(this.authClient, this.tableConfig, allRows, RETRY_COUNT);
  }
  _convertToTableRow(benchpressRow, stableSample) {
    return {
      insertId: this.benchmarkConfig.runId+'#'+benchpressRow.index,
      json: {
        runId: this.benchmarkConfig.runId,
        benchmarkId: this.benchmarkConfig.id,
        index: benchpressRow.index,
        creationTime: new Date(),
        browser: this.browserUserAgent,
        forceGc: benchpressRow.forceGc,
        stable: stableSample.indexOf(benchpressRow) >= 0,
        params: this.benchmarkConfig.params.map(function(param) {
          if (typeof param.value === 'number') {
            return {
              name: param.name,
              numvalue: param.value
            };
          } else {
            return {
              name: param.name,
              strvalue: ''+param.value
            }
          }
        }),
        metrics: this.benchmarkConfig.metrics.map(function(metricName, index) {
          return {
            name: metricName,
            value: benchpressRow.values[index]
          };
        })
      }<commit_msg>fix(perf): cloud reporter, calculate insertId correctly so that we don’t loose rows!<commit_after>var google = require('googleapis');
var bigquery = google.bigquery('v2');
var webdriver = require('protractor/node_modules/selenium-webdriver');

var TABLE_FIELDS = [
  {
    "name": 'runId',
    "type": 'STRING',
    "description": 'git SHA and uuid for the benchmark run'
  },
  {
    "name": 'benchmarkId',
    "type": 'STRING',
    "description": 'id of the benchmark'
  },
  {
    "name": 'index',
    "type": 'INTEGER',
    "description": 'index within the sample'
  },
  {
    "name": 'creationTime',
    "type": 'TIMESTAMP'
  },
  {
    "name": 'browser',
    "type": 'STRING',
    "description": 'navigator.platform'
  },
  {
    "name": 'forceGc',
    "type": 'BOOLEAN',
    "description": 'whether gc was forced at end of action'
  },
  {
    "name": 'stable',
    "type": 'BOOLEAN',
    "description": 'whether this entry was part of the stable sample'
  },
  {
    "name": 'params',
    "type": 'RECORD',
    "description": 'parameters of the benchmark',
    "mode": 'REPEATED',
    "fields": [
      {
        "name": 'name',
        "type": 'STRING',
        "description": 'param name'
      },
      {
        "name": 'strvalue',
        "type": 'STRING',
        "description": 'param value for strings'
      },
      {
        "name": 'numvalue',
        "type": 'FLOAT',
        "description": 'param value for numbers'
      }
    ]
  },
  {
    "name": 'metrics',
    "type": 'RECORD',
    "description": 'metrics of the benchmark',
    "mode": 'REPEATED',
    "fields": [
      {
        "name": 'name',
        "type": 'STRING',
        "description": 'metric name'
      },
      {
        "name": 'value',
        "type": 'FLOAT',
        "description": 'metric value'
      }
    ]
  }
];

var RETRY_COUNT = 3;

class CloudReporter {
  constructor(benchmarkConfig) {
    this.tableConfig = createTableConfig(benchmarkConfig);
    this.authConfig = benchmarkConfig.cloudReporter.auth;
    this.benchmarkConfig = benchmarkConfig;
    this.allSample = [];
    var self = this;
    browser.executeScript('return navigator.userAgent').then(function(userAgent) {
      self.browserUserAgent = userAgent;
    });
  }
  begin() {
    var self = this;
    var flow = browser.driver.controlFlow();
    flow.execute(function() {
      return authenticate(self.authConfig, RETRY_COUNT).then(function(authClient) {
        self.authClient = authClient;
      });
    });
    flow.execute(function() {
      return getOrCreateTable(self.authClient, self.tableConfig, RETRY_COUNT);
    });
  }
  add(data) {
    this.allSample.push(data);
  }
  end(stableSample) {
    var self = this;
    var flow = browser.driver.controlFlow();
    var allRows = this.allSample.map(function(data) {
      return self._convertToTableRow(data, stableSample);
    });
    return insertRows(this.authClient, this.tableConfig, allRows, RETRY_COUNT);
  }
  _convertToTableRow(benchpressRow, stableSample) {
    return {
      insertId: this.benchmarkConfig.runId+'#'+this.benchmarkConfig.id+'#'+benchpressRow.index,
      json: {
        runId: this.benchmarkConfig.runId,
        benchmarkId: this.benchmarkConfig.id,
        index: benchpressRow.index,
        creationTime: new Date(),
        browser: this.browserUserAgent,
        forceGc: benchpressRow.forceGc,
        stable: stableSample.indexOf(benchpressRow) >= 0,
        params: this.benchmarkConfig.params.map(function(param) {
          if (typeof param.value === 'number') {
            return {
              name: param.name,
              numvalue: param.value
            };
          } else {
            return {
              name: param.name,
              strvalue: ''+param.value
            }
          }
        }),
        metrics: this.benchmarkConfig.metrics.map(function(metricName, index) {
          return {
            name: metricName,
            value: benchpressRow.values[index]
          };
        })
      }
{'fraction_non_alphanumeric': 0.14618993280081147, 'fraction_numerical': 0.0007607455306200076, 'mean_word_length': 2.379605826906598, 'pattern_counts': {'":': 90, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6425472', 'n_tokens_mistral': 2452, 'n_tokens_neox': 2429, 'n_words': 505}