<commit_before>/**
 * Copyright 2020 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* eslint-env mocha */
/* eslint-disable no-invalid-this */

import {strict as assert} from 'assert';

// TODO(bckenny): use createRequire() for cjs deps when supported by tsc.
import BQModule from '@google-cloud/bigquery';
const {BigQuery} = BQModule;

import HaTablesData from '../../../js/big-query/ha-tables-data.js';
import credentials from '../../../js/big-query/auth/credentials.js';
import {extractMetricsFromHaLhrs, getExtractedTableId, assertValidYear, assertValidMonth} from
  '../../../js/big-query/extract-from-ha-tables.js';

import expectedExtractedTables from './expected-extracted-tables.js';

/** @typedef {import('@google-cloud/bigquery').BigQuery} BigQuery */
/** @typedef {import('@google-cloud/bigquery').Table} Table */
/** @typedef {import('@google-cloud/bigquery').TableSchema} TableSchema */
/** @typedef {import('../../../js/big-query/ha-tables-data.js').HaTableInfo} HaTableInfo */

// Source tables for test.
const sourceOptions = {
  haProjectId: 'lh-metrics-analysis',
  haDatasetId: 'test_lighthouse',
};
// Destination dataset for test.
const extractedDatasetId = 'test_lh_extract';

/**
 * @param {BigQuery} bigQuery
 * @return {Promise<Array<string>>}
 */
async function getSourceTableIds(bigQuery) {
  const query = `SELECT table_id
    FROM \`lh-metrics-analysis.test_lighthouse.__TABLES__\`
    WHERE ENDS_WITH(table_id, 'mobile')`;

  const [tables] = await bigQuery.query({query});
  const tableIds = tables.map(t => t.table_id);

  assert.notStrictEqual(tableIds.length, 0,
      'No test tables found in \'lh-metrics-analysis.test_lighthouse.*\'');

  return tableIds;
}

/**
 * @param {Table} extractedTable
 * @return {Promise<Array<Record<string, unknown>>>}
 */
async function getExtractedTableContents(extractedTable) {
  const query = `SELECT *
    FROM \`${extractedTable.id}\`
    ORDER BY requested_url, final_url`;

  const [rows] = await extractedTable.dataset.query({query});

  return rows;
}


describe('Extraction from HTTP Archive tables', () => {
  describe('#extractMetricsFromHaLhrs', () => {
    const bigQuery = new BigQuery(credentials);
    const haTablesData = new HaTablesData(bigQuery);
    const destinationDataset = bigQuery.dataset(extractedDatasetId);

    before('initialize ha tables data', async function() {
      // First request to BQ can take a while, so give it a bit.
      this.timeout(10000);

      await haTablesData.getListOfTables();
    });

    describe('Input validation', () => {
      it('throws if not using custom source tables in a testing env (like this one)', async () => {
        const tables = await haTablesData.getListOfTables();
        await assert.rejects(async () => {<commit_msg>misc: assert valid tableId in extraction test<commit_after>/**
 * Copyright 2020 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* eslint-env mocha */
/* eslint-disable no-invalid-this */

import {strict as assert} from 'assert';

// TODO(bckenny): use createRequire() for cjs deps when supported by tsc.
import BQModule from '@google-cloud/bigquery';
const {BigQuery} = BQModule;

import HaTablesData from '../../../js/big-query/ha-tables-data.js';
import credentials from '../../../js/big-query/auth/credentials.js';
import {extractMetricsFromHaLhrs, getExtractedTableId, assertValidYear, assertValidMonth} from
  '../../../js/big-query/extract-from-ha-tables.js';
import {assertValidBigQueryId} from '../../../js/big-query/bq-utils.js';

import expectedExtractedTables from './expected-extracted-tables.js';

/** @typedef {import('@google-cloud/bigquery').BigQuery} BigQuery */
/** @typedef {import('@google-cloud/bigquery').Table} Table */
/** @typedef {import('@google-cloud/bigquery').TableSchema} TableSchema */
/** @typedef {import('../../../js/big-query/ha-tables-data.js').HaTableInfo} HaTableInfo */

// Source tables for test.
const sourceOptions = {
  haProjectId: 'lh-metrics-analysis',
  haDatasetId: 'test_lighthouse',
};
// Destination dataset for test.
const extractedDatasetId = 'test_lh_extract';

/**
 * @param {BigQuery} bigQuery
 * @return {Promise<Array<string>>}
 */
async function getSourceTableIds(bigQuery) {
  const query = `SELECT table_id
    FROM \`lh-metrics-analysis.test_lighthouse.__TABLES__\`
    WHERE ENDS_WITH(table_id, 'mobile')`;

  const [tables] = await bigQuery.query({query});
  const tableIds = tables.map(t => t.table_id);

  assert.notStrictEqual(tableIds.length, 0,
      'No test tables found in \'lh-metrics-analysis.test_lighthouse.*\'');

  return tableIds;
}

/**
 * @param {Table} extractedTable
 * @return {Promise<Array<Record<string, unknown>>>}
 */
async function getExtractedTableContents(extractedTable) {
  const tableId = extractedTable.id || 'keep tsc happy';
  assertValidBigQueryId(tableId);

  const query = `SELECT *
    FROM \`${tableId}\`
    ORDER BY requested_url, final_url`;

  const [rows] = await extractedTable.dataset.query({query});

  return rows;
}


describe('Extraction from HTTP Archive tables', () => {
  describe('#extractMetricsFromHaLhrs', () => {
    const bigQuery = new BigQuery(credentials);
    const haTablesData = new HaTablesData(bigQuery);
    const destinationDataset = bigQuery.dataset(extractedDatasetId);

    before('initialize ha tables data', async function() {
      // First request to BQ can take a while, so give it a bit.
      this.timeout(10000);

      await haTablesData.getListOfTables();
    });

    describe('Input validation', () => {
      it('throws if not using custom source tables in a testing env (like this one)', async () => {
        const tables = await haTablesData.getListOfTables();
        await assert.rejects(async () => {
{'fraction_non_alphanumeric': 0.13877009290665093, 'fraction_numerical': 0.004129184486064003, 'mean_word_length': 4.005166051660517, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6881672', 'n_tokens_mistral': 2038, 'n_tokens_neox': 1983, 'n_words': 575}