<commit_before>/* eslint-env mocha */
/* eslint-disable max-nested-callbacks */

import { createReadStream, existsSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import parse from 'csv-parse';
import should from 'should';

import { openDb, closeDb } from '../../lib/db.js';
import { unzip } from '../../lib/file-utils.js';
import config from '../test-config.js';
import { importGtfs, getRoutes } from '../../index.js';
import models from '../../models/models.js';

let db;

const agenciesFixturesRemote = [{
  url: 'http://transitfeeds.com/p/caltrain/122/20160406/download'
}];

const agenciesFixturesLocal = [{
  path: path.join(path.dirname(fileURLToPath(import.meta.url)), '../fixture/caltrain_20160406.zip')
}];

describe('importGtfs():', function () {
  before(async () => {
    db = await openDb(config);
  });

  after(async () => {
    await closeDb();
  });

  this.timeout(10000);
  describe('Download and import from different GTFS sources', () => {
    it('should be able to download and import from HTTP', async () => {
      await importGtfs({
        ...config,
        agencies: agenciesFixturesRemote
      });

      const routes = await getRoutes();
      should.exist(routes);
      routes.length.should.equal(4);
    });

    it('should be able to download and import from local filesystem', async () => {
      await importGtfs({
        ...config,
        agencies: agenciesFixturesLocal
      });

      const routes = await getRoutes();
      should.exist(routes);
      routes.length.should.equal(4);
    });
  });

  describe('Verify data imported into database', () => {
    const countData = {};
    const temporaryDir = path.join(path.dirname(fileURLToPath(import.meta.url)), '../fixture/tmp/');

    before(async () => {
      await unzip(agenciesFixturesLocal[0].path, temporaryDir);

      await Promise.all(models.map(model => {
        const filePath = path.join(temporaryDir, `${model.filenameBase}.txt`);

        // GTFS has optional files
        if (!existsSync(filePath)) {
          countData[model.filenameBase] = 0;
          return false;
        }

        const parser = parse({
          columns: true,
          relax: true,
          trim: true
        }, (error, data) => {
          if (error) {
            throw new Error(error);
          }

          countData[model.filenameBase] = data.length;
        });<commit_msg>Add test for missing path<commit_after>/* eslint-env mocha */
/* eslint-disable max-nested-callbacks */

import { createReadStream, existsSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import parse from 'csv-parse';
import should from 'should';

import { openDb, closeDb } from '../../lib/db.js';
import { unzip } from '../../lib/file-utils.js';
import config from '../test-config.js';
import { importGtfs, getRoutes } from '../../index.js';
import models from '../../models/models.js';

let db;

const agenciesFixturesRemote = [{
  url: 'http://transitfeeds.com/p/caltrain/122/20160406/download'
}];

const agenciesFixturesLocal = [{
  path: path.join(path.dirname(fileURLToPath(import.meta.url)), '../fixture/caltrain_20160406.zip')
}];

describe('importGtfs():', function () {
  before(async () => {
    db = await openDb(config);
  });

  after(async () => {
    await closeDb();
  });

  this.timeout(10000);
  describe('Download and import from different GTFS sources', () => {
    it('should be able to download and import from HTTP', async () => {
      await importGtfs({
        ...config,
        agencies: agenciesFixturesRemote
      });

      const routes = await getRoutes();
      should.exist(routes);
      routes.length.should.equal(4);
    });

    it('should be able to download and import from local filesystem', async () => {
      await importGtfs({
        ...config,
        agencies: agenciesFixturesLocal
      });

      const routes = await getRoutes();
      should.exist(routes);
      routes.length.should.equal(4);
    });

    it('should throw an error when importing from local filesystem which doesn\'t exist', async () => {
      return importGtfs({
        ...config,
        agencies: [
          {
            path: '/does/not/exist'
          }
        ]
      }).should.be.rejected();
    });
  });

  describe('Verify data imported into database', () => {
    const countData = {};
    const temporaryDir = path.join(path.dirname(fileURLToPath(import.meta.url)), '../fixture/tmp/');

    before(async () => {
      await unzip(agenciesFixturesLocal[0].path, temporaryDir);

      await Promise.all(models.map(model => {
        const filePath = path.join(temporaryDir, `${model.filenameBase}.txt`);

        // GTFS has optional files
        if (!existsSync(filePath)) {
          countData[model.filenameBase] = 0;
          return false;
        }

        const parser = parse({
          columns: true,
          relax: true,
          trim: true
        }, (error, data) => {
          if (error) {
            throw new Error(error);
          }

          countData[model.filenameBase] = data.length;
        });
{'fraction_non_alphanumeric': 0.1552093476144109, 'fraction_numerical': 0.010905550146056476, 'mean_word_length': 2.566666666666667, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6169331', 'n_tokens_mistral': 1552, 'n_tokens_neox': 1525, 'n_words': 360}