<commit_before>const path = require('path');
const {promisify} = require('util');

const _ = require('lodash');
const extract = require('extract-zip');
const fetch = require('node-fetch');
const fs = require('fs-extra');
const parse = require('csv-parse');
const proj4 = require('proj4');
const untildify = require('untildify');

const extractAsync = promisify(extract);

const models = require('../models/models');
const utils = require('./utils');

const downloadFiles = async task => {
  task.log(`Downloading GTFS from ${task.agency_url}`);

  task.path = `${task.downloadDir}/${task.agency_key}-gtfs.zip`;

  const res = await fetch(task.agency_url);

  if (res.status !== 200) {
    throw new Error('Couldn\'t download files');
  }

  const buffer = await res.buffer();

  await fs.writeFile(task.path, buffer);
  task.log('Download successful');
};

const readFiles = async task => {
  const gtfsPath = untildify(task.path);
  task.log(`Importing GTFS from ${task.path}\r`);
  if (path.extname(gtfsPath) === '.zip') {
    try {
      await extractAsync(gtfsPath, {dir: task.downloadDir});
    } catch (err) {
      console.error(err);
      throw new Error(`Unable to unzip file ${task.path}`);
    }
  } else {
    // Local file is unzipped, just copy it from there.
    await fs.copy(gtfsPath, task.downloadDir);<commit_msg>Better error for zip files with subfolders<commit_after>const path = require('path');
const {promisify} = require('util');

const _ = require('lodash');
const extract = require('extract-zip');
const fetch = require('node-fetch');
const fs = require('fs-extra');
const parse = require('csv-parse');
const proj4 = require('proj4');
const untildify = require('untildify');

const extractAsync = promisify(extract);

const models = require('../models/models');
const utils = require('./utils');

const downloadFiles = async task => {
  task.log(`Downloading GTFS from ${task.agency_url}`);

  task.path = `${task.downloadDir}/${task.agency_key}-gtfs.zip`;

  const res = await fetch(task.agency_url);

  if (res.status !== 200) {
    throw new Error('Couldn\'t download files');
  }

  const buffer = await res.buffer();

  await fs.writeFile(task.path, buffer);
  task.log('Download successful');
};

const readFiles = async task => {
  const gtfsPath = untildify(task.path);
  task.log(`Importing GTFS from ${task.path}\r`);
  if (path.extname(gtfsPath) === '.zip') {
    try {
      await extractAsync(gtfsPath, {dir: task.downloadDir});
    } catch (err) {
      console.error(err);
      throw new Error(`Unable to unzip file ${task.path}`);
    }

    const files = await fs.readdir(task.downloadDir);
    const textFiles = files.filter(file => file.slice(-3) === 'txt');

    if (textFiles.length === 0) {
      throw new Error(`No .txt files found in ${task.path}. Ensure that .txt files are in the top level of the zip file, not in a subdirectory.`);
    }
  } else {
    // Local file is unzipped, just copy it from there.
    await fs.copy(gtfsPath, task.downloadDir);
{'fraction_non_alphanumeric': 0.16422872340425532, 'fraction_numerical': 0.003989361702127659, 'mean_word_length': 3.9983388704318936, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4958832', 'n_tokens_mistral': 969, 'n_tokens_neox': 956, 'n_words': 258}