<commit_before>/*
* To run this script, just open a terminal and run it with node.
* Then just follow the instructions on the screen.
*/

process.env['NODE_ENV'] = 'production'
require('colors')
const CanvasApi = require('kth-canvas-api')
const inquirer = require('inquirer')
const moment = require('moment')
const request = require('request-promise')

async function listErrors () {
  try {
    const {apiUrl} = await inquirer.prompt(
      {
        message: 'Vilken miljö?',
        name: 'apiUrl',
        choices: [
          {name: 'prod', value: 'https://kth.instructure.com/api/v1'},
          {name: 'test', value: 'https://kth.test.instructure.com/api/v1'},
          {name: 'beta', value: 'https://kth.beta.instructure.com/api/v1'}
        ],
        type: 'list'
      })

    const {apiKey} = await inquirer.prompt({
      message: 'Klistra in api nyckel till Canvas här',
      name: 'apiKey',
      type: 'string'
    })

    const canvasApi = new CanvasApi(apiUrl, apiKey)

    const {numOfDays} = await inquirer.prompt({
      message: 'Hur många dagar bakåt?',
      name: 'numOfDays',
      type: 'number',
      default: 7
    })

    const from = moment().subtract(numOfDays, 'days').utc().toDate().toISOString()

    const allSisImports = await canvasApi.recursePages(`${apiUrl}/accounts/1/sis_imports?created_since=${from}&per_page=100`)

    const flattenedSisImports = allSisImports
    .reduce((a, b) => a.concat(b.sis_imports), []) // Flatten every page

    flattenedSisImports.map(_sisObj => (_sisObj.errors_attachment && _sisObj.errors_attachment.url) || [])
    .reduce((a, b) => a.concat(b), [])
    .map(_url => { request(_url)
      .then(function (allWarnings) {
        console.log('Warnings and errors:'.green)
        console.log(allWarnings)
      })
      .catch(function (err) {
          // Crawling failed...
      });
    })

    // const filteredWarnings = allWarnings
    // .filter(([fileName, warning]) => !warning.startsWith('Neither course nor section existed'))
    // // .filter(([fileName, warning]) => !/There were [\d,]+ more warnings/.test(warning))
    // .filter(([fileName, warning]) => !warning.startsWith('An enrollment referenced a non-existent section'))
  } catch (e) {
    console.error(e)
  }
}
listErrors()
<commit_msg>Refactored async call for urls<commit_after>/*
* To run this script, just open a terminal and run it with node.
* Then just follow the instructions on the screen.
*/

process.env['NODE_ENV'] = 'production'
require('colors')
const CanvasApi = require('kth-canvas-api')
const inquirer = require('inquirer')
const moment = require('moment')
const request = require('request-promise')

async function listErrors () {
  try {
    const {apiUrl} = await inquirer.prompt(
      {
        message: 'Vilken miljö?',
        name: 'apiUrl',
        choices: [
          {name: 'prod', value: 'https://kth.instructure.com/api/v1'},
          {name: 'test', value: 'https://kth.test.instructure.com/api/v1'},
          {name: 'beta', value: 'https://kth.beta.instructure.com/api/v1'}
        ],
        type: 'list'
      })

    const {apiKey} = await inquirer.prompt({
      message: 'Klistra in api nyckel till Canvas här',
      name: 'apiKey',
      type: 'string'
    })

    const canvasApi = new CanvasApi(apiUrl, apiKey)

    const {numOfDays} = await inquirer.prompt({
      message: 'Hur många dagar bakåt?',
      name: 'numOfDays',
      type: 'number',
      default: 7
    })

    const from = moment().subtract(numOfDays, 'days').utc().toDate().toISOString()

    const allSisImports = await canvasApi.recursePages(`${apiUrl}/accounts/1/sis_imports?created_since=${from}&per_page=100`)

    const flattenedSisImports = allSisImports
    .reduce((a, b) => a.concat(b.sis_imports), []) // Flatten every page
    
    const reportUrls = flattenedSisImports.map(_sisObj => (_sisObj.errors_attachment && _sisObj.errors_attachment.url) || [])
    .reduce((a, b) => a.concat(b), [])

    console.log('Warnings and errors:'.green)

    for (let url of reportUrls) {
      const warnings = await request(url)
      console.log(warnings)
    }

    // const filteredWarnings = allWarnings
    // .filter(([fileName, warning]) => !warning.startsWith('Neither course nor section existed'))
    // // .filter(([fileName, warning]) => !/There were [\d,]+ more warnings/.test(warning))
    // .filter(([fileName, warning]) => !warning.startsWith('An enrollment referenced a non-existent section'))
  } catch (e) {
    console.error(e)
  }
}
listErrors()

{'fraction_non_alphanumeric': 0.15963788915875468, 'fraction_numerical': 0.0035327886950761757, 'mean_word_length': 3.1107078039927405, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 16, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1780403', 'n_tokens_mistral': 1506, 'n_tokens_neox': 1464, 'n_words': 337}