<commit_before>'use strict'

const log = require('./util/log');
const slugify = require('./util/slugify');
const firstline = require('./util/firstline');

const Promise = require('bluebird');
const writeFile = Promise.promisify(require('fs').writeFile);
const readFile = Promise.promisify(require('fs').readFile);
const access = Promise.promisify(require('fs').access);
const readdir = Promise.promisify(require('fs').readdir);
const readdirSync = require('fs').readdirSync;
const mkdirp = Promise.promisify(require('mkdirp'));
const async = require('async');
const EOL = require('os').EOL;
const isJSON = require('is-json');
const _ = require('lodash');

var convodir = './spec/convo/';
var suffix = '.convo.txt';

function readConvos() {
  return new Promise(function(readConvosResolve, readConvosReject) {
    access(convodir).then(() => {
      readdir(convodir).then(function(filenames) {
        
        var convos = [];
        
        async.each(filenames, 
          (filename, callback) => {
            if (!filename.endsWith(suffix)) {
              callback();
              return;
            }
            firstline.firstline(convodir + filename).then(
              (header) => {
                convos.push(createConvoEntry(filename, header));
                callback();
              }).catch(
              (err) => {
                log.warn('error reading first line of file ' + filename + ': ' + err);
                callback();
              });
          },
          (err) => {
            if (err) {<commit_msg>Use unix eol in convo files<commit_after>'use strict'

const log = require('./util/log');
const slugify = require('./util/slugify');
const firstline = require('./util/firstline');

const Promise = require('bluebird');
const writeFile = Promise.promisify(require('fs').writeFile);
const readFile = Promise.promisify(require('fs').readFile);
const access = Promise.promisify(require('fs').access);
const readdir = Promise.promisify(require('fs').readdir);
const readdirSync = require('fs').readdirSync;
const mkdirp = Promise.promisify(require('mkdirp'));
const async = require('async');
const EOL = "\n";
const isJSON = require('is-json');
const _ = require('lodash');

var convodir = './spec/convo/';
var suffix = '.convo.txt';

function readConvos() {
  return new Promise(function(readConvosResolve, readConvosReject) {
    access(convodir).then(() => {
      readdir(convodir).then(function(filenames) {
        
        var convos = [];
        
        async.each(filenames, 
          (filename, callback) => {
            if (!filename.endsWith(suffix)) {
              callback();
              return;
            }
            firstline.firstline(convodir + filename).then(
              (header) => {
                convos.push(createConvoEntry(filename, header));
                callback();
              }).catch(
              (err) => {
                log.warn('error reading first line of file ' + filename + ': ' + err);
                callback();
              });
          },
          (err) => {
            if (err) {
{'fraction_non_alphanumeric': 0.15559157212317667, 'fraction_numerical': 0.0, 'mean_word_length': 2.4674157303370787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7521136', 'n_tokens_mistral': 895, 'n_tokens_neox': 872, 'n_words': 177}