<commit_before>import fs from 'fs';
import * as constants from './constants/file';

export const readFile = filePath => new Promise((fulfill, reject) => {
  fs.readFile(filePath, (error, data) => {
    if (error) {
      reject(error);
    } else {
      fulfill(data);
    }
  });
});

export const sanityCheck = data => new Promise((fulfill, reject) => {
  let newLineRegex = /(\r\n|\n|\r)+/gm;
  let trimData = data.replace(newLineRegex, '');
  let cleanData = data.replace(newLineRegex, '\n');
  let lineLength = (trimData.length) % constants.LINE_LENGTH;
  let lineCount = (trimData.length) % constants.LINE_COUNT;
  if (lineLength === 0 && lineCount === 0) {
    fulfill(cleanData);
  } else {
    reject(new RangeError(`Invalid input data.`));
  }
});<commit_msg>little bit more cleanup to sanitycheck<commit_after>import fs from 'fs';
import * as constants from './constants/file';

export const readFile = filePath => new Promise((fulfill, reject) => {
  fs.readFile(filePath, (error, data) => {
    if (error) {
      reject(error);
    } else {
      fulfill(data);
    }
  });
});

export const sanityCheck = data => new Promise((fulfill, reject) => {
  let newLineRegex = /(\r\n|\n|\r)+/gm;
  let trimData = data.replace(newLineRegex, '');
  let cleanData = data.replace(newLineRegex, '\n');
  let lineLength = (trimData.length) % constants.LINE_LENGTH;
  let lineCount = (trimData.length) % constants.LINE_COUNT;
  if (cleanData.length > 0 && lineLength === 0 && lineCount === 0) {
    fulfill(cleanData);
  } else {
    reject(new Error(`Invalid input data.`));
  }
});
{'fraction_non_alphanumeric': 0.1691919191919192, 'fraction_numerical': 0.0031565656565656565, 'mean_word_length': 3.6755162241887906, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3039956', 'n_tokens_mistral': 524, 'n_tokens_neox': 518, 'n_words': 127}