<commit_before>'use strict';

/*
 * A set of functions that manipulate and read files and filepaths
 */

module.exports = {
  read: read,
  find_closest_match: find_closest_match,
  is_markdown: is_markdown,
  is_dot: is_dot,
  has_extension: has_extension
};

var grunt = require('grunt');

function read (filename, folder) {
  console.log("Reading " + filename + " from " + folder);
  if (folder) filename = find_closest_match(folder, filename);
  return grunt.file.read(filename);
}

function find_closest_match (folder, filename) {
  if (!grunt.file.isDir(folder)) grunt.warn.log("The folder " + folder + " does not exist.");

  var pattern = build_pattern(folder, filename);
  var file = grunt.file.expand(pattern);

  // Warn again if the resulting file doesn't exist
  if (!file.length) grunt.warn.log("The file " + pattern + " cannot be found.");

  return file[0];
};

function build_pattern (folder, filename) {
  var separator = '';
  if (folder) {
    separator = folder.length ? '/' : '';
  }
  var extension = has_extension(filename) ? '' : '.*';
  return folder + separator + filename + extension;
}

function is_markdown (file) {
  return /\.md$|\.md\..*$/.test(file);
};

function is_dot (file) {
  return /\.dot.html$/.test(file.toString());
};

function has_extension (file) {
  return file.toString().match(/\.[0-9a-z]+$/i);
};<commit_msg>Correct use of grunt warn. More comprehensive test for dot extension<commit_after>'use strict';

/*
 * A set of functions that manipulate and read files and filepaths
 */

module.exports = {
  read: read,
  find_closest_match: find_closest_match,
  is_markdown: is_markdown,
  is_dot: is_dot,
  has_extension: has_extension
};

var grunt = require('grunt');

function read (filename, folder) {
  if (folder) filename = find_closest_match(folder, filename);
  return grunt.file.read(filename);
}

function find_closest_match (folder, filename) {
  if (!grunt.file.isDir(folder)) grunt.warn.log("The folder " + folder + " does not exist.");

  var pattern = build_pattern(folder, filename);
  var file = grunt.file.expand(pattern);

  // Warn again if the resulting file doesn't exist
  if (!file.length) grunt.fail.warn("The file " + pattern + " cannot be found.");

  return file[0];
};

function build_pattern (folder, filename) {
  var separator = '';
  if (folder) {
    separator = folder.length ? '/' : '';
  }
  var extension = has_extension(filename) ? '' : '.*';
  return folder + separator + filename + extension;
}

function is_markdown (file) {
  return /\.md$|\.md\..*$/.test(file);
};

function is_dot (file) {
  return /\.dot$|\.dot\..*$/.test(file.toString());
};

function has_extension (file) {
  return file.toString().match(/\.[0-9a-z]+$/i);
};
{'fraction_non_alphanumeric': 0.15907421013960324, 'fraction_numerical': 0.002204261572373255, 'mean_word_length': 3.941923774954628, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3039587', 'n_tokens_mistral': 921, 'n_tokens_neox': 890, 'n_words': 232}