<commit_before>var parser = require('js-yaml')
var optionalByteOrderMark = '\\ufeff?'
var platform = typeof process !== 'undefined' ? process.platform : ''
var pattern = '^(' +
  optionalByteOrderMark +
  '(= yaml =|---)' +
  '$([\\s\\S]*?)' +
  '^(?:\\2|\\.\\.\\.)\\s*' +
  '$' +
  (platform === 'win32' ? '\\r?' : '') +
  '(?:\\n)?)'
// NOTE: If this pattern uses the 'g' flag the `regex` variable definition will
// need to be moved down into the functions that use it.
var regex = new RegExp(pattern, 'm')

module.exports = extractor
module.exports.test = test

function extractor (string, options) {
  string = string || ''
  var defaultOptions = { allowUnsafe: false }
  options = options instanceof Object ? {...defaultOptions, ...options} : defaultOptions
  options.allowunsafe = Boolean(options.allowUnsafe)
  var lines = string.split(/(\r?\n)/)
  if (lines[0] && /= yaml =|---/.test(lines[0])) {
    return parse(string, options.allowUnsafe)
  } else {
    return {
      attributes: {},
      body: string,
      bodyBegin: 1
    }<commit_msg>Fix case of `allowUnsafe` in boolean coercion<commit_after>var parser = require('js-yaml')
var optionalByteOrderMark = '\\ufeff?'
var platform = typeof process !== 'undefined' ? process.platform : ''
var pattern = '^(' +
  optionalByteOrderMark +
  '(= yaml =|---)' +
  '$([\\s\\S]*?)' +
  '^(?:\\2|\\.\\.\\.)\\s*' +
  '$' +
  (platform === 'win32' ? '\\r?' : '') +
  '(?:\\n)?)'
// NOTE: If this pattern uses the 'g' flag the `regex` variable definition will
// need to be moved down into the functions that use it.
var regex = new RegExp(pattern, 'm')

module.exports = extractor
module.exports.test = test

function extractor (string, options) {
  string = string || ''
  var defaultOptions = { allowUnsafe: false }
  options = options instanceof Object ? {...defaultOptions, ...options} : defaultOptions
  options.allowUnsafe = Boolean(options.allowUnsafe)
  var lines = string.split(/(\r?\n)/)
  if (lines[0] && /= yaml =|---/.test(lines[0])) {
    return parse(string, options.allowUnsafe)
  } else {
    return {
      attributes: {},
      body: string,
      bodyBegin: 1
    }
{'fraction_non_alphanumeric': 0.19626168224299065, 'fraction_numerical': 0.005607476635514018, 'mean_word_length': 3.1572815533980583, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3445278', 'n_tokens_mistral': 701, 'n_tokens_neox': 694, 'n_words': 188}