<commit_before>part of validator;


// convert the input to a string
String toString(input) {
  if (input == null || (input is List && input.length == 0)) {
    input = '';
  }
  return input.toString();
}


// convert the input to a date, or null if the input is not a date
DateTime toDate(String str) {
  try {
    return DateTime.parse(str);
  } catch(e) {
    return null;
  }
}


// convert the input to a float, or NaN if the input is not a float
double toFloat(String str) {
  try{
    return double.parse(str);
  } catch (e) {
    return double.NAN;
  }
}

// convert the input to a float, or NaN if the input is not a float
double toDouble(String str) {
  return toFloat(str);
}


// convert the input to an integer, or NaN if the input is not an integer
num toInt(String str, {int radix:10}) {
  try {
    return int.parse(str, radix:radix);
  } catch (e) {
    try {
      return double.parse(str).toInt();
    } catch (e) {
      return double.NAN;
    }
  }
}


// convert the input to a boolean
bool toBoolean(String str, [bool strict]) {
  if (strict == true) {
    return str == '1' || str == 'true';
  }
  return str != '0' && str != 'false' && str != '';
}


// trim characters (whitespace by default) from both sides of the input
String trim(String str, [String chars]) {
  RegExp pattern = (chars != null) ? new RegExp('^[$chars]+|[$chars]+\$') : new RegExp(r'^\s+|\s+$');
  return str.replaceAll(pattern, '');
}


// trim characters from the left-side of the input
String ltrim(String str, [String chars]) {
  var pattern = chars != null ? new RegExp('^[$chars]+') : new RegExp(r'^\s+');
  return str.replaceAll(pattern, '');
}


// trim characters from the right-side of the input
String rtrim(String str, [String chars]) {
  var pattern = chars != null ? new RegExp('[$chars]+\$') : new RegExp(r'\s+$');
  return str.replaceAll(pattern, '');
}


// remove characters that do not appear in the whitelist
String whitelist(String str, String chars) {
  return str.replaceAll(new RegExp('[^' + chars + ']+'), '');
}


// remove characters that appear in the blacklist
String blacklist(String str, String chars) {
  return str.replaceAll(new RegExp('[' + chars + ']+'), '');
}<commit_msg>implement striplow<commit_after>part of validator;


// convert the input to a string
String toString(input) {
  if (input == null || (input is List && input.length == 0)) {
    input = '';
  }
  return input.toString();
}


// convert the input to a date, or null if the input is not a date
DateTime toDate(String str) {
  try {
    return DateTime.parse(str);
  } catch(e) {
    return null;
  }
}


// convert the input to a float, or NaN if the input is not a float
double toFloat(String str) {
  try{
    return double.parse(str);
  } catch (e) {
    return double.NAN;
  }
}

// convert the input to a float, or NaN if the input is not a float
double toDouble(String str) {
  return toFloat(str);
}


// convert the input to an integer, or NaN if the input is not an integer
num toInt(String str, {int radix:10}) {
  try {
    return int.parse(str, radix:radix);
  } catch (e) {
    try {
      return double.parse(str).toInt();
    } catch (e) {
      return double.NAN;
    }
  }
}


// convert the input to a boolean
bool toBoolean(String str, [bool strict]) {
  if (strict == true) {
    return str == '1' || str == 'true';
  }
  return str != '0' && str != 'false' && str != '';
}


// trim characters (whitespace by default) from both sides of the input
String trim(String str, [String chars]) {
  RegExp pattern = (chars != null) ? new RegExp('^[$chars]+|[$chars]+\$') : new RegExp(r'^\s+|\s+$');
  return str.replaceAll(pattern, '');
}


// trim characters from the left-side of the input
String ltrim(String str, [String chars]) {
  var pattern = chars != null ? new RegExp('^[$chars]+') : new RegExp(r'^\s+');
  return str.replaceAll(pattern, '');
}


// trim characters from the right-side of the input
String rtrim(String str, [String chars]) {
  var pattern = chars != null ? new RegExp('[$chars]+\$') : new RegExp(r'\s+$');
  return str.replaceAll(pattern, '');
}


// remove characters that do not appear in the whitelist
String whitelist(String str, String chars) {
  return str.replaceAll(new RegExp('[^' + chars + ']+'), '');
}


// remove characters that appear in the blacklist
String blacklist(String str, String chars) {
  return str.replaceAll(new RegExp('[' + chars + ']+'), '');
}


// remove characters with a numerical value < 32 and 127
String stripLow(String str, [bool keep_new_lines]) {
  String chars = keep_new_lines == true ? '\x00-\x09\x0B\x0C\x0E-\x1F\x7F' : '\x00-\x1F\x7F';
  return blacklist(str, chars);
}
{'fraction_non_alphanumeric': 0.15567962207429675, 'fraction_numerical': 0.006012454369765944, 'mean_word_length': 3.3370577281191807, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5794325', 'n_tokens_mistral': 1594, 'n_tokens_neox': 1526, 'n_words': 534}