<commit_before>/*
 * TSV
 * Module for parsing TSV (and eventually other formats)
 */

const normalizeEOL = str => str.replace('\r\n', '\n').replace('\r', '\n')
const isContentfulRow = row => row && !/^\s*$/.test(row)

function parseTSV(contents) {
  const content = {
    headers: [],
    rows: [],
  }
  content.rows = normalizeEOL(contents)
    .split('\n')
    .filter(isContentfulRow)
    .map(str => str.split('\t'))
  content.headers = content.rows.length ? content.rows[0] : [] 
  return content
}

export default parseTSV<commit_msg>some tsv files have &#65279 at the start of the file, remove this when checking header names<commit_after>/*
 * TSV
 * Module for parsing TSV (and eventually other formats)
 */

const normalizeEOL = str =>
  str
    .replace('\r\n', '\n')
    .replace('\r', '\n')
    .replace(/\uFEFF/g, '')
const isContentfulRow = row => row && !/^\s*$/.test(row)

function parseTSV(contents) {
  const content = {
    headers: [],
    rows: [],
  }
  console.log('------------------')
  content.rows = normalizeEOL(contents)
    .split('\n')
    .filter(isContentfulRow)
    .map(str => str.split('\t'))
  console.log(content.rows[0][0][0])
  if (content.rows[0][0][0]) {
    console.log(content.rows[0][0][0].charCodeAt())
  }
  content.headers = content.rows.length ? content.rows[0] : []
  return content
}

export default parseTSV
{'fraction_non_alphanumeric': 0.19750367107195302, 'fraction_numerical': 0.011747430249632892, 'mean_word_length': 3.1809815950920246, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4217606', 'n_tokens_mistral': 475, 'n_tokens_neox': 472, 'n_words': 105}