<commit_before>import {load as parseContent} from 'cheerio';
import {readFile, writeFile} from 'fs';
import Tag from './tag';

function input(file) {
  return new Promise((resolve, reject) => {
    readFile(file, (err, content) => {
      if (err) {
        reject(err);
      } else {
        resolve(content);
      }
    });
  });
}

function createTag(doc) {
  let name = doc.get(0).tagName;
  let template = doc.find('> template').html();
  let script = doc.find('> script').text();
  return new Tag(name, template, script);
}

function createTags($) {
  let docs = $(':root');
  docs = docs.map((index, doc) => createTag($(doc)));
  return Array.from(docs);
}

function compileTags(tags) {
  return tags.map((tag) => tag.compile());
}

function output(to) {
  return (contents) => {
    let content = contents.join('\n');
    return new Promise((resolve, reject) => {
      writeFile(to, content, (err) => {
        if (err) {
          reject(err);
        } else {
          resolve();
        }
      });
    });
  };
}

export function parseFile(file, to, options) {
  options = options || {};
  return input(file)
    .then((content) => parseContent(content, options))
    .then(createTags)
    .then(compileTags)
    .then(output(to))
    .catch((error) => console.error(error.stack));
}
<commit_msg>Added content parser to the compiler.<commit_after>import {load as loadHTML} from 'cheerio';
import {readFile, writeFile} from 'fs';
import Tag from './tag';

function input(file) {
  return new Promise((resolve, reject) => {
    readFile(file, (err, content) => {
      if (err) {
        reject(err);
      } else {
        resolve(content);
      }
    });
  });
}

function createTag(doc) {
  let name = doc.get(0).tagName;
  let template = doc.find('> template').html();
  let script = doc.find('> script').text();
  return new Tag(name, template, script);
}

function createTags($) {
  let docs = $(':root');
  docs = docs.map((index, doc) => createTag($(doc)));
  return Array.from(docs);
}

function compileTags(tags) {
  return tags.map((tag) => tag.compile());
}

function output(to) {
  return (contents) => {
    let content = contents.join('\n');
    return new Promise((resolve, reject) => {
      writeFile(to, content, (err) => err ? reject(err) : resolve());
    });
  };
}

export function parse(content, options={}) {
  let $ = loadHTML(content, options);
  let tags = createTags($);
  return compileTags(tags);
}

export function parseFile(file, to, options={}) {
  return input(file)
    .then((content) => parse(content, options))
    .then(output(to))
    .catch((error) => console.error(error.stack));
}

{'fraction_non_alphanumeric': 0.18333333333333332, 'fraction_numerical': 0.0007575757575757576, 'mean_word_length': 3.259677419354839, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2810034', 'n_tokens_mistral': 860, 'n_tokens_neox': 853, 'n_words': 209}