<commit_before>/* eslint-disable no-console */
/**
 * Lemmatizer for English using TreeTagger
 */
import TreeTagger from 'treetagger';
import path from 'path';

// This env. variable will be used by treetagger package.
process.env.TREETAGGER_HOME =
  path.join(__dirname, '..', '..', 'vendor', 'treetagger', 'cmd');

const tagger = new TreeTagger();

export default function (text) {
  return new Promise((resolve, reject) => {
    tagger.tag(text, (err, results) => {
      if (err) {
        reject(err);
        return;
      }

      const sentence = results.map((parsed, i) => {
        // Get a context around the processing word.
        const getPart = () => {
          const prev = results[i - 1] ? results[i - 1].t : '';
          const next = results[i + 1] ? results[i + 1].t : '';

          return `${prev} ${parsed.t} ${next}`;
        };

        let shouldSpaced = true;
        let lemma;

        // Warn if an extra period is inserted
        // because it will affect the sentence separation process.
        if (!parsed.t.includes('.') && parsed.l.includes('.')) {
          console.warn('"." is added!:', getPart());
        }

        // A word TreeTagger failed to determine the lemma is usually
        // a proper noun. But sometimes inappropriate unicode characters
        // are also reported as <unknown>, which should be removed.
        if (parsed.l === '<unknown>') {
          const isSign = /^\W+$/.test(parsed.t);

          if (isSign) {
            console.log(parsed, `sign: ${isSign}, part: ${getPart()}`);
            lemma = '';
          } else {
            lemma = parsed.t;
          }
        } else {
          switch (parsed.pos) {
            case 'CD':
              // cardinal numbers have no lemma.
              // (TreeTagger put '@CARD@' as a lemma of CD)
              lemma = parsed.t;<commit_msg>Exclude error handlings from isparta coverage<commit_after>/* eslint-disable no-console */
/**
 * Lemmatizer for English using TreeTagger
 */
import TreeTagger from 'treetagger';
import path from 'path';

// This env. variable will be used by treetagger package.
process.env.TREETAGGER_HOME =
  path.join(__dirname, '..', '..', 'vendor', 'treetagger', 'cmd');

const tagger = new TreeTagger();

export default function (text) {
  return new Promise((resolve, reject) => {
    tagger.tag(text, (err, results) => {
      /* istanbul ignore if */
      if (err) {
        reject(err);
        return;
      }

      const sentence = results.map((parsed, i) => {
        // Get a context around the processing word.
        const getPart = () => {
          const prev = results[i - 1] ? results[i - 1].t : '';
          const next = results[i + 1] ? results[i + 1].t : '';

          return `${prev} ${parsed.t} ${next}`;
        };

        let shouldSpaced = true;
        let lemma;

        // Warn if an extra period is inserted
        // because it will affect the sentence separation process.
        /* istanbul ignore if */
        if (!parsed.t.includes('.') && parsed.l.includes('.')) {
          console.warn('"." is added!:', getPart());
        }

        // A word TreeTagger failed to determine the lemma is usually
        // a proper noun. But sometimes inappropriate unicode characters
        // are also reported as <unknown>, which should be removed.
        if (parsed.l === '<unknown>') {
          const isSign = /^\W+$/.test(parsed.t);

          /* istanbul ignore if */
          if (isSign) {
            console.warn(
              'Unknown sign is found!!: ',
              parsed,
              `sign: ${isSign}, part: ${getPart()}`
            );
            lemma = '';
          } else {
            lemma = parsed.t;
          }
        } else {
          switch (parsed.pos) {
            case 'CD':
              // cardinal numbers have no lemma.
              // (TreeTagger put '@CARD@' as a lemma of CD)
              lemma = parsed.t;
{'fraction_non_alphanumeric': 0.14289346554792778, 'fraction_numerical': 0.0020340706839562673, 'mean_word_length': 1.8342939481268012, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6233796', 'n_tokens_mistral': 1147, 'n_tokens_neox': 1119, 'n_words': 356}