<commit_before>'use strict';
const Text = require('./index');
const tokenize = require('./tokenize');
const Terms = require('./paths').Terms;
const normalize = require('../term/methods/normalize/normalize').normalize;
const tagArr = require('../tags');


//basically really dirty and stupid.
const normalizeLex = function(lex) {
  lex = lex || {};
  return Object.keys(lex).reduce((h, k) => {
    //add natural form
    h[k] = lex[k];
    let normal = normalize(k);
    if (k !== normal) {
      //add it too
      h[normal] = lex[k];
    }
    return h;
  }, {});
};

const extendTags = function(newTags) {
  console.log(newTags);
  console.log(tagArr);
};

const fromString = (str, lexicon, tagSet, customRules) => {
  
  let sentences = str.constructor==Array ? str : tokenize(str);

  //LS 13-03-17: include multiword lexicon entries in lexicon without white spaces or hypens
  for (var key in lexicon) {
    var noSpaceOrHypenKey = key.replace(/-|\s|[.]/g,"");

    if(noSpaceOrHypenKey != key)
    {
      lexicon[noSpaceOrHypenKey] = lexicon[key];
    }
  }

  if (tagSet) {
    extendTags(tagSet);
  }

  //make sure lexicon obeys standards
  lexicon = normalizeLex(lexicon);
  let list = sentences.map((s) => Terms.fromString(s, lexicon, skipTagging));<commit_msg>use nlp on already tokenised corpus<commit_after>'use strict';
const Text = require('./index');
const tokenize = require('./tokenize');
const Terms = require('./paths').Terms;
const normalize = require('../term/methods/normalize/normalize').normalize;
const tagArr = require('../tags');


//basically really dirty and stupid.
const normalizeLex = function(lex) {
  lex = lex || {};
  return Object.keys(lex).reduce((h, k) => {
    //add natural form
    h[k] = lex[k];
    let normal = normalize(k);
    if (k !== normal) {
      //add it too
      h[normal] = lex[k];
    }
    return h;
  }, {});
};

const extendTags = function(newTags) {
  console.log(newTags);
  console.log(tagArr);
};

const fromString = (str, lexicon, tagSet, skipTagging) => {
  
  let sentences = str !== null && str.constructor===Array ? str : tokenize(str);

  //LS 13-03-17: include multiword lexicon entries in lexicon without white spaces or hypens
  for (var key in lexicon) {
    var noSpaceOrHypenKey = key.replace(/-|\s|[.]/g,"");

    if(noSpaceOrHypenKey != key)
    {
      lexicon[noSpaceOrHypenKey] = lexicon[key];
    }
  }

  if (tagSet) {
    extendTags(tagSet);
  }

  //make sure lexicon obeys standards
  lexicon = normalizeLex(lexicon);
  let list = sentences.map((s) => Terms.fromString(s, lexicon, skipTagging));
{'fraction_non_alphanumeric': 0.16479690522243715, 'fraction_numerical': 0.0046421663442940036, 'mean_word_length': 3.383050847457627, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3667470', 'n_tokens_mistral': 916, 'n_tokens_neox': 852, 'n_words': 216}