<commit_before>const fs = require('fs');
const _ = require('lodash');

import clean from './clean';
import words from './../../data/words.json';
import aftonbladet from './../../data/aftonbladet.json';
import expressen from './../../data/expressen.json';

const match = (source) => {

  let results = [];
  let obj;

  words.forEach(function(val, index, arr) {
    obj = _(source == 'aftonbladet' ? aftonbladet : expressen)
      .filter(function(s) {
        if (s.title) {
          // s.title = clean(s.title);
          return JSON.stringify(s).includes(val)
        }
      }).value();
    results = results.concat(obj);
  });

  fs.writeFileSync(`../data/${source.substring(0,2)}.json`, JSON.stringify(results), 'utf-8');

}

export default match;<commit_msg>Clean titles<commit_after>const fs = require('fs');
const _ = require('lodash');

import clean from './clean';
import words from './../../data/words.json';
import aftonbladet from './../../data/aftonbladet.json';
import expressen from './../../data/expressen.json';

const match = (source) => {

  let results = [];
  let obj;

  words.forEach(function(val, index, arr) {
    obj = _(source == 'aftonbladet' ? aftonbladet : expressen)
      .filter(function(s) {
        if (s.title) {
          s.title = clean(s.title);
          return JSON.stringify(s).includes(val)
        }
      }).value();
    results = results.concat(obj);
  });

  fs.writeFileSync(`../data/${source.substring(0,2)}.json`, JSON.stringify(results), 'utf-8');

}

export default match;
{'fraction_non_alphanumeric': 0.19790301441677588, 'fraction_numerical': 0.003931847968545216, 'mean_word_length': 2.885496183206107, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3923874', 'n_tokens_mistral': 505, 'n_tokens_neox': 497, 'n_words': 104}