<commit_before>import {parse, getFingerprint} from './abp-filter-parser.js';
import fs from 'fs';

function discoverMatchingPrefix(bloomFilter, str, prefixLen = 8) {
  for (var i = 0; i < str.length - prefixLen + 1; i++) {
    let sub = str.substring(i, i + prefixLen);
    let cleaned = sub.replace(/^https?:\/\//, '');
    if (bloomFilter.exists(cleaned)) {
      console.log('bad-fingerprint:', sub, 'for url:', str);
    }
  }
}

let sitesToCheck = [
  'http://static.digg.com/static/fe/229ff0/images/reader/top-setting@2x.png'
];

fs.readFile('./test/data/easylist.txt', 'utf8', function (err,data) {
  if (err) {
    return console.log(err);
  }

  let parserData = {};
  parse(data, parserData);

  // Write out the bloom filter data files
  fs.writeFileSync('bloomFilterData', new Buffer(new Uint8Array(parserData.bloomFilter.toJSON())));
  fs.writeFileSync('hostBloomFilterData', new Buffer(new Uint8Array(parserData.hostBloomFilter.toJSON())));
  fs.writeFileSync('exceptionBloomFilterData', new Buffer(new Uint8Array(parserData.exceptionBloomFilter.toJSON())));

  //console.log('Number of filters processed: ', parserData.filterCount);

  console.log('-------');
  sitesToCheck.forEach(s =>
    discoverMatchingPrefix(parserData.bloomFilter, s));

  // WRite out the POD cached filter data JSM
  delete parserData.bloomFilter;
  delete parserData.hostBloomFilter;
  delete parserData.exceptionBloomFilter;
  let cachedFilterDataJSM = 'dump("######Loaded cached-rules.jsm\\n");\nthis.EXPORTED_SYMBOLS = ["parserData"];\nthis.parserData = ' + JSON.stringify(parserData) + ';\n';
  fs.writeFileSync('cachedFilterData.jsm', cachedFilterDataJSM);
});<commit_msg>Add substring not exists check and add serialize example<commit_after>import {parse, getFingerprint} from './abp-filter-parser.js';
import fs from 'fs';
import {BloomFilter} from 'bloom-filter-js';

function discoverMatchingPrefix(bloomFilter, str, prefixLen = 8) {
  if (!bloomFilter.substringExists(str, prefixLen)) {
    console.log('no substring exists for url:', str);
  }
  for (var i = 0; i < str.length - prefixLen + 1; i++) {
    let sub = str.substring(i, i + prefixLen);
    let cleaned = sub.replace(/^https?:\/\//, '');
    if (bloomFilter.exists(cleaned)) {
      console.log('bad-fingerprint:', sub, 'for url:', str);
    }
  }
}

let sitesToCheck = [
  'http://c.s-microsoft.com/en-ca/CMSImages/store_symbol.png?version=e2eecca5-4550-10c6-57b1-5114804a4c01',
];

fs.readFile('./test/data/easylist.txt', 'utf8', function (err,data) {
  if (err) {
    return console.log(err);
  }

  let parserData = {};
  parse(data, parserData);

  // Write out the bloom filter data files
  fs.writeFileSync('bloomFilterData', new Buffer(new Uint8Array(parserData.bloomFilter.toJSON())));
  fs.writeFileSync('hostBloomFilterData', new Buffer(new Uint8Array(parserData.hostBloomFilter.toJSON())));
  fs.writeFileSync('exceptionBloomFilterData', new Buffer(new Uint8Array(parserData.exceptionBloomFilter.toJSON())));

  let readData = fs.readFileSync('./bloomFilterData');
  let bloomData = new BloomFilter(new Uint8Array(readData));
  console.log(bloomData);
  let bloomFilter = new BloomFilter(bloomData);

  //console.log('Number of filters processed: ', parserData.filterCount);


  console.log('-------');
  sitesToCheck.forEach(s =>
    discoverMatchingPrefix(bloomFilter/*parserData.bloomFilter*/, s));

  // WRite out the POD cached filter data JSM
  delete parserData.bloomFilter;
  delete parserData.hostBloomFilter;
  delete parserData.exceptionBloomFilter;
  let cachedFilterDataJSM = 'dump("######Loaded cached-rules.jsm\\n");\nthis.EXPORTED_SYMBOLS = ["parserData"];\nthis.parserData = ' + JSON.stringify(parserData) + ';\n';
  fs.writeFileSync('cachedFilterData.jsm', cachedFilterDataJSM);
});
{'fraction_non_alphanumeric': 0.1596927966101695, 'fraction_numerical': 0.011122881355932203, 'mean_word_length': 4.740121580547113, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1633730', 'n_tokens_mistral': 1297, 'n_tokens_neox': 1257, 'n_words': 242}