<commit_before>'use strict';

var moment = require('moment-timezone');
var utils = require('./utils');
var whitelistEvents = require('./whitelistEvents');
var blacklistEvents = require('./blacklistEvents');
var overlap = require('word-overlap');
var clc = require('cli-color');
var API = {
  getFacebookEvents: require('./facebookEvents').get,
  getMeetupEvents: require('./meetupEvents').get,
  getEventbriteEvents: require('./eventbriteEvents').get,
  getIcsEvents: require('./icsEvents').get
};
var clc = require('cli-color');

function isDuplicateEvent(event1, event2) {
  var options = {
    ignoreCase: true,
    ignoreCommonWords: true,
    ignoreNumber: true,
    common: [
      'singapore',

      'meetup',
      'group',
      'event',

      'centre', 'center', 'tower', 'road',
      'avenue', 'ave',
      'building', 'city',
      'jalan', 'jln',
      'lane', 'ln',
      'street', 'st',
      'plaza',
      'town',
      'new',
      'level', 'floor',

      'first',
      'second',
      'third',

      'jan', 'january',
      'feb', 'february',
      'mar', 'march',
      'apr', 'april',
      'may',
      'jun', 'june',
      'jul', 'july',
      'aug', 'august',
      'sep', 'sept', 'september',
      'oct', 'october',
      'nov', 'november',
      'dec', 'december',
      '-',

      'topic', 'server', 'create', 'talk', 'session', 'workshop',

      'a', 'i', 'will', 'be', 'who', 'want', 'or', 'have', 'if', 'go'
    ],
    depluralize: true
  };

  var overlappedEventName = overlap(event1.name, event2.name, options);
  var overlappedEventLocation = overlap(event1.location, event2.location, options);
  var overlappedEventDescription = overlap(event1.description, event2.description, options);

  if ((event1.formatted_time === event2.formatted_time) &&
      (overlappedEventLocation.length > 0)) {
    if (overlappedEventName.length > 0 || overlappedEventDescription.length > 2) {
      console.log(clc.magenta('Info: Overlapped Event A (removed): ' + event1.url));
      console.log(clc.magenta('Info: Overlapped Event B: ' + event2.url));
      console.log(clc.magenta('Info: Overlapped Words: ' + (overlappedEventName.length + overlappedEventDescription.length)));
      return true;
    }
  }

  return false;
}

function afterToday(evt) {
  return moment(evt.formatted_time, utils.timeformat) > moment();
}

function removeDuplicates(feed) {
  var uniqueEvents = [];
  var isDuplicate;

  feed.forEach(function(thisEvent) {
    isDuplicate = uniqueEvents.some(function(thatEvent) {
      return isDuplicateEvent(thisEvent, thatEvent);<commit_msg>:racehorse: refactor: remove logging to improve performance #150<commit_after>'use strict';

var moment = require('moment-timezone');
var utils = require('./utils');
var whitelistEvents = require('./whitelistEvents');
var blacklistEvents = require('./blacklistEvents');
var overlap = require('word-overlap');
var clc = require('cli-color');
var API = {
  getFacebookEvents: require('./facebookEvents').get,
  getMeetupEvents: require('./meetupEvents').get,
  getEventbriteEvents: require('./eventbriteEvents').get,
  getIcsEvents: require('./icsEvents').get
};
var clc = require('cli-color');

function isDuplicateEvent(event1, event2) {
  var options = {
    ignoreCase: true,
    ignoreCommonWords: true,
    ignoreNumber: true,
    common: [
      'singapore',

      'meetup',
      'group',
      'event',

      'centre', 'center', 'tower', 'road',
      'avenue', 'ave',
      'building', 'city',
      'jalan', 'jln',
      'lane', 'ln',
      'street', 'st',
      'plaza',
      'town',
      'new',
      'level', 'floor',

      'first',
      'second',
      'third',

      'jan', 'january',
      'feb', 'february',
      'mar', 'march',
      'apr', 'april',
      'may',
      'jun', 'june',
      'jul', 'july',
      'aug', 'august',
      'sep', 'sept', 'september',
      'oct', 'october',
      'nov', 'november',
      'dec', 'december',
      '-',

      'topic', 'server', 'create', 'talk', 'session', 'workshop',

      'a', 'i', 'will', 'be', 'who', 'want', 'or', 'have', 'if', 'go'
    ],
    depluralize: true
  };

  var overlappedEventName = overlap(event1.name, event2.name, options);
  var overlappedEventLocation = overlap(event1.location, event2.location, options);
  var overlappedEventDescription = overlap(event1.description, event2.description, options);

  if ((event1.formatted_time === event2.formatted_time) &&
      (overlappedEventLocation.length > 0)) {
    if (overlappedEventName.length > 0 || overlappedEventDescription.length > 2) {
      console.log(clc.magenta('Info: Duplicate event removed [' + overlappedEventDescription.length + ']: ' + event1.url));
      return true;
    }
  }

  return false;
}

function afterToday(evt) {
  return moment(evt.formatted_time, utils.timeformat) > moment();
}

function removeDuplicates(feed) {
  var uniqueEvents = [];
  var isDuplicate;

  feed.forEach(function(thisEvent) {
    isDuplicate = uniqueEvents.some(function(thatEvent) {
      return isDuplicateEvent(thisEvent, thatEvent);
{'fraction_non_alphanumeric': 0.17423046566692976, 'fraction_numerical': 0.006314127861089187, 'mean_word_length': 3.450395083406497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6047198', 'n_tokens_mistral': 1685, 'n_tokens_neox': 1644, 'n_words': 348}