<commit_before>'use strict';

angular.module('memexLinkerApp')
.controller('MainCtrl', function ($scope, $http, $q, socket, lodash) {

  var _ = lodash;
//Start -- Trying to add in accordion
    $scope.oneAtATime = true;

    $scope.groups = [
    {
      title: 'Dynamic Group Header - 1',
      content: 'Dynamic Group Body - 1'
    },
    {
      title: 'Dynamic Group Header - 2',
      content: 'Dynamic Group Body - 2'
    }
    ];

    $scope.items = ['Item 1', 'Item 2', 'Item 3'];

    $scope.addItem = function() {
    var newItemNo = $scope.items.length + 1;
    $scope.items.push('Item ' + newItemNo);
    };

    $scope.status = {
    isFirstOpen: true,
    isFirstDisabled: false
    };

//End -- Trying to add in accordion
  var source_map = {
    1 : 'Backpage',
    2 : 'Craigslist',
    3 : 'Classivox',
    4 : 'MyProviderGuide',
    5 : 'NaughtyReviews',
    6 : 'RedBook',
    7 : 'CityVibe',
    8 : 'MassageTroll',
    9 : 'RedBookForum',
    10 : 'CityXGuide',
    11 : 'CityXGuideForum',
    12 : 'RubAds',
    13 : 'Anunico',
    14 : 'SipSap',
    15 : 'EscortsInCollege',
    16 : 'EscortPhoneList',
    17 : 'EroticMugshots',
    18 : 'EscortsAdsXXX',
    19 : 'EscortsinCA',
    20 : 'EscortsintheUS',
    21 : 'LiveEscortReviws',
    22 : 'MyProviderGuideForum',
    23 : 'USASexGuide',
    24 : 'EroticReview',
    25 : 'AdultSearch',
    26 : 'HappyMassage',
    27: 'UtopiaGuide',
    28 : 'MissingKids'
  };

  $scope.logo = 'http://icons.iconarchive.com/icons/icons8/ios7/256/Very-Basic-Paper-Clip-icon.png';
  $scope.blur = true;
  $scope.entities1 = [];
  // $scope.searchAggregates = {
  //   nEntites: 0,
  //   nAds: 0,
  //   nPictures: 0,
  //   phones: [],
  //   websites: [],
  //   ages: [],
  //   socialMediaAccounts: [],
  //   prices: [],
  //   cities: [],
  //   ethnicities: []
  // };

  function initAggregates() {
    // var aggregates = new HashMap();
    // aggregates.set('entityIds', []);
    // aggregates.set('websites', []);
    // aggregates.set('names', []);
    // aggregates.set('nAds', 0);
    // aggregates.set('nPictures', 0);
    // aggregates.set('phones', []);
    // aggregates.set('ages', []);
    // aggregates.set('socialMediaAccounts', []);
    // aggregates.set('cities', []);
    // aggregates.set('prices', []);
    // aggregates.set('ethnicities', []);
    // aggregates.set('twitters', []);
    // aggregates.set('instagrams', []);
    // return aggregates;
    return {
      'entityIds': [],
      'websites': [],
      'nAds': 0,
      'nPictures': 0,
      'phones': [],
      'ages': [],
      'socialMediaAccounts': [],
      'cities': [],
      'prices': [],
      'ethnicities': [],
      set: function(key, value) {
        this[key] = value
        return this
      },
      get: function(key) {
        return this[key]
      }
    };
  }

  // Temporary hashmap for computing search result aggregate statistics.
  $scope.aggregates = initAggregates();

    /*
    * Returns the set of unique, flattened items, and removes undefined values.
    */

    function uniqueFlatAndDefined(items) {
      return _.filter(_.uniq(_.flattenDeep(items)), function(item) {
        return ! _.isUndefined(item);
      });
    }

    function collectAdProperty(ads, propertyName) {
      return _.map(ads, function(ad) {
        return ad.properties[propertyName];
      });
    }<commit_msg>Fixing field names for entities<commit_after>'use strict';

angular.module('memexLinkerApp')
.controller('MainCtrl', function ($scope, $http, $q, socket, lodash) {

  var _ = lodash;
//Start -- Trying to add in accordion
    $scope.oneAtATime = true;

    $scope.groups = [
    {
      title: 'Dynamic Group Header - 1',
      content: 'Dynamic Group Body - 1'
    },
    {
      title: 'Dynamic Group Header - 2',
      content: 'Dynamic Group Body - 2'
    }
    ];

    $scope.items = ['Item 1', 'Item 2', 'Item 3'];

    $scope.addItem = function() {
    var newItemNo = $scope.items.length + 1;
    $scope.items.push('Item ' + newItemNo);
    };

    $scope.status = {
    isFirstOpen: true,
    isFirstDisabled: false
    };

//End -- Trying to add in accordion
  var source_map = {
    1 : 'Backpage',
    2 : 'Craigslist',
    3 : 'Classivox',
    4 : 'MyProviderGuide',
    5 : 'NaughtyReviews',
    6 : 'RedBook',
    7 : 'CityVibe',
    8 : 'MassageTroll',
    9 : 'RedBookForum',
    10 : 'CityXGuide',
    11 : 'CityXGuideForum',
    12 : 'RubAds',
    13 : 'Anunico',
    14 : 'SipSap',
    15 : 'EscortsInCollege',
    16 : 'EscortPhoneList',
    17 : 'EroticMugshots',
    18 : 'EscortsAdsXXX',
    19 : 'EscortsinCA',
    20 : 'EscortsintheUS',
    21 : 'LiveEscortReviws',
    22 : 'MyProviderGuideForum',
    23 : 'USASexGuide',
    24 : 'EroticReview',
    25 : 'AdultSearch',
    26 : 'HappyMassage',
    27: 'UtopiaGuide',
    28 : 'MissingKids'
  };

  $scope.logo = 'http://icons.iconarchive.com/icons/icons8/ios7/256/Very-Basic-Paper-Clip-icon.png';
  $scope.blur = true;
  $scope.entities1 = [];
  // $scope.searchAggregates = {
  //   nEntites: 0,
  //   nAds: 0,
  //   nPictures: 0,
  //   phones: [],
  //   websites: [],
  //   ages: [],
  //   socialMediaAccounts: [],
  //   prices: [],
  //   cities: [],
  //   ethnicities: []
  // };

  function initAggregates() {
    // var aggregates = new HashMap();
    // aggregates.set('entityIds', []);
    // aggregates.set('websites', []);
    // aggregates.set('names', []);
    // aggregates.set('nAds', 0);
    // aggregates.set('nPictures', 0);
    // aggregates.set('phones', []);
    // aggregates.set('ages', []);
    // aggregates.set('socialMediaAccounts', []);
    // aggregates.set('cities', []);
    // aggregates.set('prices', []);
    // aggregates.set('ethnicities', []);
    // aggregates.set('twitters', []);
    // aggregates.set('instagrams', []);
    // return aggregates;
    return {
      'entityIds': [],
      'websites': [],
      'names': [],
      'nAds': 0,
      'nPictures': 0,
      'phones': [],
      'ages': [],
      'socialMediaAccounts': [],
      'cities': [],
      'prices': [],
      'ethnicities': [],
      'twitters': [],
      'instagrams': [],
      
      set: function(key, value) {
        this[key] = value
        return this
      },
      get: function(key) {
        return this[key]
      }
    };
  }

  // Temporary hashmap for computing search result aggregate statistics.
  $scope.aggregates = initAggregates();

    /*
    * Returns the set of unique, flattened items, and removes undefined values.
    */

    function uniqueFlatAndDefined(items) {
      return _.filter(_.uniq(_.flattenDeep(items)), function(item) {
        return ! _.isUndefined(item);
      });
    }

    function collectAdProperty(ads, propertyName) {
      return _.map(ads, function(ad) {
        return ad.properties[propertyName];
      });
    }
{'fraction_non_alphanumeric': 0.18133957297455397, 'fraction_numerical': 0.01988885639075753, 'mean_word_length': 2.3263618677042803, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5196778', 'n_tokens_mistral': 2498, 'n_tokens_neox': 2326, 'n_words': 461}