<commit_before>/**
 * Created by Tommy on 1/28/2016.
 */
var async = require('async');

function indexProperty(propertyKey, propertyValue, indexedCommunities,
                       callback) {
  async.each(propertyValue, function(communityInfo, cb) {
    if (!indexedCommunities[communityInfo.communityID]) {
      var satisfyingPreference = {};
      satisfyingPreference[propertyKey] = communityInfo[propertyKey];
      indexedCommunities[communityInfo.communityID] =
          {
            communityID: communityInfo.communityID,
            name: communityInfo.name,
            latLng: communityInfo.latLng,
            radius: communityInfo.radius,
            satisfyingPreferences: satisfyingPreference
          };
    } else {
      indexedCommunities[communityInfo.communityID]
          .satisfyingPreferences[propertyKey] = communityInfo[propertyKey];
    }
    cb();
  }, function(err) {
    if (err) {
      callback(err);
    } else {
      callback(null, indexedCommunities);
    }
  });
}

function sortLengthArr(a, b) {
  var aSatisfyingPreferencesLen = Object.keys(a.satisfyingPreferences).length;
  var bSatisfyingPreferencesLen = Object.keys(b.satisfyingPreferences).length;
  if (aSatisfyingPreferencesLen < bSatisfyingPreferencesLen) {
    return 1;
  } else if (aSatisfyingPreferencesLen > bSatisfyingPreferencesLen) {
    return -1;
  } else {
    return 0;
  }
}

function removeTrailingNull(arr) {
  for (var i = 0; i < arr.length; ++i) {
    if (arr[i] === undefined) {
      return arr.slice(0, i);
    }
  }
}

function findCommunity(communities, communityID) {
  for (var i = 0; i < communities.length; ++i) {
    if (communities[i].communityID === communityID) {
      return communities[i];
    }
  }
  return undefined;
}

function getPctOfAvgFromPreferences(satisfyingPreferences, community,
                                    preference) {
  var communityAllInfo =
      findCommunity(
        satisfyingPreferences[Object.keys(community.satisfyingPreferences)[0]],
        community.communityID);
  if (communityAllInfo) {
    if (!communityAllInfo[preference]) {
      // data could not be found for community
      return -1;
    } else {
      return communityAllInfo[preference];
    }
  } else {
    return undefined;
  }
}

function addBadPreferencesCommunity(reqPreferences, satisfyingPreferences,
                                    community, callback) {
  async.forEachOf(satisfyingPreferences, function(preferenceVal, preferenceKey,
                                                  cb) {
    if (!community.satisfyingPreferences[preferenceKey]) {
      var pctOfAvg = getPctOfAvgFromPreferences(satisfyingPreferences,
                                                community, preferenceKey);
      if (!pctOfAvg) {
        return callback(community.communityID +
                        ' community: error adding bad preferences');
      }
      if (reqPreferences[preferenceKey].op === '<' &&
          pctOfAvg < 1) {
        community.satisfyingPreferences[preferenceKey] = pctOfAvg;
      } else if (reqPreferences[preferenceKey].op === '>' &&
                  pctOfAvg > 1) {
        community.satisfyingPreferences[preferenceKey] = pctOfAvg;
      } else {
        community.badPreferences[preferenceKey] = pctOfAvg;
      }
    }
    cb();
  }, function(err) {
    if (err) {
      callback(err);
    } else {
      callback(null, community);
    }
  });
}

function addBadPreferences(reqPreferences, indexedCommunities,
                           satisfyingPreferences, callback) {
  var newIndexedCommunities = [];
  async.each(indexedCommunities, function(satisfyingCommunity, cb) {
    satisfyingCommunity.badPrefences = {};
    if (Object.keys(satisfyingCommunity.satisfyingPreferences).length !==
        Object.keys(satisfyingPreferences).length) {
      addBadPreferencesCommunity(reqPreferences, satisfyingPreferences,
                                  satisfyingCommunity, function(err, result) {
        if (err) {
          console.error(err);
          newIndexedCommunities.push(satisfyingCommunity);
        } else {
          newIndexedCommunities.push(result);
          cb();
        }
      });<commit_msg>stupid bug in last commi<commit_after>/**
 * Created by Tommy on 1/28/2016.
 */
var async = require('async');

function indexProperty(propertyKey, propertyValue, indexedCommunities,
                       callback) {
  async.each(propertyValue, function(communityInfo, cb) {
    if (!indexedCommunities[communityInfo.communityID]) {
      var satisfyingPreference = {};
      satisfyingPreference[propertyKey] = communityInfo[propertyKey];
      indexedCommunities[communityInfo.communityID] =
          {
            communityID: communityInfo.communityID,
            name: communityInfo.name,
            latLng: communityInfo.latLng,
            radius: communityInfo.radius,
            satisfyingPreferences: satisfyingPreference
          };
    } else {
      indexedCommunities[communityInfo.communityID]
          .satisfyingPreferences[propertyKey] = communityInfo[propertyKey];
    }
    cb();
  }, function(err) {
    if (err) {
      callback(err);
    } else {
      callback(null, indexedCommunities);
    }
  });
}

function sortLengthArr(a, b) {
  var aSatisfyingPreferencesLen = Object.keys(a.satisfyingPreferences).length;
  var bSatisfyingPreferencesLen = Object.keys(b.satisfyingPreferences).length;
  if (aSatisfyingPreferencesLen < bSatisfyingPreferencesLen) {
    return 1;
  } else if (aSatisfyingPreferencesLen > bSatisfyingPreferencesLen) {
    return -1;
  } else {
    return 0;
  }
}

function removeTrailingNull(arr) {
  for (var i = 0; i < arr.length; ++i) {
    if (arr[i] === undefined) {
      return arr.slice(0, i);
    }
  }
}

function findCommunity(communities, communityID) {
  for (var i = 0; i < communities.length; ++i) {
    if (communities[i].communityID === communityID) {
      return communities[i];
    }
  }
  return undefined;
}

function getPctOfAvgFromPreferences(satisfyingPreferences, community,
                                    preference) {
  var communityAllInfo =
      findCommunity(
        satisfyingPreferences[Object.keys(community.satisfyingPreferences)[0]],
        community.communityID);
  if (communityAllInfo) {
    if (!communityAllInfo[preference]) {
      // data could not be found for community
      return -1;
    } else {
      return communityAllInfo[preference];
    }
  } else {
    return undefined;
  }
}

function addBadPreferencesCommunity(reqPreferences, satisfyingPreferences,
                                    community, callback) {
  async.forEachOf(satisfyingPreferences, function(preferenceVal, preferenceKey,
                                                  cb) {
    if (!community.satisfyingPreferences[preferenceKey]) {
      var pctOfAvg = getPctOfAvgFromPreferences(satisfyingPreferences,
                                                community, preferenceKey);
      if (!pctOfAvg) {
        return callback(community.communityID +
                        ' community: error adding bad preferences');
      }
      if (reqPreferences[preferenceKey].op === '<' &&
          pctOfAvg < 1) {
        community.satisfyingPreferences[preferenceKey] = pctOfAvg;
      } else if (reqPreferences[preferenceKey].op === '>' &&
                  pctOfAvg > 1) {
        community.satisfyingPreferences[preferenceKey] = pctOfAvg;
      } else {
        community.badPreferences[preferenceKey] = pctOfAvg;
      }
    }
    cb();
  }, function(err) {
    if (err) {
      callback(err);
    } else {
      callback(null, community);
    }
  });
}

function addBadPreferences(reqPreferences, indexedCommunities,
                           satisfyingPreferences, callback) {
  var newIndexedCommunities = [];
  async.each(indexedCommunities, function(satisfyingCommunity, cb) {
    satisfyingCommunity.badPreferences = {};
    if (Object.keys(satisfyingCommunity.satisfyingPreferences).length !==
        Object.keys(satisfyingPreferences).length) {
      addBadPreferencesCommunity(reqPreferences, satisfyingPreferences,
                                  satisfyingCommunity, function(err, result) {
        if (err) {
          console.error(err);
          newIndexedCommunities.push(satisfyingCommunity);
        } else {
          newIndexedCommunities.push(result);
          cb();
        }
      });
{'fraction_non_alphanumeric': 0.10182425181829022, 'fraction_numerical': 0.004053892929533802, 'mean_word_length': 2.5497249259415997, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3588130', 'n_tokens_mistral': 2374, 'n_tokens_neox': 2289, 'n_words': 412}