<commit_before>import R from 'ramda'
import _ from 'lodash-es'
import ss from 'simple-statistics'
import { buildAttributeDataFetcher } from '../../../lib/attribute/data'
import expression from '../../../lib/attribute/expression'
import { buildIndexFetcher } from '../../../lib/attribute/index'

export default function dataService($http, $q) {
  // Tests to determine if child region is in parent
  var subregionTests = (function() {
    function commonNumericPrefix(sourceRegion) {
      function removeNonNumericCharacters(s) {
        return s.replace(/[^\d]/g, '');
      }
      return _.flow(
        _.property('code'),
        removeNonNumericCharacters,
        _.partial(_.startsWith, _,
          removeNonNumericCharacters(sourceRegion.code)));
    }
    return {
      'country': {
        'state': _.constant(_.constant(true))
      },
      'state': {
        'ireg_2011': commonNumericPrefix,
        'sa4_2011': commonNumericPrefix,
        'lga_2011': commonNumericPrefix,
        'sa4_2016': commonNumericPrefix,
        'lga_2016': commonNumericPrefix
      },
      'ireg_2011': {
        'iare_2011': commonNumericPrefix
      },
      'iare_2011': {
        'iloc_2011': commonNumericPrefix
      },
      'sa4_2011': {
        'sa3_2011': commonNumericPrefix
      },
      'sa3_2011': {
        'sa2_2011': commonNumericPrefix
      },
      'sa4_2016': {
        'sa3_2016': commonNumericPrefix
      },
      'sa3_2016': {
        'sa2_2016': commonNumericPrefix
      }
    };
  }());

  // Map child back to parent
  var parentRegionTypes = _.chain(subregionTests)
    .map(function(fMap, parentType) {
      return _.map(
        fMap,
        function(f, childType) {
          return [childType, parentType];
        });
    })
    .flatten()
    .fromPairs()
    .value();

  function jsonDataFileFetcher(accessType, regionType, filename) {
    return $http
      .get(`./data/${accessType}/${regionType}/${filename}`)
      .then(R.prop('data'))
  }

  const getMetadataFromRemote = buildIndexFetcher(jsonDataFileFetcher)
  const getMetadata = R.memoize(getMetadataFromRemote);
  const getAttribute = buildAttributeDataFetcher(
    jsonDataFileFetcher,
    getCorrespondences,
    getMetadata<commit_msg>Cache attribute data in memory<commit_after>import R from 'ramda'
import _ from 'lodash-es'
import ss from 'simple-statistics'
import { buildAttributeDataFetcher } from '../../../lib/attribute/data'
import expression from '../../../lib/attribute/expression'
import { buildIndexFetcher } from '../../../lib/attribute/index'

export default function dataService($http, $q) {
  // Tests to determine if child region is in parent
  var subregionTests = (function() {
    function commonNumericPrefix(sourceRegion) {
      function removeNonNumericCharacters(s) {
        return s.replace(/[^\d]/g, '');
      }
      return _.flow(
        _.property('code'),
        removeNonNumericCharacters,
        _.partial(_.startsWith, _,
          removeNonNumericCharacters(sourceRegion.code)));
    }
    return {
      'country': {
        'state': _.constant(_.constant(true))
      },
      'state': {
        'ireg_2011': commonNumericPrefix,
        'sa4_2011': commonNumericPrefix,
        'lga_2011': commonNumericPrefix,
        'sa4_2016': commonNumericPrefix,
        'lga_2016': commonNumericPrefix
      },
      'ireg_2011': {
        'iare_2011': commonNumericPrefix
      },
      'iare_2011': {
        'iloc_2011': commonNumericPrefix
      },
      'sa4_2011': {
        'sa3_2011': commonNumericPrefix
      },
      'sa3_2011': {
        'sa2_2011': commonNumericPrefix
      },
      'sa4_2016': {
        'sa3_2016': commonNumericPrefix
      },
      'sa3_2016': {
        'sa2_2016': commonNumericPrefix
      }
    };
  }());

  // Map child back to parent
  var parentRegionTypes = _.chain(subregionTests)
    .map(function(fMap, parentType) {
      return _.map(
        fMap,
        function(f, childType) {
          return [childType, parentType];
        });
    })
    .flatten()
    .fromPairs()
    .value();

  function jsonDataFileFetcher(accessType, regionType, filename) {
    return $http
      .get(`./data/${accessType}/${regionType}/${filename}`)
      .then(R.prop('data'))
  }

  const getMetadataFromRemote = buildIndexFetcher(jsonDataFileFetcher)
  const getMetadata = R.memoize(getMetadataFromRemote);
  const getAttribute = R.memoize(
    buildAttributeDataFetcher(
      jsonDataFileFetcher,
      getCorrespondences,
      getMetadata
    )
{'fraction_non_alphanumeric': 0.13030973451327435, 'fraction_numerical': 0.034513274336283185, 'mean_word_length': 2.6110223642172525, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '348731', 'n_tokens_mistral': 1476, 'n_tokens_neox': 1393, 'n_words': 256}