<commit_before>import md5 from 'md5';
import config from 'config';
import debug from 'debug';
import LRU from 'lru-cache';
import memjs from 'memjs';
import { has, get } from 'lodash';

import logger from './logger';
import models from '../models';

const debugCache = debug('cache');

const lruCache = new LRU({ max: 1000 });

const oneDayInSeconds = 60 * 60 * 24;

let memcache;
if (has(config, 'memcache.servers')) {
  debugCache('Memcache configuration detected, using memcache as cache backend.');
  const options = {};
  if (has(config, 'memcache.username') && has(config, 'memcache.password')) {
    options.username = get(config, 'memcache.username');
    options.password = get(config, 'memcache.password');
  }
  memcache = memjs.Client.create(get(config, 'memcache.servers'), options);
}

async function cacheGet(key, { unserialize = JSON.parse } = {}) {
  try {
    debugCache(`get ${key}`);
    if (memcache) {
      const data = await memcache.get(key);
      if (data.value) {
        const value = data.value.toString();
        try {
          return unserialize(value);
        } catch (err) {
          debugCache(`Invalid JSON (${value}): ${err}`);
        }
      }
    } else if (lruCache) {
      return lruCache.get(key);
    }
  } catch (err) {
    logger.warn(`Error while fetching from cache: ${err.message}`);
  }
}

async function cacheSet(key, value, maxAgeInSeconds = 0, { serialize = JSON.stringify } = {}) {
  try {
    debugCache(`set ${key}`);
    // debugCache(`set ${key} ${value}`);
    if (memcache) {
      if (value !== undefined) {
        memcache.set(key, serialize(value), { expires: maxAgeInSeconds });
      }
    } else if (lruCache) {
      lruCache.set(key, value, maxAgeInSeconds * 1000);
    }
  } catch (err) {
    logger.warn(`Error while writing to cache: ${err.message}`);
  }
}

async function cacheDel(key) {
  try {
    debugCache(`del ${key}`);
    if (memcache) {
      memcache.delete(key);
    } else if (lruCache) {
      lruCache.del(key);
    }
  } catch (err) {
    logger.warn(`Error while deleting from cache: ${err.message}`);
  }
}

async function cacheClear() {
  try {
    debugCache('clear');
    if (memcache) {
      memcache.flush();
    } else if (lruCache) {
      lruCache.reset();
    }
  } catch (err) {
    logger.warn(`Error while clearing cache: ${err.message}`);
  }
}

async function cacheHas(key) {
  try {
    debugCache(`has ${key}`);
    if (memcache) {
      const value = await memcache.get(key);
      return value !== undefined;
    } else if (lruCache) {
      return lruCache.has(key);
    }
  } catch (err) {
    logger.warn(`Error while checking from cache: ${err.message}`);
  }
  return false;
}

export async function fetchCollectiveId(collectiveSlug) {
  const cacheKey = `collective_id_with_slug_${collectiveSlug}`;
  const collectiveId = await cacheGet(cacheKey);
  if (collectiveId) {
    return collectiveId;
  }
  const collective = await models.Collective.findOne({
    attributes: ['id'],
    where: { slug: collectiveSlug.toLowerCase() },
  });
  cacheSet(cacheKey, collective.id, oneDayInSeconds);
  return collective.id;
}

export function memoize(func, { key, maxAge = 0, serialize, unserialize }) {
  const cacheKey = args => {
    return args.length ? `${key}_${md5(JSON.stringify(args))}` : key;
  };

  const memoizedFunction = async function() {
    let value = await cacheGet(cacheKey(arguments), { unserialize });
    if (value === undefined) {
      value = await func(...arguments);
      cacheSet(cacheKey(arguments), value, maxAge, { serialize });
    }
    return value;
  };

  memoizedFunction.refresh = async function() {
    const value = await func(...arguments);
    cacheSet(cacheKey(arguments), value, maxAge, { serialize });
  };

  memoizedFunction.clear = async function() {
    cacheDel(cacheKey(arguments));
  };

  return memoizedFunction;
}

export default {
  get: cacheGet,
  set: cacheSet,
  del: cacheDel,
  has: cacheHas,
  clear: cacheClear,
};<commit_msg>refactor(cache): Use cacheProvider abstraction when instantiating<commit_after>import md5 from 'md5';
import { getProvider } from './cacheProvider';
import models from '../models';

const cache = getProvider();
const oneDayInSeconds = 60 * 60 * 24;

export async function fetchCollectiveId(collectiveSlug) {
  const cacheKey = `collective_id_with_slug_${collectiveSlug}`;
  const collectiveId = await cache.get(cacheKey);
  if (collectiveId) {
    return collectiveId;
  }
  const collective = await models.Collective.findOne({
    attributes: ['id'],
    where: { slug: collectiveSlug.toLowerCase() },
  });
  cache.set(cacheKey, collective.id, oneDayInSeconds);
  return collective.id;
}

export function memoize(func, { key, maxAge = 0, serialize, unserialize }) {
  const cacheKey = args => {
    return args.length ? `${key}_${md5(JSON.stringify(args))}` : key;
  };

  const memoizedFunction = async function() {
    let value = await cache.get(cacheKey(arguments), { unserialize });
    if (value === undefined) {
      value = await func(...arguments);
      cache.set(cacheKey(arguments), value, maxAge, { serialize });
    }
    return value;
  };

  memoizedFunction.refresh = async function() {
    const value = await func(...arguments);
    cache.set(cacheKey(arguments), value, maxAge, { serialize });
  };

  memoizedFunction.clear = async function() {
    cache.del(cacheKey(arguments));
  };

  return memoizedFunction;
}

export default cache;
{'fraction_non_alphanumeric': 0.14784995425434583, 'fraction_numerical': 0.005306495882891125, 'mean_word_length': 3.4987654320987653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4375554', 'n_tokens_mistral': 1751, 'n_tokens_neox': 1732, 'n_words': 430}