<commit_before>package cloudcmd.common.engine;

import cloudcmd.common.adapters.Adapter;
import cloudcmd.common.adapters.FileAdapter;
import cloudcmd.common.config.ConfigStorageService;
import org.json.JSONException;

import java.io.File;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;

public class LocalBlockCache implements BlockCache
{
  private Adapter _cacheAdapter = null;
  private Map<String, List<Adapter>> _hashProviders = null;

  public void init() throws Exception
  {
    URI adapterUri = new URI("file:///" + ConfigStorageService.instance().getConfigRoot() + File.separator + "cache");
    _cacheAdapter = new FileAdapter();
    _cacheAdapter.init(0, FileAdapter.class.getName(), new HashSet<String>(), adapterUri);
//    loadCache(Integer.MAX_VALUE);
  }

  @Override
  public void shutdown()
  {
  }

  @Override
  public Adapter getBlockCache()
  {
    return _cacheAdapter;
  }

  @Override
  public void refreshCache(int maxTier) throws Exception
  {
    List<Adapter> adapters = new ArrayList<Adapter>(ConfigStorageService.instance().getAdapters());
    adapters.add(_cacheAdapter);

    for (final Adapter adapter : adapters)
    {
      try
      {
        adapter.refreshCache();
      }
      catch (Exception e)
      {
        e.printStackTrace();
      }
    }

    loadCache(maxTier);
  }

  @Override
  public void loadCache(int maxTier) throws Exception
  {
    List<Adapter> adapters = new ArrayList<Adapter>(ConfigStorageService.instance().getAdapters());
    adapters.add(_cacheAdapter);

    _hashProviders = new HashMap<String, List<Adapter>>();

    for (final Adapter adapter : adapters)
    {
      if (adapter.Tier > maxTier) continue;

      Set<String> adapterDescription = adapter.describe();

      for (final String hash : adapterDescription)
      {
        if (!_hashProviders.containsKey(hash))
        {
          _hashProviders.put(hash, new ArrayList<Adapter>());
        }
        _hashProviders.get(hash).add(adapter);
      }
    }
  }

  @Override
  public synchronized Map<String, List<Adapter>> getHashProviders()
  {
    if (_hashProviders == null)
    {
      throw new RuntimeException("call loadCache first");<commit_msg>remove sync<commit_after>package cloudcmd.common.engine;

import cloudcmd.common.adapters.Adapter;
import cloudcmd.common.adapters.FileAdapter;
import cloudcmd.common.config.ConfigStorageService;
import org.json.JSONException;

import java.io.File;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;

public class LocalBlockCache implements BlockCache
{
  private Adapter _cacheAdapter = null;
  private Map<String, List<Adapter>> _hashProviders = null;

  public void init() throws Exception
  {
    URI adapterUri = new URI("file:///" + ConfigStorageService.instance().getConfigRoot() + File.separator + "cache");
    _cacheAdapter = new FileAdapter();
    _cacheAdapter.init(0, FileAdapter.class.getName(), new HashSet<String>(), adapterUri);
//    loadCache(Integer.MAX_VALUE);
  }

  @Override
  public void shutdown()
  {
  }

  @Override
  public Adapter getBlockCache()
  {
    return _cacheAdapter;
  }

  @Override
  public void refreshCache(int maxTier) throws Exception
  {
    List<Adapter> adapters = new ArrayList<Adapter>(ConfigStorageService.instance().getAdapters());
    adapters.add(_cacheAdapter);

    for (final Adapter adapter : adapters)
    {
      try
      {
        adapter.refreshCache();
      }
      catch (Exception e)
      {
        e.printStackTrace();
      }
    }

    loadCache(maxTier);
  }

  @Override
  public void loadCache(int maxTier) throws Exception
  {
    List<Adapter> adapters = new ArrayList<Adapter>(ConfigStorageService.instance().getAdapters());
    adapters.add(_cacheAdapter);

    _hashProviders = new HashMap<String, List<Adapter>>();

    for (final Adapter adapter : adapters)
    {
      if (adapter.Tier > maxTier) continue;

      Set<String> adapterDescription = adapter.describe();

      for (final String hash : adapterDescription)
      {
        if (!_hashProviders.containsKey(hash))
        {
          _hashProviders.put(hash, new ArrayList<Adapter>());
        }
        _hashProviders.get(hash).add(adapter);
      }
    }
  }

  @Override
  public Map<String, List<Adapter>> getHashProviders()
  {
    if (_hashProviders == null)
    {
      throw new RuntimeException("call loadCache first");
{'fraction_non_alphanumeric': 0.11244615733393788, 'fraction_numerical': 0.0004534119247336205, 'mean_word_length': 3.4431017119838874, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '937050', 'n_tokens_mistral': 1321, 'n_tokens_neox': 1270, 'n_words': 279}