<commit_before>"""
Interface for configuring modules
"""

from __future__ import absolute_import, division, print_function

import logging

from statsd import StatsClient
import requests

logger = logging.getLogger('module')

class FakeStatsClient(object):
    def __init__(self, *args, **kwargs):
        self._prefix = ''
    def __getattr__(self, name):
        def foo(*args, **kwargs):
            pass
        return foo

class ElasticClient(object):
    def __init__(self, hostname, basename='iceprod'):
        self.session = requests.Session()
        # handle auth
        if '@' in hostname:
            self.session.auth = tuple(hostname.split('@')[0].split('://')[1].split(':'))
            hostname = hostname.split('://',1)[0]+'://'+hostname.split('@',1)[1]
        # try a connection
        r = self.session.get(hostname, timeout=5)
        r.raise_for_status()
        # concat hostname and basename
        self.hostname = hostname+'/'+basename+'/'
    def head(self, name, index_name):
        try:
            r = self.session.head(self.hostname+name+'/'+index_name, timeout=5)
            r.raise_for_status()
        except Exception:
            return False
        else:
            return True
    def get(self, name, index_name):
        r = self.session.get(self.hostname+name+'/'+index_name, timeout=5)
        r.raise_for_status()
        return r.json()
    def post(self, name, index_name, data):
        r = self.session.post(self.hostname+name+'/'+index_name, timeout=5)
        r.raise_for_status()
        return r.json()
    def put(self, name, index_name, data):
        r = None
        try:
            kwargs = {'timeout':5}
            if isinstance(data,dict):
                kwargs['json'] = data
            else:
                kwargs['data'] = data
            r = self.session.put(self.hostname+name+'/'+index_name, **kwargs)
            r.raise_for_status()
        except Exception:
            logger.warning('cannot put %s/%s to elasticsearch at %r', name,
                         index_name, self.hostname, exc_info=True)
            if r:
                logger.info('%r',r.content)

class module(object):<commit_msg>refactor elasticsearch to stop using mappings, and instead write to different indexes<commit_after>"""
Interface for configuring modules
"""

from __future__ import absolute_import, division, print_function

import logging

from statsd import StatsClient
import requests

logger = logging.getLogger('module')

class FakeStatsClient(object):
    def __init__(self, *args, **kwargs):
        self._prefix = ''
    def __getattr__(self, name):
        def foo(*args, **kwargs):
            pass
        return foo

class ElasticClient(object):
    def __init__(self, hostname, basename='iceprod'):
        self.session = requests.Session()
        # handle auth
        if '@' in hostname:
            self.session.auth = tuple(hostname.split('@')[0].split('://')[1].split(':'))
            hostname = hostname.split('://',1)[0]+'://'+hostname.split('@',1)[1]
        # try a connection
        r = self.session.get(hostname, timeout=5)
        r.raise_for_status()
        # concat hostname and basename
        self.hostname = hostname+'/'+basename+'_'
    def head(self, name, index_name):
        try:
            r = self.session.head(self.hostname+name+'/item/'+index_name, timeout=5)
            r.raise_for_status()
        except Exception:
            return False
        else:
            return True
    def get(self, name, index_name):
        r = self.session.get(self.hostname+name+'/item/'+index_name, timeout=5)
        r.raise_for_status()
        return r.json()
    def post(self, name, index_name, data):
        r = self.session.post(self.hostname+name+'/item/'+index_name, timeout=5)
        r.raise_for_status()
        return r.json()
    def put(self, name, index_name, data):
        r = None
        try:
            kwargs = {'timeout':5}
            if isinstance(data,dict):
                kwargs['json'] = data
            else:
                kwargs['data'] = data
            r = self.session.put(self.hostname+name+'/item/'+index_name, **kwargs)
            r.raise_for_status()
        except Exception:
            logger.warning('cannot put to elasticsearch: %s%s/%s',
                           self.hostname, name, index_name, exc_info=True)
            if r:
                logger.info('%r',r.content)

class module(object):
{'fraction_non_alphanumeric': 0.13397883359603693, 'fraction_numerical': 0.004953839225399685, 'mean_word_length': 2.263776634827333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1844337', 'n_tokens_mistral': 1302, 'n_tokens_neox': 1307, 'n_words': 294}