<commit_before>from collections import OrderedDict


class Endpoints:
    host = 'http://cobalt.qas.im/api/1.0'

    @staticmethod
    def run(api, get, endpoint=None, params=None, map=None):
        endpoint = Endpoints._parse_endpoint(endpoint)

        url, params = Endpoints._parse_url(api, endpoint, params), \
            Endpoints._parse_params(endpoint, params, map)

        return get(url=url, params=params)

    @staticmethod
    def _process_filter(queries, map):
        # [[(), (), ()], [(), (), ()], [()]])

        if (type(queries) == str):
            return queries

        a = []
        for filter in queries:
            o = []
            for k, v in filter:
                if k.lower() not in map:
                    continue
                o.append('%s:%s' % (k, v))
            a.append(' OR '.join(o))
        return ' AND '.join(a)

    @staticmethod
    def _parse_endpoint(endpoint):
        if not endpoint:
            endpoint = ''

        endpoint = endpoint.lower().strip().replace('/', '')
        return endpoint if endpoint in ('search', 'filter') else ''

    @staticmethod
    def _parse_url(api, endpoint, params):
        url = '%s/%s/%s' % (Endpoints.host, api, endpoint)

        if endpoint == '' and ('date' in params or 'id' in params):
            url += params['id'] if 'id' in params else params['date']
<commit_msg>Default map to None incase unavailable (i.e. food)<commit_after>from collections import OrderedDict


class Endpoints:
    host = 'http://cobalt.qas.im/api/1.0'

    @staticmethod
    def run(api, get, endpoint=None, params=None, map=None):
        endpoint = Endpoints._parse_endpoint(endpoint)

        url, params = Endpoints._parse_url(api, endpoint, params), \
            Endpoints._parse_params(endpoint, params, map)

        return get(url=url, params=params)

    @staticmethod
    def _process_filter(queries, map=None):
        # [[(), (), ()], [(), (), ()], [()]])

        if (type(queries) == str):
            return queries

        a = []
        for filter in queries:
            o = []
            for k, v in filter:
                if map and k.lower() not in map:
                    continue
                o.append('%s:%s' % (k, v))
            a.append(' OR '.join(o))
        return ' AND '.join(a)

    @staticmethod
    def _parse_endpoint(endpoint):
        if not endpoint:
            endpoint = ''

        endpoint = endpoint.lower().strip().replace('/', '')
        return endpoint if endpoint in ('search', 'filter') else ''

    @staticmethod
    def _parse_url(api, endpoint, params):
        url = '%s/%s/%s' % (Endpoints.host, api, endpoint)

        if endpoint == '' and ('date' in params or 'id' in params):
            url += params['id'] if 'id' in params else params['date']

{'fraction_non_alphanumeric': 0.1484263233190272, 'fraction_numerical': 0.001430615164520744, 'mean_word_length': 2.0803964757709252, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '223794', 'n_tokens_mistral': 837, 'n_tokens_neox': 804, 'n_words': 225}