<commit_before># Copyright (c) 2014 Scopely, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

import time
import logging
import os

import elasticsearch
import skew


LOG = logging.getLogger(__name__)


class Skewer(object):

    def __init__(self, host, port):
        self.host = host
        self.port = port
        self.ts = int(time.time())
        self.es = elasticsearch.Elasticsearch(
            hosts=[{'host': host, 'port': port}])

    def _index_name(self):
        return 'skewer-%d' % self.ts

    def create_template(self):
        template_path = os.path.join(
            os.path.dirname(__file__),
            'skewer-es-template.json'
        )
        with open(template_path, 'r') as fh:
            template_body = fh.read()
        self.es.indices.put_template(
            name="skewer",
            body=template_body
        )

    def clear_index(self):
        LOG.debug('Deleting existing indices')
        self.es.indices.delete('skewer-*')

    def index_aws(self, arn_pattern='arn:aws:*:*:*:*/*'):
        self.create_template()
        all_services = set()
        all_regions = set()
        all_accounts = set()

        new_index_name = self._index_name()

        LOG.debug('using ARN: %s', arn_pattern)

        i = 0
        arn = skew.scan(arn_pattern)

        for resource in arn:
            _, _, service, region, acct_id, _ = str(resource).split(':', 5)
            resource.data['service'] = service
            resource.data['region'] = region
            resource.data['account_id'] = acct_id
            resource.data['arn'] = resource.arn
            all_services.add(service)
            all_regions.add(region)
            all_accounts.add(acct_id)
            self.es.index(new_index_name, doc_type=resource.resourcetype,
                          id=str(resource), body=resource.data)
            i += 1
            LOG.debug('indexed %s', resource.arn)

        # Delete old indexes if they exist and create new aliases
        if self.es.indices.exists(['skewer']):
            self.es.indices.delete(['skewer'])
        if self.es.indices.exists([new_index_name]):
            self.es.indices.put_alias(index=[new_index_name],
                                      name='skewer')

        # Write updated metadata to ES
        metadata = {
            'services': list(all_services),
            'regions': list(all_regions),
            'accounts': list(all_accounts)}
        self.es.index('skewer-meta', doc_type='skewermeta',
                      id='skewermeta', body=metadata)<commit_msg>Add a really simple Query class.<commit_after># Copyright (c) 2014 Scopely, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

import os

__version__ = open(os.path.join(os.path.dirname(__file__), '_version')).read()

import time
import logging

import elasticsearch
import skew

escape_chars = ['+', '-', '&&', '||', '!', '(', ')', '{', '}', '[', ']',
                '^', '"', '~', '*', '?', ':', '\\']

LOG = logging.getLogger(__name__)


class Query(object):

    def __init__(self, host, port=9200):
        self._es = elasticsearch.Elasticsearch(
            hosts=[{'host': host, 'port': port}])

    def search(self, query_string=None, body=None):
        results = self._es.search(
            q=query_string, body=body, analyze_wildcard=True)
        return results['hits']['hits']


class Skewer(object):

    def __init__(self, host, port):
        self.host = host
        self.port = port
        self.ts = int(time.time())
        self.es = elasticsearch.Elasticsearch(
            hosts=[{'host': host, 'port': port}])

    def _index_name(self):
        return 'skewer-%d' % self.ts

    def create_template(self):
        template_path = os.path.join(
            os.path.dirname(__file__),
            'skewer-es-template.json'
        )
        with open(template_path, 'r') as fh:
            template_body = fh.read()
        self.es.indices.put_template(
            name="skewer",
            body=template_body
        )

    def clear_index(self):
        LOG.debug('Deleting existing indices')
        self.es.indices.delete('skewer-*')

    def index_aws(self, arn_pattern='arn:aws:*:*:*:*/*'):
        self.create_template()
        all_services = set()
        all_regions = set()
        all_accounts = set()

        new_index_name = self._index_name()

        LOG.debug('using ARN: %s', arn_pattern)

        i = 0
        arn = skew.scan(arn_pattern)

        for resource in arn:
            _, _, service, region, acct_id, _ = str(resource).split(':', 5)
            resource.data['service'] = service
            resource.data['region'] = region
            resource.data['account_id'] = acct_id
            resource.data['arn'] = resource.arn
            all_services.add(service)
            all_regions.add(region)
            all_accounts.add(acct_id)
            self.es.index(new_index_name, doc_type=resource.resourcetype,
                          id=str(resource), body=resource.data)
            i += 1
            LOG.debug('indexed %d resources', i)

        # Delete old indexes if they exist and create new aliases
        if self.es.indices.exists(['skewer']):
            self.es.indices.delete(['skewer'])
        if self.es.indices.exists([new_index_name]):
            self.es.indices.put_alias(index=[new_index_name],
                                      name='skewer')

        # Write updated metadata to ES
        metadata = {
            'services': list(all_services),
            'regions': list(all_regions),
            'accounts': list(all_accounts)}
        self.es.index('skewer-meta', doc_type='skewermeta',
                      id='skewermeta', body=metadata)
{'fraction_non_alphanumeric': 0.12762397322786737, 'fraction_numerical': 0.003954974140553696, 'mean_word_length': 2.267892644135189, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '722509', 'n_tokens_mistral': 1982, 'n_tokens_neox': 1908, 'n_words': 466}