<commit_before># Copyright 2016, <NAME>
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.


import logging

from collections import deque
from datetime import timedelta

from elasticsearch.helpers import scan as scanner
from elasticsearch.helpers import bulk

logger = logging.getLogger(__name__)


class Commits(object):
    def __init__(self, connector=None):
        self.es = connector.es
        self.ic = connector.ic
        self.index = connector.index
        self.dbname = 'commits'
        if not self.ic.exists_type(index=self.index,
                                   doc_type=self.dbname):
            self.mapping = {
                self.dbname: {
                    "properties": {
                        "sha": {"type": "string", "index": "not_analyzed"},
                        "author_date": {"type": "date",
                                        "format": "epoch_second"},
                        "committer_date": {"type": "date",
                                           "format": "epoch_second"},
                        "ttl": {"type": "integer",
                                "index": "not_analyzed"},
                        "author_name": {"type": "string"},
                        "committer_name": {"type": "string"},
                        "author_email": {"type": "string",
                                         "index": "not_analyzed"},
                        "committer_email": {"type": "string",
                                            "index": "not_analyzed"},
                        "projects": {"type": "string",
                                     "index": "not_analyzed"},
                        "line_modifieds": {"type": "integer",
                                           "index": "not_analyzed"},
                        "merge_commit": {"type": "boolean"},
                        "commit_msg": {"type": "string"}
                    }
                }
            }
            self.ic.put_mapping(index=self.index, doc_type=self.dbname,
                                body=self.mapping)

    def add_commits(self, source_it):
        def gen(it):
            for source in it:
                d = {}
                d['_index'] = self.index
                d['_type'] = self.dbname
                d['_op_type'] = 'create'
                d['_id'] = source['sha']
                d['_source'] = source
                yield d
        bulk(self.es, gen(source_it))
        self.es.indices.refresh(index=self.index)

    def update_commits(self, source_it):
        """ Take the sha from each doc and use
        it to reference the doc id to delete then
        create the doc. The doc needs to be already updated.
        """
        def gen(it):
            for source in it:
                d = {}
                d['_index'] = self.index
                d['_type'] = self.dbname
                d['_op_type'] = 'delete'
                d['_id'] = source['sha']
                yield d
                d = {}
                d['_index'] = self.index
                d['_type'] = self.dbname
                d['_op_type'] = 'create'
                d['_id'] = source['sha']
                d['_source'] = source
                yield d
        bulk(self.es, gen(source_it))
        self.es.indices.refresh(index=self.index)

    def get_commit(self, sha):
        try:
            res = self.es.get(index=self.index,
                              doc_type=self.dbname,
                              id=sha)
            return res['_source']
        except Exception, e:<commit_msg>Improve update phase at indexation of GIT objects<commit_after># Copyright 2016, <NAME>
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.


import logging

from collections import deque
from datetime import timedelta

from elasticsearch.helpers import scan as scanner
from elasticsearch.helpers import bulk

logger = logging.getLogger(__name__)


class Commits(object):
    def __init__(self, connector=None):
        self.es = connector.es
        self.ic = connector.ic
        self.index = connector.index
        self.dbname = 'commits'
        if not self.ic.exists_type(index=self.index,
                                   doc_type=self.dbname):
            self.mapping = {
                self.dbname: {
                    "properties": {
                        "sha": {"type": "string", "index": "not_analyzed"},
                        "author_date": {"type": "date",
                                        "format": "epoch_second"},
                        "committer_date": {"type": "date",
                                           "format": "epoch_second"},
                        "ttl": {"type": "integer",
                                "index": "not_analyzed"},
                        "author_name": {"type": "string"},
                        "committer_name": {"type": "string"},
                        "author_email": {"type": "string",
                                         "index": "not_analyzed"},
                        "committer_email": {"type": "string",
                                            "index": "not_analyzed"},
                        "projects": {"type": "string",
                                     "index": "not_analyzed"},
                        "line_modifieds": {"type": "integer",
                                           "index": "not_analyzed"},
                        "merge_commit": {"type": "boolean"},
                        "commit_msg": {"type": "string"}
                    }
                }
            }
            self.ic.put_mapping(index=self.index, doc_type=self.dbname,
                                body=self.mapping)

    def add_commits(self, source_it):
        def gen(it):
            for source in it:
                d = {}
                d['_index'] = self.index
                d['_type'] = self.dbname
                d['_op_type'] = 'create'
                d['_id'] = source['sha']
                d['_source'] = source
                yield d
        bulk(self.es, gen(source_it))
        self.es.indices.refresh(index=self.index)

    def update_commits(self, source_it, field='projects'):
        """ Take the sha from each doc and use
        it to reference the doc to update. This method only
        support updating a single field for now. The default one
        is projects because that's the only one to make sense in
        this context.
        """
        def gen(it):
            for source in it:
                d = {}
                d['_index'] = self.index
                d['_type'] = self.dbname
                d['_op_type'] = 'update'
                d['_id'] = source['sha']
                d['_source'] = {'doc': {field: source[field]}}
                yield d
        bulk(self.es, gen(source_it))
        self.es.indices.refresh(index=self.index)

    def get_commit(self, sha):
        try:
            res = self.es.get(index=self.index,
                              doc_type=self.dbname,
                              id=sha)
            return res['_source']
        except Exception, e:
{'fraction_non_alphanumeric': 0.1118803313141303, 'fraction_numerical': 0.0019779948077636297, 'mean_word_length': 1.3927832002366163, 'pattern_counts': {'":': 66, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6410821', 'n_tokens_mistral': 2092, 'n_tokens_neox': 2001, 'n_words': 572}