<commit_before># Copyright 2016, <NAME>
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os
import re
import copy
import logging
import subprocess
import multiprocessing as mp

from pecan import conf
from pecan import configuration

from repoxplorer import index
from repoxplorer.index.tags import Tags
from repoxplorer.index.tags import PROPERTIES as T_PROPERTIES
from repoxplorer.index.commits import Commits
from repoxplorer.index.commits import PROPERTIES as C_PROPERTIES

logger = logging.getLogger(__name__)

METADATA_RE = re.compile('^([a-zA-Z-0-9_-]+):([^//].+)$')
AUTHOR_RE = re.compile('author (.*) <(.*)> (.*) (.*)')
COMMITTER_RE = re.compile('committer (.*) <(.*)> (.*) (.*)')
STATSL_RE = re.compile('(.*)\t(.*)\t(.*)')
FILE_RENAME_RE = re.compile("(.*){(.*)\s=>\s(.*)}(.*)")


def run(cmd, path):
    process = subprocess.Popen(cmd,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               cwd=path)
    out, err = process.communicate()
    if process.returncode != 0:
        logger.debug(err)
        raise Exception('%s exited with code %s' % (cmd, process.returncode))
    return out


def parse_commit_line(line, re):
    reserved_metadata_keys = C_PROPERTIES.keys() + T_PROPERTIES.keys()
    m = re.match(line)
    if m:
        key = m.groups()[0]
        if key not in reserved_metadata_keys:
            value = m.groups()[1]
            # Remove space before and after the string and remove
            # the \# that will cause trouble when metadata are queried
            # via the URL arguments
            return key.strip(), value.strip().replace('#', '')


def parse_commit_msg(msg, extra_parsers=None):
    metadatas = []
    parsers = [METADATA_RE, ]
    if extra_parsers:
        for p in extra_parsers:
            parsers.append(p)
    lines = msg.split('\n')
    subject = lines[0]
    for line in lines[1:]:
        for parser in parsers:
            metadata = parse_commit_line(line, parser)
            if metadata:
                metadatas.append(metadata)
    return subject, metadatas


def get_all_shas(path):<commit_msg>Check EL reserved fields check when adding dynamic discovered fields<commit_after># Copyright 2016, <NAME>
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os
import re
import copy
import logging
import subprocess
import multiprocessing as mp

from pecan import conf
from pecan import configuration

from repoxplorer import index
from repoxplorer.index.tags import Tags
from repoxplorer.index.tags import PROPERTIES as T_PROPERTIES
from repoxplorer.index.commits import Commits
from repoxplorer.index.commits import PROPERTIES as C_PROPERTIES

logger = logging.getLogger(__name__)

METADATA_RE = re.compile('^([a-zA-Z-0-9_-]+):([^//].+)$')
AUTHOR_RE = re.compile('author (.*) <(.*)> (.*) (.*)')
COMMITTER_RE = re.compile('committer (.*) <(.*)> (.*) (.*)')
STATSL_RE = re.compile('(.*)\t(.*)\t(.*)')
FILE_RENAME_RE = re.compile("(.*){(.*)\s=>\s(.*)}(.*)")

EL_RESERVED_FIELDS = [
    '_index',
    '_uid',
    '_type',
    '_id',
    '_source',
    '_size',
    '_all',
    '_field_names',
    '_timestamp',
    '_ttl',
    '_parent',
    '_routing',
    '_meta',
]

RESERVED_METADATA_KEYS = (
    C_PROPERTIES.keys() +
    T_PROPERTIES.keys() +
    EL_RESERVED_FIELDS
)


def run(cmd, path):
    process = subprocess.Popen(cmd,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               cwd=path)
    out, err = process.communicate()
    if process.returncode != 0:
        logger.debug(err)
        raise Exception('%s exited with code %s' % (cmd, process.returncode))
    return out


def parse_commit_line(line, re):
    m = re.match(line)
    if m:
        key = m.groups()[0]
        if key not in RESERVED_METADATA_KEYS:
            value = m.groups()[1]
            # Remove space before and after the string and remove
            # the \# that will cause trouble when metadata are queried
            # via the URL arguments
            return key.strip(), value.strip().replace('#', '')


def parse_commit_msg(msg, extra_parsers=None):
    metadatas = []
    parsers = [METADATA_RE, ]
    if extra_parsers:
        for p in extra_parsers:
            parsers.append(p)
    lines = msg.split('\n')
    subject = lines[0]
    for line in lines[1:]:
        for parser in parsers:
            metadata = parse_commit_line(line, parser)
            if metadata:
                metadatas.append(metadata)
    return subject, metadatas


def get_all_shas(path):
{'fraction_non_alphanumeric': 0.11928888888888889, 'fraction_numerical': 0.005333333333333333, 'mean_word_length': 2.773306505700872, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5900496', 'n_tokens_mistral': 1792, 'n_tokens_neox': 1724, 'n_words': 513}