<commit_before># Copyright 2017-2019 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import ast
import logging
import os
import re
import types

try:
    # pylint: disable=ungrouped-imports
    from ast import NameConstant
except ImportError:
    class FakeType(object):
        pass
    NameConstant = FakeType

log = logging.getLogger("guild")

class Script(object):

    def __init__(self, src):
        self.src = src
        self.name = _script_name(src)
        self._parsed = False
        self._imports = []
        self._calls = []
        self._params = {}

    def __lt__(self, x):
        return self.__cmp__(x) < 0

    def __cmp__(self, x):
        return (self.src > x.src) - (self.src < x.src)

    @property
    def imports(self):
        self._ensure_parsed()
        return self._imports

    @property
    def calls(self):
        self._ensure_parsed()
        return self._calls

    @property
    def params(self):
        self._ensure_parsed()
        return self._params

    def _ensure_parsed(self):
        if not self._parsed:
            try:
                parsed = ast.parse(open(self.src, "r").read())
            except SyntaxError:
                log.exception("parsing %s", self.src)
            else:
                for node in ast.walk(parsed):
                    self._apply_node(node)
            self._parsed = True

    def _apply_node(self, node):
        if isinstance(node, ast.ImportFrom):
            self._apply_import_from(node)
        elif isinstance(node, ast.Import):
            self._apply_import(node)
        elif isinstance(node, ast.Call):
            self._maybe_apply_call(node)
        elif isinstance(node, ast.Assign):
            self._apply_assign(node)

    def _apply_import_from(self, node):
        self._imports.append(node.module)

    def _apply_import(self, node):
        for name in node.names:
            if isinstance(name, ast.alias):
                self._imports.append(name.name)

    def _maybe_apply_call(self, node):
        call = Call(node)
        if call.name:
            self._calls.append(call)

    def _apply_assign(self, node):
        if node.col_offset == 0:
            self._try_apply_param(node)

    def _try_apply_param(self, node):
        try:
            val = _try_param_val(node.value)
        except TypeError:
            pass
        else:
            for target in node.targets:
                self._params[target.id] = val

def _try_param_val(val):
    if isinstance(val, ast.Num):
        return val.n
    elif isinstance(val, ast.Str):
        return val.s
    elif isinstance(val, NameConstant):
        return val.value
    elif isinstance(val, ast.Name):
        if val.id == "True":
            return True
        elif val.id == "False":
            return False
        elif val.id == "None":
            return None
    raise TypeError()

class Call(object):

    def __init__(self, node):
        self.node = node
        self.name = self._func_name(node.func)<commit_msg>Bug in reading flag values for non-name assigns<commit_after># Copyright 2017-2019 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import ast
import logging
import os
import re
import types

try:
    # pylint: disable=ungrouped-imports
    from ast import NameConstant
except ImportError:
    class FakeType(object):
        pass
    NameConstant = FakeType

log = logging.getLogger("guild")

class Script(object):

    def __init__(self, src):
        self.src = src
        self.name = _script_name(src)
        self._parsed = False
        self._imports = []
        self._calls = []
        self._params = {}

    def __lt__(self, x):
        return self.__cmp__(x) < 0

    def __cmp__(self, x):
        return (self.src > x.src) - (self.src < x.src)

    @property
    def imports(self):
        self._ensure_parsed()
        return self._imports

    @property
    def calls(self):
        self._ensure_parsed()
        return self._calls

    @property
    def params(self):
        self._ensure_parsed()
        return self._params

    def _ensure_parsed(self):
        if not self._parsed:
            try:
                parsed = ast.parse(open(self.src, "r").read())
            except SyntaxError:
                log.exception("parsing %s", self.src)
            else:
                for node in ast.walk(parsed):
                    self._safe_apply_node(node)
            self._parsed = True

    def _safe_apply_node(self, node):
        try:
            self._apply_node(node)
        except Exception as e:
            self._handle_apply_node_error(e, node)

    def _handle_apply_node_error(self, e, node):
        msg = "error applying AST node %s from %s:%s:" % (
            node.__class__.__name__,
            self.src,
            node.lineno)
        if log.getEffectiveLevel() <= logging.DEBUG:
            log.exception(msg)
        else:
            msg += " %s (use 'guild --debug ...' for more information)" % e
            log.warning(msg)

    def _apply_node(self, node):
        if isinstance(node, ast.ImportFrom):
            self._apply_import_from(node)
        elif isinstance(node, ast.Import):
            self._apply_import(node)
        elif isinstance(node, ast.Call):
            self._maybe_apply_call(node)
        elif isinstance(node, ast.Assign):
            self._apply_assign(node)

    def _apply_import_from(self, node):
        self._imports.append(node.module)

    def _apply_import(self, node):
        for name in node.names:
            if isinstance(name, ast.alias):
                self._imports.append(name.name)

    def _maybe_apply_call(self, node):
        call = Call(node)
        if call.name:
            self._calls.append(call)

    def _apply_assign(self, node):
        if node.col_offset == 0:
            self._try_apply_param(node)

    def _try_apply_param(self, node):
        try:
            val = _try_param_val(node.value)
        except TypeError:
            pass
        else:
            for target in node.targets:
                if not isinstance(target, ast.Name):
                    continue
                self._params[target.id] = val

def _try_param_val(val):
    if isinstance(val, ast.Num):
        return val.n
    elif isinstance(val, ast.Str):
        return val.s
    elif isinstance(val, NameConstant):
        return val.value
    elif isinstance(val, ast.Name):
        if val.id == "True":
            return True
        elif val.id == "False":
            return False
        elif val.id == "None":
            return None
    raise TypeError(val)

class Call(object):

    def __init__(self, node):
        self.node = node
        self.name = self._func_name(node.func)
{'fraction_non_alphanumeric': 0.09397682414363938, 'fraction_numerical': 0.0035655163631733095, 'mean_word_length': 2.1104950495049506, 'pattern_counts': {'":': 6, '<': 8, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2807538', 'n_tokens_mistral': 2283, 'n_tokens_neox': 2234, 'n_words': 644}