<commit_before>#  Copyright 2008-2009 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os
import copy
import operator

from robotide import utils
from robotide.spec import UserKeywordContent, VariableSpec
from robotide.errors import NoRideError, DataError, SerializationError
from robotide.publish import RideSaving, RideSaved
from robotide.robotapi import TestSuiteData, ResourceFileData, InitFileData,\
    UserErrorHandler
from robotide.writer import FileWriter

from tables import InitFileSettingTable, SuiteSettingTable,\
        ResourceSettingTable, VariableTable, TestCaseTable, UserKeywordTable
from cache import LIBRARYCACHE, VARIABLEFILECACHE, RESOURCEFILECACHE
from tcuk import UserKeyword


def TestSuiteFactory(path):
    if not os.path.exists(path):
        if '__init__' in path:
            return _TestSuiteFactory(_EmptyInitFile(path))
        return _TestSuiteFactory(_EmptyTestSuite(path))
    if _is_empty_dir(path):
        return _TestSuiteFactory(_EmptyInitFile(path))
    if _is_empty_file(path):
        return _TestSuiteFactory(_EmptyTestSuite(path))
    return _TestSuiteFactory(TestSuiteData(path))

def _is_empty_dir(path):
    return os.path.isdir(path) and not os.listdir(path)

def _is_empty_file(path):
    return os.path.isfile(path) and os.path.getsize(path) == 0

def _TestSuiteFactory(data, parent=None):
    if hasattr(data, 'initfile'):
        return InitFile(data, parent)
    return TestCaseFile(data, parent)


def ResourceFileFactory(path, create_empty=True):
    if path and os.path.isfile(path):
        data = ResourceFileData(path)
    elif create_empty:
        data = _EmptyResourceFile(path)
    else:
        return None
    return ResourceFile(data)


class _AbstractDataFile(object):
    imports = property(lambda self: self.settings.imports)
    is_directory_suite = False

    def __init__(self, data):
        self.source = data.source.decode('UTF-8')
        self._stat = self._get_stat(self.source)
        self.variables = VariableTable(self, data.variables)
        kws = [ kw for kw in data.user_keywords
                if not isinstance(kw, UserErrorHandler)]
        self.keywords = UserKeywordTable(self, kws)
        self.dirty = False
        self.suites = []
        self.tests = []
        self.datafile = self # Needed by editors that edit suites and tc/uks

    def _get_stat(self, path):
        if os.path.isfile(path):
            stat = os.stat(path)
            return (stat.st_mtime, stat.st_size)
        return (0, 0)

    def get_resources(self):
        resources = []
        for name in self.get_resource_imports():
            #TODO: Extract variable resolving
            name = self.variables.replace_scalar(name)
            for res in resources:
                name = res.variables.replace_scalar(name)
            resource = RESOURCEFILECACHE.get_resource_file(self.source, name)
            if resource:
                resources.append(resource)
        return resources

    def set_dirty(self):
        self.dirty = True

    def get_user_keyword(self, name):
        kws = self._filter(self.get_user_keywords(), name)
        return kws and kws[0] or None

    def get_user_keywords(self):
        kws = copy.copy(self.keywords)
        for res in self.get_resources():
            kws.extend(res.get_user_keywords())
        return kws

    def get_keywords(self):
        return self._get_keywords(self.name)

    def content_assist_values(self):
        return self._sort(self._get_keywords('<this file>'))

    def get_keyword_details(self, name):
        kws = self._filter(self.content_assist_values(), name)
        return kws and kws[0].get_details() or None

    def is_library_keyword(self, name):
        kws = self._filter(self.content_assist_values(), name)
        return kws and kws[0].is_library_keyword() or False

    def _get_keywords(self, source_for_own_kws):
        kws =  self._get_own_keywords(source_for_own_kws) + \
               self.imports.get_keywords() + \
               LIBRARYCACHE.get_default_keywords()# TODO: should be elsewhere
        return self._remove_duplicates(kws)

    def _get_own_keywords(self, source):
        return [ UserKeywordContent(kw, source, self.type) for kw in self.keywords ]

    def _sort(self, keywords):
        keywords.sort(key=operator.attrgetter('name'))
        return keywords

    def _filter(self, keywords, name):
        if name is not None:
            keywords = [ kw for kw in keywords if utils.eq(kw.name, name) or
                                                  utils.eq(kw.longname, name) ]
        return keywords

    def _remove_duplicates(self, keywords):
        return list(set(keywords))

    def replace_variables(self, value):
        value = self.variables.replace_scalar(value)
        for res in self.get_resources():
            value = res.replace_variables(value)
        for vars in self._get_variable_files():
            try:
                value = vars.replace_scalar(value)
            except DataError:
                pass
        return value

    def get_variables_for_content_assist(self):
        return [ VariableSpec(self.name, var) for var in self.variables ] +\
                 self._get_resource_variables() + self._get_variable_file_variables()

    def validate_keyword_name(self, value):
        return self.keywords.validate_name(value)

    def _get_resource_variables(self):
        vars = []
        for resource in self.get_resources():
            vars.extend(resource.get_variables_for_content_assist())
        return vars

    def _get_variable_file_variables(self):
        vars = []
        for varfile in self._get_variable_files():
            vars.extend([ VariableSpec(varfile.source, var)
                         for var in varfile.keys() ])
        return vars<commit_msg>renamed method for clarity<commit_after>#  Copyright 2008-2009 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os
import copy
import operator

from robotide import utils
from robotide.spec import UserKeywordContent, VariableSpec
from robotide.errors import NoRideError, DataError, SerializationError
from robotide.publish import RideSaving, RideSaved
from robotide.robotapi import TestSuiteData, ResourceFileData, InitFileData,\
    UserErrorHandler
from robotide.writer import FileWriter

from tables import InitFileSettingTable, SuiteSettingTable,\
        ResourceSettingTable, VariableTable, TestCaseTable, UserKeywordTable
from cache import LIBRARYCACHE, VARIABLEFILECACHE, RESOURCEFILECACHE
from tcuk import UserKeyword


def TestSuiteFactory(path):
    if not os.path.exists(path):
        if '__init__' in path:
            return _TestSuiteFactory(_EmptyInitFile(path))
        return _TestSuiteFactory(_EmptyTestSuite(path))
    if _is_empty_dir(path):
        return _TestSuiteFactory(_EmptyInitFile(path))
    if _is_empty_file(path):
        return _TestSuiteFactory(_EmptyTestSuite(path))
    return _TestSuiteFactory(TestSuiteData(path))

def _is_empty_dir(path):
    return os.path.isdir(path) and not os.listdir(path)

def _is_empty_file(path):
    return os.path.isfile(path) and os.path.getsize(path) == 0

def _TestSuiteFactory(data, parent=None):
    if hasattr(data, 'initfile'):
        return InitFile(data, parent)
    return TestCaseFile(data, parent)


def ResourceFileFactory(path, create_empty=True):
    if path and os.path.isfile(path):
        data = ResourceFileData(path)
    elif create_empty:
        data = _EmptyResourceFile(path)
    else:
        return None
    return ResourceFile(data)


class _AbstractDataFile(object):
    imports = property(lambda self: self.settings.imports)
    is_directory_suite = False

    def __init__(self, data):
        self.source = data.source.decode('UTF-8')
        self._stat = self._get_stat(self.source)
        self.variables = VariableTable(self, data.variables)
        kws = [ kw for kw in data.user_keywords
                if not isinstance(kw, UserErrorHandler)]
        self.keywords = UserKeywordTable(self, kws)
        self.dirty = False
        self.suites = []
        self.tests = []
        self.datafile = self # Needed by editors that edit suites and tc/uks

    def _get_stat(self, path):
        if os.path.isfile(path):
            stat = os.stat(path)
            return (stat.st_mtime, stat.st_size)
        return (0, 0)

    def get_resources(self):
        resources = []
        for name in self.get_resource_imports():
            #TODO: Extract variable resolving
            name = self.variables.replace_scalar(name)
            for res in resources:
                name = res.variables.replace_scalar(name)
            resource = RESOURCEFILECACHE.get_resource_file(self.source, name)
            if resource:
                resources.append(resource)
        return resources

    def set_dirty(self):
        self.dirty = True

    def get_user_keyword(self, name):
        kws = self._filter(self.get_user_keywords(), name)
        return kws and kws[0] or None

    def get_user_keywords(self):
        kws = copy.copy(self.keywords)
        for res in self.get_resources():
            kws.extend(res.get_user_keywords())
        return kws

    def get_keywords(self):
        return self._get_keywords(self.name)

    def content_assist_values(self):
        return self._sort(self._get_keywords('<this file>'))

    def get_keyword_details(self, name):
        kws = self._filter(self.content_assist_values(), name)
        return kws and kws[0].get_details() or None

    def is_library_keyword(self, name):
        kws = self._filter(self.content_assist_values(), name)
        return kws and kws[0].is_library_keyword() or False

    def _get_keywords(self, source_for_own_kws):
        kws =  self._get_own_keywords(source_for_own_kws) + \
               self.imports.get_keywords() + \
               LIBRARYCACHE.get_default_keywords()# TODO: should be elsewhere
        return self._remove_duplicates(kws)

    def _get_own_keywords(self, source):
        return [ UserKeywordContent(kw, source, self.type) for kw in self.keywords ]

    def _sort(self, keywords):
        keywords.sort(key=operator.attrgetter('name'))
        return keywords

    def _filter(self, keywords, name):
        if name is not None:
            keywords = [ kw for kw in keywords if utils.eq(kw.name, name) or
                                                  utils.eq(kw.longname, name) ]
        return keywords

    def _remove_duplicates(self, keywords):
        return list(set(keywords))

    def replace_variables(self, value):
        value = self.variables.replace_scalar(value)
        for res in self.get_resources():
            value = res.replace_variables(value)
        for vars in self._get_variable_files():
            try:
                value = vars.replace_scalar(value)
            except DataError:
                pass
        return value

    def get_variables(self):
        return [ VariableSpec(self.name, var) for var in self.variables ] +\
                 self._get_resource_variables() + self._get_variable_file_variables()

    def validate_keyword_name(self, value):
        return self.keywords.validate_name(value)

    def _get_resource_variables(self):
        vars = []
        for resource in self.get_resources():
            vars.extend(resource.get_variables())
        return vars

    def _get_variable_file_variables(self):
        vars = []
        for varfile in self._get_variable_files():
            vars.extend([ VariableSpec(varfile.source, var)
                         for var in varfile.keys() ])
        return vars
{'fraction_non_alphanumeric': 0.0841025641025641, 'fraction_numerical': 0.002998027613412229, 'mean_word_length': 2.808894230769231, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6931241', 'n_tokens_mistral': 3685, 'n_tokens_neox': 3558, 'n_words': 954}