<commit_before>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import time
from robot import utils

from robot.result import json
from robot.result.parsingcontext import TextIndex


# TODO: Rename to match the responsibility - this isn't really a model but a class writing model to outputs
class DataModel(object):

    def __init__(self, robot_data, split_results=None):
        self._robot_data = robot_data
        self._split_results = split_results
        self._settings = None
        self._set_generated(time.localtime())

    def _set_generated(self, timetuple):
        genMillis = long(time.mktime(timetuple) * 1000) -\
                        self._robot_data['baseMillis']
        self._set_attr('generatedMillis', genMillis)
        self._set_attr('generatedTimestamp',
                       utils.format_time(timetuple, gmtsep=' '))

    def _set_attr(self, name, value):
        self._robot_data[name] = value

    def set_settings(self, settings):
        self._settings = settings

    def write_to(self, output, separator='', split_threshold=9500):
        writer = _SeparatingWriter(output, separator)
        writer.write('window.output = {};\n')
        writer.separator()
        for key, value in self._robot_data.items():
            self._write_output_element(key, split_threshold, value, writer)
            writer.separator()
        writer.dump_json('window.settings = ', self._settings)

    def _write_output_element(self, key, split_threshold, value, writer):
        if key == 'suite':
            splitWriter = SplittingSuiteWriter(writer, split_threshold)
            data, mapping = splitWriter.write(self._robot_data['suite'])
            writer.dump_json('window.output["suite"] = ', data, mapping=mapping)
        elif key == 'strings':
            self._dump_and_split_strings(split_threshold, writer)
        else:
            writer.dump_json('window.output["%s"] = ' % key, value)

    def _dump_and_split_strings(self, split_threshold, writer):
        strings = self._robot_data['strings']
        writer.write('window.output["strings"] = [];\n')
        while strings:
            writer.separator()
            writer.dump_json('window.output["strings"] = window.output["strings"].concat(', strings[:split_threshold], ');\n')
            strings = strings[split_threshold:]

    def remove_keywords(self):
        self._robot_data['suite'] = self._remove_keywords_from(self._robot_data['suite'])
        self._prune_unused_indices()

    # TODO: this and remove_keywords should be removed
    # instead there should be a reportify or write_for_report_to method
    def remove_errors(self):
        self._robot_data.pop('errors')

    def _remove_keywords_from(self, data):
        if not isinstance(data, list):
            return data
        return list(self._remove_keywords_from(item) for item in data
                    if not self._is_keyword(item))

    def _is_keyword(self, item):
        # FIXME: This is a hack.
        return isinstance(item, list) and len(item) == 8 and not isinstance(item[0], TextIndex)

    def _prune_unused_indices(self):
        used = self._collect_used_indices(self._robot_data['suite'], set())
        remap = {}
        self._robot_data['strings'] = \
            list(self._prune(self._robot_data['strings'], used, remap))
        self._remap_indices(self._robot_data['suite'], remap)

    def _prune(self, data, used, index_remap, map_index=None, offset_increment=1):
        offset = 0
        for index, text in enumerate(data):
            index = map_index(index) if map_index else index
            if index in used:
                index_remap[index] = index - offset
                yield text
            else:
                offset += offset_increment

    def _remap_indices(self, data, remap):
        for i, item in enumerate(data):
            if isinstance(item, TextIndex):
                data[i] = remap[item]<commit_msg>somewhat cleaner algorithm for pruning keywords<commit_after>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import time
from robot import utils

from robot.result import json
from robot.result.parsingcontext import TextIndex


# TODO: Rename to match the responsibility - this isn't really a model but a class writing model to outputs
class DataModel(object):

    def __init__(self, robot_data, split_results=None):
        self._robot_data = robot_data
        self._split_results = split_results
        self._settings = None
        self._set_generated(time.localtime())

    def _set_generated(self, timetuple):
        genMillis = long(time.mktime(timetuple) * 1000) -\
                        self._robot_data['baseMillis']
        self._set_attr('generatedMillis', genMillis)
        self._set_attr('generatedTimestamp',
                       utils.format_time(timetuple, gmtsep=' '))

    def _set_attr(self, name, value):
        self._robot_data[name] = value

    def set_settings(self, settings):
        self._settings = settings

    def write_to(self, output, separator='', split_threshold=9500):
        writer = _SeparatingWriter(output, separator)
        writer.write('window.output = {};\n')
        writer.separator()
        for key, value in self._robot_data.items():
            self._write_output_element(key, split_threshold, value, writer)
            writer.separator()
        writer.dump_json('window.settings = ', self._settings)

    def _write_output_element(self, key, split_threshold, value, writer):
        if key == 'suite':
            splitWriter = SplittingSuiteWriter(writer, split_threshold)
            data, mapping = splitWriter.write(self._robot_data['suite'])
            writer.dump_json('window.output["suite"] = ', data, mapping=mapping)
        elif key == 'strings':
            self._dump_and_split_strings(split_threshold, writer)
        else:
            writer.dump_json('window.output["%s"] = ' % key, value)

    def _dump_and_split_strings(self, split_threshold, writer):
        strings = self._robot_data['strings']
        writer.write('window.output["strings"] = [];\n')
        while strings:
            writer.separator()
            writer.dump_json('window.output["strings"] = window.output["strings"].concat(', strings[:split_threshold], ');\n')
            strings = strings[split_threshold:]

    def remove_keywords(self):
        self._remove_keywords_from_suite(self._robot_data['suite'])
        self._prune_unused_indices()

    # TODO: this and remove_keywords should be removed
    # instead there should be a reportify or write_for_report_to method
    def remove_errors(self):
        self._robot_data.pop('errors')

    def _remove_keywords_from_suite(self, suite):
        suite[8] = []
        for subsuite in suite[6]:
            self._remove_keywords_from_suite(subsuite)
        for test in suite[7]:
            test[-1] = []

    def _prune_unused_indices(self):
        used = self._collect_used_indices(self._robot_data['suite'], set())
        remap = {}
        self._robot_data['strings'] = \
            list(self._prune(self._robot_data['strings'], used, remap))
        self._remap_indices(self._robot_data['suite'], remap)

    def _prune(self, data, used, index_remap, map_index=None, offset_increment=1):
        offset = 0
        for index, text in enumerate(data):
            index = map_index(index) if map_index else index
            if index in used:
                index_remap[index] = index - offset
                yield text
            else:
                offset += offset_increment

    def _remap_indices(self, data, remap):
        for i, item in enumerate(data):
            if isinstance(item, TextIndex):
                data[i] = remap[item]
{'fraction_non_alphanumeric': 0.10153148043108338, 'fraction_numerical': 0.005672149744753261, 'mean_word_length': 2.861585632939115, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6592873', 'n_tokens_mistral': 2537, 'n_tokens_neox': 2441, 'n_words': 657}