<commit_before>#!/usr/bin/env python

import fileinput
from os.path import join, dirname, abspath
import sys
import os
from robot.result.datamodel import JSModelCreator


BASEDIR = dirname(abspath(__file__))
OUTPUT = join(BASEDIR, 'output.xml')

sys.path.insert(0, join(BASEDIR, '..', '..', '..', '..', 'src'))

import robot
from robot.reporting.jswriter import SeparatingWriter, DataModelWriter


def run_robot(testdata, loglevel='INFO'):
    robot.run(testdata, log='NONE', report='NONE',
              tagstatlink=['force:http://google.com:<kuukkeli&gt;',
                           'i*:http://%1/:Title of i%1'],
              tagdoc=['test:this_is_*my_bold*_test',
                      'IX:*Combined* & escaped <&lt; tag doc'],
              tagstatcombine=['fooANDi*:zap', 'i?:IX'],
              critical=[], noncritical=[], outputdir=BASEDIR, loglevel=loglevel)


def create_jsdata(outxml, target, split_log):
    result = robot.result.builders.ResultFromXml(outxml)
    visitor = JSModelCreator(split_log=split_log)
    result.visit(visitor)
    model = DataModelWriter(visitor.datamodel, visitor._context.split_results)
    model.set_settings({'logURL': 'log.html',
                        'reportURL': 'report.html',
                        'background': {'fail': 'DeepPink'}})
    with open(target, 'w') as output:
        model.write_to(output)
        for index, (keywords, strings) in enumerate(model._split_results):
            writer = SeparatingWriter(output, '')
            writer.dump_json('window.outputKeywords%d = ' % index, keywords)
            writer.dump_json('window.outputStrings%d = ' % index, strings)

def replace_all(file,searchExp,replaceExp):
    for line in fileinput.input(file, inplace=1):
        if searchExp in line:
            line = line.replace(searchExp,replaceExp)
        sys.stdout.write(line)


def create(input, target, targetName, loglevel='INFO', split_log=False):
    input = join(BASEDIR, input)
    target = join(BASEDIR, target)
    run_robot(input, loglevel)
    create_jsdata(OUTPUT, target, split_log)
    replace_all(target, 'window.output', 'window.' + targetName)
<commit_msg>working data generator<commit_after>#!/usr/bin/env python

import fileinput
from os.path import join, dirname, abspath
import sys
import os
from robot.result.datamodel import JSModelCreator
from robot.result.executionresult import ResultFromXml


BASEDIR = dirname(abspath(__file__))
OUTPUT = join(BASEDIR, 'output.xml')

sys.path.insert(0, join(BASEDIR, '..', '..', '..', '..', 'src'))

import robot
from robot.reporting.jswriter import SeparatingWriter, ScriptBlockWriter


def run_robot(testdata, loglevel='INFO'):
    robot.run(testdata, log='NONE', report='NONE',
              tagstatlink=['force:http://google.com:<kuukkeli&gt;',
                           'i*:http://%1/:Title of i%1'],
              tagdoc=['test:this_is_*my_bold*_test',
                      'IX:*Combined* & escaped <&lt; tag doc'],
              tagstatcombine=['fooANDi*:zap', 'i?:IX'],
              critical=[], noncritical=[], outputdir=BASEDIR, loglevel=loglevel)


def create_jsdata(outxml, target, split_log):
    result = ResultFromXml(outxml)
    visitor = JSModelCreator(split_log=split_log)
    result.visit(visitor)
    model = visitor.datamodel
    with open(target, 'w') as output:
        ScriptBlockWriter('\n').write_to(output, model, {'logURL': 'log.html',
                                                        'reportURL': 'report.html',
                                                        'background': {'fail': 'DeepPink'}})
        for index, (keywords, strings) in enumerate(model.split_results):
            writer = SeparatingWriter(output, '')
            writer.dump_json('window.outputKeywords%d = ' % index, keywords)
            writer.dump_json('window.outputStrings%d = ' % index, strings)

def replace_all(file,searchExp,replaceExp):
    for line in fileinput.input(file, inplace=1):
        if searchExp in line:
            line = line.replace(searchExp,replaceExp)
        sys.stdout.write(line)


def create(input, target, targetName, loglevel='INFO', split_log=False):
    input = join(BASEDIR, input)
    target = join(BASEDIR, target)
    run_robot(input, loglevel)
    create_jsdata(OUTPUT, target, split_log)
    replace_all(target, 'window.output', 'window.' + targetName)

{'fraction_non_alphanumeric': 0.13858195211786373, 'fraction_numerical': 0.001841620626151013, 'mean_word_length': 3.1145833333333335, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7655633', 'n_tokens_mistral': 1349, 'n_tokens_neox': 1340, 'n_words': 264}