<commit_before>#!/usr/bin/env python

#  Copyright 2008-2009 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.


"""Tool for cleaning up Robot Framework test data.

Usage:  robotidy.py [options] inputfile outputfile
  or:   robotidy.py --inplace [options] inputfile [more input files]
  or:   robotidy.py --recursive [options] directory

Options:
 -I --inplace      Tidy given file(s) so that original file(s) are overwritten
                   When this option is used, it is possible to give multiple
                   files at once. Examples:
                   robotidy.py --inplace tests.html
                   robotidy.py --inplace --fixcomments *.html
 -R --recursive    Process given directory recursively. Files in the directory
                   are processed in place similarly as when '--inplace'
                   option is used.
 -X --fixcomments  Fix comments in the test data.
 -F --format html|tsv  
                   Format to use for output. Possible values are HTML and TSV.
                   If this option is not used, the format is got from the
                   extension of the output file.     
 -T --title text   Title to use in the test data. By default the title is got 
                   from the name of the output file. If the output file is
                   HTML the title is used with 'h1' and 'title' tags, and with
                   TSV it is simply printed before the first table. Possible
                   underscores in the given title are converted to spaces.
 -S --style path   Read styles from an external style sheet file and replace
                   default styles with them. If the path has a special value
                   'NONE', no styles are used. This setting is applicable only
                   when the output format is HTML.
 -h --help         Print this help.


This multipurpose tool has three main usages listed below.

1) Clean up the test data.

Source code created by most HTML editors is quite ugly. With this tool the 
source can be formatted nicely and it is even possible to specify a custom
style sheet. Additionally test data itself is cleaned up so that settings and
metadata is always in same predefined order.

Examples:
  robotidy.py messed_up_tests.html cleaned_tests.html 
  robotidy.py --style new_styles.css my_tests.html my_tests.html

2) Change format between HTML and TSV.

Robot Framework supports test data in HTML and TSV formats and this tools makes
changing between formats trivial. Input format is always determined from the
extension of the input file. Output format is also got from the output file
extension by default but it can also be set explicitly with '--format' option.

Examples:
  robotidy.py tests_in_html.html tests_in_tsv.tsv
  robotidy.py --format html tests.tsv tests.xxx

3) Fix comments.

Robot IDE is a great tool for editing the test data but at least currently it
totally ignores comments. This tool can be used to convert comments so that 
they are not lost when the test data is opened to the IDE. Comments in test 
case and user keyword tables are changed so that instead of '#' a built-in 
'Comment' keyword is used. In variable tables comments are converted to '@{#}'
or '${#}'. Comments in setting tables are not fixed. 

Examples:
  robotidy.py --fixcomments orig.html fixed.html
  robotidy.py --fixcomments --inplace *.html


Outputs are always written using UTF-8 encoding.
"""


import sys
import os
import glob

from robot.parsing import RawData, rawdatatables, rawdata
from robot.output import LOGGER
from robot.errors import DataError, Information
from robot.variables import is_scalar_var
from robot import utils

rawdata.PROCESS_CURDIR = False

# Rows having comment in the first cell need to be handled differently because
# otherwise they'd start a new tc or uk. Such rows are simply indented one column
# right if they are after first real tc/uk has started. If there is a comment
# before first element, a new tc/uk is created with name '#' and it's contents
# are moved to the firts real tc/uk later if comments are fixed. 
def _monkey_patched_add_row(self, name, data):
    if name.startswith('#'):
        data = [name] + data
        if self._item is not None:
            name = ''
        else:
            name = '#'  
    self._orig_add_row(name, data)

rawdatatables.ComplexTable._orig_add_row = rawdatatables.ComplexTable._add_row
rawdatatables.ComplexTable._add_row = _monkey_patched_add_row

# Order of names is important because settings/metadata are written in this
# order (except for 'Teardown' and 'Return')
_import_names = ['Variables', 'Resource', 'Library']
_setting_names = ['Documentation', 'Suite Setup', 'Suite Teardown', 'Test Setup',
                  'Test Teardown', 'Default Tags', 'Force Tags', 'Test Timeout']
_test_names = ['Documentation', 'Setup', 'Teardown', 'Tags', 'Timeout']
_keyword_names = ['Arguments', 'Documentation', 'Return', 'Timeout']

def _create_mapping(names):
    mapping = utils.NormalizedDict()
    for name in names:
        mapping[name] = name
    if 'Documentation' in names:
        mapping['Document'] = 'Documentation'
    return mapping
    
_import_map = _create_mapping(_import_names)
_setting_map = _create_mapping(_setting_names)
_test_map = _create_mapping(_test_names)
_keyword_map = _create_mapping(_keyword_names)
_valid_extensions = ['TSV','HTML','HTM','XHTML']
_default_styles = '''
<style type="text/css">
html {
  font-family: Arial,Helvetica,sans-serif;
  background-color: white;
  color: black;
}
table {
  border-collapse: collapse;
  empty-cells: show;
  margin: 1em 0em;
  border: 0.1em solid black;
}
th, td {
  border-style: solid;
  border-width: 0.05em 0.1em;
  border-color: black;
  padding: 0.1em 0.2em;
  height: 1.5em;
}
th {
  background-color: rgb(192, 192, 192);
  color: black;
  border-width: 0.1em;
  font-weight: bold;
  text-align: center;
  text-transform: capitalize;
  letter-spacing: 0.1em;
}
.col_name, .col_value {
  width: 12em;
}
td.col_name {
  background-color: rgb(240, 240, 240);
  text-transform: capitalize;
  letter-spacing: 0.1em;
}
</style>
'''

class Tidy:

    def process_file(self, infile, outfile, opts):
        if not os.path.isfile(infile):
            LOGGER.error("'%s' is not a regular file" % infile)
            return
        if outfile:
            print '%s -> %s' % (infile, outfile)
        else:
            print infile
            outfile = infile
        try:
            data = TestData(infile, opts['fixcomments'])
            data.serialize(outfile, opts['format'], opts['title'], opts['style'])
        except:
            LOGGER.error(utils.get_error_message())
            
    def process_directory(self, indir, opts):
        for name in os.listdir(indir):
            path = os.path.join(indir, name)
            if os.path.isdir(path):
                self.process_directory(path, opts)
            elif os.path.isfile(path) and self._is_valid_robot_file(name):
                self.process_file(path, None, opts)
                        
    def _is_valid_robot_file(self, name):
        base, ext = os.path.splitext(name)
        if not base or (base[0] in ['.', '_'] and base.lower() != '__init__'):
            return False
        return ext.upper()[1:] in _valid_extensions


class TestData:
    
    def __init__(self, path, fix_comments=False):
        if not os.path.isfile(path) or os.path.splitext(path)[1][1:].upper() \
                not in _valid_extensions:
            raise DataError("Input format must be either HTML or TSV.")
        raw = RawData(path, strip_comments=False)
        if raw.is_empty():
            raise DataError("'%s' contains no test data" % path)
        self.settings = Settings(raw.settings, fix_comments)
        self.variables = Variables(raw.variables, fix_comments)
        self.testcases = TestCases(raw.testcases, fix_comments)
        self.keywords = UserKeywords(raw.keywords, fix_comments)

    def serialize(self, path, format=None, title=None, style=None):
        serializer = self._get_serializer(path, format, title, style)
        self.settings.serialize(serializer)
        self.variables.serialize(serializer)
        self.testcases.serialize(serializer)
        self.keywords.serialize(serializer)
        serializer.close()
        
    def _get_serializer(self, path, format, title, style):
        if format is None:
            format = os.path.splitext(path)[1][1:]
        format = format.upper()
        if format not in _valid_extensions:
            raise DataError("Invalid output format '%s'. Only HTML and TSV "
                            "are supported." % format)<commit_msg>Changed so that extension is changed if format is changed in case --inplace or --recursive options are used. See issue 304 for more details.<commit_after>#!/usr/bin/env python

#  Copyright 2008-2009 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.


"""Tool for cleaning up Robot Framework test data.

Usage:  robotidy.py [options] inputfile outputfile
  or:   robotidy.py --inplace [options] inputfile [more input files]
  or:   robotidy.py --recursive [options] directory

Options:
 -I --inplace      Tidy given file(s) so that original file(s) are overwritten
                   When this option is used, it is possible to give multiple
                   files at once. Examples:
                   robotidy.py --inplace tests.html
                   robotidy.py --inplace --fixcomments *.html
 -R --recursive    Process given directory recursively. Files in the directory
                   are processed in place similarly as when '--inplace'
                   option is used.
 -X --fixcomments  Fix comments in the test data.
 -F --format html|tsv  
                   Format to use for output. Possible values are HTML and TSV.
                   If this option is not used, the format is got from the
                   extension of the output file.     
 -T --title text   Title to use in the test data. By default the title is got 
                   from the name of the output file. If the output file is
                   HTML the title is used with 'h1' and 'title' tags, and with
                   TSV it is simply printed before the first table. Possible
                   underscores in the given title are converted to spaces.
 -S --style path   Read styles from an external style sheet file and replace
                   default styles with them. If the path has a special value
                   'NONE', no styles are used. This setting is applicable only
                   when the output format is HTML.
 -h --help         Print this help.


This multipurpose tool has three main usages listed below.

1) Clean up the test data.

Source code created by most HTML editors is quite ugly. With this tool the 
source can be formatted nicely and it is even possible to specify a custom
style sheet. Additionally test data itself is cleaned up so that settings and
metadata is always in same predefined order.

Examples:
  robotidy.py messed_up_tests.html cleaned_tests.html 
  robotidy.py --style new_styles.css my_tests.html my_tests.html

2) Change format between HTML and TSV.

Robot Framework supports test data in HTML and TSV formats and this tools makes
changing between formats trivial. Input format is always determined from the
extension of the input file. Output format is also got from the output file
extension by default but it can also be set explicitly with '--format' option.

Examples:
  robotidy.py tests_in_html.html tests_in_tsv.tsv
  robotidy.py --format html tests.tsv tests.xxx

3) Fix comments.

Robot IDE is a great tool for editing the test data but at least currently it
totally ignores comments. This tool can be used to convert comments so that 
they are not lost when the test data is opened to the IDE. Comments in test 
case and user keyword tables are changed so that instead of '#' a built-in 
'Comment' keyword is used. In variable tables comments are converted to '@{#}'
or '${#}'. Comments in setting tables are not fixed. 

Examples:
  robotidy.py --fixcomments orig.html fixed.html
  robotidy.py --fixcomments --inplace *.html


Outputs are always written using UTF-8 encoding.
"""


import sys
import os
import glob

from robot.parsing import RawData, rawdatatables, rawdata
from robot.output import LOGGER
from robot.errors import DataError, Information
from robot.variables import is_scalar_var
from robot import utils

rawdata.PROCESS_CURDIR = False

# Rows having comment in the first cell need to be handled differently because
# otherwise they'd start a new tc or uk. Such rows are simply indented one column
# right if they are after first real tc/uk has started. If there is a comment
# before first element, a new tc/uk is created with name '#' and it's contents
# are moved to the firts real tc/uk later if comments are fixed. 
def _monkey_patched_add_row(self, name, data):
    if name.startswith('#'):
        data = [name] + data
        if self._item is not None:
            name = ''
        else:
            name = '#'  
    self._orig_add_row(name, data)

rawdatatables.ComplexTable._orig_add_row = rawdatatables.ComplexTable._add_row
rawdatatables.ComplexTable._add_row = _monkey_patched_add_row

# Order of names is important because settings/metadata are written in this
# order (except for 'Teardown' and 'Return')
_import_names = ['Variables', 'Resource', 'Library']
_setting_names = ['Documentation', 'Suite Setup', 'Suite Teardown', 'Test Setup',
                  'Test Teardown', 'Default Tags', 'Force Tags', 'Test Timeout']
_test_names = ['Documentation', 'Setup', 'Teardown', 'Tags', 'Timeout']
_keyword_names = ['Arguments', 'Documentation', 'Return', 'Timeout']

def _create_mapping(names):
    mapping = utils.NormalizedDict()
    for name in names:
        mapping[name] = name
    if 'Documentation' in names:
        mapping['Document'] = 'Documentation'
    return mapping
    
_import_map = _create_mapping(_import_names)
_setting_map = _create_mapping(_setting_names)
_test_map = _create_mapping(_test_names)
_keyword_map = _create_mapping(_keyword_names)
_valid_formats = utils.NormalizedDict({'TSV':'TSV','HTML':'HTML', 
                                          'HTM':'HTML','XHTML':'HTML'})
_default_styles = '''
<style type="text/css">
html {
  font-family: Arial,Helvetica,sans-serif;
  background-color: white;
  color: black;
}
table {
  border-collapse: collapse;
  empty-cells: show;
  margin: 1em 0em;
  border: 0.1em solid black;
}
th, td {
  border-style: solid;
  border-width: 0.05em 0.1em;
  border-color: black;
  padding: 0.1em 0.2em;
  height: 1.5em;
}
th {
  background-color: rgb(192, 192, 192);
  color: black;
  border-width: 0.1em;
  font-weight: bold;
  text-align: center;
  text-transform: capitalize;
  letter-spacing: 0.1em;
}
.col_name, .col_value {
  width: 12em;
}
td.col_name {
  background-color: rgb(240, 240, 240);
  text-transform: capitalize;
  letter-spacing: 0.1em;
}
</style>
'''

class Tidy:

    def process_file(self, infile, outfile, opts):
        if not os.path.isfile(infile):
            LOGGER.error("'%s' is not a regular file" % infile)
            return
        print infile, 
        try:
            data = TestData(infile, opts['fixcomments'])
            format = self._get_format(data, opts['format'])
            outfile, remove_infile = self._get_outfile(infile, outfile, format, data)
            data.serialize(outfile, format, opts['title'], opts['style']) 
        except:
            LOGGER.error(utils.get_error_message())
        else:
            if infile != outfile:
                print '-> %s' % outfile
            else:
                print
            if remove_infile:
                os.remove(infile)

    def _get_format(self, data, format):
        if format is None:
            format = data.format
        elif format not in _valid_formats:
            raise DataError("Invalid output format '%s'. Only HTML and TSV "
                            "are supported." % format)
        return _valid_formats[format]

    def _get_outfile(self, infile, outfile, format, data):
        if outfile:
            return outfile, False
        if format != data.format: # inplace and format changed
            return infile[:-len(data.format)] + format.lower(), True
        return infile, False
            
    def process_directory(self, indir, opts):
        for name in os.listdir(indir):
            path = os.path.join(indir, name)
            if os.path.isdir(path):
                self.process_directory(path, opts)
            elif os.path.isfile(path) and self._is_valid_robot_file(name):
                self.process_file(path, None, opts)
                        
    def _is_valid_robot_file(self, name):
        base, ext = os.path.splitext(name)
        if not base or (base[0] in ['.', '_'] and base.lower() != '__init__'):
            return False
        return ext[1:] in _valid_formats


class TestData:
    
    def __init__(self, path, fix_comments=False):
        try:
            self.format = _valid_formats[os.path.splitext(path)[1][1:]]
        except:
            raise DataError("Input format must be either HTML or TSV.")
        raw = RawData(path, strip_comments=False)
        if raw.is_empty():
            raise DataError("'%s' contains no test data" % path)
        self.settings = Settings(raw.settings, fix_comments)
        self.variables = Variables(raw.variables, fix_comments)
        self.testcases = TestCases(raw.testcases, fix_comments)
        self.keywords = UserKeywords(raw.keywords, fix_comments)

    def serialize(self, path, format=None, title=None, style=None):
        serializer = self._get_serializer(path, format, title, style)
        self.settings.serialize(serializer)
        self.variables.serialize(serializer)
        self.testcases.serialize(serializer)
        self.keywords.serialize(serializer)
        serializer.close()
        
    def _get_serializer(self, path, format, title, style):
{'fraction_non_alphanumeric': 0.08668031711030609, 'fraction_numerical': 0.006772720113403686, 'mean_word_length': 2.7525610717100077, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7290236', 'n_tokens_mistral': 5547, 'n_tokens_neox': 5362, 'n_words': 2035}