<commit_before>#!/usr/bin/env python

#  Copyright 2008-2009 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.


"""Robot Framework Library and Resource File Documentation Generator

Usage:  libdoc.py [options] library_or_resource

This script can generate keyword documentation in HTML or XML. The former is
for humans and the latter mainly for RIDE and other tools.
Documentation can be created for both test libraries and resource files.

***************  TODO! Document uploading *********************

Options:
 -a --argument value *   Possible arguments that a library needs.         
 -f --format HTML|XML    Specifies whether to generate HTML or XML output.
                         The default value is HTML.
 -o --output path        Where to write the generated documentation. Can be
                         either a directory or a file or server address in which
                         case the file is uploaded there. The default value is
                         the directory where the script is executed from.
 -N --name newname       Sets the name of the documented library or resource.
 -T --title title        Sets the title of the generated HTML documentation.
                         Underscores in the given title are automatically
                         converted to spaces.
 -P --pythonpath path *  Additional path(s) to insert into PYTHONPATH.
 -E --escape what:with *  Escapes characters which are problematic in console.
                         'what' is the name of the character to escape and
                         'with' is the string to escape it with.
                         <--------------------ESCAPES------------------------>
 -h -? --help            Print this help.

See more information from 'tools/libdoc/doc/libdoc.html folder in case of 
source distribution or from 
http://code.google.com/p/robotframework/wiki/LibraryDocumentationTool 
"""

import sys
import os
import re
import tempfile
from httplib import HTTPConnection
from HTMLParser import HTMLParser

from robot.running import TestLibrary, UserLibrary
from robot.serializing import Template, Namespace
from robot.errors import DataError, Information
from robot.parsing import rawdata
from robot import utils


rawdata.PROCESS_CURDIR = False


def main(args):
    try:
        opts, libname = process_arguments(args)
    except Information, msg:
        exit(msg=str(msg))
    except DataError, err:
        exit(error=str(err))
    try:
        library = LibraryDoc(libname, opts['argument'], opts['name'])
    except DataError, err:
        exit(error=str(err))
    outpath = get_outpath(opts['output'], library.name, opts['format'])
    try:
        if opts['format'] == 'HTML':
            create_html_doc(library, outpath, opts['title'])
        else:
            create_xml_doc(library, outpath)
    except Exception, err:
        exit(error=str(err))
    exit(outpath)


def process_arguments(args_list):
    argparser = utils.ArgumentParser(__doc__)
    opts, args = argparser.parse_args(args_list, pythonpath='pythonpath',
                                      help='help', unescape='escape',
                                      check_args=True)
    if not opts['output']:
        opts['output'] = '.'
    if not opts['format']:
        opts['format'] = _uploading(opts['output']) and 'XML' or 'HTML'
    opts['format'] = opts['format'].upper()
    if opts['title']:
        opts['title'] = opts['title'].replace('_', ' ')
    return opts, args[0]

def _uploading(output):
    return output.startswith('http://')

def get_outpath(path, libname, format):
    if _uploading(path):
        return path
    path = os.path.abspath(path)
    if os.path.isdir(path):
        path = os.path.join(path, '%s.%s' % (libname, format.lower()))
        if os.path.exists(path):
            path = get_unique_path(path)
    return path


def get_unique_path(base, index=1):
    body, ext = os.path.splitext(base)
    path = '%s-%d%s' % (body, index, ext)
    if os.path.exists(path):
        path = get_unique_path(base, index+1)
    return path


def create_html_doc(lib, outpath, title=None):
    if not title:
        title = '%s - Documentation' % lib.name
    generated = utils.get_timestamp(daysep='-', millissep=None)
    namespace = Namespace(LIB=lib, TITLE=title, GENERATED=generated)
    doc = Template(template=DOCUMENT_TEMPLATE).generate(namespace) + '\n'
    outfile = open(outpath, 'w')
    outfile.write(doc.encode('UTF-8'))
    outfile.close()


def create_xml_doc(lib, outpath):
    if _uploading(outpath):
        uploadurl = outpath
        outpath = os.path.join(tempfile.gettempdir(), 'libdoc_upload.xml')
    else:
        uploadurl = None
    _create_xml_doc(lib, outpath)
    if uploadurl:
        RFDocUploader().upload(outpath, uploadurl)
        os.remove(outpath)

def _create_xml_doc(lib, outpath):
    writer = utils.XmlWriter(outpath)
    writer.start('keywordspec', {'name': lib.name, 'type': lib.type, 'generated': utils.get_timestamp(millissep=None)})
    writer.element('version', lib.version)
    writer.element('doc', lib.doc)
    _write_keywords_to_xml(writer, 'init', lib.inits)
    _write_keywords_to_xml(writer, 'kw', lib.keywords)
    writer.end('keywordspec')
    writer.close()

def _write_keywords_to_xml(writer, kwtype, keywords):
    for kw in keywords:
        attrs = kwtype == 'kw' and {'name': kw.name} or {}
        writer.start(kwtype, attrs)
        writer.element('doc', kw.doc)
        writer.start('arguments')
        for arg in kw.args:
            writer.element('arg', arg)
        writer.end('arguments')
        writer.end(kwtype)


def exit(msg=None, error=None):
    if msg:
        if _uploading(msg):
            msg = 'Library successfully uploaded to ' + msg
        sys.stdout.write(msg + '\n')
    if error:
        sys.stderr.write(error + '\n\nTry --help for usage information.\n')
        sys.exit(1)
    sys.exit(0)


def LibraryDoc(libname, arguments=None, newname=None):
    ext = os.path.splitext(libname)[1].lower()
    if  ext in ('.html', '.htm', '.xhtml', '.tsv'):
        return ResourceDoc(libname, arguments, newname)
    elif ext == '.java':
        if not utils.is_jython:
            exit(error='Documenting Java test libraries requires Jython.')
        return JavaLibraryDoc(libname, newname)
    else:
        return PythonLibraryDoc(libname, arguments, newname)


class _DocHelper:
    _name_regexp = re.compile("`(.+?)`")
    _list_or_table_regexp = re.compile('^(\d+\.|[-*|]|\[\d+\]) .')

    def __getattr__(self, name):
        if name == 'htmldoc':
            return self._get_htmldoc(self.doc)
        if name == 'htmlshortdoc':
            return utils.html_attr_escape(self.shortdoc)
        if name == 'htmlname':
            return utils.html_attr_escape(self.name)
        raise AttributeError("Non-existing attribute '%s'" % name)

    def _process_doc(self, doc):
        ret = ['']
        for line in doc.splitlines():
            line = line.strip()
            ret.append(self._get_doc_line_separator(line, ret[-1]))<commit_msg>Added txt resource files support<commit_after>#!/usr/bin/env python

#  Copyright 2008-2009 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.


"""Robot Framework Library and Resource File Documentation Generator

Usage:  libdoc.py [options] library_or_resource

This script can generate keyword documentation in HTML or XML. The former is
for humans and the latter mainly for RIDE and other tools.
Documentation can be created for both test libraries and resource files.

***************  TODO! Document uploading *********************

Options:
 -a --argument value *   Possible arguments that a library needs.         
 -f --format HTML|XML    Specifies whether to generate HTML or XML output.
                         The default value is HTML.
 -o --output path        Where to write the generated documentation. Can be
                         either a directory or a file or server address in which
                         case the file is uploaded there. The default value is
                         the directory where the script is executed from.
 -N --name newname       Sets the name of the documented library or resource.
 -T --title title        Sets the title of the generated HTML documentation.
                         Underscores in the given title are automatically
                         converted to spaces.
 -P --pythonpath path *  Additional path(s) to insert into PYTHONPATH.
 -E --escape what:with *  Escapes characters which are problematic in console.
                         'what' is the name of the character to escape and
                         'with' is the string to escape it with.
                         <--------------------ESCAPES------------------------>
 -h -? --help            Print this help.

See more information from 'tools/libdoc/doc/libdoc.html folder in case of 
source distribution or from 
http://code.google.com/p/robotframework/wiki/LibraryDocumentationTool 
"""

import sys
import os
import re
import tempfile
from httplib import HTTPConnection
from HTMLParser import HTMLParser

from robot.running import TestLibrary, UserLibrary
from robot.serializing import Template, Namespace
from robot.errors import DataError, Information
from robot.parsing import rawdata
from robot import utils


rawdata.PROCESS_CURDIR = False


def main(args):
    try:
        opts, libname = process_arguments(args)
    except Information, msg:
        exit(msg=str(msg))
    except DataError, err:
        exit(error=str(err))
    try:
        library = LibraryDoc(libname, opts['argument'], opts['name'])
    except DataError, err:
        exit(error=str(err))
    outpath = get_outpath(opts['output'], library.name, opts['format'])
    try:
        if opts['format'] == 'HTML':
            create_html_doc(library, outpath, opts['title'])
        else:
            create_xml_doc(library, outpath)
    except Exception, err:
        exit(error=str(err))
    exit(outpath)


def process_arguments(args_list):
    argparser = utils.ArgumentParser(__doc__)
    opts, args = argparser.parse_args(args_list, pythonpath='pythonpath',
                                      help='help', unescape='escape',
                                      check_args=True)
    if not opts['output']:
        opts['output'] = '.'
    if not opts['format']:
        opts['format'] = _uploading(opts['output']) and 'XML' or 'HTML'
    opts['format'] = opts['format'].upper()
    if opts['title']:
        opts['title'] = opts['title'].replace('_', ' ')
    return opts, args[0]

def _uploading(output):
    return output.startswith('http://')

def get_outpath(path, libname, format):
    if _uploading(path):
        return path
    path = os.path.abspath(path)
    if os.path.isdir(path):
        path = os.path.join(path, '%s.%s' % (libname, format.lower()))
        if os.path.exists(path):
            path = get_unique_path(path)
    return path


def get_unique_path(base, index=1):
    body, ext = os.path.splitext(base)
    path = '%s-%d%s' % (body, index, ext)
    if os.path.exists(path):
        path = get_unique_path(base, index+1)
    return path


def create_html_doc(lib, outpath, title=None):
    if not title:
        title = '%s - Documentation' % lib.name
    generated = utils.get_timestamp(daysep='-', millissep=None)
    namespace = Namespace(LIB=lib, TITLE=title, GENERATED=generated)
    doc = Template(template=DOCUMENT_TEMPLATE).generate(namespace) + '\n'
    outfile = open(outpath, 'w')
    outfile.write(doc.encode('UTF-8'))
    outfile.close()


def create_xml_doc(lib, outpath):
    if _uploading(outpath):
        uploadurl = outpath
        outpath = os.path.join(tempfile.gettempdir(), 'libdoc_upload.xml')
    else:
        uploadurl = None
    _create_xml_doc(lib, outpath)
    if uploadurl:
        RFDocUploader().upload(outpath, uploadurl)
        os.remove(outpath)

def _create_xml_doc(lib, outpath):
    writer = utils.XmlWriter(outpath)
    writer.start('keywordspec', {'name': lib.name, 'type': lib.type, 'generated': utils.get_timestamp(millissep=None)})
    writer.element('version', lib.version)
    writer.element('doc', lib.doc)
    _write_keywords_to_xml(writer, 'init', lib.inits)
    _write_keywords_to_xml(writer, 'kw', lib.keywords)
    writer.end('keywordspec')
    writer.close()

def _write_keywords_to_xml(writer, kwtype, keywords):
    for kw in keywords:
        attrs = kwtype == 'kw' and {'name': kw.name} or {}
        writer.start(kwtype, attrs)
        writer.element('doc', kw.doc)
        writer.start('arguments')
        for arg in kw.args:
            writer.element('arg', arg)
        writer.end('arguments')
        writer.end(kwtype)


def exit(msg=None, error=None):
    if msg:
        if _uploading(msg):
            msg = 'Library successfully uploaded to ' + msg
        sys.stdout.write(msg + '\n')
    if error:
        sys.stderr.write(error + '\n\nTry --help for usage information.\n')
        sys.exit(1)
    sys.exit(0)


def LibraryDoc(libname, arguments=None, newname=None):
    ext = os.path.splitext(libname)[1].lower()
    if  ext in ('.html', '.htm', '.xhtml', '.tsv', '.txt'):
        return ResourceDoc(libname, arguments, newname)
    elif ext == '.java':
        if not utils.is_jython:
            exit(error='Documenting Java test libraries requires Jython.')
        return JavaLibraryDoc(libname, newname)
    else:
        return PythonLibraryDoc(libname, arguments, newname)


class _DocHelper:
    _name_regexp = re.compile("`(.+?)`")
    _list_or_table_regexp = re.compile('^(\d+\.|[-*|]|\[\d+\]) .')

    def __getattr__(self, name):
        if name == 'htmldoc':
            return self._get_htmldoc(self.doc)
        if name == 'htmlshortdoc':
            return utils.html_attr_escape(self.shortdoc)
        if name == 'htmlname':
            return utils.html_attr_escape(self.name)
        raise AttributeError("Non-existing attribute '%s'" % name)

    def _process_doc(self, doc):
        ret = ['']
        for line in doc.splitlines():
            line = line.strip()
            ret.append(self._get_doc_line_separator(line, ret[-1]))
{'fraction_non_alphanumeric': 0.1208248237696818, 'fraction_numerical': 0.0026352197114434418, 'mean_word_length': 2.685360524399126, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '734487', 'n_tokens_mistral': 4460, 'n_tokens_neox': 4367, 'n_words': 1312}