<commit_before>#!/usr/bin/env python
# Copyright (C) 2013 eNovance SAS <<EMAIL>>
#
# Author: <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from docutils.nodes import SparseNodeVisitor, StopTraversal
import json
import os
from sphinx.builders import Builder
from xml.dom import minidom
import xml.etree.ElementTree as ET


output_file = None


class MyNodeVisitor(SparseNodeVisitor):
    def __init__(self, document):
        SparseNodeVisitor.__init__(self, document)

        self.must_parse = False
        self.methods = []
        self.paths = {}
        self.current_method = None
        self.root = None
        self.in_method_definition = False
        self.needs_method_description = False
        self.in_bullet_list = False
        self.in_request = False
        self.in_response = False
        self.current_request_example = []
        self.current_response_example = []

    def depart_document(self, node):
        if not self.must_parse:
            # We've probably raised StopTraversal, but walkabout() will keep
            # running the depart_* functions. Here, this is only
            # depart_document(), so we have to leave early.
            return

        # We are done parsing this document, let's print everything we need.
        resources = ET.SubElement(self.root, 'resources', {
            'base': 'http://www.example.com'
        })

        # Create 'd', a dict of the form:
        # {
        #     'foo': {
        #         'bar': {},
        #         'baz': {}
        #     }
        # }
        # If the paths are 'foo/bar' and 'foo/baz'.
        d = {}
        for path, methods in self.paths.iteritems():
            cd = d
            l = path[1:-1].replace('(', '{').replace(')', '}').split('/')
            for e in l:
                if e not in cd:
                    cd[e] = {}
                cd = cd[e]

        def build_resources(root, d, path=''):
            for k, v in d.iteritems():
                tmp = ET.SubElement(root, 'resource', {'path': k})
                if path + '/' + k + '/' in self.paths:
                    for method in self.paths[path + '/' + k + '/']:
                        ET.SubElement(tmp, 'method', {'href': '#' + method})
                build_resources(tmp, v, path + '/' + k)

        build_resources(resources, d)

        # Now, add all the methods we've gathered.
        for method in self.methods:
            self.root.append(method)

        # Finally, write the output.
        with open(output_file, 'w+') as f:
            parsed_string = minidom.parseString(ET.tostring(self.root))
            f.write(parsed_string.toprettyxml("    "))

    # If we're inside a bullet list, all the "paragraph" elements will be
    # parameters description, so we need to know whether we currently are in a
    # bullet list.
    def depart_bullet_list(self, node):
        self.in_bullet_list = False

    def visit_bullet_list(self, node):
        self.in_bullet_list = True

    def visit_comment(self, node):
        # If this .rst file is meant to be parsed by
        # sphinxcontrib-docbookrestapi, then it must have a comment, before the
        # first section, that just reads 'docbookrestapi'.
        if node.astext() == 'docbookrestapi':
            self.must_parse = True

    def visit_document(self, node):
        # This is where we should start. Let's build the root.
        attrs = {
            'xmlns': 'http://wadl.dev.java.net/2009/02',
            'xmlns:xsdxt': 'http://docs.rackspacecloud.com/xsd-ext/v1.0',
            'xmlns:wadl': 'http://wadl.dev.java.net/2009/02',
            'xmlns:xsd': 'http://docs.rackspacecloud.com/xsd/v1.0'
        }
        self.root = ET.Element('application', attrs)

    # If we are in a 'desc' node with the 'domain' attribute equal to 'http',
    # we're in a method definition. The next 'paragraph' element will be the
    # description of this method.
    def depart_desc(self, node):
        self.in_method_definition = False

    def visit_desc(self, node):
        attrs = {k: v for (k, v) in node.attlist()}
        if attrs['domain'] == 'http':
            self.in_method_definition = True

    def visit_desc_signature(self, node):
        attrs = {k: v for (k, v) in node.attlist()}
        if 'method' in attrs and 'path' in attrs:
            method_id = attrs['method'] + attrs['path'].replace('/', '_') + \
                "method"
            self.current_method = ET.Element('method', {
                'id': method_id,
                'name': attrs['method'].upper()
            })
            self.methods.append(self.current_method)
            path = attrs['path'].replace('(', '{').replace(')', '}')
            self.paths.setdefault(path, []).append(method_id)
            self.current_wadl_doc = ET.SubElement(self.current_method, 'wadl:doc', {
                'xmlns': 'http://www.w3.org/1999/xhtml',
                'xml:lang': 'EN',
                'title': ''
            })
            self.current_request = ET.SubElement(self.current_method,
                                                 'request')
            self.current_response = ET.SubElement(self.current_method,
                                                  'response',
                                                  {'status': '200'})
            self.needs_method_description = True

    def visit_paragraph(self, node):
        if self.in_method_definition and self.needs_method_description:
            text = node.astext()

            # Remove ":type data: foo" and ":return type: bar".
            type_index = text.find(':type data:')
            return_index = text.find(':return type:')
            min_index = min(type_index, return_index)
            if min_index > -1:
                text = text[0:min_index]

            # Create the doc node in the method.
            ET.SubElement(self.current_wadl_doc, 'para', {<commit_msg>Generated human-readable method ids and titles<commit_after>#!/usr/bin/env python
# Copyright (C) 2013 eNovance SAS <<EMAIL>>
#
# Author: <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from docutils.nodes import SparseNodeVisitor, StopTraversal
import json
import os
from sphinx.builders import Builder
from xml.dom import minidom
import xml.etree.ElementTree as ET


output_file = None


def generate_id(path, method):
    path = path.replace('(', '')
    path = path.replace(')', '')
    elems = path.split('/')
    elems = filter(lambda x: x, elems)  # Remove empty strings
    elems = elems[1:]  # Remove "vx" (v1, v2...)

    n_elems = len(elems)
    name = ""

    if method == 'delete':
        if elems[-1].endswith('_id'):
            name += "delete" + elems[-1][0:-3].capitalize()
    elif method == 'get':
        if elems[-1].endswith('_id'):
            name += "show" + elems[-1][0:-3].capitalize()
        elif n_elems > 2:
            if elems[-3][:-1] + '_id' == elems[-2]:
                name += 'show' + elems[-3][:-1].capitalize()
                name += elems[-1].capitalize()
        else:
            name += "list" + elems[-1].capitalize()
    elif method == 'post':
        if elems[-1].endswith('_id'):
            name += "create" + elems[-1][0:-3].capitalize()
        else:
            name += "create" + elems[-1][:-1].capitalize()
    elif method == 'update':
        if elems[-1].endswith('_id'):
            name += "update" + elems[-1][0:-3].capitalize()

    if not name:
        name = raw_input('id for %s (%s)' % (path, method))

    return name


def generate_title_from_id(id_):
    words = []
    current_start = 0
    for i, char in enumerate(id_):
        if not char.islower():
            words.append(id_[current_start:i].lower())
            current_start = i

    if i > current_start:
        words.append(id_[current_start:])

    return ' '.join(words).capitalize()


class MyNodeVisitor(SparseNodeVisitor):
    def __init__(self, document):
        SparseNodeVisitor.__init__(self, document)

        self.must_parse = False
        self.methods = []
        self.paths = {}
        self.current_method = None
        self.root = None
        self.in_method_definition = False
        self.needs_method_description = False
        self.in_bullet_list = False
        self.in_request = False
        self.in_response = False
        self.current_request_example = []
        self.current_response_example = []

    def depart_document(self, node):
        if not self.must_parse:
            # We've probably raised StopTraversal, but walkabout() will keep
            # running the depart_* functions. Here, this is only
            # depart_document(), so we have to leave early.
            return

        # We are done parsing this document, let's print everything we need.
        resources = ET.SubElement(self.root, 'resources', {
            'base': 'http://www.example.com'
        })

        # Create 'd', a dict of the form:
        # {
        #     'foo': {
        #         'bar': {},
        #         'baz': {}
        #     }
        # }
        # If the paths are 'foo/bar' and 'foo/baz'.
        d = {}
        for path, methods in self.paths.iteritems():
            cd = d
            l = path[1:-1].replace('(', '{').replace(')', '}').split('/')
            for e in l:
                if e not in cd:
                    cd[e] = {}
                cd = cd[e]

        def build_resources(root, d, path=''):
            for k, v in d.iteritems():
                tmp = ET.SubElement(root, 'resource', {'path': k})
                if path + '/' + k + '/' in self.paths:
                    for method in self.paths[path + '/' + k + '/']:
                        ET.SubElement(tmp, 'method', {'href': '#' + method})
                build_resources(tmp, v, path + '/' + k)

        build_resources(resources, d)

        # Now, add all the methods we've gathered.
        for method in self.methods:
            self.root.append(method)

        # Finally, write the output.
        with open(output_file, 'w+') as f:
            parsed_string = minidom.parseString(ET.tostring(self.root))
            f.write(parsed_string.toprettyxml("    "))

    # If we're inside a bullet list, all the "paragraph" elements will be
    # parameters description, so we need to know whether we currently are in a
    # bullet list.
    def depart_bullet_list(self, node):
        self.in_bullet_list = False

    def visit_bullet_list(self, node):
        self.in_bullet_list = True

    def visit_comment(self, node):
        # If this .rst file is meant to be parsed by
        # sphinxcontrib-docbookrestapi, then it must have a comment, before the
        # first section, that just reads 'docbookrestapi'.
        if node.astext() == 'docbookrestapi':
            self.must_parse = True

    def visit_document(self, node):
        # This is where we should start. Let's build the root.
        attrs = {
            'xmlns': 'http://wadl.dev.java.net/2009/02',
            'xmlns:xsdxt': 'http://docs.rackspacecloud.com/xsd-ext/v1.0',
            'xmlns:wadl': 'http://wadl.dev.java.net/2009/02',
            'xmlns:xsd': 'http://docs.rackspacecloud.com/xsd/v1.0'
        }
        self.root = ET.Element('application', attrs)

    # If we are in a 'desc' node with the 'domain' attribute equal to 'http',
    # we're in a method definition. The next 'paragraph' element will be the
    # description of this method.
    def depart_desc(self, node):
        self.in_method_definition = False

    def visit_desc(self, node):
        attrs = {k: v for (k, v) in node.attlist()}
        if attrs['domain'] == 'http':
            self.in_method_definition = True

    def visit_desc_signature(self, node):
        attrs = {k: v for (k, v) in node.attlist()}
        if 'method' in attrs and 'path' in attrs:
            method_id = generate_id(attrs['path'], attrs['method'])
            self.current_method = ET.Element('method', {
                'id': method_id,
                'name': attrs['method'].upper()
            })
            self.methods.append(self.current_method)
            path = attrs['path'].replace('(', '{').replace(')', '}')
            self.paths.setdefault(path, []).append(method_id)
            self.current_wadl_doc = ET.SubElement(
                self.current_method, 'wadl:doc', {
                    'xmlns': 'http://www.w3.org/1999/xhtml',
                    'xml:lang': 'EN',
                    'title': generate_title_from_id(method_id)
                }
            )
            self.current_request = ET.SubElement(self.current_method,
                                                 'request')
            self.current_response = ET.SubElement(self.current_method,
                                                  'response',
                                                  {'status': '200'})
            self.needs_method_description = True

    def visit_paragraph(self, node):
        if self.in_method_definition and self.needs_method_description:
            text = node.astext()

            # Remove ":type data: foo" and ":return type: bar".
            type_index = text.find(':type data:')
            return_index = text.find(':return type:')
            min_index = min(type_index, return_index)
            if min_index > -1:
                text = text[0:min_index]

            # Create the doc node in the method.
            ET.SubElement(self.current_wadl_doc, 'para', {
{'fraction_non_alphanumeric': 0.12474876983851965, 'fraction_numerical': 0.007069096957516114, 'mean_word_length': 1.96, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5550005', 'n_tokens_mistral': 4285, 'n_tokens_neox': 4115, 'n_words': 1164}