<commit_before>import os
import re
import sys
import shutil
from docs.docs_writer import DocsWriter

# Small trick so importing telethon_generator works
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))

from telethon_generator.parser import TLParser, TLObject


# TLObject -> Python class name
def get_class_name(tlobject):
    """Gets the class name following the Python style guidelines, in ThisClassFormat"""
    # Courtesy of http://stackoverflow.com/a/31531797/4759433
    name = tlobject.name if isinstance(tlobject, TLObject) else tlobject
    result = re.sub(r'_([a-z])', lambda m: m.group(1).upper(), name)

    # Replace '_' with '' once again to make sure it doesn't appear on the name
    result = result[:1].upper() + result[1:].replace('_', '')

    # If it's a function, let it end with "Request" to identify them more easily
    if isinstance(tlobject, TLObject) and tlobject.is_function:
        result += 'Request'

    return result


# TLObject -> filename
def get_file_name(tlobject, add_extension=False):
    """Gets the file name in file_name_format.html for the given TLObject.
       Only its name may also be given if the full TLObject is not available"""
    if isinstance(tlobject, TLObject):
        name = tlobject.name
    else:
        name = tlobject

    # Courtesy of http://stackoverflow.com/a/1176023/4759433
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    result = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
    if add_extension:
        return result + '.html'
    else:
        return result


def get_create_path_for(tlobject):
    """Gets the file path (and creates the parent directories)
       for the given 'tlobject', relative to nothing; only its local path"""

    # Determine the output directory
    out_dir = 'methods' if tlobject.is_function else 'constructors'

    if tlobject.namespace:
        out_dir = os.path.join(out_dir, tlobject.namespace)

    # Ensure that it exists
    os.makedirs(out_dir, exist_ok=True)

    # Return the resulting filename
    return os.path.join(out_dir, get_file_name(tlobject, add_extension=True))


def get_path_for_type(type, relative_to='.'):
    """Similar to getting the path for a TLObject, it might not be possible
       to have the TLObject itself but rather its name (the type);
       this method works in the same way, returning a relative path"""
    if type.lower() in {'int', 'long', 'int128', 'int256', 'double',
                        'vector', 'string', 'bool', 'true', 'bytes', 'date'}:
        path = 'index.html#%s' % type.lower()

    elif '.' in type:
        # If it's not a core type, then it has to be a custom Telegram type
        namespace, name = type.split('.')
        path = 'types/%s/%s' % (namespace, get_file_name(name, add_extension=True))
    else:
        path = 'types/%s' % get_file_name(type, add_extension=True)

    return get_relative_path(path, relative_to)


# Destination path from the current position -> relative to the given path
def get_relative_path(destination, relative_to):
    if os.path.isfile(relative_to):
        relative_to = os.path.dirname(relative_to)

    return os.path.relpath(destination, start=relative_to)


def get_relative_paths(original, relative_to):
    """Converts the dictionary of 'original' paths to relative paths
       starting from the given 'relative_to' file"""
    return {k: get_relative_path(v, relative_to) for k, v in original.items()}


# Generate a index.html file for the given folder
def find_title(html_file):
    """Finds the <title> for the given HTML file, returns (Unknown) if not found"""
    with open(html_file) as handle:
        for line in handle:
            if '<title>' in line:
                # + 7 to skip len('<title>')
                return line[line.index('<title>') + 7:line.index('</title>')]

    return '(Unknown)'


def build_menu(docs, filename, relative_main_index):
    """Builds the menu using the given DocumentWriter up to 'filename',
       which must be a file (it cannot be a directory)"""
    # TODO Maybe this could be part of DocsWriter itself, "build path menu"
    docs.add_menu('API', relative_main_index)

    items = filename.split('/')
    for i in range(len(items) - 1):
        item = items[i]
        link = '../' * (len(items) - (i + 2))
        link += 'index.html'
        docs.add_menu(item.title(), link=link)

    if items[-1] != 'index.html':
        docs.add_menu(os.path.splitext(items[-1])[0])
    docs.end_menu()


def generate_index(folder, original_paths):
    """Generates the index file for the specified folder"""

    # Determine the namespaces listed here (as subfolders)
    # and the files (.html files) that we should link to
    namespaces = []
    files = []
    for item in os.listdir(folder):
        if os.path.isdir(os.path.join(folder, item)):
            namespaces.append(item)
        elif item != 'index.html':
            files.append(item)

    # We work with relative paths
    paths = get_relative_paths(original_paths, relative_to=folder)

    # Now that everything is setup, write the index.html file
    filename = os.path.join(folder, 'index.html')
    with DocsWriter(filename, type_to_path_function=get_path_for_type) as docs:
        # Title should be the current folder name
        docs.write_head(folder.title(), relative_css_path=paths['css'])

        docs.set_menu_separator(paths['arrow'])
        build_menu(docs, filename, relative_main_index=paths['index_all'])

        docs.write_title(folder.title())

        if namespaces:
            docs.write_title('Namespaces', level=3)
            docs.begin_table(4)
            for namespace in namespaces:
                # For every namespace, also write the index of it
                generate_index(os.path.join(folder, namespace), original_paths)
                docs.add_row(namespace.title(),
                             link=os.path.join(namespace, 'index.html'))

            docs.end_table()

        docs.write_title('Available items')
        docs.begin_table(2)
        for file in files:
            docs.add_row(find_title(os.path.join(folder, file)),
                         link=file)
        docs.end_table()
        docs.end_body()


def generate_documentation(scheme_file):
    """Generates the documentation HTML files from from scheme.tl to /methods and /constructors, etc."""
    original_paths = {
        'css': 'css/docs.css',
        'arrow': 'img/arrow.svg',
        'index_all': 'index.html',
        'index_types': 'types/index.html',
        'index_methods': 'methods/index.html',
        'index_constructors': 'constructors/index.html'
    }

    tlobjects = tuple(TLParser.parse_file(scheme_file))

    print('Generating constructors and functions documentation...')
    for tlobject in tlobjects:
        filename = get_create_path_for(tlobject)

        # Determine the relative paths for this file
        paths = get_relative_paths(original_paths, relative_to=filename)
<commit_msg>Sort namespaces and types when generating the docs (#87)<commit_after>import os
import re
import sys
import shutil
from docs.docs_writer import DocsWriter

# Small trick so importing telethon_generator works
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))

from telethon_generator.parser import TLParser, TLObject


# TLObject -> Python class name
def get_class_name(tlobject):
    """Gets the class name following the Python style guidelines, in ThisClassFormat"""
    # Courtesy of http://stackoverflow.com/a/31531797/4759433
    name = tlobject.name if isinstance(tlobject, TLObject) else tlobject
    result = re.sub(r'_([a-z])', lambda m: m.group(1).upper(), name)

    # Replace '_' with '' once again to make sure it doesn't appear on the name
    result = result[:1].upper() + result[1:].replace('_', '')

    # If it's a function, let it end with "Request" to identify them more easily
    if isinstance(tlobject, TLObject) and tlobject.is_function:
        result += 'Request'

    return result


# TLObject -> filename
def get_file_name(tlobject, add_extension=False):
    """Gets the file name in file_name_format.html for the given TLObject.
       Only its name may also be given if the full TLObject is not available"""
    if isinstance(tlobject, TLObject):
        name = tlobject.name
    else:
        name = tlobject

    # Courtesy of http://stackoverflow.com/a/1176023/4759433
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    result = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
    if add_extension:
        return result + '.html'
    else:
        return result


def get_create_path_for(tlobject):
    """Gets the file path (and creates the parent directories)
       for the given 'tlobject', relative to nothing; only its local path"""

    # Determine the output directory
    out_dir = 'methods' if tlobject.is_function else 'constructors'

    if tlobject.namespace:
        out_dir = os.path.join(out_dir, tlobject.namespace)

    # Ensure that it exists
    os.makedirs(out_dir, exist_ok=True)

    # Return the resulting filename
    return os.path.join(out_dir, get_file_name(tlobject, add_extension=True))


def get_path_for_type(type, relative_to='.'):
    """Similar to getting the path for a TLObject, it might not be possible
       to have the TLObject itself but rather its name (the type);
       this method works in the same way, returning a relative path"""
    if type.lower() in {'int', 'long', 'int128', 'int256', 'double',
                        'vector', 'string', 'bool', 'true', 'bytes', 'date'}:
        path = 'index.html#%s' % type.lower()

    elif '.' in type:
        # If it's not a core type, then it has to be a custom Telegram type
        namespace, name = type.split('.')
        path = 'types/%s/%s' % (namespace, get_file_name(name, add_extension=True))
    else:
        path = 'types/%s' % get_file_name(type, add_extension=True)

    return get_relative_path(path, relative_to)


# Destination path from the current position -> relative to the given path
def get_relative_path(destination, relative_to):
    if os.path.isfile(relative_to):
        relative_to = os.path.dirname(relative_to)

    return os.path.relpath(destination, start=relative_to)


def get_relative_paths(original, relative_to):
    """Converts the dictionary of 'original' paths to relative paths
       starting from the given 'relative_to' file"""
    return {k: get_relative_path(v, relative_to) for k, v in original.items()}


# Generate a index.html file for the given folder
def find_title(html_file):
    """Finds the <title> for the given HTML file, returns (Unknown) if not found"""
    with open(html_file) as handle:
        for line in handle:
            if '<title>' in line:
                # + 7 to skip len('<title>')
                return line[line.index('<title>') + 7:line.index('</title>')]

    return '(Unknown)'


def build_menu(docs, filename, relative_main_index):
    """Builds the menu using the given DocumentWriter up to 'filename',
       which must be a file (it cannot be a directory)"""
    # TODO Maybe this could be part of DocsWriter itself, "build path menu"
    docs.add_menu('API', relative_main_index)

    items = filename.split('/')
    for i in range(len(items) - 1):
        item = items[i]
        link = '../' * (len(items) - (i + 2))
        link += 'index.html'
        docs.add_menu(item.title(), link=link)

    if items[-1] != 'index.html':
        docs.add_menu(os.path.splitext(items[-1])[0])
    docs.end_menu()


def generate_index(folder, original_paths):
    """Generates the index file for the specified folder"""

    # Determine the namespaces listed here (as subfolders)
    # and the files (.html files) that we should link to
    namespaces = []
    files = []
    for item in os.listdir(folder):
        if os.path.isdir(os.path.join(folder, item)):
            namespaces.append(item)
        elif item != 'index.html':
            files.append(item)

    # We work with relative paths
    paths = get_relative_paths(original_paths, relative_to=folder)

    # Now that everything is setup, write the index.html file
    filename = os.path.join(folder, 'index.html')
    with DocsWriter(filename, type_to_path_function=get_path_for_type) as docs:
        # Title should be the current folder name
        docs.write_head(folder.title(), relative_css_path=paths['css'])

        docs.set_menu_separator(paths['arrow'])
        build_menu(docs, filename, relative_main_index=paths['index_all'])

        docs.write_title(folder.title())

        if namespaces:
            docs.write_title('Namespaces', level=3)
            docs.begin_table(4)
            namespaces.sort()
            for namespace in namespaces:
                # For every namespace, also write the index of it
                generate_index(os.path.join(folder, namespace), original_paths)
                docs.add_row(namespace.title(),
                             link=os.path.join(namespace, 'index.html'))

            docs.end_table()

        docs.write_title('Available items')
        docs.begin_table(2)

        files = [(f, find_title(os.path.join(folder, f))) for f in files]
        files.sort(key=lambda t: t[1])

        for file, title in files:
            docs.add_row(title, link=file)

        docs.end_table()
        docs.end_body()


def generate_documentation(scheme_file):
    """Generates the documentation HTML files from from scheme.tl to /methods and /constructors, etc."""
    original_paths = {
        'css': 'css/docs.css',
        'arrow': 'img/arrow.svg',
        'index_all': 'index.html',
        'index_types': 'types/index.html',
        'index_methods': 'methods/index.html',
        'index_constructors': 'constructors/index.html'
    }

    tlobjects = tuple(TLParser.parse_file(scheme_file))

    print('Generating constructors and functions documentation...')
    for tlobject in tlobjects:
        filename = get_create_path_for(tlobject)

        # Determine the relative paths for this file
        paths = get_relative_paths(original_paths, relative_to=filename)

{'fraction_non_alphanumeric': 0.11681315565636519, 'fraction_numerical': 0.008293166997448257, 'mean_word_length': 3.003688989784336, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6369240', 'n_tokens_mistral': 4370, 'n_tokens_neox': 4180, 'n_words': 1250}