<commit_before>#!/usr/bin/python
# Copyright 2013 <NAME>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.

# this is all ugly.. but a start

import sys
sys.path.insert(0, "../")


from xml.dom.minidom import parseString
import xml.sax.saxutils as saxutils
import types
import os
import re
import inspect
import subprocess
import shutil

NAMESPACE = ""
VERSION = ""


TYPES = {}
PARAMETERS = {}
RETURNS = {}
FUNCTIONS = {}
CLASSES = {}


def fixup_docs(namespace, d):

    d = saxutils.unescape(d)

    def fixup_code(match):
        code = match.group(1)
        lines = code.splitlines()
        return "\n::\n\n%s" % ("\n".join(["    %s" % l for l in lines]))
    d = re.sub('\|\[(.*?)\]\|', fixup_code, d, flags=re.MULTILINE | re.DOTALL)
    d = re.sub('<programlisting>(.*?)</programlisting>', fixup_code, d, flags=re.MULTILINE | re.DOTALL)

    d = re.sub('<literal>(.*?)</literal>', '`\\1`', d)
    d = re.sub('<[^<]+?>', '', d)

    def fixup_class_refs(match):
        x = match.group(1)
        if x in TYPES:
            local = TYPES[x]
            if "." not in local:
                local = namespace + "." + local
            return ":class:`%s` " % local
        return x
    d = re.sub('#([A-Za-z]*)', fixup_class_refs, d)
    d = re.sub('%([A-Za-z0-9_]*)', fixup_class_refs, d)

    def fixup_param_refs(match):
        return "`%s`" % match.group(1)
    d = re.sub('@([A-Za-z0-9_]*)', fixup_param_refs, d)

    d = d.replace("NULL", ":obj:`None`")
    d = d.replace("%NULL", ":obj:`None`")
    d = d.replace("%TRUE", ":obj:`True`")
    d = d.replace("TRUE", ":obj:`True`")
    d = d.replace("%FALSE", ":obj:`False`")
    d = d.replace("FALSE", ":obj:`False`")
    return d


def init():
    gir_path = "/usr/share/gir-1.0/%s-%s.gir" % (NAMESPACE, VERSION)

    handle = open(gir_path, "rb")
    data = handle.read()
    handle.close()

    dom = parseString(data)

    # get a mapping of inline references and real classes
    for t in dom.getElementsByTagName("type"):
        local_name = t.getAttribute("name")
        c_name = t.getAttribute("c:type").rstrip("*")
        TYPES[c_name] = local_name

    for t in dom.getElementsByTagName("member"):
        parent = t.parentNode
        if parent.tagName == "bitfield" or parent.tagName == "enumeration":
            c_name = t.getAttribute("c:identifier")
            local_name = NAMESPACE + "." + parent.getAttribute("name") + "." + t.getAttribute("name").upper()
            TYPES[c_name] = local_name

    for doc in dom.getElementsByTagName("doc"):
        parent = doc.parentNode
        docs = fixup_docs(NAMESPACE, doc.firstChild.nodeValue)

        parent_name = parent.tagName
        if parent_name == "parameter":
            up = parent.parentNode.parentNode
            if up.tagName == "function":
                namespace = up.parentNode.getAttribute("name")
                func_name = up.getAttribute("name")
                param_name = parent.getAttribute("name")
                name = namespace + "." + func_name + "." + param_name
                PARAMETERS[name] = docs
        elif parent_name == "function":
            up = parent.parentNode
            if up.tagName == "namespace":
                namespace = up.getAttribute("name")
                func_name = parent.getAttribute("name")<commit_msg>docs: reference toplevel functions<commit_after>#!/usr/bin/python
# Copyright 2013 <NAME>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.

# this is all ugly.. but a start

import sys
sys.path.insert(0, "../")


from xml.dom.minidom import parseString
import xml.sax.saxutils as saxutils
import types
import os
import re
import inspect
import subprocess
import shutil

NAMESPACE = ""
VERSION = ""


TYPES = {}

PARAMETERS = {}
RETURNS = {}
FUNCTIONS = {}
CLASSES = {}


def fixup_docs(namespace, d):

    d = saxutils.unescape(d)

    def fixup_code(match):
        code = match.group(1)
        lines = code.splitlines()
        return "\n::\n\n%s" % ("\n".join(["    %s" % l for l in lines]))
    d = re.sub('\|\[(.*?)\]\|', fixup_code, d, flags=re.MULTILINE | re.DOTALL)
    d = re.sub('<programlisting>(.*?)</programlisting>', fixup_code, d, flags=re.MULTILINE | re.DOTALL)

    d = re.sub('<literal>(.*?)</literal>', '`\\1`', d)
    d = re.sub('<[^<]+?>', '', d)

    def fixup_class_refs(match):
        x = match.group(1)
        if x in TYPES:
            local = TYPES[x]
            if "." not in local:
                local = namespace + "." + local
            return ":class:`%s` " % local
        return x
    d = re.sub('#([A-Za-z]*)', fixup_class_refs, d)
    d = re.sub('%([A-Za-z0-9_]*)', fixup_class_refs, d)

    def fixup_param_refs(match):
        return "`%s`" % match.group(1)
    d = re.sub('@([A-Za-z0-9_]*)', fixup_param_refs, d)

    def fixup_function_refs(match):
        x = match.group(1)
        if x in TYPES:
            return ":func:`%s`" % TYPES[x]
        return x
    d = re.sub('([a-z0-9_]+)\(\)', fixup_function_refs, d)

    d = d.replace("NULL", ":obj:`None`")
    d = d.replace("%NULL", ":obj:`None`")
    d = d.replace("%TRUE", ":obj:`True`")
    d = d.replace("TRUE", ":obj:`True`")
    d = d.replace("%FALSE", ":obj:`False`")
    d = d.replace("FALSE", ":obj:`False`")
    return d


def init():
    gir_path = "/usr/share/gir-1.0/%s-%s.gir" % (NAMESPACE, VERSION)

    handle = open(gir_path, "rb")
    data = handle.read()
    handle.close()

    dom = parseString(data)

    # get a mapping of inline references and real classes
    for t in dom.getElementsByTagName("type"):
        local_name = t.getAttribute("name")
        c_name = t.getAttribute("c:type").rstrip("*")
        TYPES[c_name] = local_name

    # gtk_main -> Gtk.main
    for t in dom.getElementsByTagName("function"):
        local_name = t.getAttribute("name")
        namespace = t.parentNode.getAttribute("name")
        c_name = t.getAttribute("c:identifier")
        name = namespace + "." + local_name
        TYPES[c_name] = name

    for t in dom.getElementsByTagName("member"):
        parent = t.parentNode
        if parent.tagName == "bitfield" or parent.tagName == "enumeration":
            c_name = t.getAttribute("c:identifier")
            local_name = NAMESPACE + "." + parent.getAttribute("name") + "." + t.getAttribute("name").upper()
            TYPES[c_name] = local_name

    for doc in dom.getElementsByTagName("doc"):
        parent = doc.parentNode
        docs = fixup_docs(NAMESPACE, doc.firstChild.nodeValue)

        parent_name = parent.tagName
        if parent_name == "parameter":
            up = parent.parentNode.parentNode
            if up.tagName == "function":
                namespace = up.parentNode.getAttribute("name")
                func_name = up.getAttribute("name")
                param_name = parent.getAttribute("name")
                name = namespace + "." + func_name + "." + param_name
                PARAMETERS[name] = docs
        elif parent_name == "function":
            up = parent.parentNode
            if up.tagName == "namespace":
                namespace = up.getAttribute("name")
                func_name = parent.getAttribute("name")
{'fraction_non_alphanumeric': 0.16185812599258867, 'fraction_numerical': 0.004896770778189518, 'mean_word_length': 2.5083565459610027, 'pattern_counts': {'":': 25, '<': 17, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4070644', 'n_tokens_mistral': 2529, 'n_tokens_neox': 2489, 'n_words': 590}