<commit_before>#!/usr/bin/env python
# -*- Mode: Python; py-indent-offset: 4 -*-
#
# Copyright (C) 2006-2008 <NAME>.
#
# Scan the given public .h files of a GTK module (or module using
# GTK object conventions) and generates a set of scheme defs.
#
# defsgen uses the ctypes module to extract information from the installed
# module library (or libraries) to generate the object, interface, function,
# method, virtual, enum and flags defs. defsgen uses the gobject library
# g_type_* functions. defsgen will try to open the "libgobject-2.0.so" library
# if one is not specified on the command line.
#
# Basically the operation of defsgen is:
#
# - open and initialize the gobject and module libraries
# - read each .h file into a buffer which is scrubbed of extraneous data
# - find all *_get_type() functions prototypes
# - look in the module libraries for the get_type symbols
# - if found run the get_type() function to retrieve the GType
# - find the parent type name and save the object info
# - find each function prototypes in the file and check if it has a symbol in
#   the module libraries - save the info if found
# - extract the virtual prototypes from the Class or Iface structs and save
# - write out the various defs.
#
# The command line options are:
#
#   -l --modulelib   Adds the given module library name to the list to be used
#                    for finding symbols. Mor ethan one modulefile may be
#                    specified. (required)
#   -L --libgobject  Specifies the name of the gobject library (optional but
#                    must be specified if "libgobject-2.0.so" is not availble)
#   -s --separate    Create separate files for objects and function/method defs
#                    using the given name as the base name (optional). If this
#                    is not specified the combined object and function defs
#                    will be output to sys.stdout.
#   -f --defsfile    Extract defs from the given file to filter the output defs
#                    that is don't output defs that are defined in the
#                    defsfile. More than one deffile may be specified.
#   -D --defines     Include portions of the defsfile defs that are conditional
#                    on the given define, for example GDK_TARGET_X11. Only
#                    useful with the --defsfile option
#   -m --modulename  The prefix to be stripped from the front of function names
#                    for the given module
#   -p --useprefix   Use the modulename prefix as a hint to split names into
#                    module and name for object and enum defs. Also used for
#                    generating type codes.
#   --onlyenums      Only produce defs for enums and flags
#   --onlyobjdefs    Only produce defs for objects
#   --onlyvirtuals   Only produce defs for virtuals
#   --genpropgetsets Experimental option to generate prop-getset annotations.
#                    Not supported by codegen.py and friends.
#
# Examples:
#
# python defsgen.py -m pango -l libpango-1.0.so \
#            /usr/local/include/pango-1.0/pango/*.h >/tmp/pango.defs
#
# - Outputs all defs for the pango module.using the library module
#   libpango-1.0.so.
#
# python defsgen.py -m gdk -DGDK_TARGET_X11 -l libgdk-x11-2.0.so \
#            -l libgdk_pixbuf-2.0.so -s /tmp/gdk-2.10 \
#            -f /usr/tmp/pygtk/gtk/gdk-base.defs \
#            /usr/local/include/gtk-2.0/gdk/*.h \
#            /usr/local/include/gtk-2.0/gdk-pixbuf/*.h
#
# - Outputs the gdk module defs that are not contained in the defs file
#   /usr/tmp/pygtk/gtk/gdk-base.defs. Two output files are created:
#   /tmp/gdk-2.10-types.defs and /tmp/gdk-2.10.defs.
#
# Based on the original h2def.py program by
# <NAME> <<EMAIL>> and
# modified by <NAME> <<EMAIL>> to output stuff in
# Havoc's new defs format.  Info on this format can be seen at:
# http://www.gnome.org/mailing-lists/archives/gtk-devel-list/2000-January/0085.shtml
# Updated to be PEP-8 compatible and refactored to use OOP
# Extensively modified by <NAME> to use ctypes module to extract GType
# info from the given library and to create virtual defines.
#

import getopt
import os
import re, string
import sys
import ctypes
import defsparser

#------------------ utility defs -----------------

_upperstr_pat1 = re.compile(r'([^A-Z])([A-Z])')
_upperstr_pat2 = re.compile(r'([A-Z][A-Z])([A-Z][0-9a-z])')
_upperstr_pat3 = re.compile(r'^([A-Z])([A-Z])')

def to_upper_str(name):
    """Converts a typename to the equivalent uppercase and underscores
    name.  This is used to form the type conversion macros and enum/flag
    name variables"""
    name = _upperstr_pat1.sub(r'\1_\2', name)
    name = _upperstr_pat2.sub(r'\1_\2', name)
    name = _upperstr_pat3.sub(r'\1_\2', name, count=1)
    return name.upper()

def typecode(typename, prefix, use_prefix):
    """create a typecode (eg. GTK_TYPE_WIDGET) from a typename"""
    tcode = to_upper_str(typename)
    if (use_prefix and prefix and tcode.lower() != prefix
        and tcode.lower().startswith(prefix)):
        l = len(prefix)
        tcode = tcode[:l] + '_TYPE' + tcode[l:]
    else:
        tcode = tcode.replace('_', '_TYPE_', 1)
    return tcode

_class_iface_pat = re.compile(r'\w+(Class|Iface)')

def class_iface_sub(mobj):
    '''Returns matched string if it matches a Class or Iface struct
    otherwise returns the empty string'''
    if not _class_iface_pat.match(mobj.group(1)):
        return ''
    return mobj.group(0)

clean_patterns = [
    # strip comments
    (re.compile(r'/\*.*?\*/', re.DOTALL), ''),
    # compact continued lines
    (re.compile(r"\\\n", re.MULTILINE), ''),
    # remove preprocess directives
    (re.compile(r"""^[#].*?$""", re.MULTILINE), ''),
    # strip DECLS macros
    (re.compile(r"""G_BEGIN_DECLS|BEGIN_LIBGTOP_DECLS|G_END_DECLS""",
                re.MULTILINE), ''),
    # remove extern "C"
    (re.compile(r'^\s*(extern)\s+"C"\s+{', re.MULTILINE), ''),
    # remove singleline typedefs of stucts
    (re.compile(r'^typedef\s+struct\s*[^{;\n]*;\s*$', re.MULTILINE), ''),
    # remove all struct definitons but those for object classes and interfaces
    (re.compile(r'^struct\s+(\w+)\s+{[^}]+}\s*;\s*$', re.MULTILINE),
     class_iface_sub),<commit_msg>        * codegen/defsgen.py (clean_patterns): Strip out Windows DLL API macros.<commit_after>#!/usr/bin/env python
# -*- Mode: Python; py-indent-offset: 4 -*-
#
# Copyright (C) 2006-2009 <NAME>.
#
# Scan the given public .h files of a GTK module (or module using
# GTK object conventions) and generates a set of scheme defs.
#
# defsgen uses the ctypes module to extract information from the installed
# module library (or libraries) to generate the object, interface, function,
# method, virtual, enum and flags defs. defsgen uses the gobject library
# g_type_* functions. defsgen will try to open the "libgobject-2.0.so" library
# if one is not specified on the command line.
#
# Basically the operation of defsgen is:
#
# - open and initialize the gobject and module libraries
# - read each .h file into a buffer which is scrubbed of extraneous data
# - find all *_get_type() functions prototypes
# - look in the module libraries for the get_type symbols
# - if found run the get_type() function to retrieve the GType
# - find the parent type name and save the object info
# - find each function prototypes in the file and check if it has a symbol in
#   the module libraries - save the info if found
# - extract the virtual prototypes from the Class or Iface structs and save
# - write out the various defs.
#
# The command line options are:
#
#   -l --modulelib   Adds the given module library name to the list to be used
#                    for finding symbols. Mor ethan one modulefile may be
#                    specified. (required)
#   -L --libgobject  Specifies the name of the gobject library (optional but
#                    must be specified if "libgobject-2.0.so" is not availble)
#   -s --separate    Create separate files for objects and function/method defs
#                    using the given name as the base name (optional). If this
#                    is not specified the combined object and function defs
#                    will be output to sys.stdout.
#   -f --defsfile    Extract defs from the given file to filter the output defs
#                    that is don't output defs that are defined in the
#                    defsfile. More than one deffile may be specified.
#   -D --defines     Include portions of the defsfile defs that are conditional
#                    on the given define, for example GDK_TARGET_X11. Only
#                    useful with the --defsfile option
#   -m --modulename  The prefix to be stripped from the front of function names
#                    for the given module
#   -p --useprefix   Use the modulename prefix as a hint to split names into
#                    module and name for object and enum defs. Also used for
#                    generating type codes.
#   --onlyenums      Only produce defs for enums and flags
#   --onlyobjdefs    Only produce defs for objects
#   --onlyvirtuals   Only produce defs for virtuals
#   --genpropgetsets Experimental option to generate prop-getset annotations.
#                    Not supported by codegen.py and friends.
#
# Examples:
#
# python defsgen.py -m pango -l libpango-1.0.so \
#            /usr/local/include/pango-1.0/pango/*.h >/tmp/pango.defs
#
# - Outputs all defs for the pango module.using the library module
#   libpango-1.0.so.
#
# python defsgen.py -m gdk -DGDK_TARGET_X11 -l libgdk-x11-2.0.so \
#            -l libgdk_pixbuf-2.0.so -s /tmp/gdk-2.10 \
#            -f /usr/tmp/pygtk/gtk/gdk-base.defs \
#            /usr/local/include/gtk-2.0/gdk/*.h \
#            /usr/local/include/gtk-2.0/gdk-pixbuf/*.h
#
# - Outputs the gdk module defs that are not contained in the defs file
#   /usr/tmp/pygtk/gtk/gdk-base.defs. Two output files are created:
#   /tmp/gdk-2.10-types.defs and /tmp/gdk-2.10.defs.
#
# Based on the original h2def.py program by
# <NAME> <<EMAIL>> and
# modified by <NAME> <<EMAIL>> to output stuff in
# Havoc's new defs format.  Info on this format can be seen at:
# http://www.gnome.org/mailing-lists/archives/gtk-devel-list/2000-January/0085.shtml
# Updated to be PEP-8 compatible and refactored to use OOP
# Extensively modified by <NAME> to use ctypes module to extract GType
# info from the given library and to create virtual defines.
#

import getopt
import os
import re, string
import sys
import ctypes
import defsparser

#------------------ utility defs -----------------

_upperstr_pat1 = re.compile(r'([^A-Z])([A-Z])')
_upperstr_pat2 = re.compile(r'([A-Z][A-Z])([A-Z][0-9a-z])')
_upperstr_pat3 = re.compile(r'^([A-Z])([A-Z])')

def to_upper_str(name):
    """Converts a typename to the equivalent uppercase and underscores
    name.  This is used to form the type conversion macros and enum/flag
    name variables"""
    name = _upperstr_pat1.sub(r'\1_\2', name)
    name = _upperstr_pat2.sub(r'\1_\2', name)
    name = _upperstr_pat3.sub(r'\1_\2', name, count=1)
    return name.upper()

def typecode(typename, prefix, use_prefix):
    """create a typecode (eg. GTK_TYPE_WIDGET) from a typename"""
    tcode = to_upper_str(typename)
    if (use_prefix and prefix and tcode.lower() != prefix
        and tcode.lower().startswith(prefix)):
        l = len(prefix)
        tcode = tcode[:l] + '_TYPE' + tcode[l:]
    else:
        tcode = tcode.replace('_', '_TYPE_', 1)
    return tcode

_class_iface_pat = re.compile(r'\w+(Class|Iface)')

def class_iface_sub(mobj):
    '''Returns matched string if it matches a Class or Iface struct
    otherwise returns the empty string'''
    if not _class_iface_pat.match(mobj.group(1)):
        return ''
    return mobj.group(0)

clean_patterns = [
    # strip comments
    (re.compile(r'/\*.*?\*/', re.DOTALL), ''),
    # compact continued lines
    (re.compile(r"\\\n", re.MULTILINE), ''),
    # remove preprocess directives
    (re.compile(r"""^[#].*?$""", re.MULTILINE), ''),
    # strip DECLS macros and Windows DLL API macros
    (re.compile(r"""G_BEGIN_DECLS|BEGIN_LIBGTOP_DECLS|G_END_DECLS|[A-Z]+_API """,
                re.MULTILINE), ''),
    # remove extern "C"
    (re.compile(r'^\s*(extern)\s+"C"\s+{', re.MULTILINE), ''),
    # remove singleline typedefs of stucts
    (re.compile(r'^typedef\s+struct\s*[^{;\n]*;\s*$', re.MULTILINE), ''),
    # remove all struct definitons but those for object classes and interfaces
    (re.compile(r'^struct\s+(\w+)\s+{[^}]+}\s*;\s*$', re.MULTILINE),
     class_iface_sub),
{'fraction_non_alphanumeric': 0.11936972783702053, 'fraction_numerical': 0.011141174598121917, 'mean_word_length': 2.7081735025081146, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4779217', 'n_tokens_mistral': 4164, 'n_tokens_neox': 4002, 'n_words': 1384}