<commit_before>/*  parsermodule.c
 *
 *  Copyright 1995-1996 by <NAME>, Jr. and Virginia Polytechnic
 *  Institute and State University, Blacksburg, Virginia, USA.
 *  Portions copyright 1991-1995 by Stichting Mathematisch Centrum,
 *  Amsterdam, The Netherlands.  Copying is permitted under the terms
 *  associated with the main Python distribution, with the additional
 *  restriction that this additional notice be included and maintained
 *  on all distributed copies.
 *
 *  This module serves to replace the original parser module written
 *  by Guido.  The functionality is not matched precisely, but the
 *  original may be implemented on top of this.  This is desirable
 *  since the source of the text to be parsed is now divorced from
 *  this interface.
 *
 *  Unlike the prior interface, the ability to give a parse tree
 *  produced by Python code as a tuple to the compiler is enabled by
 *  this module.  See the documentation for more details.
 */

#include "Python.h"			/* general Python API		  */
#include "graminit.h"			/* symbols defined in the grammar */
#include "node.h"			/* internal parser structure	  */
#include "token.h"			/* token definitions		  */
					/* ISTERMINAL() / ISNONTERMINAL() */
#include "compile.h"			/* PyNode_Compile()		  */

/*
 *  All the "fudge" declarations are here:
 *
 *  This isn't part of the Python runtime, but it's in the library somewhere.
 *  Where it is varies a bit, so just declare it.  Don't use any prototype;
 *  different systems declare it a little differently, and we don't need the
 *  extra warnings.
 */
extern char* strdup();


/*  String constants used to initialize module attributes.
 *
 */
static char*
parser_copyright_string
= "Copyright 1995-1996 by Virginia Polytechnic Institute & State\n\
University, Blacksburg, Virginia, USA, and <NAME>, Jr., Reston,\n\
Virginia, USA.  Portions copyright 1991-1995 by Stichting Mathematisch\n\
Centrum, Amsterdam, The Netherlands.";


static char*
parser_doc_string
= "This is an interface to Python's internal parser.";

static char*
parser_version_string = "0.4";


typedef PyObject* (*SeqMaker) Py_PROTO((int length));
typedef void (*SeqInserter) Py_PROTO((PyObject* sequence,
				      int index,
				      PyObject* element));

/*  The function below is copyrigthed by Stichting Mathematisch Centrum.  The
 *  original copyright statement is included below, and continues to apply
 *  in full to the function immediately following.  All other material is
 *  original, copyrighted by <NAME>, Jr. and Virginia Polytechnic
 *  Institute and State University.  Changes were made to comply with the
 *  new naming conventions.  Added arguments to provide support for creating
 *  lists as well as tuples, and optionally including the line numbers.
 */

/***********************************************************
Copyright 1991-1995 by Stichting Mathematisch Centrum, Amsterdam,
The Netherlands.

                        All Rights Reserved

Permission to use, copy, modify, and distribute this software and its
documentation for any purpose and without fee is hereby granted,
provided that the above copyright notice appear in all copies and that
both that copyright notice and this permission notice appear in
supporting documentation, and that the names of Stichting Mathematisch
Centrum or CWI or Corporation for National Research Initiatives or
CNRI not be used in advertising or publicity pertaining to
distribution of the software without specific, written prior
permission.

While CWI is the initial source for this software, a modified version
is made available by the Corporation for National Research Initiatives
(CNRI) at the Internet address ftp://ftp.python.org.

STICHTING MATHEMATISCH CENTRUM AND CNRI DISCLAIM ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH
CENTRUM OR CNRI BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.

******************************************************************/

static PyObject*
node2tuple(n, mkseq, addelem, lineno)
     node *n;				/* node to convert		 */
     SeqMaker mkseq;			/* create sequence		 */
     SeqInserter addelem;		/* func. to add elem. in seq.	 */
     int lineno;			/* include line numbers?	 */
{
    if (n == NULL) {
	Py_INCREF(Py_None);
	return Py_None;
    }
    if (ISNONTERMINAL(TYPE(n))) {
	int i;
	PyObject *v, *w;
	v = mkseq(1 + NCH(n));
	if (v == NULL)
	    return v;
	w = PyInt_FromLong(TYPE(n));
	if (w == NULL) {
	    Py_DECREF(v);
	    return NULL;
	}
	addelem(v, 0, w);
	for (i = 0; i < NCH(n); i++) {
	    w = node2tuple(CHILD(n, i), mkseq, addelem, lineno);
	    if (w == NULL) {
		Py_DECREF(v);
		return NULL;
	    }
	    addelem(v, i+1, w);
	}
	return (v);
    }
    else if (ISTERMINAL(TYPE(n))) {
	PyObject *result = mkseq(2 + lineno);
	if (result != NULL) {
	    addelem(result, 0, PyInt_FromLong(TYPE(n)));
	    addelem(result, 1, PyString_FromString(STR(n)));
	    if (lineno == 1)
		addelem(result, 2, PyInt_FromLong(n->n_lineno));
	}
	return (result);
    }
    else {
	PyErr_SetString(PyExc_SystemError,
			"unrecognized parse tree node type");
	return NULL;
    }
}   /* node2tuple() */
/*
 *  End of material copyrighted by Stichting Mathematisch Centrum.
 */



/*  There are two types of intermediate objects we're interested in:
 *  'eval' and 'exec' types.  These constants can be used in the ast_type
 *  field of the object type to identify which any given object represents.
 *  These should probably go in an external header to allow other extensions
 *  to use them, but then, we really should be using C++ too.  ;-)
 *
 *  The PyAST_FRAGMENT type is not currently supported.  Maybe not useful?
 *  Haven't decided yet.
 */

#define PyAST_EXPR	1
#define PyAST_SUITE	2
#define PyAST_FRAGMENT	3


/*  These are the internal objects and definitions required to implement the
 *  AST type.  Most of the internal names are more reminiscent of the 'old'
 *  naming style, but the code uses the new naming convention.
 */

static PyObject*
parser_error = 0;


typedef struct _PyAST_Object {

    PyObject_HEAD			/* standard object header	    */
    node* ast_node;			/* the node* returned by the parser */
    int	  ast_type;			/* EXPR or SUITE ?		    */

} PyAST_Object;


staticforward void parser_free Py_PROTO((PyAST_Object *ast));
staticforward int  parser_compare Py_PROTO((PyAST_Object *left,
					    PyAST_Object *right));


/* static */
PyTypeObject PyAST_Type = {

    PyObject_HEAD_INIT(NULL)
    0,
    "ast",				/* tp_name		*/
    sizeof(PyAST_Object),		/* tp_basicsize		*/
    0,					/* tp_itemsize		*/
    (destructor)parser_free,		/* tp_dealloc		*/
    0,					/* tp_print		*/
    0,					/* tp_getattr		*/
    0,					/* tp_setattr		*/
    (cmpfunc)parser_compare,		/* tp_compare		*/
    0,					/* tp_repr		*/
    0,					/* tp_as_number		*/
    0,					/* tp_as_sequence	*/
    0,					/* tp_as_mapping	*/
    0,					/* tp_hash		*/
    0,					/* tp_call		*/
    0					/* tp_str		*/
<commit_msg>Add doc string to type object.<commit_after>/*  parsermodule.c
 *
 *  Copyright 1995-1996 by <NAME>, Jr. and Virginia Polytechnic
 *  Institute and State University, Blacksburg, Virginia, USA.
 *  Portions copyright 1991-1995 by Stichting Mathematisch Centrum,
 *  Amsterdam, The Netherlands.  Copying is permitted under the terms
 *  associated with the main Python distribution, with the additional
 *  restriction that this additional notice be included and maintained
 *  on all distributed copies.
 *
 *  This module serves to replace the original parser module written
 *  by Guido.  The functionality is not matched precisely, but the
 *  original may be implemented on top of this.  This is desirable
 *  since the source of the text to be parsed is now divorced from
 *  this interface.
 *
 *  Unlike the prior interface, the ability to give a parse tree
 *  produced by Python code as a tuple to the compiler is enabled by
 *  this module.  See the documentation for more details.
 */

#include "Python.h"			/* general Python API		  */
#include "graminit.h"			/* symbols defined in the grammar */
#include "node.h"			/* internal parser structure	  */
#include "token.h"			/* token definitions		  */
					/* ISTERMINAL() / ISNONTERMINAL() */
#include "compile.h"			/* PyNode_Compile()		  */

/*
 *  All the "fudge" declarations are here:
 *
 *  This isn't part of the Python runtime, but it's in the library somewhere.
 *  Where it is varies a bit, so just declare it.  Don't use any prototype;
 *  different systems declare it a little differently, and we don't need the
 *  extra warnings.
 */
extern char* strdup();


/*  String constants used to initialize module attributes.
 *
 */
static char*
parser_copyright_string
= "Copyright 1995-1996 by Virginia Polytechnic Institute & State\n\
University, Blacksburg, Virginia, USA, and <NAME>, Jr., Reston,\n\
Virginia, USA.  Portions copyright 1991-1995 by Stichting Mathematisch\n\
Centrum, Amsterdam, The Netherlands.";


static char*
parser_doc_string
= "This is an interface to Python's internal parser.";

static char*
parser_version_string = "0.4";


typedef PyObject* (*SeqMaker) Py_PROTO((int length));
typedef void (*SeqInserter) Py_PROTO((PyObject* sequence,
				      int index,
				      PyObject* element));

/*  The function below is copyrigthed by Stichting Mathematisch Centrum.  The
 *  original copyright statement is included below, and continues to apply
 *  in full to the function immediately following.  All other material is
 *  original, copyrighted by <NAME>, Jr. and Virginia Polytechnic
 *  Institute and State University.  Changes were made to comply with the
 *  new naming conventions.  Added arguments to provide support for creating
 *  lists as well as tuples, and optionally including the line numbers.
 */

/***********************************************************
Copyright 1991-1995 by Stichting Mathematisch Centrum, Amsterdam,
The Netherlands.

                        All Rights Reserved

Permission to use, copy, modify, and distribute this software and its
documentation for any purpose and without fee is hereby granted,
provided that the above copyright notice appear in all copies and that
both that copyright notice and this permission notice appear in
supporting documentation, and that the names of Stichting Mathematisch
Centrum or CWI or Corporation for National Research Initiatives or
CNRI not be used in advertising or publicity pertaining to
distribution of the software without specific, written prior
permission.

While CWI is the initial source for this software, a modified version
is made available by the Corporation for National Research Initiatives
(CNRI) at the Internet address ftp://ftp.python.org.

STICHTING MATHEMATISCH CENTRUM AND CNRI DISCLAIM ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH
CENTRUM OR CNRI BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.

******************************************************************/

static PyObject*
node2tuple(n, mkseq, addelem, lineno)
     node *n;				/* node to convert		 */
     SeqMaker mkseq;			/* create sequence		 */
     SeqInserter addelem;		/* func. to add elem. in seq.	 */
     int lineno;			/* include line numbers?	 */
{
    if (n == NULL) {
	Py_INCREF(Py_None);
	return Py_None;
    }
    if (ISNONTERMINAL(TYPE(n))) {
	int i;
	PyObject *v, *w;
	v = mkseq(1 + NCH(n));
	if (v == NULL)
	    return v;
	w = PyInt_FromLong(TYPE(n));
	if (w == NULL) {
	    Py_DECREF(v);
	    return NULL;
	}
	addelem(v, 0, w);
	for (i = 0; i < NCH(n); i++) {
	    w = node2tuple(CHILD(n, i), mkseq, addelem, lineno);
	    if (w == NULL) {
		Py_DECREF(v);
		return NULL;
	    }
	    addelem(v, i+1, w);
	}
	return (v);
    }
    else if (ISTERMINAL(TYPE(n))) {
	PyObject *result = mkseq(2 + lineno);
	if (result != NULL) {
	    addelem(result, 0, PyInt_FromLong(TYPE(n)));
	    addelem(result, 1, PyString_FromString(STR(n)));
	    if (lineno == 1)
		addelem(result, 2, PyInt_FromLong(n->n_lineno));
	}
	return (result);
    }
    else {
	PyErr_SetString(PyExc_SystemError,
			"unrecognized parse tree node type");
	return NULL;
    }
}   /* node2tuple() */
/*
 *  End of material copyrighted by Stichting Mathematisch Centrum.
 */



/*  There are two types of intermediate objects we're interested in:
 *  'eval' and 'exec' types.  These constants can be used in the ast_type
 *  field of the object type to identify which any given object represents.
 *  These should probably go in an external header to allow other extensions
 *  to use them, but then, we really should be using C++ too.  ;-)
 *
 *  The PyAST_FRAGMENT type is not currently supported.  Maybe not useful?
 *  Haven't decided yet.
 */

#define PyAST_EXPR	1
#define PyAST_SUITE	2
#define PyAST_FRAGMENT	3


/*  These are the internal objects and definitions required to implement the
 *  AST type.  Most of the internal names are more reminiscent of the 'old'
 *  naming style, but the code uses the new naming convention.
 */

static PyObject*
parser_error = 0;


typedef struct _PyAST_Object {

    PyObject_HEAD			/* standard object header	    */
    node* ast_node;			/* the node* returned by the parser */
    int	  ast_type;			/* EXPR or SUITE ?		    */

} PyAST_Object;


staticforward void parser_free Py_PROTO((PyAST_Object *ast));
staticforward int  parser_compare Py_PROTO((PyAST_Object *left,
					    PyAST_Object *right));


/* static */
PyTypeObject PyAST_Type = {

    PyObject_HEAD_INIT(NULL)
    0,
    "ast",				/* tp_name		*/
    sizeof(PyAST_Object),		/* tp_basicsize		*/
    0,					/* tp_itemsize		*/
    (destructor)parser_free,		/* tp_dealloc		*/
    0,					/* tp_print		*/
    0,					/* tp_getattr		*/
    0,					/* tp_setattr		*/
    (cmpfunc)parser_compare,		/* tp_compare		*/
    0,					/* tp_repr		*/
    0,					/* tp_as_number		*/
    0,					/* tp_as_sequence	*/
    0,					/* tp_as_mapping	*/
    0,					/* tp_hash		*/
    0,					/* tp_call		*/
    0,					/* tp_str		*/
    0,					/* tp_getattro		*/
    0,					/* tp_setattro		*/

    /* Functions to access object as input/output buffer */
    0,					/* tp_as_buffer		*/

    /* Space for future expansion */
    0,					/* tp_xxx4		*/

    /* __doc__ */
    "Intermediate representation of a Python parse tree."

{'fraction_non_alphanumeric': 0.10383655855977345, 'fraction_numerical': 0.009776818825433214, 'mean_word_length': 3.278050187481973, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4082298', 'n_tokens_mistral': 4860, 'n_tokens_neox': 4497, 'n_words': 1740}