<commit_before>/*************************************************************************/
/* Copyright (c) 2004                                                    */
/* <NAME>, <NAME>, and <NAME>                    */
/* Copyright (c) 2013 <NAME>                                      */
/* Copyright (c) 2014 <NAME>                                     */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/

#ifndef _TOK_STRUCTURES_H_
#define _TOK_STRUCTURES_H_

/* conditional compiling flags */
#define INFIX_NOTATION
    /* If defined, then we're using infix notation for the dictionary */
    /* otherwise we're using prefix notation */

/* An ordered set of gword pointers, used to indicate the source gword
 * (Wordgraph word) of disjuncts and connectors. Usually it contains only
 * one element.  However, when a duplicate disjunct is eliminated (see
 * eliminate_duplicate_disjuncts()) and it originated from a different
 * gword (a relatively rare event) its gword is added to the gword_set of
 * the remaining disjunct. A set of 3 elements is extremely rare. The
 * original order is preserved, in a hope for better caching on
 * alternatives match checks in fast-match.c.
 *
 * Memory management:
 * A copy-on-write semantics is used when constructing a new gword_set.  It
 * means that all the gword sets with one element are shared.  These gword
 * sets are part of the Gword structure. Copied and added element are
 * alloc'ed and chained. The result is that the chain_next of the gword
 * sets that are part of each gword contains the list of alloc'ed elements -
 * to be used in gword_set_delete() called *only* in sentence_delete().
 * This ensures that the gword_set of connectors doesn't get stale when
 * their disjuncts are deleted and later restored in one-step parse when
 * min_null_count=0 and max_null count>0 (see classic_parse()).
 */
struct gword_set
{
	Gword *o_gword;
	struct gword_set *next;
	struct gword_set *chain_next;
};


typedef enum
{
	MT_INVALID,            /* Zero, to be changed to the correct type */
	MT_WORD,               /* Regular word */
	MT_FEATURE,            /* Pseudo morpheme, currently capitalization marks */
	MT_INFRASTRUCTURE,     /* Start and end Wordgraph pseudo-words */
	MT_WALL,               /* The LEFT-WALL and RIGHT-WALL pseudo-words */
	MT_EMPTY,              /* Empty word FIXME: Remove it. */
	MT_UNKNOWN,            /* Unknown word (FIXME? Unused) */
	/* Experimental for Semitic languages (yet unused) */
	MT_TEMPLATE,
	MT_ROOT,
	/* Experimental - for display purposes.
	 * MT_CONTR is now used in the tokenization step, see the comments there. */
	MT_CONTR,              /* Contracted part of a contraction (e.g. y', 's) */
	MT_PUNC,               /* Punctuation (yet unused) */
	/* We are not going to have >63 types up to here. */
	MT_STEM    = 1<<6,     /* Stem */
	MT_PREFIX  = 1<<7,     /* Prefix */
	MT_MIDDLE  = 1<<8,     /* Middle morpheme (yet unused) */
	MT_SUFFIX  = 1<<9      /* Suffix */
} Morpheme_type;
#define IS_REG_MORPHEME (MT_STEM|MT_PREFIX|MT_MIDDLE|MT_SUFFIX)

/* Word status */
/* - Tokenization */
#define WS_UNKNOWN (1<<0) /* Unknown word */
#define WS_REGEX   (1<<1) /* Matches a regex */
#define WS_SPELL   (1<<2) /* Result of a spell guess */
#define WS_RUNON   (1<<3) /* Separated from words run-on */
#define WS_HASALT  (1<<4) /* Has alternatives (one or more)*/
#define WS_UNSPLIT (1<<5) /* It's an alternative to itself as an unsplit word */
#define WS_INDICT  (1<<6) /* boolean_dictionary_lookup() is true */
#define WS_FIRSTUPPER (1<<7) /* Subword is the lc version of its unsplit_word.
                              The idea of marking subwords this way, in order to
                              enable restoring their original capitalization,
                              may be wrong in general, since in some languages
                              the process is not always reversible. Instead,
                              the original word may be saved. */
/* - Post linkage stage. XXX Experimental. */
#define WS_PL      (1<<14) /* Post-Linkage, not belonging to tokenization */

#define WS_GUESS (WS_SPELL|WS_RUNON|WS_REGEX)

/* XXX Only TS_ANYSPLIT and TS_DONE are actually used. */
typedef enum
{
	TS_INITIAL,
	TS_LR_STRIP,
	TS_AFFIX_SPLIT,
	TS_REGEX,
	TS_RUNON,
	TS_SPELL,
	TS_ANYSPLIT,             /* After anysplit */
	TS_DONE                  /* Tokenization done */
} Tokenizing_step;

/* For the "guess" field of Gword_struct. */
typedef enum
{
	GM_REGEX = '!',
	GM_SPELL = '~',
	GM_RUNON = '&',
	GM_UNKNOWN = '?'
} Guess_mark;

#define MAX_SPLITS 10   /* See split_counter below */

struct Gword_struct
{
	const char *subword;

	Gword *unsplit_word; /* Upward-going co-tree */
	Gword **next;        /* Right-going tree */
	Gword **prev;        /* Left-going tree */
	Gword *chain_next;   /* Next word in the chain of all words */<commit_msg>Add word position to Gword_struct<commit_after>/*************************************************************************/
/* Copyright (c) 2004                                                    */
/* <NAME>, <NAME>, and <NAME>                    */
/* Copyright (c) 2013 <NAME>                                      */
/* Copyright (c) 2014 <NAME>                                     */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/

#ifndef _TOK_STRUCTURES_H_
#define _TOK_STRUCTURES_H_

/* conditional compiling flags */
#define INFIX_NOTATION
    /* If defined, then we're using infix notation for the dictionary */
    /* otherwise we're using prefix notation */

/* An ordered set of gword pointers, used to indicate the source gword
 * (Wordgraph word) of disjuncts and connectors. Usually it contains only
 * one element.  However, when a duplicate disjunct is eliminated (see
 * eliminate_duplicate_disjuncts()) and it originated from a different
 * gword (a relatively rare event) its gword is added to the gword_set of
 * the remaining disjunct. A set of 3 elements is extremely rare. The
 * original order is preserved, in a hope for better caching on
 * alternatives match checks in fast-match.c.
 *
 * Memory management:
 * A copy-on-write semantics is used when constructing a new gword_set.  It
 * means that all the gword sets with one element are shared.  These gword
 * sets are part of the Gword structure. Copied and added element are
 * alloc'ed and chained. The result is that the chain_next of the gword
 * sets that are part of each gword contains the list of alloc'ed elements -
 * to be used in gword_set_delete() called *only* in sentence_delete().
 * This ensures that the gword_set of connectors doesn't get stale when
 * their disjuncts are deleted and later restored in one-step parse when
 * min_null_count=0 and max_null count>0 (see classic_parse()).
 */
struct gword_set
{
	Gword *o_gword;
	struct gword_set *next;
	struct gword_set *chain_next;
};


typedef enum
{
	MT_INVALID,            /* Zero, to be changed to the correct type */
	MT_WORD,               /* Regular word */
	MT_FEATURE,            /* Pseudo morpheme, currently capitalization marks */
	MT_INFRASTRUCTURE,     /* Start and end Wordgraph pseudo-words */
	MT_WALL,               /* The LEFT-WALL and RIGHT-WALL pseudo-words */
	MT_EMPTY,              /* Empty word FIXME: Remove it. */
	MT_UNKNOWN,            /* Unknown word (FIXME? Unused) */
	/* Experimental for Semitic languages (yet unused) */
	MT_TEMPLATE,
	MT_ROOT,
	/* Experimental - for display purposes.
	 * MT_CONTR is now used in the tokenization step, see the comments there. */
	MT_CONTR,              /* Contracted part of a contraction (e.g. y', 's) */
	MT_PUNC,               /* Punctuation (yet unused) */
	/* We are not going to have >63 types up to here. */
	MT_STEM    = 1<<6,     /* Stem */
	MT_PREFIX  = 1<<7,     /* Prefix */
	MT_MIDDLE  = 1<<8,     /* Middle morpheme (yet unused) */
	MT_SUFFIX  = 1<<9      /* Suffix */
} Morpheme_type;
#define IS_REG_MORPHEME (MT_STEM|MT_PREFIX|MT_MIDDLE|MT_SUFFIX)

/* Word status */
/* - Tokenization */
#define WS_UNKNOWN (1<<0) /* Unknown word */
#define WS_REGEX   (1<<1) /* Matches a regex */
#define WS_SPELL   (1<<2) /* Result of a spell guess */
#define WS_RUNON   (1<<3) /* Separated from words run-on */
#define WS_HASALT  (1<<4) /* Has alternatives (one or more)*/
#define WS_UNSPLIT (1<<5) /* It's an alternative to itself as an unsplit word */
#define WS_INDICT  (1<<6) /* boolean_dictionary_lookup() is true */
#define WS_FIRSTUPPER (1<<7) /* Subword is the lc version of its unsplit_word.
                              The idea of marking subwords this way, in order to
                              enable restoring their original capitalization,
                              may be wrong in general, since in some languages
                              the process is not always reversible. Instead,
                              the original word may be saved. */
/* - Post linkage stage. XXX Experimental. */
#define WS_PL      (1<<14) /* Post-Linkage, not belonging to tokenization */

#define WS_GUESS (WS_SPELL|WS_RUNON|WS_REGEX)

/* XXX Only TS_ANYSPLIT and TS_DONE are actually used. */
typedef enum
{
	TS_INITIAL,
	TS_LR_STRIP,
	TS_AFFIX_SPLIT,
	TS_REGEX,
	TS_RUNON,
	TS_SPELL,
	TS_ANYSPLIT,             /* After anysplit */
	TS_DONE                  /* Tokenization done */
} Tokenizing_step;

/* For the "guess" field of Gword_struct. */
typedef enum
{
	GM_REGEX = '!',
	GM_SPELL = '~',
	GM_RUNON = '&',
	GM_UNKNOWN = '?'
} Guess_mark;

#define MAX_SPLITS 10   /* See split_counter below */

struct Gword_struct
{
	const char *subword;
	const char *start;   /* subword start position. */
	const char *end;     /* subword end position. */

	Gword *unsplit_word; /* Upward-going co-tree */
	Gword **next;        /* Right-going tree */
	Gword **prev;        /* Left-going tree */
	Gword *chain_next;   /* Next word in the chain of all words */
{'fraction_non_alphanumeric': 0.119533527696793, 'fraction_numerical': 0.008381924198250729, 'mean_word_length': 2.3213313161875946, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4724534', 'n_tokens_mistral': 3288, 'n_tokens_neox': 3101, 'n_words': 1168}