<commit_before>/* string.h
 *  Copyright: (When this is determined...it will go here)
 *  CVS Info
 *     $Id$
 *  Overview:
 *     This is the api header for the string subsystem
 *  Data Structure and Algorithms:
 *  History:
 *  Notes:
 *  References:
 */

#if !defined(PARROT_STRING_H_GUARD)
#define PARROT_STRING_H_GUARD

typedef struct parrot_string STRING;
typedef struct string_vtable STRING_VTABLE;

typedef enum {
    enc_native,
    enc_utf8,
    enc_utf16,
    enc_utf32,
    enc_foreign,
    enc_max
} encoding_t;


/* String vtable functions */

typedef INTVAL (*string_to_iv_t)(STRING *);
typedef STRING* (*string_iv_to_string_t)(STRING *, INTVAL);
typedef STRING* (*two_strings_iv_to_string_t)(STRING *, STRING *, INTVAL);
typedef STRING* (*substr_t)(STRING*, INTVAL, INTVAL, STRING*);
typedef INTVAL (*iv_to_iv_t)(INTVAL);
typedef INTVAL (*two_strings_to_iv_t)(STRING*, STRING*);

struct string_vtable {
    encoding_t which;                   /* What sort of encoding is this? */
    string_to_iv_t compute_strlen;      /* How long is a piece of string? */
    iv_to_iv_t max_bytes;               /* I have n characters - how many bytes should I allocate? */
    two_strings_iv_to_string_t concat;  /* Append string b to the end of string a */
    string_iv_to_string_t chopn;        /* Remove n characters from the end of a string */
    substr_t substr;                    /* Substring operation */
    two_strings_to_iv_t compare;        /* Compare operation */
};

struct parrot_string {
    void *bufstart;
    INTVAL buflen;
    INTVAL bufused;
    INTVAL flags;
    INTVAL strlen;
    STRING_VTABLE* encoding;
    INTVAL type;
    INTVAL unused;
};


/* Declarations of accessors */

INTVAL
string_compute_strlen(STRING*);
INTVAL
string_max_bytes(STRING*, INTVAL);
STRING*
string_concat(STRING*, STRING*, INTVAL);
STRING*
string_chopn(STRING*, INTVAL);
STRING*
string_substr(STRING*, INTVAL, INTVAL, STRING**);
INTVAL
string_compare(STRING*, STRING*);

/* Declarations of other functions */
INTVAL
string_length(STRING*);<commit_msg>Changed string 'unused' field to the language field.<commit_after>/* string.h
 *  Copyright: (When this is determined...it will go here)
 *  CVS Info
 *     $Id$
 *  Overview:
 *     This is the api header for the string subsystem
 *  Data Structure and Algorithms:
 *  History:
 *  Notes:
 *  References:
 */

#if !defined(PARROT_STRING_H_GUARD)
#define PARROT_STRING_H_GUARD

typedef struct parrot_string STRING;
typedef struct string_vtable STRING_VTABLE;

typedef enum {
    enc_native,
    enc_utf8,
    enc_utf16,
    enc_utf32,
    enc_foreign,
    enc_max
} encoding_t;


/* String vtable functions */

typedef INTVAL (*string_to_iv_t)(STRING *);
typedef STRING* (*string_iv_to_string_t)(STRING *, INTVAL);
typedef STRING* (*two_strings_iv_to_string_t)(STRING *, STRING *, INTVAL);
typedef STRING* (*substr_t)(STRING*, INTVAL, INTVAL, STRING*);
typedef INTVAL (*iv_to_iv_t)(INTVAL);
typedef INTVAL (*two_strings_to_iv_t)(STRING*, STRING*);

struct string_vtable {
    encoding_t which;                   /* What sort of encoding is this? */
    string_to_iv_t compute_strlen;      /* How long is a piece of string? */
    iv_to_iv_t max_bytes;               /* I have n characters - how many bytes should I allocate? */
    two_strings_iv_to_string_t concat;  /* Append string b to the end of string a */
    string_iv_to_string_t chopn;        /* Remove n characters from the end of a string */
    substr_t substr;                    /* Substring operation */
    two_strings_to_iv_t compare;        /* Compare operation */
};

struct parrot_string {
    void *bufstart;
    INTVAL buflen;
    INTVAL bufused;
    INTVAL flags;
    INTVAL strlen;
    STRING_VTABLE* encoding;
    INTVAL type;
    INTVAL lanugage;
};


/* Declarations of accessors */

INTVAL
string_compute_strlen(STRING*);
INTVAL
string_max_bytes(STRING*, INTVAL);
STRING*
string_concat(STRING*, STRING*, INTVAL);
STRING*
string_chopn(STRING*, INTVAL);
STRING*
string_substr(STRING*, INTVAL, INTVAL, STRING**);
INTVAL
string_compare(STRING*, STRING*);

/* Declarations of other functions */
INTVAL
string_length(STRING*);
{'fraction_non_alphanumeric': 0.10127495790233342, 'fraction_numerical': 0.0024055809477988932, 'mean_word_length': 3.6148723640399556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6779057', 'n_tokens_mistral': 1418, 'n_tokens_neox': 1360, 'n_words': 380}