<commit_before>/**
 * tree.c
 *
 * Copyright (c) 1999, 2000, 2001
 *      <NAME> and <NAME>.
 *      All rights reserved.
 *
 * Copyright (c) 2004-2006, 2008, 2010-2014
 *      libchewing Core Team. See ChangeLog for details.
 *
 * See the file "COPYING" for information on usage and redistribution
 * of this file.
 */

/**
 * @file tree.c
 * @brief API for accessing the phrase tree.
 */
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "taigi-private.h"
#include "taigi-utf8-util.h"
#include "userphrase-private.h"
#include "global.h"
#include "global-private.h"
#include "dict-private.h"
#include "memory-private.h"
#include "tree-private.h"
#include "private.h"
#include "plat_mmap.h"
#include "taigiutil.h"
#include "key2pho-private.h"

#define INTERVAL_SIZE ( ( MAX_PHONE_SEQ_LEN + 1 ) * MAX_PHONE_SEQ_LEN / 2 )

#ifndef LOG_API_TREE
#undef LOG_API
#undef LOG_VERBOSE
#define LOG_API(fmt, ...) 
#define LOG_VERBOSE(fmt, ...) 
#endif

typedef struct PhraseIntervalType {
    int from, to, source;
    Phrase *p_phr;
} PhraseIntervalType;

typedef struct RecordNode {
    int *arrIndex;              /* the index array of the things in "interval" */
    int nInter, score;
    struct RecordNode *next;
    int nMatchCnnct;            /* match how many Cnnct. */
} RecordNode;

typedef struct TreeDataType {
    int leftmost[MAX_PHONE_SEQ_LEN + 1];
    char graph[MAX_PHONE_SEQ_LEN + 1][MAX_PHONE_SEQ_LEN + 1];
    PhraseIntervalType interval[MAX_INTERVAL];
    int nInterval;
    RecordNode *phList;
    int nPhListLen;
} TreeDataType;

static int IsContain(IntervalType in1, IntervalType in2)
{
    return (in1.from <= in2.from && in1.to >= in2.to);
}

int IsIntersect(IntervalType in1, IntervalType in2)
{
    return (max(in1.from, in2.from) < min(in1.to, in2.to));
}

static int PhraseIntervalContain(PhraseIntervalType in1, PhraseIntervalType in2)
{
    return (in1.from <= in2.from && in1.to >= in2.to);
}

static int PhraseIntervalIntersect(PhraseIntervalType in1, PhraseIntervalType in2)
{
    return (max(in1.from, in2.from) < min(in1.to, in2.to));
}

void TerminateTree(ChewingData *pgdata)
{
    pgdata->static_data.tree = NULL;
    plat_mmap_close(&pgdata->static_data.tree_mmap);
}


int InitTree(ChewingData *pgdata, const char *prefix)
{
    char filename[PATH_MAX];
    size_t len;
    size_t offset;

    len = snprintf(filename, sizeof(filename), "%s" PLAT_SEPARATOR "%s", prefix, PHONE_TREE_FILE);
    if (len + 1 > sizeof(filename))
        return -1;

    plat_mmap_set_invalid(&pgdata->static_data.tree_mmap);
    pgdata->static_data.tree_size = plat_mmap_create(&pgdata->static_data.tree_mmap, filename, FLAG_ATTRIBUTE_READ);
    if (pgdata->static_data.tree_size <= 0)
        return -1;

    offset = 0;
    pgdata->static_data.tree =
        (const TreeType *) plat_mmap_set_view(&pgdata->static_data.tree_mmap, &offset, &pgdata->static_data.tree_size);
    if (!pgdata->static_data.tree)
        return -1;

    return 0;
}

static int CheckBreakpoint(int from, int to, int bArrBrkpt[])
{
    int i;

    for (i = from + 1; i < to; i++)
        if (bArrBrkpt[i])
            return 0;
    return 1;
}

static int CheckUserChoose(ChewingData *pgdata,
                           uint32_t *new_phoneSeq, int from, int to,
                           Phrase **pp_phr,
                           char selectStr[][MAX_PHONE_SEQ_LEN * MAX_UTF8_SIZE + 1],
                           IntervalType selectInterval[], int nSelect)
{
    IntervalType inte, c;
    int chno, len;
    int user_alloc;
    UserPhraseData *pUserPhraseData;
    Phrase *p_phr = ALC(Phrase, 1);

    assert(p_phr);
    inte.from = from;
    inte.to = to;
    *pp_phr = NULL;

    /* pass 1
     * if these exist one selected interval which is not contained by inte
     * but has intersection with inte, then inte is an unacceptable interval
     */
    for (chno = 0; chno < nSelect; chno++) {
        c = selectInterval[chno];
        if (IsIntersect(inte, c) && !IsContain(inte, c)) {
            free(p_phr);
            return 0;
        }
    }

    /* pass 2
     * if there exist one phrase satisfied all selectStr then return 1, else return 0.
     * also store the phrase with highest freq
     */
    pUserPhraseData = UserGetPhraseFirst(pgdata, new_phoneSeq);
    if (pUserPhraseData == NULL)
      goto end;
    p_phr->freq = -1;
    do {
        for (chno = 0; chno < nSelect; chno++) {
            c = selectInterval[chno];

            if (IsContain(inte, c)) {
                /*
                 * find a phrase of ph_id where the text contains
                 * 'selectStr[chno]' test if not ok then return 0,
                 * if ok then continue to test. */
                len = c.to - c.from;
                if (memcmp(ueStrSeek(pUserPhraseData->wordSeq, c.from - from),
                           selectStr[chno], ueStrNBytes(selectStr[chno], len)))
                    break;
            }

        }
        if (chno == nSelect) {
            /* save phrase data to "pp_phr" */
            if (pUserPhraseData->userfreq > p_phr->freq) {
                if ((user_alloc = (to - from)) > 0) {
                    ueStrNCpy(p_phr->phrase, pUserPhraseData->wordSeq, user_alloc, 1);
                }
                p_phr->freq = pUserPhraseData->userfreq;
                *pp_phr = p_phr;
            }
        }
    } while ((pUserPhraseData = UserGetPhraseNext(pgdata, new_phoneSeq)) != NULL);
    UserGetPhraseEnd(pgdata, new_phoneSeq);

    if (p_phr->freq != -1)
        return 1;
  end:
    free(p_phr);
    return 0;
}

/*
 * phrase is said to satisfy a choose interval if
 * their intersections are the same */
static int CheckChoose(ChewingData *pgdata,
                       const TreeType *phrase_parent, int from, int to, Phrase **pp_phr,
                       char selectStr[][MAX_PHONE_SEQ_LEN * MAX_UTF8_SIZE + 1],
                       IntervalType selectInterval[], int nSelect)
{
    IntervalType inte, c;
    int chno, len;
    Phrase *phrase = ALC(Phrase, 1);

    assert(phrase);<commit_msg>The FillPreeditBuf can get the correct type<commit_after>/**
 * tree.c
 *
 * Copyright (c) 1999, 2000, 2001
 *      <NAME> and <NAME>.
 *      All rights reserved.
 *
 * Copyright (c) 2004-2006, 2008, 2010-2014
 *      libchewing Core Team. See ChangeLog for details.
 *
 * See the file "COPYING" for information on usage and redistribution
 * of this file.
 */

/**
 * @file tree.c
 * @brief API for accessing the phrase tree.
 */
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "taigi-private.h"
#include "taigi-utf8-util.h"
#include "userphrase-private.h"
#include "global.h"
#include "global-private.h"
#include "dict-private.h"
#include "memory-private.h"
#include "tree-private.h"
#include "private.h"
#include "plat_mmap.h"
#include "taigiutil.h"
#include "key2pho-private.h"

#define INTERVAL_SIZE ( ( MAX_PHONE_SEQ_LEN + 1 ) * MAX_PHONE_SEQ_LEN / 2 )

#ifndef LOG_API_TREE
#undef LOG_API
#undef LOG_VERBOSE
#define LOG_API(fmt, ...) 
#define LOG_VERBOSE(fmt, ...) 
#endif

typedef struct PhraseIntervalType {
    int from, to, source;
    Phrase *p_phr;
} PhraseIntervalType;

typedef struct RecordNode {
    int *arrIndex;              /* the index array of the things in "interval" */
    int nInter, score;
    struct RecordNode *next;
    int nMatchCnnct;            /* match how many Cnnct. */
} RecordNode;

typedef struct TreeDataType {
    int leftmost[MAX_PHONE_SEQ_LEN + 1];
    char graph[MAX_PHONE_SEQ_LEN + 1][MAX_PHONE_SEQ_LEN + 1];
    PhraseIntervalType interval[MAX_INTERVAL];
    int nInterval;
    RecordNode *phList;
    int nPhListLen;
} TreeDataType;

static int IsContain(IntervalType in1, IntervalType in2)
{
    return (in1.from <= in2.from && in1.to >= in2.to);
}

int IsIntersect(IntervalType in1, IntervalType in2)
{
    return (max(in1.from, in2.from) < min(in1.to, in2.to));
}

static int PhraseIntervalContain(PhraseIntervalType in1, PhraseIntervalType in2)
{
    return (in1.from <= in2.from && in1.to >= in2.to);
}

static int PhraseIntervalIntersect(PhraseIntervalType in1, PhraseIntervalType in2)
{
    return (max(in1.from, in2.from) < min(in1.to, in2.to));
}

void TerminateTree(ChewingData *pgdata)
{
    pgdata->static_data.tree = NULL;
    plat_mmap_close(&pgdata->static_data.tree_mmap);
}


int InitTree(ChewingData *pgdata, const char *prefix)
{
    char filename[PATH_MAX];
    size_t len;
    size_t offset;

    len = snprintf(filename, sizeof(filename), "%s" PLAT_SEPARATOR "%s", prefix, PHONE_TREE_FILE);
    if (len + 1 > sizeof(filename))
        return -1;

    plat_mmap_set_invalid(&pgdata->static_data.tree_mmap);
    pgdata->static_data.tree_size = plat_mmap_create(&pgdata->static_data.tree_mmap, filename, FLAG_ATTRIBUTE_READ);
    if (pgdata->static_data.tree_size <= 0)
        return -1;

    offset = 0;
    pgdata->static_data.tree =
        (const TreeType *) plat_mmap_set_view(&pgdata->static_data.tree_mmap, &offset, &pgdata->static_data.tree_size);
    if (!pgdata->static_data.tree)
        return -1;

    return 0;
}

static int CheckBreakpoint(int from, int to, int bArrBrkpt[])
{
    int i;

    for (i = from + 1; i < to; i++)
        if (bArrBrkpt[i])
            return 0;
    return 1;
}

static int CheckUserChoose(ChewingData *pgdata,
                           uint32_t *new_phoneSeq, int from, int to,
                           Phrase **pp_phr,
                           char selectStr[][MAX_PHONE_SEQ_LEN * MAX_UTF8_SIZE + 1],
                           IntervalType selectInterval[], int nSelect)
{
    IntervalType inte, c;
    int chno, len;
    int user_alloc;
    UserPhraseData *pUserPhraseData;
    Phrase *p_phr = ALC(Phrase, 1);

    assert(p_phr);
    inte.from = from;
    inte.to = to;
    *pp_phr = NULL;

    /* pass 1
     * if these exist one selected interval which is not contained by inte
     * but has intersection with inte, then inte is an unacceptable interval
     */
    for (chno = 0; chno < nSelect; chno++) {
        c = selectInterval[chno];
        if (IsIntersect(inte, c) && !IsContain(inte, c)) {
            free(p_phr);
            return 0;
        }
    }

    /* pass 2
     * if there exist one phrase satisfied all selectStr then return 1, else return 0.
     * also store the phrase with highest freq
     */
    pUserPhraseData = UserGetPhraseFirst(pgdata, new_phoneSeq);
    if (pUserPhraseData == NULL)
      goto end;
    p_phr->freq = -1;
    do {
        for (chno = 0; chno < nSelect; chno++) {
            c = selectInterval[chno];

            if (IsContain(inte, c)) {
                /*
                 * find a phrase of ph_id where the text contains
                 * 'selectStr[chno]' test if not ok then return 0,
                 * if ok then continue to test. */
                len = c.to - c.from;
                if (memcmp(ueStrSeek(pUserPhraseData->wordSeq, c.from - from),
                           selectStr[chno], ueStrNBytes(selectStr[chno], len)))
                    break;
            }

        }
        if (chno == nSelect) {
            /* save phrase data to "pp_phr" */
            if (pUserPhraseData->userfreq > p_phr->freq) {
                if ((user_alloc = (to - from)) > 0) {
                    ueStrNCpy(p_phr->phrase, pUserPhraseData->wordSeq, user_alloc, 1);
                }
                p_phr->freq = pUserPhraseData->userfreq;
		p_phr->type = pUserPhraseData->type;
                *pp_phr = p_phr;
            }
        }
    } while ((pUserPhraseData = UserGetPhraseNext(pgdata, new_phoneSeq)) != NULL);
    UserGetPhraseEnd(pgdata, new_phoneSeq);

    if (p_phr->freq != -1)
        return 1;
  end:
    free(p_phr);
    return 0;
}

/*
 * phrase is said to satisfy a choose interval if
 * their intersections are the same */
static int CheckChoose(ChewingData *pgdata,
                       const TreeType *phrase_parent, int from, int to, Phrase **pp_phr,
                       char selectStr[][MAX_PHONE_SEQ_LEN * MAX_UTF8_SIZE + 1],
                       IntervalType selectInterval[], int nSelect)
{
    IntervalType inte, c;
    int chno, len;
    Phrase *phrase = ALC(Phrase, 1);

    assert(phrase);
{'fraction_non_alphanumeric': 0.1157214174646043, 'fraction_numerical': 0.01554955397332024, 'mean_word_length': 2.453928773318259, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2295175', 'n_tokens_mistral': 4380, 'n_tokens_neox': 4128, 'n_words': 1047}