<commit_before>/**
 * char.c
 *
 * Copyright (c) 1999, 2000, 2001
 *	<NAME> and <NAME>.
 *	All rights reserved.
 *
 * Copyright (c) 2004, 2005, 2006, 2008
 *	libchewing Core Team. See ChangeLog for details.
 *
 * See the file "COPYING" for information on usage and redistribution
 * of this file.
 */

/** 
 * @file char.c
 * @brief word data file
 */
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <string.h>

#include "global-private.h"
#include "chewing-definition.h"
#include "char-private.h"
#include "private.h"
#include "plat_mmap.h"

#if ! defined(USE_BINARY_DATA)
static char *fgettab( char *buf, int maxlen, FILE *fp )
{
	int i;

	for ( i = 0; i < maxlen; i++ ) {
		buf[ i ] = (char) fgetc( fp );
		if ( feof( fp ) )
			break;
		if ( buf[ i ] == '\t' )
			break;
	}
	if ( feof( fp ) )
		return 0;
	buf[ i ] = '\0';
	return buf;
}
#endif

static int CompUint16( const uint16 *pa, const uint16 *pb )
{
	return ( (*pa) - (*pb) );
}

void TerminateChar( ChewingData *pgdata )
{
#ifdef USE_BINARY_DATA
	pgdata->arrPhone = NULL;
	plat_mmap_close( &pgdata->char_phone_mmap );

	pgdata->begin = NULL;
	plat_mmap_close( &pgdata->char_begin_mmap );

	pgdata->dict = NULL;
	plat_mmap_close( &pgdata->dict_mmap );

	pgdata->phone_num = 0;
#else
	if ( pgdata->dictfile )
		fclose( dictfile );
	free( pgdata->begin );
	free( pgdata->arrPhone );
	pgdata->phone_num = 0;
#endif
}

int InitChar( ChewingData *pgdata , const char * prefix )
{
#ifdef USE_BINARY_DATA
	char filename[ PATH_MAX ];
	int len;
	size_t offset = 0;
	size_t file_size;
	size_t csize;

	len = snprintf( filename, sizeof( filename ), "%s" PLAT_SEPARATOR "%s", prefix, CHAR_FILE );
	if ( len + 1 > sizeof( filename ) )
		return -1;

	plat_mmap_set_invalid( &pgdata->dict_mmap );
	file_size = plat_mmap_create( &pgdata->dict_mmap, filename, FLAG_ATTRIBUTE_READ );
	if ( file_size <= 0 )
		return -1;

	csize = file_size;
	offset = 0;
	pgdata->dict = plat_mmap_set_view( &pgdata->dict_mmap, &offset, &csize );
	if ( !pgdata->dict )
		return -1;

	len = snprintf( filename, sizeof( filename ), "%s" PLAT_SEPARATOR "%s", prefix, CHAR_INDEX_BEGIN_FILE );
	if ( len + 1 > sizeof( filename ) )
		return -1;

	plat_mmap_set_invalid( &pgdata->char_begin_mmap );
	file_size = plat_mmap_create( &pgdata->char_begin_mmap, filename, FLAG_ATTRIBUTE_READ );
	if ( file_size <= 0 )
		return -1;

	pgdata->phone_num = file_size / sizeof( int );

	csize = file_size;
	pgdata->begin = plat_mmap_set_view( &pgdata->char_begin_mmap, &offset, &csize );
	if ( !pgdata->begin )
		return -1;

	len = snprintf( filename, sizeof( filename ), "%s" PLAT_SEPARATOR "%s", prefix, CHAR_INDEX_PHONE_FILE );
	if ( len + 1 > sizeof( filename ) )
		return -1;

	plat_mmap_set_invalid( &pgdata->char_phone_mmap );
	file_size = plat_mmap_create( &pgdata->char_phone_mmap, filename, FLAG_ATTRIBUTE_READ );
	if ( file_size <= 0 )
		return -1;

	if ( pgdata->phone_num != file_size / sizeof( uint16 ))
		return -1;

	offset = 0;
	csize = file_size;
	pgdata->arrPhone = plat_mmap_set_view( &pgdata->char_phone_mmap, &offset, &csize );
	if ( !pgdata->arrPhone )
		return -1;

	return 0;
#else
	int len;
	FILE *indexfile = NULL;

	pgdata->phone_num = PHONE_NUM;

	pgdata->arrPhone = calloc( pgdata->phone_num, sizeof( *pgdata->arrPhone ) );
	if ( !pgdata->arrPhone )
	    return -1;

	pgdata->begin = calloc( pgdata->phone_num, sizeof( *pgdata->begin ) );
	if ( !pgdata->begin )
	    return -1;

	len = snprintf( filename, sizeof( filename ), "%s" PLAT_SEPARATOR "%s", prefix, CHAR_FILE );
	if ( len + 1 > sizeof( filename ) )
		return -1;

	pgdata->dictfile = fopen( filename, "r" );
	if ( !pgdata->dictfile )
		return -1;

	len = snprintf( filename, sizeof( filename ), "%s" PLAT_SEPARATOR "%s", prefix, CHAR_INDEX_FILE );
	if ( len + 1 > sizeof( filename ) )
		return -1;

	indexfile = fopen( filename, "r" );
	if ( !indexfile )
		return -1;

	for ( i = 0; i < pgdata->phone_num; ++i )
		fscanf( indexfile, "%hu %d", &pgdata->arrPhone[i], &pgdata->begin[i] );

	fclose( indexfile );
	return 0;
#endif
}

static void Str2Word( ChewingData *pgdata, Word *wrd_ptr )
{
#ifndef USE_BINARY_DATA
	char buf[ 1000 ];
	uint16 sh;

	fgettab( buf, 1000, dictfile );
	/* only read 6 bytes to wrd_ptr->word avoid buffer overflow */
	sscanf( buf, "%hu %6[^ ]", &sh, wrd_ptr->word );
	assert( wrd_ptr->word != '\0' );
#else
	unsigned char size;
	size = *(unsigned char *) pgdata->cur_pos;
	pgdata->cur_pos = (unsigned char*) pgdata->cur_pos + sizeof(unsigned char);
	memcpy( wrd_ptr->word, pgdata->cur_pos, size );
	pgdata->cur_pos = (unsigned char*) pgdata->cur_pos + size;
	wrd_ptr->word[ size ] = '\0';
#endif
}

int GetCharFirst( ChewingData *pgdata, Word *wrd_ptr, uint16 phoneid )
{
	uint16 *pinx;

	pinx = (uint16 *) bsearch(
		&phoneid, pgdata->arrPhone, pgdata->phone_num,<commit_msg>Fix compiling error when --enable-binary-data=no<commit_after>/**
 * char.c
 *
 * Copyright (c) 1999, 2000, 2001
 *	<NAME> and <NAME>.
 *	All rights reserved.
 *
 * Copyright (c) 2004, 2005, 2006, 2008
 *	libchewing Core Team. See ChangeLog for details.
 *
 * See the file "COPYING" for information on usage and redistribution
 * of this file.
 */

/** 
 * @file char.c
 * @brief word data file
 */
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <string.h>

#include "global-private.h"
#include "chewing-definition.h"
#include "char-private.h"
#include "private.h"
#include "plat_mmap.h"

#if ! defined(USE_BINARY_DATA)
static char *fgettab( char *buf, int maxlen, FILE *fp )
{
	int i;

	for ( i = 0; i < maxlen; i++ ) {
		buf[ i ] = (char) fgetc( fp );
		if ( feof( fp ) )
			break;
		if ( buf[ i ] == '\t' )
			break;
	}
	if ( feof( fp ) )
		return 0;
	buf[ i ] = '\0';
	return buf;
}
#endif

static int CompUint16( const uint16 *pa, const uint16 *pb )
{
	return ( (*pa) - (*pb) );
}

void TerminateChar( ChewingData *pgdata )
{
#ifdef USE_BINARY_DATA
	pgdata->arrPhone = NULL;
	plat_mmap_close( &pgdata->char_phone_mmap );

	pgdata->begin = NULL;
	plat_mmap_close( &pgdata->char_begin_mmap );

	pgdata->dict = NULL;
	plat_mmap_close( &pgdata->dict_mmap );

	pgdata->phone_num = 0;
#else
	if ( pgdata->dictfile )
		fclose( pgdata->dictfile );
	free( pgdata->begin );
	free( pgdata->arrPhone );
	pgdata->phone_num = 0;
#endif
}

int InitChar( ChewingData *pgdata , const char * prefix )
{
#ifdef USE_BINARY_DATA
	char filename[ PATH_MAX ];
	int len;
	size_t offset = 0;
	size_t file_size;
	size_t csize;

	len = snprintf( filename, sizeof( filename ), "%s" PLAT_SEPARATOR "%s", prefix, CHAR_FILE );
	if ( len + 1 > sizeof( filename ) )
		return -1;

	plat_mmap_set_invalid( &pgdata->dict_mmap );
	file_size = plat_mmap_create( &pgdata->dict_mmap, filename, FLAG_ATTRIBUTE_READ );
	if ( file_size <= 0 )
		return -1;

	csize = file_size;
	offset = 0;
	pgdata->dict = plat_mmap_set_view( &pgdata->dict_mmap, &offset, &csize );
	if ( !pgdata->dict )
		return -1;

	len = snprintf( filename, sizeof( filename ), "%s" PLAT_SEPARATOR "%s", prefix, CHAR_INDEX_BEGIN_FILE );
	if ( len + 1 > sizeof( filename ) )
		return -1;

	plat_mmap_set_invalid( &pgdata->char_begin_mmap );
	file_size = plat_mmap_create( &pgdata->char_begin_mmap, filename, FLAG_ATTRIBUTE_READ );
	if ( file_size <= 0 )
		return -1;

	pgdata->phone_num = file_size / sizeof( int );

	csize = file_size;
	pgdata->begin = plat_mmap_set_view( &pgdata->char_begin_mmap, &offset, &csize );
	if ( !pgdata->begin )
		return -1;

	len = snprintf( filename, sizeof( filename ), "%s" PLAT_SEPARATOR "%s", prefix, CHAR_INDEX_PHONE_FILE );
	if ( len + 1 > sizeof( filename ) )
		return -1;

	plat_mmap_set_invalid( &pgdata->char_phone_mmap );
	file_size = plat_mmap_create( &pgdata->char_phone_mmap, filename, FLAG_ATTRIBUTE_READ );
	if ( file_size <= 0 )
		return -1;

	if ( pgdata->phone_num != file_size / sizeof( uint16 ))
		return -1;

	offset = 0;
	csize = file_size;
	pgdata->arrPhone = plat_mmap_set_view( &pgdata->char_phone_mmap, &offset, &csize );
	if ( !pgdata->arrPhone )
		return -1;

	return 0;
#else
	char filename[ PATH_MAX ];
	int len;
	int i;
	FILE *indexfile = NULL;

	pgdata->phone_num = PHONE_NUM;

	pgdata->arrPhone = calloc( pgdata->phone_num, sizeof( *pgdata->arrPhone ) );
	if ( !pgdata->arrPhone )
	    return -1;

	pgdata->begin = calloc( pgdata->phone_num, sizeof( *pgdata->begin ) );
	if ( !pgdata->begin )
	    return -1;

	len = snprintf( filename, sizeof( filename ), "%s" PLAT_SEPARATOR "%s", prefix, CHAR_FILE );
	if ( len + 1 > sizeof( filename ) )
		return -1;

	pgdata->dictfile = fopen( filename, "r" );
	if ( !pgdata->dictfile )
		return -1;

	len = snprintf( filename, sizeof( filename ), "%s" PLAT_SEPARATOR "%s", prefix, CHAR_INDEX_FILE );
	if ( len + 1 > sizeof( filename ) )
		return -1;

	indexfile = fopen( filename, "r" );
	if ( !indexfile )
		return -1;

	for ( i = 0; i < pgdata->phone_num; ++i )
		fscanf( indexfile, "%hu %d", &pgdata->arrPhone[i], &pgdata->begin[i] );

	fclose( indexfile );
	return 0;
#endif
}

static void Str2Word( ChewingData *pgdata, Word *wrd_ptr )
{
#ifndef USE_BINARY_DATA
	char buf[ 1000 ];
	uint16 sh;

	fgettab( buf, 1000, pgdata->dictfile );
	/* only read 6 bytes to wrd_ptr->word avoid buffer overflow */
	sscanf( buf, "%hu %6[^ ]", &sh, wrd_ptr->word );
	assert( wrd_ptr->word != '\0' );
#else
	unsigned char size;
	size = *(unsigned char *) pgdata->cur_pos;
	pgdata->cur_pos = (unsigned char*) pgdata->cur_pos + sizeof(unsigned char);
	memcpy( wrd_ptr->word, pgdata->cur_pos, size );
	pgdata->cur_pos = (unsigned char*) pgdata->cur_pos + size;
	wrd_ptr->word[ size ] = '\0';
#endif
}

int GetCharFirst( ChewingData *pgdata, Word *wrd_ptr, uint16 phoneid )
{
	uint16 *pinx;

	pinx = (uint16 *) bsearch(
		&phoneid, pgdata->arrPhone, pgdata->phone_num,
{'fraction_non_alphanumeric': 0.156368870155238, 'fraction_numerical': 0.018916418217333197, 'mean_word_length': 3.9255696202531647, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 137, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6830256', 'n_tokens_mistral': 4072, 'n_tokens_neox': 3809, 'n_words': 970}