<commit_before>/**
 * sort_word.c
 *
 * Copyright (c) 1999, 2000, 2001
 *	<NAME> and <NAME>.
 *	All rights reserved.
 *
 * Copyright (c) 2004
 *	libchewing Core Team. See ChangeLog for details.
 *
 * See the file "COPYING" for information on usage and redistribution
 * of this file.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "global.h"
#include "zuin.h"

#define PHONE_CIN_FILE	"phone.cin"

#define CHARDEF_BEGIN	"%chardef  begin"
#define CHARDEF_END	"%chardef  end"
#define DO_WORD_ERROR (1)
#define MAX_WORD	(20000)
#define MAX_NUMBER	(12000)
#define MAX_BUF_LEN	(4096)

typedef struct {
	uint16 num;
	char word[ 3 ];
} WORD_DATA;

WORD_DATA word_data[ MAX_WORD ];
int nWord;

int SortWord( const WORD_DATA *a, const WORD_DATA *b )
{
	return ( a->num - b->num );
}

int DoWord( char *buf )
{
	char keyBuf[ 128 ], phoneBuf[ 128 ];
	int phoneInx[ 4 ];

	memset( phoneInx, 0, sizeof( phoneInx ) );
	sscanf( buf, "%s %s", keyBuf, word_data[ nWord ].word );
	if ( strlen( keyBuf ) > 4 )
		return DO_WORD_ERROR;

	Key2Pho( phoneBuf, keyBuf, KB_DEFAULT, 1 );
	word_data[ nWord ].num = PhoneBg2Uint( phoneBuf );
	nWord++ ;
	return 0;
}

void Output()
{
	FILE *indexfile, *datafile;
	int i;
	uint16 previous;
<commit_msg> r10888@ib:  gugod | 2005-02-19 00:50:05 +0800  * Constant 4 to ZUIN_SIZE<commit_after>/**
 * sort_word.c
 *
 * Copyright (c) 1999, 2000, 2001
 *	<NAME> and <NAME>.
 *	All rights reserved.
 *
 * Copyright (c) 2004
 *	libchewing Core Team. See ChangeLog for details.
 *
 * See the file "COPYING" for information on usage and redistribution
 * of this file.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "global.h"
#include "zuin.h"

#define PHONE_CIN_FILE	"phone.cin"

#define CHARDEF_BEGIN	"%chardef  begin"
#define CHARDEF_END	"%chardef  end"
#define DO_WORD_ERROR (1)
#define MAX_WORD	(20000)
#define MAX_NUMBER	(12000)
#define MAX_BUF_LEN	(4096)

typedef struct {
	uint16 num;
	char word[ 3 ];
} WORD_DATA;

WORD_DATA word_data[ MAX_WORD ];
int nWord;

int SortWord( const WORD_DATA *a, const WORD_DATA *b )
{
	return ( a->num - b->num );
}

int DoWord( char *buf )
{
	char keyBuf[ 128 ], phoneBuf[ 128 ];
	int phoneInx[ ZUIN_SIZE ];

	memset( phoneInx, 0, sizeof( phoneInx ) );
	sscanf( buf, "%s %s", keyBuf, word_data[ nWord ].word );
	if ( strlen( keyBuf ) > ZUIN_SIZE )
		return DO_WORD_ERROR;

	Key2Pho( phoneBuf, keyBuf, KB_DEFAULT, 1 );
	word_data[ nWord ].num = PhoneBg2Uint( phoneBuf );
	nWord++ ;
	return 0;
}

void Output()
{
	FILE *indexfile, *datafile;
	int i;
	uint16 previous;

{'fraction_non_alphanumeric': 0.13813229571984437, 'fraction_numerical': 0.04669260700389105, 'mean_word_length': 4.041176470588235, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '621609', 'n_tokens_mistral': 1142, 'n_tokens_neox': 1029, 'n_words': 301}