<commit_before>/* -*- Mode: C++ -*- */

/*
 * This file contains some classes, typedefs and constant common to all
 * hfst-optimized-lookup stuff. This is just to get them out of the way
 * in the actual ospell code.
 */

#ifndef HFST_OSPELL_HFST_OL_H_
#define HFST_OSPELL_HFST_OL_H_

#include <vector>
#include <map>
#include <climits>
#include <cstdio>
#include <cstdlib>
#include <iostream>
#include <cstring>
#include <set>
#include <utility>

namespace hfst_ol {

typedef unsigned short SymbolNumber;
typedef unsigned int TransitionTableIndex;
typedef std::vector<SymbolNumber> SymbolVector;
typedef std::vector<std::string> KeyTable;
typedef std::map<std::string, SymbolNumber> StringSymbolMap;
typedef short ValueNumber;
typedef float Weight;

// Forward declarations to typedef some more containers
class TransitionIndex;
class Transition;
class FlagDiacriticOperation;

typedef std::vector<TransitionIndex*> TransitionIndexVector;
typedef std::vector<Transition*> TransitionVector;
typedef std::map<SymbolNumber, FlagDiacriticOperation> OperationMap;

const SymbolNumber NO_SYMBOL = USHRT_MAX;
const TransitionTableIndex NO_TABLE_INDEX = UINT_MAX;
const Weight INFINITE_WEIGHT = static_cast<float>(NO_TABLE_INDEX);

// This is 2^31, hopefully equal to UINT_MAX/2 rounded up.
// For some profound reason it can't be replaced with (UINT_MAX+1)/2.
const TransitionTableIndex TARGET_TABLE = 2147483648u;

// the flag diacritic operators as given in
// Beesley & Karttunen, Finite State Morphology (U of C Press 2003)
enum FlagDiacriticOperator {P, N, R, D, C, U};

enum HeaderFlag {Weighted, Deterministic, Input_deterministic, Minimized,
		 Cyclic, Has_epsilon_epsilon_transitions,
		 Has_input_epsilon_transitions, Has_input_epsilon_cycles,
		 Has_unweighted_input_epsilon_cycles};

class TransducerParsingException: public std::exception
{
public:
    virtual const char* what() const throw()
	{
	    return("Parsing error while reading alphabet");
	}
};

class UnweightedSpellerException: public std::exception
{
public:
    virtual const char* what() const throw()
	{
	    return("Transducers for use in spellers must be weighted");
	}
};

class TransducerHeader
{
private:
    SymbolNumber number_of_symbols;
    SymbolNumber number_of_input_symbols;
    TransitionTableIndex size_of_transition_index_table;
    TransitionTableIndex size_of_transition_target_table;

    TransitionTableIndex number_of_states;
    TransitionTableIndex number_of_transitions;

    bool weighted;
    bool deterministic;
    bool input_deterministic;
    bool minimized;
    bool cyclic;
    bool has_epsilon_epsilon_transitions;
    bool has_input_epsilon_transitions;
    bool has_input_epsilon_cycles;
    bool has_unweighted_input_epsilon_cycles;

    void read_property(bool &property, FILE * f)
	{
	    unsigned int prop;
	    unsigned int ret;
	    ret = fread(&prop,sizeof(unsigned int),1,f);
	    if (prop == 0)
	    {
		property = false;
		return;
	    }
	    else
	    {
		property = true;
		return;
	    }
	    throw TransducerParsingException();
	}

public:
    TransducerHeader(FILE * f)
	{
	    // The silly compiler complains about not catching the return value
	    // of fread(). Hence this dummy variable is needed.
	    size_t val;

	    val = fread(&number_of_input_symbols,sizeof(SymbolNumber),1,f);
	    val = fread(&number_of_symbols,sizeof(SymbolNumber),1,f);

	    val = fread(&size_of_transition_index_table,
			sizeof(TransitionTableIndex),1,f);
	    val = fread(&size_of_transition_target_table,
			sizeof(TransitionTableIndex),1,f);

	    val = fread(&number_of_states,sizeof(TransitionTableIndex),1,f);
	    val = fread(&number_of_transitions,sizeof(TransitionTableIndex),1,f);

	    read_property(weighted,f);

	    read_property(deterministic,f);
	    read_property(input_deterministic,f);
	    read_property(minimized,f);
	    read_property(cyclic,f);
	    read_property(has_epsilon_epsilon_transitions,f);
	    read_property(has_input_epsilon_transitions,f);
	    read_property(has_input_epsilon_cycles,f);
	    read_property(has_unweighted_input_epsilon_cycles,f);

	    // For ospell: demand weightedness
	    if (!weighted) {
		throw UnweightedSpellerException();
	    }
	}

    SymbolNumber symbol_count(void)
	{ return number_of_symbols; }<commit_msg>Fixed some comments<commit_after>/* -*- Mode: C++ -*- */

/*
 * This file contains some classes, typedefs and constant common to all
 * hfst-optimized-lookup stuff. This is just to get them out of the way
 * of the actual ospell code.
 */

#ifndef HFST_OSPELL_HFST_OL_H_
#define HFST_OSPELL_HFST_OL_H_

#include <vector>
#include <map>
#include <climits>
#include <cstdio>
#include <cstdlib>
#include <iostream>
#include <cstring>
#include <set>
#include <utility>

namespace hfst_ol {

typedef unsigned short SymbolNumber;
typedef unsigned int TransitionTableIndex;
typedef std::vector<SymbolNumber> SymbolVector;
typedef std::vector<std::string> KeyTable;
typedef std::map<std::string, SymbolNumber> StringSymbolMap;
typedef short ValueNumber;
typedef float Weight;

// Forward declarations to typedef some more containers
class TransitionIndex;
class Transition;
class FlagDiacriticOperation;

typedef std::vector<TransitionIndex*> TransitionIndexVector;
typedef std::vector<Transition*> TransitionVector;
typedef std::map<SymbolNumber, FlagDiacriticOperation> OperationMap;

const SymbolNumber NO_SYMBOL = USHRT_MAX;
const TransitionTableIndex NO_TABLE_INDEX = UINT_MAX;
const Weight INFINITE_WEIGHT = static_cast<float>(NO_TABLE_INDEX);

// This is 2^31, hopefully equal to UINT_MAX/2 rounded up.
// For some profound reason it can't be replaced with (UINT_MAX+1)/2.
const TransitionTableIndex TARGET_TABLE = 2147483648u;

// the flag diacritic operators as given in
// Beesley & Karttunen, Finite State Morphology (U of C Press 2003)
enum FlagDiacriticOperator {P, N, R, D, C, U};

enum HeaderFlag {Weighted, Deterministic, Input_deterministic, Minimized,
		 Cyclic, Has_epsilon_epsilon_transitions,
		 Has_input_epsilon_transitions, Has_input_epsilon_cycles,
		 Has_unweighted_input_epsilon_cycles};

class TransducerParsingException: public std::exception
{
public:
    virtual const char* what() const throw()
	{
	    return("Parsing error while reading alphabet");
	}
};

class UnweightedSpellerException: public std::exception
{
public:
    virtual const char* what() const throw()
	{
	    return("Transducers for use in spellers must be weighted");
	}
};

class TransducerHeader
{
private:
    SymbolNumber number_of_symbols;
    SymbolNumber number_of_input_symbols;
    TransitionTableIndex size_of_transition_index_table;
    TransitionTableIndex size_of_transition_target_table;

    TransitionTableIndex number_of_states;
    TransitionTableIndex number_of_transitions;

    bool weighted;
    bool deterministic;
    bool input_deterministic;
    bool minimized;
    bool cyclic;
    bool has_epsilon_epsilon_transitions;
    bool has_input_epsilon_transitions;
    bool has_input_epsilon_cycles;
    bool has_unweighted_input_epsilon_cycles;

    void read_property(bool &property, FILE * f)
	{
	    unsigned int prop;
	    unsigned int ret;
	    ret = fread(&prop,sizeof(unsigned int),1,f);
	    if (prop == 0)
	    {
		property = false;
		return;
	    }
	    else
	    {
		property = true;
		return;
	    }
	    throw TransducerParsingException();
	}

public:
    TransducerHeader(FILE * f)
	{
	    // The silly compiler complains about not catching the return value
	    // of fread(). Hence this dummy variable is needed.
	    size_t val;

	    val = fread(&number_of_input_symbols,sizeof(SymbolNumber),1,f);
	    val = fread(&number_of_symbols,sizeof(SymbolNumber),1,f);

	    val = fread(&size_of_transition_index_table,
			sizeof(TransitionTableIndex),1,f);
	    val = fread(&size_of_transition_target_table,
			sizeof(TransitionTableIndex),1,f);

	    val = fread(&number_of_states,sizeof(TransitionTableIndex),1,f);
	    val = fread(&number_of_transitions,sizeof(TransitionTableIndex),1,f);

	    read_property(weighted,f);

	    read_property(deterministic,f);
	    read_property(input_deterministic,f);
	    read_property(minimized,f);
	    read_property(cyclic,f);
	    read_property(has_epsilon_epsilon_transitions,f);
	    read_property(has_input_epsilon_transitions,f);
	    read_property(has_input_epsilon_cycles,f);
	    read_property(has_unweighted_input_epsilon_cycles,f);

	    // For ospell: demand weightedness FIXME: do this somewhere else
	    if (!weighted) {
		throw UnweightedSpellerException();
	    }
	}

    SymbolNumber symbol_count(void)
	{ return number_of_symbols; }
{'fraction_non_alphanumeric': 0.08753778191118344, 'fraction_numerical': 0.006510113926993723, 'mean_word_length': 4.933103448275862, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6659852', 'n_tokens_mistral': 2897, 'n_tokens_neox': 2607, 'n_words': 657}