<commit_before><?php
// shouldn't enable, since there are in-loop-body notices due to known closure issues
// error_reporting( E_ALL | E_STRICT );
error_reporting( E_ALL );
ini_set("memory_limit","256M");

/**
 * spelling-suggester, a simple spelling corrector written for PHP 5.3, based on
 * Peter Norvig's python implementation.
 *
 * see the following article for more information on PHP's memory usage for arrays:
 * http://www.pankaj-k.net/weblog/2008/03/did_you_know_that_each_integer.html
 *
 * see the following article for Peter Norvig's original implementation in python:
 * http://norvig.com/spell-correct.html
 *
 * at the expense of proper encapsulation, several members are public where otherwise 
 * i would prefer they be private. this is to accomodate shortcomings in PHP 5.3's 
 * handling of closures (namely that they don't close over $this in object context).
 */

class Suggest {
	public $freq = array();
	private $alphabet = "abcdefghijklmnopqrstuvwxyz";
	
	public function __construct( $dict_path, $force = false ) {
		$this->alphabet = str_split( $this->alphabet );
			
		if( is_array( $dict_path ) ) {
			$this->freq = $this->train( $dict_path );
			return;
		}

		$path_serialized = $dict_path . ".freq";
		if ( file_exists( $path_serialized ) && !$force ) {
			$this->freq = unserialize( file_get_contents( $path_serialized ) );
			return;
		}

		$words = $this->load_dict( $dict_path );
		$this->freq = $this->train( $words );

		if ( is_writable( dirname( $path_serialized ) ) ) {
			file_put_contents( $path_serialized, serialize( $this->freq ) );
		}
	}
	
	public function __invoke( $word ) {
		$candidates = $this->known( array( $word ) );
		if( empty( $candidates ) ) $candidates = $this->known( $this->edits( $word ) );
		if( empty( $candidates ) ) $candidates = $this->known_edits( $word );
		if( empty( $candidates ) ) $candidates = array( $word );
		$that = $this;
		return $this->argmax( $candidates, function ( $w ) use ( $that ) {
			return $that->freq[ $w ];
		});
	}
	
	private function argmax( array $array, $cbe ) {
		$tmp = array();
		foreach( $array as $a ) {
			$tmp[ $cbe( $a ) ] = $a;
		}
		return $tmp[ max( array_flip( $tmp ) ) ];
	}
	
	private function load_dict( $path ) {
		if( !file_exists( $path ) || !is_readable( $path ) ) throw new Exception("cannot read {$path}");
		$file = file( $path );
		$that = $this;
		$allwords = array();
		array_walk( $file, function( $line ) use ( &$allwords, $that ) {
			// E_STRICT doesn't like us using &$allwords in this closure
			array_walk( $that->words( $line ), function( $word ) use ( &$allwords ) {
				$allwords[] = $word;
			});
		});
		return $allwords;
	}
	
	public function words( $text ) {
		return preg_split( '/(\w+)/', strtolower( $text ), null, PREG_SPLIT_DELIM_CAPTURE );
	}
	
	// passing this by reference saves a lot of memory at peak
	private function train( &$features ) {
		$model = array();
		foreach( $features as $f ) {
			$model[ $f ] = !isset( $model[ $f ] ) ? 1 : $model[ $f ] + 1;
		}
		return $model;
	}
	
	private function edits( $word ) {
		$splits = $deletes = $transposes = $inserts = array();
		for( $i = 0; $i < strlen( $word ) + 1; $i++ ) {
			$splits[ substr( $word, 0, $i ) ] = substr( $word, $i );<commit_msg>moved wordtrainer out into its own class<commit_after><?php
// shouldn't enable, since there are in-loop-body notices due to known closure issues
// error_reporting( E_ALL | E_STRICT );
error_reporting( E_ALL );
ini_set( "memory_limit", "256M" );

require_once dirname(__FILE__).'/lib/wordtrainer.php';

/**
 * spelling-suggester, a simple spelling corrector written for PHP 5.3, based on
 * Peter Norvig's python implementation.
 *
 * see the following article for more information on PHP's memory usage for arrays:
 * http://www.pankaj-k.net/weblog/2008/03/did_you_know_that_each_integer.html
 *
 * see the following article for Peter Norvig's original implementation in python:
 * http://norvig.com/spell-correct.html
 *
 * at the expense of proper encapsulation, several members are public where otherwise 
 * i would prefer they be private. this is to accomodate shortcomings in PHP 5.3's 
 * handling of closures (namely that they don't close over $this in object context).
 */

class Suggest {
	public $freq = array();
	private $alphabet = "abcdefghijklmnopqrstuvwxyz";
	
	/**
	 * decoupled from the type of dictionary provided at the expense
	 * of no longer being a "one-file" program
	 */
	public function __construct( $dict_path, $force = false ) {
		$this->alphabet = str_split( $this->alphabet );
		$words = new WordTrainer( $dict_path, $force );
		$this->freq = $words->getFrequencies();
	}
	
	public function __invoke( $word ) {
		$candidates = $this->known( array( $word ) );
		if( empty( $candidates ) ) $candidates = $this->known( $this->edits( $word ) );
		if( empty( $candidates ) ) $candidates = $this->known_edits( $word );
		if( empty( $candidates ) ) $candidates = array( $word );
		$that = $this;
		return $this->argmax( $candidates, function ( $w ) use ( $that ) {
			return $that->freq[ $w ];
		});
	}
	
	private function argmax( array $array, $cbe ) {
		$tmp = array();
		foreach( $array as $a ) {
			$tmp[ $cbe( $a ) ] = $a;
		}
		return $tmp[ max( array_flip( $tmp ) ) ];
	}
	
	private function edits( $word ) {
		$splits = $deletes = $transposes = $inserts = array();
		for( $i = 0; $i < strlen( $word ) + 1; $i++ ) {
			$splits[ substr( $word, 0, $i ) ] = substr( $word, $i );
{'fraction_non_alphanumeric': 0.15407190022010273, 'fraction_numerical': 0.0062362435803374906, 'mean_word_length': 3.708981001727116, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1940232', 'n_tokens_mistral': 2053, 'n_tokens_neox': 1845, 'n_words': 560}