<commit_before><?php

/*

    TextStatistics Class
    https://github.com/DaveChild/Text-Statistics

    Released under New BSD license
    http://www.opensource.org/licenses/bsd-license.php

    Calculates following readability scores (formulae can be found in wiki):
      * Flesch Kincaid Reading Ease
      * Flesch Kincaid Grade Level
      * Gunning Fog Score
      * Coleman Liau Index
      * SMOG Index
      * Automated Reability Index

    Will also give:
      * String length
      * Letter count
      * Syllable count
      * Sentence count
      * Average words per sentence
      * Average syllables per word

    Sample Code
    ----------------
    $statistics = new TextStatistics;
    $text = 'The quick brown fox jumped over the lazy dog.';
    echo 'Flesch-Kincaid Reading Ease: ' . $statistics->flesch_kincaid_reading_ease($text);

*/

class TextStatistics
{
    protected $strEncoding = ''; // Used to hold character encoding to be used by object, if set

    protected $blnMbstring = true; // Efficiency: Is the MB String extension loaded ?

    protected static $blnBcmath = true; // Efficiency: Is the BC Math extension loaded ?

    public $normalize = true;

    /**
     * Constructor.
     *
     * @param  string  $strEncoding Optional character encoding.
     * @return void
     */
    public function __construct($strEncoding = '')
    {
        if ($strEncoding <> '') {
            // Encoding is given. Use it!
            $this->strEncoding = $strEncoding;
        }

        $this->blnMbstring = extension_loaded('mbstring');
        self::$blnBcmath = extension_loaded('bcmath');
    }

    /**
     * Gives the Flesch-Kincaid Reading Ease of text entered rounded to one digit
     * @param   string  $strText         Text to be checked
     * @return  int|float
     */
    public function flesch_kincaid_reading_ease($strText)
    {
        $strText = $this->clean_text($strText);

        $score = self::bc_calc(self::bc_calc(206.835, '-', self::bc_calc(1.015, '*', $this->average_words_per_sentence($strText))), '-', self::bc_calc(84.6, '*', $this->average_syllables_per_word($strText)));

        return $this->normalize_score($score, 0, 100);
    }

    /**
     * Gives the Flesch-Kincaid Grade level of text entered rounded to one digit
     * @param   string  $strText         Text to be checked
     * @return  int|float
     */
    public function flesch_kincaid_grade_level($strText)
    {
        $strText = $this->clean_text($strText);

        $score = self::bc_calc(self::bc_calc(0.39, '*', $this->average_words_per_sentence($strText)), '+', self::bc_calc(self::bc_calc(11.8, '*', $this->average_syllables_per_word($strText)), '-', 15.59), true, 1);

        return $this->normalize_score($score, 0, 12);
    }

    /**
     * Gives the Gunning-Fog score of text entered rounded to one digit
     * @param   string  $strText         Text to be checked
     * @return  int|float
     */
    public function gunning_fog_score($strText)
    {
        $strText = $this->clean_text($strText);

        $score = self::bc_calc(self::bc_calc($this->average_words_per_sentence($strText), '+', $this->percentage_words_with_three_syllables($strText, false)), '*', '0.4');

        return $this->normalize_score($score, 0, 19);
    }

    /**
     * Gives the Coleman-Liau Index of text entered rounded to one digit
     * @param   string  $strText         Text to be checked
     * @return  int|float
     */
    public function coleman_liau_index($strText)
    {
        $strText = $this->clean_text($strText);

        $score = self::bc_calc(self::bc_calc(self::bc_calc(5.89, '*', self::bc_calc($this->letter_count($strText), '/', $this->word_count($strText))), '-', self::bc_calc(0.3, '*', self::bc_calc($this->sentence_count($strText), '/', $this->word_count($strText)))), '-', 15.8);

        return $this->normalize_score($score, 0, 12);
    }

    /**
     * Gives the SMOG Index of text entered rounded to one digit
     * @param   string  $strText         Text to be checked
     * @return  int|float
     */
    public function smog_index($strText)
    {
        $strText = $this->clean_text($strText);

        $score = self::bc_calc(1.043, '*', sqrt(self::bc_calc(self::bc_calc($this->words_with_three_syllables($strText), '*', self::bc_calc(30, '/', $this->sentence_count($strText))), '+', 3.1291)), true, 1);

        return $this->normalize_score($score, 0, 12);
    }

    /**
     * Gives the Automated Readability Index of text entered rounded to one digit
     * @param   string  $strText         Text to be checked
     * @return  int|float
     */
    public function automated_readability_index($strText)
    {
        $strText = $this->clean_text($strText);

        $score = self::bc_calc(self::bc_calc(4.71, '*', self::bc_calc($this->letter_count($strText), '/', $this->word_count($strText))), '+', self::bc_calc(self::bc_calc(0.5, '*', self::bc_calc($this->word_count($strText), '/', $this->sentence_count($strText))), '-', 21.43), true, 1);

        return $this->normalize_score($score, 0, 12);
    }

    /**
     * Gives string length. Tries mb_strlen and if that fails uses regular strlen.
     * @param   string  $strText      Text to be measured
     * @return  int
     */
    public function text_length($strText)
    {
        $intTextLength = 0;
        try {

            if (!$this->blnMbstring) {
                throw new Exception('The extension mbstring is not loaded.');
            }

            if ($this->strEncoding == '') {
                $intTextLength = mb_strlen($strText);
            } else {
                $intTextLength = mb_strlen($strText, $this->strEncoding);
            }
        } catch (Exception $e) {
            $intTextLength = strlen($strText);
        }

        return $intTextLength;
    }

    /**
     * Gives letter count (ignores all non-letters). Tries mb_strlen and if that fails uses regular strlen.
     * @param   string  $strText      Text to be measured
     * @return  int
     */
    public function letter_count($strText)
    {
        $strText = $this->clean_text($strText); // To clear out newlines etc
        $intTextLength = 0;
        $strText = preg_replace('/[^A-Za-z]+/', '', $strText);
        try {

            if (!$this->blnMbstring) {
                throw new Exception('The extension mbstring is not loaded.');
            }

            if ($this->strEncoding == '') {
                $intTextLength = mb_strlen($strText);
            } else {
                $intTextLength = mb_strlen($strText, $this->strEncoding);
            }
        } catch (Exception $e) {
            $intTextLength = strlen($strText);
        }

        return $intTextLength;
    }

    /**
     * Trims, removes line breaks, multiple spaces and generally cleans text before processing.
     * @param   string  $strText      Text to be transformed
     * @return  string
     */
    protected function clean_text($strText)
    {
        static $clean = array();

		$key = sha1($strText);

		if (isset($clean[$key])) {
			return $clean[$key];
		}

        // all these tags should be preceeded by a full stop.
        $fullStopTags = array('li', 'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'dd');
        foreach ($fullStopTags as $tag) {
            $strText = str_ireplace('</'.$tag.'>', '.', $strText);
        }
        $strText = strip_tags($strText);
        $strText = preg_replace('/[",:;()-]/', ' ', $strText); // Replace commas, hyphens, quotes etc (count them as spaces)<commit_msg>spaces not tabs<commit_after><?php

/*

    TextStatistics Class
    https://github.com/DaveChild/Text-Statistics

    Released under New BSD license
    http://www.opensource.org/licenses/bsd-license.php

    Calculates following readability scores (formulae can be found in wiki):
      * Flesch Kincaid Reading Ease
      * Flesch Kincaid Grade Level
      * Gunning Fog Score
      * Coleman Liau Index
      * SMOG Index
      * Automated Reability Index

    Will also give:
      * String length
      * Letter count
      * Syllable count
      * Sentence count
      * Average words per sentence
      * Average syllables per word

    Sample Code
    ----------------
    $statistics = new TextStatistics;
    $text = 'The quick brown fox jumped over the lazy dog.';
    echo 'Flesch-Kincaid Reading Ease: ' . $statistics->flesch_kincaid_reading_ease($text);

*/

class TextStatistics
{
    protected $strEncoding = ''; // Used to hold character encoding to be used by object, if set

    protected $blnMbstring = true; // Efficiency: Is the MB String extension loaded ?

    protected static $blnBcmath = true; // Efficiency: Is the BC Math extension loaded ?

    public $normalize = true;

    /**
     * Constructor.
     *
     * @param  string  $strEncoding Optional character encoding.
     * @return void
     */
    public function __construct($strEncoding = '')
    {
        if ($strEncoding <> '') {
            // Encoding is given. Use it!
            $this->strEncoding = $strEncoding;
        }

        $this->blnMbstring = extension_loaded('mbstring');
        self::$blnBcmath = extension_loaded('bcmath');
    }

    /**
     * Gives the Flesch-Kincaid Reading Ease of text entered rounded to one digit
     * @param   string  $strText         Text to be checked
     * @return  int|float
     */
    public function flesch_kincaid_reading_ease($strText)
    {
        $strText = $this->clean_text($strText);

        $score = self::bc_calc(self::bc_calc(206.835, '-', self::bc_calc(1.015, '*', $this->average_words_per_sentence($strText))), '-', self::bc_calc(84.6, '*', $this->average_syllables_per_word($strText)));

        return $this->normalize_score($score, 0, 100);
    }

    /**
     * Gives the Flesch-Kincaid Grade level of text entered rounded to one digit
     * @param   string  $strText         Text to be checked
     * @return  int|float
     */
    public function flesch_kincaid_grade_level($strText)
    {
        $strText = $this->clean_text($strText);

        $score = self::bc_calc(self::bc_calc(0.39, '*', $this->average_words_per_sentence($strText)), '+', self::bc_calc(self::bc_calc(11.8, '*', $this->average_syllables_per_word($strText)), '-', 15.59), true, 1);

        return $this->normalize_score($score, 0, 12);
    }

    /**
     * Gives the Gunning-Fog score of text entered rounded to one digit
     * @param   string  $strText         Text to be checked
     * @return  int|float
     */
    public function gunning_fog_score($strText)
    {
        $strText = $this->clean_text($strText);

        $score = self::bc_calc(self::bc_calc($this->average_words_per_sentence($strText), '+', $this->percentage_words_with_three_syllables($strText, false)), '*', '0.4');

        return $this->normalize_score($score, 0, 19);
    }

    /**
     * Gives the Coleman-Liau Index of text entered rounded to one digit
     * @param   string  $strText         Text to be checked
     * @return  int|float
     */
    public function coleman_liau_index($strText)
    {
        $strText = $this->clean_text($strText);

        $score = self::bc_calc(self::bc_calc(self::bc_calc(5.89, '*', self::bc_calc($this->letter_count($strText), '/', $this->word_count($strText))), '-', self::bc_calc(0.3, '*', self::bc_calc($this->sentence_count($strText), '/', $this->word_count($strText)))), '-', 15.8);

        return $this->normalize_score($score, 0, 12);
    }

    /**
     * Gives the SMOG Index of text entered rounded to one digit
     * @param   string  $strText         Text to be checked
     * @return  int|float
     */
    public function smog_index($strText)
    {
        $strText = $this->clean_text($strText);

        $score = self::bc_calc(1.043, '*', sqrt(self::bc_calc(self::bc_calc($this->words_with_three_syllables($strText), '*', self::bc_calc(30, '/', $this->sentence_count($strText))), '+', 3.1291)), true, 1);

        return $this->normalize_score($score, 0, 12);
    }

    /**
     * Gives the Automated Readability Index of text entered rounded to one digit
     * @param   string  $strText         Text to be checked
     * @return  int|float
     */
    public function automated_readability_index($strText)
    {
        $strText = $this->clean_text($strText);

        $score = self::bc_calc(self::bc_calc(4.71, '*', self::bc_calc($this->letter_count($strText), '/', $this->word_count($strText))), '+', self::bc_calc(self::bc_calc(0.5, '*', self::bc_calc($this->word_count($strText), '/', $this->sentence_count($strText))), '-', 21.43), true, 1);

        return $this->normalize_score($score, 0, 12);
    }

    /**
     * Gives string length. Tries mb_strlen and if that fails uses regular strlen.
     * @param   string  $strText      Text to be measured
     * @return  int
     */
    public function text_length($strText)
    {
        $intTextLength = 0;
        try {

            if (!$this->blnMbstring) {
                throw new Exception('The extension mbstring is not loaded.');
            }

            if ($this->strEncoding == '') {
                $intTextLength = mb_strlen($strText);
            } else {
                $intTextLength = mb_strlen($strText, $this->strEncoding);
            }
        } catch (Exception $e) {
            $intTextLength = strlen($strText);
        }

        return $intTextLength;
    }

    /**
     * Gives letter count (ignores all non-letters). Tries mb_strlen and if that fails uses regular strlen.
     * @param   string  $strText      Text to be measured
     * @return  int
     */
    public function letter_count($strText)
    {
        $strText = $this->clean_text($strText); // To clear out newlines etc
        $intTextLength = 0;
        $strText = preg_replace('/[^A-Za-z]+/', '', $strText);
        try {

            if (!$this->blnMbstring) {
                throw new Exception('The extension mbstring is not loaded.');
            }

            if ($this->strEncoding == '') {
                $intTextLength = mb_strlen($strText);
            } else {
                $intTextLength = mb_strlen($strText, $this->strEncoding);
            }
        } catch (Exception $e) {
            $intTextLength = strlen($strText);
        }

        return $intTextLength;
    }

    /**
     * Trims, removes line breaks, multiple spaces and generally cleans text before processing.
     * @param   string  $strText      Text to be transformed
     * @return  string
     */
    protected function clean_text($strText)
    {
        static $clean = array();

        $key = sha1($strText);

        if (isset($clean[$key])) {
            return $clean[$key];
        }

        // all these tags should be preceeded by a full stop.
        $fullStopTags = array('li', 'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'dd');
        foreach ($fullStopTags as $tag) {
            $strText = str_ireplace('</'.$tag.'>', '.', $strText);
        }
        $strText = strip_tags($strText);
        $strText = preg_replace('/[",:;()-]/', ' ', $strText); // Replace commas, hyphens, quotes etc (count them as spaces)
{'fraction_non_alphanumeric': 0.14371972662729746, 'fraction_numerical': 0.01114723641430562, 'mean_word_length': 2.458467186782928, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 83, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '720826', 'n_tokens_mistral': 4892, 'n_tokens_neox': 4623, 'n_words': 1113}