<commit_before><?php

namespace Documer;

Class Documer
{

    /**
     * @var Spot Spot Object
     */
    var $db;

    function __construct(\Spot\Locator $spot)
    {
        date_default_timezone_set('Europe/Athens');

        $this->db = $spot;

    }

    /**
     * This will return if a word is starting with an Uppercase letter, with UTF-8 Support
     *
     * @see http://stackoverflow.com/questions/2814880/how-to-check-if-letter-is-upper-or-lower-in-php
     *
     * @param $str String The string to examine
     */
    private function startsWithUppercase($str)
    {

        $chr = mb_substr($str, 0, 1, "UTF-8");

        return mb_strtolower($chr, "UTF-8") != $chr && mb_strlen($str, "UTF-8") > 1;
    }

    /**
     * This is our training method, that parses the text and push the keywords to DB
     *
     * @param $label
     * @param $text
     */
    public function train($label, $text)
    {

        $keywords = $this->parse($text);

        $labelMapper = $this->db->mapper('Documer\Entity\Label');
        $labelMapper->migrate();
        $labelModel       = $labelMapper->get();
        $labelModel->name = $label;
        $labelMapper->insert($labelModel);

        foreach ($keywords as $k) {

            $wordMapper = $this->db->mapper('Documer\Entity\Word');
            $wordMapper->migrate();
            $wordModel        = $wordMapper->get();
            $wordModel->label = $label;
            $wordModel->name  = $k;
            $wordMapper->insert($wordModel);

        }

    }

    /**
     * We need to parse the text from some sort of tokenization
     *
     * We keep only alphanumeric strings that starts with an uppercase
     *
     * @param $text
     *
     * @return array
     */
    private function parse($text)
    {

        $unwantedChars = array(
            ',', '!', '?', '.', ']', '[', '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', '/', ':', ';', '<',
            '=', '>', '?', '^', '{', '|', '}', '~', '-', '@', '\', ', '_', '`'
        );
        $str   = str_replace($unwantedChars, ' ', $text);
        $array = explode(" ", $str);
        $array = array_map('trim', $array);
        $array = array_unique($array);
        $array = array_values($array);

        $clean = array();
        foreach ($array as $k)
            if ($this->startsWithUppercase($k))
                $clean[ ] = $k;

        return array_values($clean);

    }

    /*
     * This is the guessing method, which uses Bayes Theorem to calculate probabilities
     */
    public function guess($text)
    {

        $scores = array();
        $words  = $this->parse($text);

        $labels = $this->getDistinctLabels();

        foreach ($labels as $label) {
            $logSum = 0;

            foreach ($words as $word) {

                $wordTotalCount = $this->getWordCount($word);

                if ($wordTotalCount == 0) {

                    continue;

                } else {

                    $wordProbability        = $this->getWordProbabilityWithLabel($word, $label);
                    $wordInverseProbability = $this->getInverseWordProbabilityWithLabel($word, $label);

                    /**
                     * Bayes Theorem using the above parameters
                     *
                     * the probability that this document is a particular LABEL
                     * given that a particular WORD is in it
                     *
                     */
                    $wordicity = $wordProbability / ($wordProbability + $wordInverseProbability);

                    /*
                     * here 0.5 is the weight, higher training data in the db means higher weight
                     */
                    $wordicity = ((10 * 0.5) + ($wordTotalCount * $wordicity)) / (10 + $wordTotalCount);

                    if ($wordicity == 0)
                        $wordicity = 0.01;
                    else if ($wordicity == 1)
                        $wordicity = 0.99;
                }

                /**
                 * logs to avoid "floating point underflow",
                 */
                $logSum += (log(1 - $wordicity) - log($wordicity));
            }


            /**
             * undo the log function and get back to 0-1 range
             */
            $scores[ $label ] = 1 / (1 + exp($logSum));
        }


        return $scores;
    }


    /**
     * Get total documents the system has been trained,
     * by counting the number of labels (not distinct)
     *
     * @return int
     */
    public function getTotalDocs()
    {
        $labelMapper = $this->db->mapper('Documer\Entity\Label');

        return count($labelMapper->all());
    }

    /**
     * Get how many documents we have seen so far for each label
     *
     * @return array
     */
    public function getTotalDocsGroupByLabel()
    {
        $labelMapper = $this->db->mapper('Documer\Entity\Label');

        $eachLabelTotal = $labelMapper->query("SELECT name, COUNT( name ) AS total FROM labels GROUP BY name");
<commit_msg>is function added check against a given label<commit_after><?php

namespace Documer;

Class Documer
{

    /**
     * @var Spot Spot Object
     */
    var $db;

    function __construct(\Spot\Locator $spot)
    {
        date_default_timezone_set('Europe/Athens');

        $this->db = $spot;

    }

    /**
     * This will return if a word is starting with an Uppercase letter, with UTF-8 Support
     *
     * @see http://stackoverflow.com/questions/2814880/how-to-check-if-letter-is-upper-or-lower-in-php
     *
     * @param $str String The string to examine
     */
    private function startsWithUppercase($str)
    {

        $chr = mb_substr($str, 0, 1, "UTF-8");

        return mb_strtolower($chr, "UTF-8") != $chr && mb_strlen($str, "UTF-8") > 1;
    }

    /**
     * This is our training method, that parses the text and push the keywords to DB
     *
     * @param $label
     * @param $text
     */
    public function train($label, $text)
    {

        $keywords = $this->parse($text);

        $labelMapper = $this->db->mapper('Documer\Entity\Label');
        $labelMapper->migrate();
        $labelModel       = $labelMapper->get();
        $labelModel->name = $label;
        $labelMapper->insert($labelModel);

        foreach ($keywords as $k) {

            $wordMapper = $this->db->mapper('Documer\Entity\Word');
            $wordMapper->migrate();
            $wordModel        = $wordMapper->get();
            $wordModel->label = $label;
            $wordModel->name  = $k;
            $wordMapper->insert($wordModel);

        }

    }

    /**
     * We need to parse the text from some sort of tokenization
     *
     * We keep only alphanumeric strings that starts with an uppercase
     *
     * @param $text
     *
     * @return array
     */
    private function parse($text)
    {

        $unwantedChars = array(
            ',', '!', '?', '.', ']', '[', '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', '/', ':', ';', '<',
            '=', '>', '?', '^', '{', '|', '}', '~', '-', '@', '\', ', '_', '`'
        );
        $str   = str_replace($unwantedChars, ' ', $text);
        $array = explode(" ", $str);
        $array = array_map('trim', $array);
        $array = array_unique($array);
        $array = array_values($array);

        $clean = array();
        foreach ($array as $k)
            if ($this->startsWithUppercase($k))
                $clean[ ] = $k;

        return array_values($clean);

    }

    /*
     * This is the guessing method, which uses Bayes Theorem to calculate probabilities
     */
    public function guess($text)
    {

        $scores = array();
        $words  = $this->parse($text);

        $labels = $this->getDistinctLabels();

        foreach ($labels as $label) {
            $logSum = 0;

            foreach ($words as $word) {

                $wordTotalCount = $this->getWordCount($word);

                if ($wordTotalCount == 0) {

                    continue;

                } else {

                    $wordProbability        = $this->getWordProbabilityWithLabel($word, $label);
                    $wordInverseProbability = $this->getInverseWordProbabilityWithLabel($word, $label);

                    /**
                     * Bayes Theorem using the above parameters
                     *
                     * the probability that this document is a particular LABEL
                     * given that a particular WORD is in it
                     *
                     */
                    $wordicity = $wordProbability / ($wordProbability + $wordInverseProbability);

                    /*
                     * here 0.5 is the weight, higher training data in the db means higher weight
                     */
                    $wordicity = ((10 * 0.5) + ($wordTotalCount * $wordicity)) / (10 + $wordTotalCount);

                    if ($wordicity == 0)
                        $wordicity = 0.01;
                    else if ($wordicity == 1)
                        $wordicity = 0.99;
                }

                /**
                 * logs to avoid "floating point underflow",
                 */
                $logSum += (log(1 - $wordicity) - log($wordicity));
            }


            /**
             * undo the log function and get back to 0-1 range
             */
            $scores[ $label ] = 1 / (1 + exp($logSum));
        }


        return $scores;
    }

    /**
     * Check if text is of the given label
     *
     * @param $label
     * @param $text
     */
    public function is($label, $text) {
        $scores = $this->guess($text);

        $value = max($scores);

        return $label == array_search($value, $scores);
    }


    /**
     * Get total documents the system has been trained,
     * by counting the number of labels (not distinct)
     *
     * @return int
     */
    public function getTotalDocs()
    {
        $labelMapper = $this->db->mapper('Documer\Entity\Label');

        return count($labelMapper->all());
    }

    /**
     * Get how many documents we have seen so far for each label
     *
     * @return array
     */
    public function getTotalDocsGroupByLabel()
    {
        $labelMapper = $this->db->mapper('Documer\Entity\Label');

        $eachLabelTotal = $labelMapper->query("SELECT name, COUNT( name ) AS total FROM labels GROUP BY name");

{'fraction_non_alphanumeric': 0.14605009633911367, 'fraction_numerical': 0.007129094412331407, 'mean_word_length': 1.597847847847848, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2706356', 'n_tokens_mistral': 3102, 'n_tokens_neox': 2877, 'n_words': 719}