<commit_before><?php
namespace phpdotnet\phd;
/* $Id$ */

abstract class Format extends ObjectStorage {
    const SDESC = 1;
    const LDESC = 2;

    private $elementmap = array();
    private $textmap = array();
    private $formatname = "UNKNOWN";
    protected $sqlite;

    protected $title;
    protected $fp = array();
    protected $ext;
    protected $outputdir;
    protected $chunked;

    /*Processing Instructions Handlers*/
    private $pihandlers = array();

    /* Indexing maps */
    protected $indexes = array();
    protected $childrens = array();
    protected $refs = array();
    protected $vars = array();
    protected $classes = array();

    private static $autogen = array();

    /* PhDHelper */
    private static $highlighters = array();

    public function __construct() {
        if (file_exists(Config::output_dir() . "index.sqlite")) {
            $this->sqlite = new \SQLite3(Config::output_dir() . 'index.sqlite');
            $this->sortIDs();
        }
    }

    abstract public function transformFromMap($open, $tag, $name, $attrs, $props);
    abstract public function UNDEF($open, $name, $attrs, $props);
    abstract public function TEXT($value);
    abstract public function CDATA($value);
    abstract public function createLink($for, &$desc = null, $type = Format::SDESC);
    abstract public function appendData($data);
    abstract public function update($event, $value = null);

    public final function parsePI($target, $data) {
        if (isset($this->pihandlers[$target])) {
            return $this->pihandlers[$target]->parse($target, $data);
        }
    }

    public final function registerPIHandlers($pihandlers) {
        foreach ($pihandlers as $target => $classname) {
            $class = __NAMESPACE__ . "\\" . $classname;
            $this->pihandlers[$target] = new $class($this);
        } 
    }

    public function getPIHandler($target) {
        return $this->pihandlers[$target];
    }

    public function sortIDs() {
        $this->sqlite->createAggregate("indexes", array($this, "SQLiteIndex"), array($this, "SQLiteFinal"), 9);
        $this->sqlite->createAggregate("childrens", array($this, "SQLiteChildrens"), array($this, "SQLiteFinal"), 2);
        $this->sqlite->createAggregate("refname", array($this, "SQLiteRefname"), array($this, "SQLiteFinal"), 2);
        $this->sqlite->createAggregate("varname", array($this, "SQLiteVarname"), array($this, "SQLiteFinal"), 2);
        $this->sqlite->createAggregate("classname", array($this, "SQLiteClassname"), array($this, "SQLiteFinal"), 2);
        $this->sqlite->query('SELECT indexes(docbook_id, filename, parent_id, sdesc, ldesc, element, previous, next, chunk) FROM ids');
        $this->sqlite->query('SELECT childrens(docbook_id, parent_id) FROM ids WHERE chunk != 0');
        $this->sqlite->query('SELECT refname(docbook_id, sdesc) FROM ids WHERE element=\'refentry\'');
        $this->sqlite->query('SELECT varname(docbook_id, sdesc) FROM ids WHERE element=\'phpdoc:varentry\'');
        $this->sqlite->query('SELECT classname(docbook_id, sdesc) FROM ids WHERE element=\'phpdoc:exceptionref\' OR element=\'phpdoc:classref\'');
    }

    public function SQLiteIndex(&$context, $index, $id, $filename, $parent, $sdesc, $ldesc, $element, $previous, $next, $chunk) {
        $this->indexes[$id] = array(
            "docbook_id" => $id,
            "filename"   => $filename,
            "parent_id"  => $parent,
            "sdesc"      => $sdesc,
            "ldesc"      => $ldesc,
            "element"    => $element,
            "previous"   => $previous,
            "next"       => $next,
            "chunk"      => $chunk,
        );
    }

    public function SQLiteChildrens(&$context, $index, $id, $parent) {
        if (!isset($this->childrens[$parent]) || !is_array($this->childrens[$parent])) {
            $this->childrens[$parent] = array();
        }
        $this->childrens[$parent][] = $id;
    }

    public function SQLiteRefname(&$context, $index, $id, $sdesc) {
        $ref = strtolower(str_replace(array("_", "::", "->"), array("-", "-", "-"), $sdesc));
        $this->refs[$ref] = $id;
    }

    public function SQLiteVarname(&$context, $index, $id, $sdesc) {
        $this->vars[$sdesc] = $id;
    }

    public function SQLiteClassname(&$context, $index, $id, $sdesc) {
        $this->classes[strtolower($sdesc)] = $id;
    }

    public static function SQLiteFinal(&$context) {
        return $context;
    }

    final public function notify($event, $val = null) {
        $this->update($event, $val);
        foreach($this as $format) {
            $format->update($event, $val);<commit_msg>Quick fix for the bug #49675. The problem is because the indexer stores the sdesc and ldesc with htmlentities. I think we should remove that function from the indexer and only use when necessary.<commit_after><?php
namespace phpdotnet\phd;
/* $Id$ */

abstract class Format extends ObjectStorage {
    const SDESC = 1;
    const LDESC = 2;

    private $elementmap = array();
    private $textmap = array();
    private $formatname = "UNKNOWN";
    protected $sqlite;

    protected $title;
    protected $fp = array();
    protected $ext;
    protected $outputdir;
    protected $chunked;

    /*Processing Instructions Handlers*/
    private $pihandlers = array();

    /* Indexing maps */
    protected $indexes = array();
    protected $childrens = array();
    protected $refs = array();
    protected $vars = array();
    protected $classes = array();

    private static $autogen = array();

    /* PhDHelper */
    private static $highlighters = array();

    public function __construct() {
        if (file_exists(Config::output_dir() . "index.sqlite")) {
            $this->sqlite = new \SQLite3(Config::output_dir() . 'index.sqlite');
            $this->sortIDs();
        }
    }

    abstract public function transformFromMap($open, $tag, $name, $attrs, $props);
    abstract public function UNDEF($open, $name, $attrs, $props);
    abstract public function TEXT($value);
    abstract public function CDATA($value);
    abstract public function createLink($for, &$desc = null, $type = Format::SDESC);
    abstract public function appendData($data);
    abstract public function update($event, $value = null);

    public final function parsePI($target, $data) {
        if (isset($this->pihandlers[$target])) {
            return $this->pihandlers[$target]->parse($target, $data);
        }
    }

    public final function registerPIHandlers($pihandlers) {
        foreach ($pihandlers as $target => $classname) {
            $class = __NAMESPACE__ . "\\" . $classname;
            $this->pihandlers[$target] = new $class($this);
        } 
    }

    public function getPIHandler($target) {
        return $this->pihandlers[$target];
    }

    public function sortIDs() {
        $this->sqlite->createAggregate("indexes", array($this, "SQLiteIndex"), array($this, "SQLiteFinal"), 9);
        $this->sqlite->createAggregate("childrens", array($this, "SQLiteChildrens"), array($this, "SQLiteFinal"), 2);
        $this->sqlite->createAggregate("refname", array($this, "SQLiteRefname"), array($this, "SQLiteFinal"), 2);
        $this->sqlite->createAggregate("varname", array($this, "SQLiteVarname"), array($this, "SQLiteFinal"), 2);
        $this->sqlite->createAggregate("classname", array($this, "SQLiteClassname"), array($this, "SQLiteFinal"), 2);
        $this->sqlite->query('SELECT indexes(docbook_id, filename, parent_id, sdesc, ldesc, element, previous, next, chunk) FROM ids');
        $this->sqlite->query('SELECT childrens(docbook_id, parent_id) FROM ids WHERE chunk != 0');
        $this->sqlite->query('SELECT refname(docbook_id, sdesc) FROM ids WHERE element=\'refentry\'');
        $this->sqlite->query('SELECT varname(docbook_id, sdesc) FROM ids WHERE element=\'phpdoc:varentry\'');
        $this->sqlite->query('SELECT classname(docbook_id, sdesc) FROM ids WHERE element=\'phpdoc:exceptionref\' OR element=\'phpdoc:classref\'');
    }

    public function SQLiteIndex(&$context, $index, $id, $filename, $parent, $sdesc, $ldesc, $element, $previous, $next, $chunk) {
        $this->indexes[$id] = array(
            "docbook_id" => $id,
            "filename"   => $filename,
            "parent_id"  => $parent,
            "sdesc"      => $sdesc,
            "ldesc"      => $ldesc,
            "element"    => $element,
            "previous"   => $previous,
            "next"       => $next,
            "chunk"      => $chunk,
        );
    }

    public function SQLiteChildrens(&$context, $index, $id, $parent) {
        if (!isset($this->childrens[$parent]) || !is_array($this->childrens[$parent])) {
            $this->childrens[$parent] = array();
        }
        $this->childrens[$parent][] = $id;
    }

    public function SQLiteRefname(&$context, $index, $id, $sdesc) {
        $ref = strtolower(str_replace(array("_", "::", "->"), array("-", "-", "-"), html_entity_decode($sdesc, ENT_QUOTES, 'UTF-8')));
        $this->refs[$ref] = $id;
    }

    public function SQLiteVarname(&$context, $index, $id, $sdesc) {
        $this->vars[$sdesc] = $id;
    }

    public function SQLiteClassname(&$context, $index, $id, $sdesc) {
        $this->classes[strtolower($sdesc)] = $id;
    }

    public static function SQLiteFinal(&$context) {
        return $context;
    }

    final public function notify($event, $val = null) {
        $this->update($event, $val);
        foreach($this as $format) {
            $format->update($event, $val);
{'fraction_non_alphanumeric': 0.16623022939143187, 'fraction_numerical': 0.0025138787053524667, 'mean_word_length': 3.05607476635514, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1969952', 'n_tokens_mistral': 2912, 'n_tokens_neox': 2863, 'n_words': 674}