<commit_before><?php
namespace php_active_record;

class WikimediaHarvester
{
    private $mysqli;
    private $resource_file;
    private $resource;

    function __construct($resource, $files_subdir)
    {
        $this->mysqli =& $GLOBALS['mysqli_connection'];
        $this->validMIMEtypes = array('image/png', 'image/jpeg', 'image/gif', 'image/svg+xml', 'image/tiff', 'image/x-xcf', 'application/ogg', 'audio/ogg', 'video/webm', 'video/ogg', 'video/webm');
        $this->resource = $resource;
        $this->base_directory_path = DOC_ROOT . $files_subdir;
        $this->part_files = array('base' => $this->base_directory_path, 'subdir' => 'wikimedia', 'prefix' => 'part_');
        $this->part_path = implode(DIRECTORY_SEPARATOR, $this->part_files);
        // allows for splitting the file into max 26^n parts, e.g. n=3, parts=300Mb, copes with xml files < 4.2Tb
        $this->part_file_suffix_chars = 3;
        $this->number_of_separate_taxa = 0;
        $this->taxa = array();
        $this->taxonav_includes = array();
        $this->galleries_for_file = array();  // key=media-filename, value = array of gallery names (usually just one name)
        $this->taxonomies_for_file = array(); // key=media-filename, value = count of taxonomies for this file (just for info)
        $this->taxonomy_pagenames = array();  // key=media-filename, value = array of redirects (used as temp name store)
        $this->map_categories = self::get_map_categories($this->base_directory_path);
        // TODO - add blacklist of "unwanted" categories, so that if an image falls into one of these (or a child thereof),
        // it is not harvested. E.g. a suggested "unwanted" category might be
        // Category:Uploaded_with_Open_Access_Media_Importer_and_needing_category_review
        $this->total_pages_in_dump = 0;
        $this->queue_of_pages_to_process = array();

        $this->resource_file_path = CONTENT_RESOURCE_LOCAL_PATH . $this->resource->id . "_temp.xml";
        $this->resource_file = new \SchemaDocument($this->resource_file_path);
    }

    public function begin_wikipedia_harvest()
    {
        //set encoding for the xml dump file - important for things like WikiParser::mb_ucfirst
        mb_internal_encoding("UTF-8");
        // delete the downloaded files
        $this->cleanup_dump();
        $this->download_dump();

        // FIRST PASS: parse TaxonavigationIncluded* pages (e.g. https://commons.wikimedia.org/wiki/Template:Aves)
        // simultaneously locate galleries and categories with potential taxonomic information (i.e. a Taxonavigation template)
        // also make a list of "gallery media" (files listed in taxonomic galleries)
        $this->iterate_files(array($this, 'locate_taxonomic_pages'));

        // SECOND PASS: parse & validate taxonomic information in galleries + categories - place in 'taxa' array
        // simutaneously link to 'taxa' any pages which redirect to these taxonomic pages
        // also if any gallery media are redirects, replace their name with the proper (redirected) name.
        $this->iterate_files(array($this, 'check_taxonomy_and_redirects'));

        // FINAL PASS: check files for categories, grab file information for scientific media pages and save to file
        $this->iterate_files(array($this, 'get_media_pages'));

        echo "\n(processing last ". count($this->queue_of_pages_to_process) ." media)\n";
        $this->process_page_queue();
        echo "\n\nTOTAL # OF MEDIA FILES: ".count($this->taxonomies_for_file)." (in ". count($this->taxa) ." taxa)\n";
        $this->check_for_unaccounted_galleries();

        @unlink(CONTENT_RESOURCE_LOCAL_PATH . $this->resource->id."_previous.xml");
        @rename(CONTENT_RESOURCE_LOCAL_PATH . $this->resource->id.".xml", CONTENT_RESOURCE_LOCAL_PATH . $this->resource->id."_previous.xml");
        rename(CONTENT_RESOURCE_LOCAL_PATH . $this->resource->id."_temp.xml", CONTENT_RESOURCE_LOCAL_PATH . $this->resource->id.".xml");

        echo "\nEND\n";
        self::print_memory_and_time();
    }

    private function download_dump()
    {
        // download latest Wikimedia Commons export
        echo "Downloading ".$this->resource->accesspoint_url." ...\n";
        shell_exec("curl ".escapeshellarg($this->resource->accesspoint_url)." -o ". escapeshellarg($this->base_directory_path . "wikimedia.xml.bz2"));
        // unzip the download
        echo "... unpacking downloaded file ...\n";
        shell_exec("bunzip2 --force ". escapeshellarg($this->base_directory_path . "wikimedia.xml.bz2"));
        // split the huge file into 300M chunks
        echo "... splitting file into parts ...\n";
        shell_exec("split -a ". $this->part_file_suffix_chars ." -b 300m ". escapeshellarg($this->base_directory_path . "wikimedia.xml")." ". escapeshellarg($this->part_path));
        echo "... done.\n";
    }

    private function cleanup_dump()
    {
        // cleaning up downloaded files
        echo "Removing old wikimedia dump files ...\n";
        shell_exec("rm -f ". escapeshellarg($this->part_path)."*");
        shell_exec("rm -f ". escapeshellarg($this->base_directory_path . "wikimedia.xml"));
        shell_exec("rm -f ". escapeshellarg($this->base_directory_path . "wikimedia.xml.bz2"));
        echo "... done.\n";
    }

    private function iterate_files($callback)
    {
        $total_pages_processed = 0;
        $this->page_iteration_left_overs = "";
        $suffix = str_repeat('a', $this->part_file_suffix_chars);
        while((strlen($suffix) == $this->part_file_suffix_chars))
        {
            $filename = $this->part_path . $suffix;
            if(!file_exists($filename))
            {
                echo "Assuming no more part files to process (as ". basename($filename) ." doesn't exist)\n";
                break;
            }
            $total_pages_processed += $this->process_file($filename, $callback);
            // auto-increment allows us to match the output of the 'split' command: aaa->aab, aaz->aba, etc
            $suffix++;
        }
        if(!preg_match('/\s*<\/mediawiki>\s*/smi', $this->page_iteration_left_overs))
        {
            echo "WARNING: THE LAST WIKI FILE APPEARS TO BE TRUNCATED. Part of the wiki download may be missing.\n";
        }
        if($this->total_pages_in_dump == 0) $this->total_pages_in_dump = $total_pages_processed;
        echo "\n---------------------(done processing $total_pages_processed pages)---------------------\n\n\n";
    }

    private function process_file($filename, $callback)
    {
        $pages_processed = 0;
        if(file_exists($filename))
        {
            echo "Processing file '". basename($filename) ."' with callback ". $callback[1] ."\n";
            self::print_memory_and_time();

            // the last file ended in the middle of a page. Use the remainder of the last file as a starting point
            if(isset($this->page_iteration_left_overs)) $current_page = $this->page_iteration_left_overs;
            else $current_page = "";
            $this->page_iteration_left_overs = "";
            foreach(new FileIterator($filename, false, false) as $line)
            {
                $current_page .= $line;
                // this is a new page so reset $current_page
                if(trim($line) == "<page>") $current_page = $line;
                elseif(trim($line) == "</page>")
                {
                    call_user_func($callback, $current_page);
                    $pages_processed++;
                    $current_page = "";
                }
            }
            // in the middle of a <page> tag. Save the current page to use as a starting point for the next file
            $this->page_iteration_left_overs = $current_page;
        }
        return $pages_processed;
    }

    private function locate_taxonomic_pages($xml)
    {
        $this->debug_locate_taxonomic_pages();
        // make sure we don't include cases with {{Taxonavigation in the comments field, etc.
        if(!\WikimediaPage::fast_is_gallery_category_or_template($xml)) return false;
        if(!($text_start = strpos($xml, "<text"))) return false;
        if(strpos($xml, "{{Taxonavigation", $text_start) === false) return false;
        $page = new \WikimediaPage($xml);
        if($page->is_template())
        {
            // should check here for template redirects
            if($page->contains_template("TaxonavigationIncluded[\w\s]*"))
            {
                $include_array = $page->taxonav_as_array("[Tt]axonavigationIncluded[\w\s]*");
                if(count($include_array))
                {
                    $this->taxonav_includes[$page->title] = array('taxo' => $include_array, 'last_mod' => strtotime($page->timestamp));
                }else echo "$page->title is not a real TaxonavigationInclude* template\n";
            }elseif($page->contains_template("Taxonavigation"))
            {
                // This is a template that itself contains the template "Taxonavigation", so we might be interested in it<commit_msg>Stamp on last warning message about strtotime not having a set timezone (wiki timestamps always have a timezone set - this is an overzealous PHP warning)<commit_after><?php
namespace php_active_record;

class WikimediaHarvester
{
    private $mysqli;
    private $resource_file;
    private $resource;

    function __construct($resource, $files_subdir)
    {
        $this->mysqli =& $GLOBALS['mysqli_connection'];
        $this->validMIMEtypes = array('image/png', 'image/jpeg', 'image/gif', 'image/svg+xml', 'image/tiff', 'image/x-xcf', 'application/ogg', 'audio/ogg', 'video/webm', 'video/ogg', 'video/webm');
        $this->resource = $resource;
        $this->base_directory_path = DOC_ROOT . $files_subdir;
        $this->part_files = array('base' => $this->base_directory_path, 'subdir' => 'wikimedia', 'prefix' => 'part_');
        $this->part_path = implode(DIRECTORY_SEPARATOR, $this->part_files);
        // allows for splitting the file into max 26^n parts, e.g. n=3, parts=300Mb, copes with xml files < 4.2Tb
        $this->part_file_suffix_chars = 3;
        $this->number_of_separate_taxa = 0;
        $this->taxa = array();
        $this->taxonav_includes = array();
        $this->galleries_for_file = array();  // key=media-filename, value = array of gallery names (usually just one name)
        $this->taxonomies_for_file = array(); // key=media-filename, value = count of taxonomies for this file (just for info)
        $this->taxonomy_pagenames = array();  // key=media-filename, value = array of redirects (used as temp name store)
        $this->map_categories = self::get_map_categories($this->base_directory_path);
        // TODO - add blacklist of "unwanted" categories, so that if an image falls into one of these (or a child thereof),
        // it is not harvested. E.g. a suggested "unwanted" category might be
        // Category:Uploaded_with_Open_Access_Media_Importer_and_needing_category_review
        $this->total_pages_in_dump = 0;
        $this->queue_of_pages_to_process = array();

        $this->resource_file_path = CONTENT_RESOURCE_LOCAL_PATH . $this->resource->id . "_temp.xml";
        $this->resource_file = new \SchemaDocument($this->resource_file_path);
    }

    public function begin_wikipedia_harvest()
    {
        //set encoding for the xml dump file - important for things like WikiParser::mb_ucfirst
        mb_internal_encoding("UTF-8");
        // delete the downloaded files
        $this->cleanup_dump();
        $this->download_dump();

        // FIRST PASS: parse TaxonavigationIncluded* pages (e.g. https://commons.wikimedia.org/wiki/Template:Aves)
        // simultaneously locate galleries and categories with potential taxonomic information (i.e. a Taxonavigation template)
        // also make a list of "gallery media" (files listed in taxonomic galleries)
        $this->iterate_files(array($this, 'locate_taxonomic_pages'));

        // SECOND PASS: parse & validate taxonomic information in galleries + categories - place in 'taxa' array
        // simutaneously link to 'taxa' any pages which redirect to these taxonomic pages
        // also if any gallery media are redirects, replace their name with the proper (redirected) name.
        $this->iterate_files(array($this, 'check_taxonomy_and_redirects'));

        // FINAL PASS: check files for categories, grab file information for scientific media pages and save to file
        $this->iterate_files(array($this, 'get_media_pages'));

        echo "\n(processing last ". count($this->queue_of_pages_to_process) ." media)\n";
        $this->process_page_queue();
        echo "\n\nTOTAL # OF MEDIA FILES: ".count($this->taxonomies_for_file)." (in ". count($this->taxa) ." taxa)\n";
        $this->check_for_unaccounted_galleries();

        @unlink(CONTENT_RESOURCE_LOCAL_PATH . $this->resource->id."_previous.xml");
        @rename(CONTENT_RESOURCE_LOCAL_PATH . $this->resource->id.".xml", CONTENT_RESOURCE_LOCAL_PATH . $this->resource->id."_previous.xml");
        rename(CONTENT_RESOURCE_LOCAL_PATH . $this->resource->id."_temp.xml", CONTENT_RESOURCE_LOCAL_PATH . $this->resource->id.".xml");

        echo "\nEND\n";
        self::print_memory_and_time();
    }

    private function download_dump()
    {
        // download latest Wikimedia Commons export
        echo "Downloading ".$this->resource->accesspoint_url." ...\n";
        shell_exec("curl ".escapeshellarg($this->resource->accesspoint_url)." -o ". escapeshellarg($this->base_directory_path . "wikimedia.xml.bz2"));
        // unzip the download
        echo "... unpacking downloaded file ...\n";
        shell_exec("bunzip2 --force ". escapeshellarg($this->base_directory_path . "wikimedia.xml.bz2"));
        // split the huge file into 300M chunks
        echo "... splitting file into parts ...\n";
        shell_exec("split -a ". $this->part_file_suffix_chars ." -b 300m ". escapeshellarg($this->base_directory_path . "wikimedia.xml")." ". escapeshellarg($this->part_path));
        echo "... done.\n";
    }

    private function cleanup_dump()
    {
        // cleaning up downloaded files
        echo "Removing old wikimedia dump files ...\n";
        shell_exec("rm -f ". escapeshellarg($this->part_path)."*");
        shell_exec("rm -f ". escapeshellarg($this->base_directory_path . "wikimedia.xml"));
        shell_exec("rm -f ". escapeshellarg($this->base_directory_path . "wikimedia.xml.bz2"));
        echo "... done.\n";
    }

    private function iterate_files($callback)
    {
        $total_pages_processed = 0;
        $this->page_iteration_left_overs = "";
        $suffix = str_repeat('a', $this->part_file_suffix_chars);
        while((strlen($suffix) == $this->part_file_suffix_chars))
        {
            $filename = $this->part_path . $suffix;
            if(!file_exists($filename))
            {
                echo "Assuming no more part files to process (as ". basename($filename) ." doesn't exist)\n";
                break;
            }
            $total_pages_processed += $this->process_file($filename, $callback);
            // auto-increment allows us to match the output of the 'split' command: aaa->aab, aaz->aba, etc
            $suffix++;
        }
        if(!preg_match('/\s*<\/mediawiki>\s*/smi', $this->page_iteration_left_overs))
        {
            echo "WARNING: THE LAST WIKI FILE APPEARS TO BE TRUNCATED. Part of the wiki download may be missing.\n";
        }
        if($this->total_pages_in_dump == 0) $this->total_pages_in_dump = $total_pages_processed;
        echo "\n---------------------(done processing $total_pages_processed pages)---------------------\n\n\n";
    }

    private function process_file($filename, $callback)
    {
        $pages_processed = 0;
        if(file_exists($filename))
        {
            echo "Processing file '". basename($filename) ."' with callback ". $callback[1] ."\n";
            self::print_memory_and_time();

            // the last file ended in the middle of a page. Use the remainder of the last file as a starting point
            if(isset($this->page_iteration_left_overs)) $current_page = $this->page_iteration_left_overs;
            else $current_page = "";
            $this->page_iteration_left_overs = "";
            foreach(new FileIterator($filename, false, false) as $line)
            {
                $current_page .= $line;
                // this is a new page so reset $current_page
                if(trim($line) == "<page>") $current_page = $line;
                elseif(trim($line) == "</page>")
                {
                    call_user_func($callback, $current_page);
                    $pages_processed++;
                    $current_page = "";
                }
            }
            // in the middle of a <page> tag. Save the current page to use as a starting point for the next file
            $this->page_iteration_left_overs = $current_page;
        }
        return $pages_processed;
    }

    private function locate_taxonomic_pages($xml)
    {
        $this->debug_locate_taxonomic_pages();
        // make sure we don't include cases with {{Taxonavigation in the comments field, etc.
        if(!\WikimediaPage::fast_is_gallery_category_or_template($xml)) return false;
        if(!($text_start = strpos($xml, "<text"))) return false;
        if(strpos($xml, "{{Taxonavigation", $text_start) === false) return false;
        $page = new \WikimediaPage($xml);
        if($page->is_template())
        {
            // should check here for template redirects
            if($page->contains_template("TaxonavigationIncluded[\w\s]*"))
            {
                $include_array = $page->taxonav_as_array("[Tt]axonavigationIncluded[\w\s]*");
                if(count($include_array))
                {
                    $this->taxonav_includes[$page->title] = array('taxo' => $include_array, 'last_mod' => @strtotime($page->timestamp));
                }else echo "$page->title is not a real TaxonavigationInclude* template\n";
            }elseif($page->contains_template("Taxonavigation"))
            {
                // This is a template that itself contains the template "Taxonavigation", so we might be interested in it
{'fraction_non_alphanumeric': 0.13578283520701948, 'fraction_numerical': 0.0028516588977241567, 'mean_word_length': 2.6377418711350487, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 179, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 38}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1774265', 'n_tokens_mistral': 5488, 'n_tokens_neox': 5204, 'n_words': 1265}