<commit_before><?php
namespace php_active_record;

include_once(dirname(__FILE__) . "/../../config/environment.php");
$mysqli = $GLOBALS['db_connection'];




require_library('connectors/NatureServeAPI');
@unlink(DOC_ROOT . "/temp/dwc_archive_test/meta.xml");
@unlink(DOC_ROOT . "/temp/dwc_archive_test/taxon.tab");
@unlink(DOC_ROOT . "/temp/dwc_archive_test/taxon_working.tab");
@unlink(DOC_ROOT . "/temp/dwc_archive_test/media_resource.tab");
@unlink(DOC_ROOT . "/temp/dwc_archive_test/media_resource_working.tab");
@unlink(DOC_ROOT . "/temp/dwc_archive_test/reference.tab");
rmdir(DOC_ROOT . "/temp/dwc_archive_test/");
$naturserveAPI = new NatureServeAPI();
$naturserveAPI->get_all_taxa();









$resource_file = fopen(CONTENT_RESOURCE_LOCAL_PATH . "263_temp.xml", "w+");

// start the resource file with the XML header
fwrite($resource_file, \SchemaDocument::xml_header());



$archive = new ContentArchiveReader(null, DOC_ROOT . "/temp/dwc_archive_test/");

$GLOBALS['data_objects'] = array();
$GLOBALS['taxon_id_media'] = array();
$GLOBALS['all_references'] = array();

print_r($archive->tables);

$archive->process_row_type("http://eol.org/schema/reference/Reference", "php_active_record\\lookup_references");
$archive->process_row_type("http://eol.org/schema/media/Document", "php_active_record\\lookup_data_objects");
$archive->process_row_type("http://rs.tdwg.org/dwc/terms/Taxon", "php_active_record\\lookup_taxa", array('resource_file' => $resource_file));


// write the resource footer
fwrite($resource_file, \SchemaDocument::xml_footer());
fclose($resource_file);

// cache the previous version and make this new version the current version
@unlink(CONTENT_RESOURCE_LOCAL_PATH . "263_previous.xml");
@rename(CONTENT_RESOURCE_LOCAL_PATH . "263.xml", CONTENT_RESOURCE_LOCAL_PATH . "263_previous.xml");
rename(CONTENT_RESOURCE_LOCAL_PATH . "263_temp.xml", CONTENT_RESOURCE_LOCAL_PATH . "263.xml");

if(filesize(CONTENT_RESOURCE_LOCAL_PATH . "263.xml") > 600)
{
    $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=".ResourceStatus::find_or_create_by_translated_label('Force Harvest')->id." WHERE id=263");
}






function lookup_taxa($taxon, $parameters)
{
    static $i=0;
    if($i%1000==0) echo "$i - ".memory_get_usage()."\n";
    $i++;
    // if($i >= 1000) return;
    
    $taxon_parameters = array();
    $taxon_parameters['identifier'] = $taxon['http://rs.tdwg.org/dwc/terms/taxonID'];
    $taxon_parameters['source'] = $taxon['http://purl.org/dc/terms/source'];
    $taxon_parameters['scientificName'] = $taxon['http://rs.tdwg.org/dwc/terms/scientificName'];
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/kingdom']) $taxon_parameters['kingdom'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/phylum']) $taxon_parameters['phylum'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/class']) $taxon_parameters['class'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/order']) $taxon_parameters['order'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/family']) $taxon_parameters['family'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/genus']) $taxon_parameters['genus'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/taxonRank']) $taxon_parameters['rank'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/vernacularName'])
    {
        $taxon_parameters['commonNames'] = array(new \SchemaCommonName(array('name' => $v, 'language' => 'en')));
    }
    
    $taxon_parameters['dataObjects'] = array();
    if($data_object_ids = @$GLOBALS['taxon_id_media'][$taxon_parameters['identifier']])
    {
        foreach($data_object_ids as $data_object_id)
        {
            if($data_object = $GLOBALS['data_objects'][$data_object_id])<commit_msg>NatureServe will not set a force-harvest if main file with names and uids is not available.<commit_after><?php
namespace php_active_record;

include_once(dirname(__FILE__) . "/../../config/environment.php");
$mysqli = $GLOBALS['db_connection'];




require_library('connectors/NatureServeAPI');
@unlink(DOC_ROOT . "/temp/dwc_archive_test/meta.xml");
@unlink(DOC_ROOT . "/temp/dwc_archive_test/taxon.tab");
@unlink(DOC_ROOT . "/temp/dwc_archive_test/taxon_working.tab");
@unlink(DOC_ROOT . "/temp/dwc_archive_test/media_resource.tab");
@unlink(DOC_ROOT . "/temp/dwc_archive_test/media_resource_working.tab");
@unlink(DOC_ROOT . "/temp/dwc_archive_test/reference.tab");
rmdir(DOC_ROOT . "/temp/dwc_archive_test/");
$naturserveAPI = new NatureServeAPI();
$naturserveAPI->get_all_taxa();









$resource_file = fopen(CONTENT_RESOURCE_LOCAL_PATH . "263_temp.xml", "w+");

// start the resource file with the XML header
fwrite($resource_file, \SchemaDocument::xml_header());



$archive = new ContentArchiveReader(null, DOC_ROOT . "/temp/dwc_archive_test/");

$GLOBALS['data_objects'] = array();
$GLOBALS['taxon_id_media'] = array();
$GLOBALS['all_references'] = array();

print_r($archive->tables);

$archive->process_row_type("http://eol.org/schema/reference/Reference", "php_active_record\\lookup_references");
$archive->process_row_type("http://eol.org/schema/media/Document", "php_active_record\\lookup_data_objects");
$archive->process_row_type("http://rs.tdwg.org/dwc/terms/Taxon", "php_active_record\\lookup_taxa", array('resource_file' => $resource_file));


// write the resource footer
fwrite($resource_file, \SchemaDocument::xml_footer());
fclose($resource_file);

// cache the previous version and make this new version the current version
@unlink(CONTENT_RESOURCE_LOCAL_PATH . "263_previous.xml");
@rename(CONTENT_RESOURCE_LOCAL_PATH . "263.xml", CONTENT_RESOURCE_LOCAL_PATH . "263_previous.xml");
rename(CONTENT_RESOURCE_LOCAL_PATH . "263_temp.xml", CONTENT_RESOURCE_LOCAL_PATH . "263.xml");

if(filesize(CONTENT_RESOURCE_LOCAL_PATH . "263.xml") > 200000)
{
    $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=".ResourceStatus::find_or_create_by_translated_label('Force Harvest')->id." WHERE id=263");
}

function lookup_taxa($taxon, $parameters)
{
    static $i=0;
    if($i%1000==0) echo "$i - ".memory_get_usage()."\n";
    $i++;
    // if($i >= 1000) return;
    
    $taxon_parameters = array();
    $taxon_parameters['identifier'] = $taxon['http://rs.tdwg.org/dwc/terms/taxonID'];
    $taxon_parameters['source'] = $taxon['http://purl.org/dc/terms/source'];
    $taxon_parameters['scientificName'] = $taxon['http://rs.tdwg.org/dwc/terms/scientificName'];
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/kingdom']) $taxon_parameters['kingdom'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/phylum']) $taxon_parameters['phylum'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/class']) $taxon_parameters['class'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/order']) $taxon_parameters['order'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/family']) $taxon_parameters['family'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/genus']) $taxon_parameters['genus'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/taxonRank']) $taxon_parameters['rank'] = $v;
    if($v = $taxon['http://rs.tdwg.org/dwc/terms/vernacularName'])
    {
        $taxon_parameters['commonNames'] = array(new \SchemaCommonName(array('name' => $v, 'language' => 'en')));
    }
    
    $taxon_parameters['dataObjects'] = array();
    if($data_object_ids = @$GLOBALS['taxon_id_media'][$taxon_parameters['identifier']])
    {
        foreach($data_object_ids as $data_object_id)
        {
            if($data_object = $GLOBALS['data_objects'][$data_object_id])
{'fraction_non_alphanumeric': 0.18969616908850725, 'fraction_numerical': 0.010171730515191546, 'mean_word_length': 5.454390451832907, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5508069', 'n_tokens_mistral': 2865, 'n_tokens_neox': 2709, 'n_words': 344}