<commit_before>#!/usr/local/bin/php
<?php
/* Antarctic Invertebrates connector */
//define("ENVIRONMENT", "development");
//define("ENVIRONMENT", "slave_32");

define("MYSQL_DEBUG", false);
define("DEBUG", true);
include_once(dirname(__FILE__) . "/../../config/start.php");

$mysqli =& $GLOBALS['mysqli_connection'];

/*
$mysqli->truncate_tables("development");
Functions::load_fixtures("development");
*/

$resource = new Resource(93); //93 exit($resource->id);

$wrap="\n";
//$wrap="<br>";

$schema_taxa = array();
$used_taxa = array();

/*
$xml_url = "http://antiz.redmon.com/admin/taxon_descriptions.cfm?rows=5";
$xml_url = "http://127.0.0.1/ai.xml";
*/
$xml_url = "http://antiz.redmon.com/admin/taxon_descriptions.cfm";

if(!($xml = @simplexml_load_file($xml_url)))
{
    print "$wrap <a href='$url'>$url</a> not accessible"; exit;
}

// loop
$k=0;
foreach($xml->taxon as $rec)
{
    print "$wrap" . $rec["name"];
    //print "$wrap" . $rec->original_description;
    
    /*
    $dwc_Kingdom = trim($dwc->Kingdom);
    $dwc_Phylum = trim($dwc->Phylum);
    $dwc_Class = trim($dwc->Class);
    $dwc_Order = trim($dwc->Order);
    $dwc_Family = trim($dwc->Family);
    */
    
    if(@$rec["rank"]=="Genus")      
    {   $dwc_Genus = trim($rec["name"]);
        $dwc_ScientificName = trim($rec["name"]);
    }
    if(@$rec["rank"]=="Species")
    {   $dwc_ScientificName = trim($rec["name"]);
        $dwc_Genus = substr($rec["name"],0,stripos($rec["name"]," "));
    }    
    
    $taxon_identifier = "AI_" . $rec["ID"];
    $agent = get_agent($rec->original_description);        
    
    print "$wrap agent = [$agent]";
    $source_url = "http://antiz.redmon.com/taxon_view.cfm?mode=advancedSearch&name=" . urlencode($dwc_ScientificName) . "&rank=&phylum=&match=substring&Submit=Search";
    
    //print " -- <a href='$source_url'>url</a>";    
    print "$wrap";    
    
    if(@$used_taxa[$taxon_identifier])
    {
        $taxon_parameters = $used_taxa[$taxon_identifier];
    }
    else
    {
        $taxon_parameters = array();
        $taxon_parameters["identifier"] = $taxon_identifier;
        /*
        $taxon_parameters["kingdom"] = $dwc_Kingdom;
        $taxon_parameters["phylum"] = $dwc_Phylum;
        $taxon_parameters["class"] = $dwc_Class;
        $taxon_parameters["order"] = $dwc_Order;
        $taxon_parameters["family"] = $dwc_Family;
        */
        $taxon_parameters["genus"] = $dwc_Genus;
        $taxon_parameters["scientificName"]= $dwc_ScientificName;        
        $taxon_parameters["source"] = $source_url;
                
        $taxon_parameters["dataObjects"]= array();        
        $used_taxa[$taxon_identifier] = $taxon_parameters;        

        // /* start first dataobject - text 
        if(isset($rec->original_description))
        {    
            $dcterms_created="";
            $dcterms_modified="";
            $copyright_text="";
        
            $dc_identifier = $taxon_identifier . "_GenDesc";
            $dc_source = $source_url;
            $agent_name = $agent;
            $license = "http://creativecommons.org/licenses/publicdomain/";    
            $desc = $rec->original_description;
            $data_object_parameters = get_data_object($dc_identifier, $dcterms_created, $dcterms_modified, $copyright_text, $license, $agent_name, $desc, "text", $source_url);       
            $taxon_parameters["dataObjects"][] = new SchemaDataObject($data_object_parameters);         
        }
        // end first dataobject - text */             
        
    }

    /* start 2nd dataobject - image */         
    /* end second dataobject - image */      
    
    $used_taxa[$taxon_identifier] = $taxon_parameters;                
    //$k++;if($k == 3)break;    //debug; to limit no. of records
}

/*
print "<pre>";
print_r($used_taxa);
print "</pre>";
exit;
*/

foreach($used_taxa as $taxon_parameters)
{
    $schema_taxa[] = new SchemaTaxon($taxon_parameters);
}<commit_msg>change license to by-nc 3.0<commit_after>#!/usr/local/bin/php
<?php
/* Antarctic Invertebrates connector */
//define("ENVIRONMENT", "development");
//define("ENVIRONMENT", "slave_32");

define("MYSQL_DEBUG", false);
define("DEBUG", true);
include_once(dirname(__FILE__) . "/../../config/start.php");

$mysqli =& $GLOBALS['mysqli_connection'];

/*
$mysqli->truncate_tables("development");
Functions::load_fixtures("development");
*/

$resource = new Resource(93); //93 exit($resource->id);

$wrap="\n";
//$wrap="<br>";

$schema_taxa = array();
$used_taxa = array();

/*
$xml_url = "http://antiz.redmon.com/admin/taxon_descriptions.cfm?rows=5";
$xml_url = "http://127.0.0.1/ai.xml";
*/
$xml_url = "http://antiz.redmon.com/admin/taxon_descriptions.cfm";

if(!($xml = @simplexml_load_file($xml_url)))
{
    print "$wrap <a href='$url'>$url</a> not accessible"; exit;
}

// loop
$k=0;
foreach($xml->taxon as $rec)
{
    print "$wrap" . $rec["name"];
    //print "$wrap" . $rec->original_description;
    
    /*
    $dwc_Kingdom = trim($dwc->Kingdom);
    $dwc_Phylum = trim($dwc->Phylum);
    $dwc_Class = trim($dwc->Class);
    $dwc_Order = trim($dwc->Order);
    $dwc_Family = trim($dwc->Family);
    */
    
    if(@$rec["rank"]=="Genus")      
    {   $dwc_Genus = trim($rec["name"]);
        $dwc_ScientificName = trim($rec["name"]);
    }
    if(@$rec["rank"]=="Species")
    {   $dwc_ScientificName = trim($rec["name"]);
        $dwc_Genus = substr($rec["name"],0,stripos($rec["name"]," "));
    }    
    
    $taxon_identifier = "AI_" . $rec["ID"];
    $agent = get_agent($rec->original_description);        
    
    print "$wrap agent = [$agent]";
    $source_url = "http://antiz.redmon.com/taxon_view.cfm?mode=advancedSearch&name=" . urlencode($dwc_ScientificName) . "&rank=&phylum=&match=substring&Submit=Search";
    
    //print " -- <a href='$source_url'>url</a>";    
    print "$wrap";    
    
    if(@$used_taxa[$taxon_identifier])
    {
        $taxon_parameters = $used_taxa[$taxon_identifier];
    }
    else
    {
        $taxon_parameters = array();
        $taxon_parameters["identifier"] = $taxon_identifier;
        /*
        $taxon_parameters["kingdom"] = $dwc_Kingdom;
        $taxon_parameters["phylum"] = $dwc_Phylum;
        $taxon_parameters["class"] = $dwc_Class;
        $taxon_parameters["order"] = $dwc_Order;
        $taxon_parameters["family"] = $dwc_Family;
        */
        $taxon_parameters["genus"] = $dwc_Genus;
        $taxon_parameters["scientificName"]= $dwc_ScientificName;        
        $taxon_parameters["source"] = $source_url;
                
        $taxon_parameters["dataObjects"]= array();        
        $used_taxa[$taxon_identifier] = $taxon_parameters;        

        // /* start first dataobject - text 
        if(isset($rec->original_description))
        {    
            $dcterms_created="";
            $dcterms_modified="";
            $copyright_text="";
        
            $dc_identifier = $taxon_identifier . "_GenDesc";
            $dc_source = $source_url;
            $agent_name = $agent;
            $license = "http://creativecommons.org/licenses/by-nc/3.0/";    
            $desc = $rec->original_description;
            $data_object_parameters = get_data_object($dc_identifier, $dcterms_created, $dcterms_modified, $copyright_text, $license, $agent_name, $desc, "text", $source_url);       
            $taxon_parameters["dataObjects"][] = new SchemaDataObject($data_object_parameters);         
        }
        // end first dataobject - text */             
        
    }

    /* start 2nd dataobject - image */         
    /* end second dataobject - image */      
    
    $used_taxa[$taxon_identifier] = $taxon_parameters;                
    //$k++;if($k == 3)break;    //debug; to limit no. of records
}

/*
print "<pre>";
print_r($used_taxa);
print "</pre>";
exit;
*/

foreach($used_taxa as $taxon_parameters)
{
    $schema_taxa[] = new SchemaTaxon($taxon_parameters);
}
{'fraction_non_alphanumeric': 0.18279298599722468, 'fraction_numerical': 0.00479374290399899, 'mean_word_length': 2.8188824662813103, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1061849', 'n_tokens_mistral': 2785, 'n_tokens_neox': 2708, 'n_words': 380}