<commit_before>#!/usr/local/bin/php  
<?php
//
/* connector for BOLD Systems */
//exit;
/*
http://www.boldsystems.org/connect/REST/getBarcodeRepForSpecies.php?taxid=26136&iwidth=600
http://www.boldsystems.org/connect/REST/getBarcodeRepForSpecies.php?taxid=111651&iwidth=600
http://www.boldsystems.org/connect/REST/getBarcodeRepForSpecies.php?taxid=127144&iwidth=600

http://www.boldsystems.org/pcontr.php?action=doPublicSequenceDownload&taxids=10325
http://www.boldsystems.org/pcontr.php?action=doPublicSequenceDownload&taxids=26136

http://www.barcodinglife.org/views/taxbrowser.php?taxon=Gadus+morhua
http://www.barcodinglife.org/views/taxbrowser.php?taxon=Bimastos+welchi

http://www.boldsystems.org/connect/REST/getSpeciesBarcodeStatus.php?phylum=Annelida

http://www.boldsystems.org/connect/REST/getSpeciesBarcodeStatus.php?phylum=Basidiomycota
http://www.barcodinglife.org/views/taxbrowser.php?taxon=Agaricus+pequinii
http://www.boldsystems.org/connect/REST/getBarcodeRepForSpecies.php?taxid=93150&iwidth=600
http://www.boldsystems.org/pcontr.php?action=doPublicSequenceDownload&taxids=93150
*/

//define("ENVIRONMENT", "development");
define("ENVIRONMENT", "slave_32");
define("MYSQL_DEBUG", false);
define("DEBUG", false);
include_once(dirname(__FILE__) . "/../../config/start.php");
$mysqli =& $GLOBALS['mysqli_connection'];

//only on local; to be deleted before going into production
/*
$mysqli->truncate_tables("development");
Functions::load_fixtures("development");
exit;
*/

////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////

set_time_limit(0);
$resource = new Resource(81); //print $resource->id; exit;

$schema_taxa = array();
$used_taxa = array();

$id_list=array();

$wrap = "\n";
//$wrap = "<br>";

$phylum_service_url = "http://www.boldsystems.org/connect/REST/getSpeciesBarcodeStatus.php?phylum=";
$species_service_url = "http://www.barcodinglife.org/views/taxbrowser.php?taxon=";

/*
$query="Select distinct taxa.taxon_phylum From taxa Where taxa.taxon_phylum Is Not Null and taxa.taxon_phylum <> '' ";
$query .= " Order By taxa.taxon_phylum Asc ";
*/

$query="Select distinct names.`string` as taxon_phylum From hierarchy_entries Inner Join ranks ON hierarchy_entries.rank_id = ranks.id
Inner Join names ON hierarchy_entries.name_id = names.id Where
ranks.id = 280 Order By names.`string` Asc ";
//rank.id 280 = phylum

//$query .= " and taxon_phylum = 'Chordata' ";
//$query .= " and taxon_phylum = 'Chaetognatha' ";
//$query .= " and taxon_phylum <> 'Annelida' ";

//$query .= " limit 1 ";
$result = $mysqli->query($query);    
print "phylum count = " . $result->num_rows . "$wrap"; //exit;

$total_taxid_count = 0;
$do_count = 0;//weird but needed here
$ctr=0;
$id_with_public_barcode=array();
while($row=$result->fetch_assoc())     
{
    $taxid_count=0;
    $taxid_count_with_barcode=0;
    
    $ctr++;   
    print "$wrap $ctr. phylum = " . $row["taxon_phylum"] . "$wrap";
        
    $url = $phylum_service_url . trim($row["taxon_phylum"]);
    
    /* for debug - to limit no. of record to process
    $url = "http://127.0.0.1/bold.xml";<commit_msg>update BOLDS connector<commit_after>#!/usr/local/bin/php  
<?php
//
/* connector for BOLD Systems */
//exit;
/*
http://www.boldsystems.org/connect/REST/getBarcodeRepForSpecies.php?taxid=26136&iwidth=600
http://www.boldsystems.org/connect/REST/getBarcodeRepForSpecies.php?taxid=111651&iwidth=600
http://www.boldsystems.org/connect/REST/getBarcodeRepForSpecies.php?taxid=127144&iwidth=600

http://www.boldsystems.org/pcontr.php?action=doPublicSequenceDownload&taxids=10325
http://www.boldsystems.org/pcontr.php?action=doPublicSequenceDownload&taxids=26136

http://www.barcodinglife.org/views/taxbrowser.php?taxon=Gadus+morhua
http://www.barcodinglife.org/views/taxbrowser.php?taxon=Bimastos+welchi

http://www.boldsystems.org/connect/REST/getSpeciesBarcodeStatus.php?phylum=Annelida

http://www.boldsystems.org/connect/REST/getSpeciesBarcodeStatus.php?phylum=Basidiomycota
http://www.barcodinglife.org/views/taxbrowser.php?taxon=Agaricus+pequinii
http://www.boldsystems.org/connect/REST/getBarcodeRepForSpecies.php?taxid=93150&iwidth=600
http://www.boldsystems.org/pcontr.php?action=doPublicSequenceDownload&taxids=93150

date            taxid   with public barcode
2010 Jan 20     16105   5594    

*/

//define("ENVIRONMENT", "development");
//define("ENVIRONMENT", "slave_32");
define("MYSQL_DEBUG", false);
define("DEBUG", false);
include_once(dirname(__FILE__) . "/../../config/start.php");
$mysqli =& $GLOBALS['mysqli_connection'];

//only on local; to be deleted before going into production
/*
$mysqli->truncate_tables("development");
Functions::load_fixtures("development");
exit;
*/

////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////

set_time_limit(0);
$resource = new Resource(81); //print $resource->id; exit;

$schema_taxa = array();
$used_taxa = array();

$id_list=array();

$wrap = "\n";
//$wrap = "<br>";

$phylum_service_url = "http://www.boldsystems.org/connect/REST/getSpeciesBarcodeStatus.php?phylum=";
$species_service_url = "http://www.barcodinglife.org/views/taxbrowser.php?taxon=";

/*
$query="Select distinct taxa.taxon_phylum From taxa Where taxa.taxon_phylum Is Not Null and taxa.taxon_phylum <> '' ";
$query .= " Order By taxa.taxon_phylum Asc ";
*/

$query="Select distinct names.`string` as taxon_phylum From hierarchy_entries Inner Join ranks ON hierarchy_entries.rank_id = ranks.id
Inner Join names ON hierarchy_entries.name_id = names.id Where
ranks.id = 280 Order By names.`string` Asc ";
//rank.id 280 = phylum
//$query .= " and taxon_phylum = 'Chordata' ";
//$query .= " and taxon_phylum = 'Chaetognatha' ";
//$query .= " and taxon_phylum <> 'Annelida' ";
//$query .= " limit 1 ";

$result = $mysqli->query($query);    
print "phylum count = " . $result->num_rows . "$wrap"; //exit;

$total_taxid_count = 0;
$do_count = 0;//weird but needed here
$ctr=0;
$id_with_public_barcode=array();
while($row=$result->fetch_assoc())     
{
    $taxid_count=0;
    $taxid_count_with_barcode=0;
    
    $ctr++;   
    print "$wrap $ctr. phylum = " . $row["taxon_phylum"] . "$wrap";
        
    $url = $phylum_service_url . trim($row["taxon_phylum"]);
    
    /* for debug - to limit no. of record to process
    $url = "http://127.0.0.1/bold.xml";
{'fraction_non_alphanumeric': 0.21194900936876057, 'fraction_numerical': 0.02442021194900937, 'mean_word_length': 3.715423606082549, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 28, 'xml': 2}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4421548', 'n_tokens_mistral': 2427, 'n_tokens_neox': 2233, 'n_words': 343}