<commit_before><?php 

/*****
 * Author: <NAME>
 * Last modified: February 2020
 * Function: Process a Google Spreadsheet of IGCH Archival Records into TEI files of facsimile images, similar to the Newell notebooks
 *****/

ini_set("allow_url_fopen", 1);

$data = generate_json('https://docs.google.com/spreadsheets/d/e/2PACX-1vTRQbc1C_jdWLdV7jehdctKs7_BJG1TrOEGllRQhge-bXkBIeyEVEpS0cS-4vjmAsmqlYsOvaFzWH5J/pub?output=csv');
$people = generate_json('xeac-people.csv');

$objects = array();

foreach($data as $row){    
    //generate object from distinct IDs
    if ($row['Public'] != 'no' && strlen($row['Archival Record ID']) > 0){
        $objects[$row['Archival Record ID']] = generate_object($data, $row);
    }
}

//var_dump($objects);

//iterate through each data object and generate a TEI file
foreach ($objects as $id=>$array){
    generate_tei($id, $array);
}

/***** FUNCTIONS *****/
function generate_tei ($id, $array){
    
    $title = generate_title($array['hoard'], $array['files']);   
    
    echo $title . "\n";
}

//parse metadata pieces in order to generate a standardized human-readable title
function generate_title($uri, $files){
    $types = array();
    $authors = array();
    $recipients = array();
    $dates = array();
    
    foreach($files as $file){
        
        if ($file['public'] == TRUE){
            if (array_key_exists('type', $file)){
                if (strlen($file['type']) > 0 && !in_array($file['type'], $types)){
                    $types[] = $file['type'];
                }
            }
            if (array_key_exists('author', $file)){
                if (strlen($file['author']) > 0 && !in_array($file['author'], $authors)){
                    $authors[] = $file['author'];
                }
            }
            if (array_key_exists('recipient', $file)){
                if (strlen($file['recipient']) > 0 && !in_array($file['recipient'], $recipients)){
                    $recipients[] = $file['recipient'];
                }
            }
            if (array_key_exists('date', $file)){
                if (strlen($file['date']) > 0 && !in_array($file['date'], $dates)){
                    $dates[] = $file['date'];
                }
            }
        }
    }
    
    $title = (count($types) == 0 ? 'Misc.' : implode('/', $types));
    
    if (count($authors) > 0 || count($recipients) > 0){
        if (count($authors) >= 1 || count($recipients) >= 1){
            $title .= ' of ';
        }
        
        if (count($authors) >= 1){
            $names = array();
            foreach ($authors as $a){
                $names[] = normalize_uri($a);
            }
            
            $title .= implode('/', $names);
            
            unset($names);
        }
        //sep
        if (count($authors) >= 1 && count($recipients) >= 1){
            $title .= ' and ';
        }
        
        if (count($recipients) >= 1) {
            $names = array();
            foreach ($recipients as $r){
                $names[] = normalize_uri($r);
            }
            
            $title .= implode('/', $names);
            
            unset($names);
        }
    }
    
    $title .= ' about ';
    
    //parse IGCH URI
    switch($uri){
        case (strpos($uri, 'igch') !== FALSE):
            $num = preg_replace('/^http:\/\/.*\/igch([0-9]{4})$/', '$1', $uri);
            $title .= 'IGCH ' . ltrim($num, '0');
            break;
        case (strpos($uri, 'ch.') !== FALSE):
            $pieces = explode('.', str_replace('http://coinhoards.org/id/', '', $uri));
            switch($pieces[1]){
                case ('1'):
                    $vol = 'I';
                    break;
                case ('2'):
                    $vol = 'II';
                    break;
                case ('3'):
                    $vol = 'III';
                    break;
                case ('4'):
                    $vol = 'IV';
                    break;
                case ('5'):
                    $vol = 'V';
                    break;
                case ('6'):
                    $vol = 'VI';
                    break;
            }
            
            $title .= "Coin Hoards {$vol} {$pieces[2]}";
    }
    
    if (count($dates) == 1){
        $title .= " (" . safe_strtotime($dates[0]) . ")";
    }
    
    return $title;
}

function normalize_uri ($uri){
    GLOBAL $people;
    
    foreach ($people as $row){
        if ($row['uri'] == $uri){
            if (strpos($row['name'], ',') !== FALSE){
                $pieces = explode(',', $row['name']);
                return $pieces[0];
            } else {
                return $row['name'];
            }
        }
    }
}


function generate_object($data, $row){
    $object = array();
    
    $id = $row['Archival Record ID'];
    
    
    $object['hoard'] = "http://coinhoards.org/id/" . $row['Coin Hoard ID'];    
    
    //re-iterate through each row looking for tif files that match the ID pattern
    foreach ($data as $page){
        if (strpos($page['File Name'], $id) !== FALSE){
            $file = array();
            $file['filename'] = $page['File Name'];<commit_msg>successfully parsing metadata into titles<commit_after><?php 

/*****
 * Author: <NAME>
 * Last modified: February 2020
 * Function: Process a Google Spreadsheet of IGCH Archival Records into TEI files of facsimile images, similar to the Newell notebooks
 *****/

ini_set("allow_url_fopen", 1);

$data = generate_json('https://docs.google.com/spreadsheets/d/e/2PACX-1vTRQbc1C_jdWLdV7jehdctKs7_BJG1TrOEGllRQhge-bXkBIeyEVEpS0cS-4vjmAsmqlYsOvaFzWH5J/pub?output=csv');
$people = generate_json('xeac-people.csv');

$objects = array();

foreach($data as $row){    
    //generate object from distinct IDs
    if ($row['Public'] != 'no' && strlen($row['Archival Record ID']) > 0){
        $objects[$row['Archival Record ID']] = generate_object($data, $row);
    }
}

//var_dump($objects);

//iterate through each data object and generate a TEI file
foreach ($objects as $id=>$array){
    generate_tei($id, $array);
}

/***** FUNCTIONS *****/
function generate_tei ($id, $array){
    
    //generate a human-readable title from file metadata
    $title = generate_title($array['hoard'], $array['files']);   
    
    $doc = new XMLWriter();
    
    $doc->openUri('php://output');
    //$doc->openUri('tei/' . $id . '.xml');
    $doc->setIndent(true);
    //now we need to define our Indent string,which is basically how many blank spaces we want to have for the indent
    $doc->setIndentString("    ");
    
    $doc->startDocument('1.0','UTF-8');
    
    //start TEI file
    $doc->startElement('TEI');
        $doc->writeAttribute('xmlns', 'http://www.tei-c.org/ns/1.0');
        $doc->writeAttribute('xmlns:xsi', 'http://www.w3.org/2001/XMLSchema-instance');
        $doc->writeAttribute('xsi:schemaLocation', 'http://www.tei-c.org/ns/1.0 http://www.tei-c.org/release/xml/tei/custom/schema/xsd/tei_all.xsd');
        $doc->writeAttribute('xml:id', $id);
        
        //TEI Header
        $doc->startElement('teiHeader');
        
        $doc->endElement();
        
    $doc->endElement();
    
    //close file
    $doc->endDocument();
    $doc->flush();
    
    echo "Wrote {$id}\n";
}

//parse metadata pieces in order to generate a standardized human-readable title
function generate_title($uri, $files){
    $types = array();
    $authors = array();
    $recipients = array();
    $dates = array();
    
    foreach($files as $file){
        
        if ($file['public'] == TRUE){
            if (array_key_exists('type', $file)){
                if (strlen($file['type']) > 0 && !in_array($file['type'], $types)){
                    $types[] = $file['type'];
                }
            }
            if (array_key_exists('author', $file)){
                if (strlen($file['author']) > 0 && !in_array($file['author'], $authors)){
                    $authors[] = $file['author'];
                }
            }
            if (array_key_exists('recipient', $file)){
                if (strlen($file['recipient']) > 0 && !in_array($file['recipient'], $recipients)){
                    $recipients[] = $file['recipient'];
                }
            }
            if (array_key_exists('date', $file)){
                if (strlen($file['date']) > 0 && !in_array($file['date'], $dates)){
                    $dates[] = $file['date'];
                }
            }
        }
    }
    
    $title = (count($types) == 0 ? 'Misc.' : implode('/', $types));
    
    if (count($authors) > 0 || count($recipients) > 0){
        if (count($authors) >= 1 || count($recipients) >= 1){
            $title .= ' of ';
        }
        
        if (count($authors) >= 1){
            $names = array();
            foreach ($authors as $a){
                $names[] = normalize_uri($a);
            }
            
            $title .= implode('/', $names);
            
            unset($names);
        }
        //sep
        if (count($authors) >= 1 && count($recipients) >= 1){
            $title .= ' and ';
        }
        
        if (count($recipients) >= 1) {
            $names = array();
            foreach ($recipients as $r){
                $names[] = normalize_uri($r);
            }
            
            $title .= implode('/', $names);
            
            unset($names);
        }
    }
    
    $title .= ' about ';
    
    //parse IGCH URI
    switch($uri){
        case (strpos($uri, 'igch') !== FALSE):
            $num = preg_replace('/^http:\/\/.*\/igch([0-9]{4})$/', '$1', $uri);
            $title .= 'IGCH ' . ltrim($num, '0');
            break;
        case (strpos($uri, 'ch.') !== FALSE):
            $pieces = explode('.', str_replace('http://coinhoards.org/id/', '', $uri));
            switch($pieces[1]){
                case ('1'):
                    $vol = 'I';
                    break;
                case ('2'):
                    $vol = 'II';
                    break;
                case ('3'):
                    $vol = 'III';
                    break;
                case ('4'):
                    $vol = 'IV';
                    break;
                case ('5'):
                    $vol = 'V';
                    break;
                case ('6'):
                    $vol = 'VI';
                    break;
            }
            
            $title .= "Coin Hoards {$vol} {$pieces[2]}";
    }
    
    if (count($dates) == 1){
        $title .= " (" . safe_strtotime($dates[0]) . ")";
    }
    
    return $title;
}

function normalize_uri ($uri){
    GLOBAL $people;
    
    foreach ($people as $row){
        if ($row['uri'] == $uri){
            if (strpos($row['name'], ',') !== FALSE){
                $pieces = explode(',', $row['name']);
                return $pieces[0];
            } else {
                return $row['name'];
            }
        }
    }
}


function generate_object($data, $row){
    $object = array();
    
    $id = $row['Archival Record ID'];
    
    $object['id'] = $id;
    $object['hoard'] = "http://coinhoards.org/id/" . $row['Coin Hoard ID'];    
    
    //re-iterate through each row looking for tif files that match the ID pattern
    foreach ($data as $page){
        if (strpos($page['File Name'], $id) !== FALSE){
            $file = array();
            $file['filename'] = $page['File Name'];
{'fraction_non_alphanumeric': 0.1837757075057128, 'fraction_numerical': 0.008788890841975743, 'mean_word_length': 1.636468952734013, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 48, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5296923', 'n_tokens_mistral': 3607, 'n_tokens_neox': 3405, 'n_words': 747}