<commit_before>#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their curie prefix (or literal object)
#  Reduce predicates to the specific term identifier
#  and the ontology namespace they are from. (curi prefix )
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology (t-box) as opposed to
#  a data uri (a-box).

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
    print "usage: ntriple2dot.awk prefix_baseurl.yaml rdf.nt > dot.gv"
}

##########################################################
# remove first and last chars from input string
function trim(str){
    return substr(str,2,length(str)-2)
}

# break URI into tokens to use a a path in
# multi-dim array pointing to a curie-prefix
function tokenize (str){
    # b/c found in local IDs leave .:
    gsub(/[?&:#=/_-]+/, SUBSEP, str)
    # discard the first token it is only low info content  http|https|ftp
    # discard any empty trailing tokens
    tail = match(str, ".*" SUBSEP "$")  # side effects RSTART & RLENGTH
    # print "tail = " tail " tail length = " RLENGTH
    finish = 0
    start = index(str, SUBSEP) + 1
    if (tail>0){
        finish = RLENGTH - start
    }
    else { # if(tail==0) {
        # print str
        finish = length(str) - start + 1
    }

    return substr(str, start, finish)
}

# remove final token from token path delimited with SUBSEP
function detail(key) {
    return substr(key, 1, match(key, SUBSEP "[[:alnum:].]+$")-1)
}

# for output, keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective (C-lang identifier rules)
function simplify(str){
    gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
    gsub(/^_*|_*$/, "",str)
    gsub(/__*/,"_",str)
    return str
}

# Replace IRI with CuriePrefix when possible
# Otherwise whinge and return a gv printable version of the original
function contract(uri,  u){
    u = tokenize(uri)
    if(index(u, SUBSEP)>0)
        u = detail(u)

    if(u == "") {
        printf("ERROR contracting %s\n", uri) > "/dev/stderr"
        printf("ERROR tokenizeing %s\n", tokenize(uri)) > "/dev/stderr"
        printf("ERROR detailing   %s\n", detail(tokenize(uri))) > "/dev/stderr"
        # exit(1)  # while testing
    }
    # shorten till longest uri in curi map is found (or not)
    while(!(u in prefix) &&  index(u, SUBSEP)>0)
        u = detail(u)

    if(u in prefix)
        return prefix[u]
    else{
        printf("WARN  %s\n", uri) > "/dev/stderr"
        printf("BASE  %s\n", detail(tokenize(uri))) > "/dev/stderr"
        return simplify(uri)
    }
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri,  b, p, anchor){
    split(uri, b, "/")
    p = b[length(b)]
    anchor = match(p, "#")
    if(anchor > 0)
        p = substr(p, anchor+1)
    return p
}

# fix curie namespace when redundant due to OBO  i.e RO:RO_123
# is this still needed circa 2020?
function deoboify(curie,  a){
    split(curie, a , ":")
    if(match(a[2], a[1]"_") > 0){
        sub("_", ":", a[2]);
        curie=a[2]
    }
    return curie
}

BEGIN{
    # exceptions
    prefix["BNODE"]="BNODE"  # is a fixed point
    prefix[tokenize("https://monarchinitiative.org/.well-known/genid")]="BNODE"
    prefix[tokenize("https://archive.monarchinitiative.org/")]="MonarchArchive"
    ############################################################
    # not expected to be added to curie map
    # in HPOA
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/bookshelf/br.fcgi?book=gene&part=")]="NCBIBook"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/books/NBK")]="NCBINBK"

    ############################################################
    # Often re-visit whether these exceptions are still necessary
    # they may have been moved into the curie prefix mapping file
    # or addressed in a local translation table
    # https://raw.githubusercontent.com/monarch-initiative/dipper/master/dipper/curie_map.yaml

    # in mgi
    prefix[tokenize("https://www.mousephenotype.org")]="IMPC"
    # in IMPC (not httpS  --- hmm all three IRI exist)
    prefix[tokenize("http://www.mousephenotype.org")]="IMPC"
    # note in curie_map IMPC is:
    # http://www.mousephenotype.org/data/genes/

    # in panther
    # prefix["http://identifiers.org/wormbase/"]="WormBase"
    # note in curie_map WormBase is
    # 'https://www.wormbase.org/get?name='

    # various, transient?
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/gene/")]="NCBIGene"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/genome/")]="NCBIGenome"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/assembly?term=")]="NCBIAssembly"
    # kegg
    prefix[tokenize("http://www.genome.jp/kegg/pathway/map/map")]="KEGG-img"
    # in EOM
    prefix[tokenize("https://elementsofmorphology.nih.gov/images/terms/")]="EOM_IMG"
    prefix[tokenize("http://elementsofmorphology.nih.gov/index.cgi?tid=")]="EOM"  # w/o httpS
<commit_msg>correct kegg-img base-uri & reframe how the last token is found<commit_after>#! /usr/bin/gawk -f

#  Reduce the subject and object of RDF triples (ntriples format)
#  down to their curie prefix (or literal object)
#  Reduce predicates to the specific term identifier
#  and the ontology namespace they are from. (curi prefix )
#  Express the subject and object as nodes with a directed edge
#  labeled with the predicate in the graphviz dot format
#  Include a tally of each combination of nodes and edge

#  This may over generalize in some cases because I do not have
#  a handy way to differentiate uri for subjects and objects
#  which may belong to a "structural" ontology (t-box) as opposed to
#  a data uri (a-box).

#  Post processing to augment predicate identifiers
#  with their labels seems to improve usefulness.

function usage(){
    print "usage: ntriple2dot.awk prefix_baseurl.yaml rdf.nt > dot.gv"
}

##########################################################
# remove first and last chars from input string
function trim(str){
    return substr(str,2,length(str)-2)
}

# break URI into tokens to use a a path in
# multi-dim array pointing to a curie-prefix
function tokenize (str){
    # b/c found in local IDs leave .:
    gsub(/[?&:#=/_-]+/, SUBSEP, str)
    # discard the first token it is only low info content  http|https|ftp
    # discard any empty trailing tokens
    tail = match(str, ".*" SUBSEP "$")  # side effects RSTART & RLENGTH
    # print "tail = " tail " tail length = " RLENGTH
    finish = 0
    start = index(str, SUBSEP) + 1
    if (tail>0){
        finish = RLENGTH - start
    }
    else { # if(tail==0) {
        # print str
        finish = length(str) - start + 1
    }
    return substr(str, start, finish)
}

# remove final token from token path delimited with SUBSEP
# this ends up matching chars included in local identifiers 
# ._-;():%#,?=@*\&+   (yea ... nope)

function detail(key,   start) {
    start = match(key, ".+" SUBSEP) 
    if(start == 1)
        return substr(key, 1, RLENGTH -1)
    else
        return key
}

# for output, keep underscore, letters & numbers
# change the rest to (a single) underscore sans leading & trailing
# this passes valid node labels from dot's perspective (C-lang identifier rules)
function simplify(str){
    gsub(/[^[:alpha:][:digit:]_]+/,"_",str)
    gsub(/^_*|_*$/, "",str)
    gsub(/__*/,"_",str)
    return str
}

# Replace IRI with CuriePrefix when possible
# Otherwise whinge and return a gv printable version of the original
function contract(uri,  u){
    u = tokenize(uri)
    if(index(u, SUBSEP)>0)
        u = detail(u)

    if(u == "") {
        printf("ERROR contracting %s\n", uri) > "/dev/stderr"
        printf("ERROR tokenizeing %s\n", tokenize(uri)) > "/dev/stderr"
        printf("ERROR detailing   %s\n", detail(tokenize(uri))) > "/dev/stderr"
        # exit(1)  # while testing
    }
    # shorten till longest uri in curi map is found (or not)
    while(!(u in prefix) &&  index(u, SUBSEP)>0)
        u = detail(u)

    if(u in prefix)
        return prefix[u]
    else{
        printf("WARN  %s\n", uri) > "/dev/stderr"
        printf("BASE  %s\n", detail(tokenize(uri))) > "/dev/stderr"
        return simplify(uri)
    }
}

# get the final (incl fragment identifier) portion of a slashed path
function final(uri,  b, p, anchor){
    split(uri, b, "/")
    p = b[length(b)]
    anchor = match(p, "#")
    if(anchor > 0)
        p = substr(p, anchor+1)
    return p
}

# fix curie namespace when redundant due to OBO  i.e RO:RO_123
# is this still needed circa 2020?
function deoboify(curie,  a){
    split(curie, a , ":")
    if(match(a[2], a[1]"_") > 0){
        sub("_", ":", a[2]);
        curie = a[2]
    }
    return curie
}

BEGIN{
    # exceptions
    prefix["BNODE"]="BNODE"  # is a fixed point
    prefix[tokenize("https://monarchinitiative.org/.well-known/genid")]="BNODE"
    prefix[tokenize("https://archive.monarchinitiative.org/")]="MonarchArchive"
    ############################################################
    # not expected to be added to curie map
    # in HPOA
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/bookshelf/br.fcgi?book=gene&part=")]="NCBIBook"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/books/NBK")]="NCBINBK"

    ############################################################
    # Often re-visit whether these exceptions are still necessary
    # they may have been moved into the curie prefix mapping file
    # or addressed in a local translation table
    # https://raw.githubusercontent.com/monarch-initiative/dipper/master/dipper/curie_map.yaml

    # in mgi
    prefix[tokenize("https://www.mousephenotype.org")]="IMPC"
    # in IMPC (not httpS  --- hmm all three IRI exist)
    prefix[tokenize("http://www.mousephenotype.org")]="IMPC"
    # note in curie_map IMPC is:
    # http://www.mousephenotype.org/data/genes/

    # in panther
    # prefix["http://identifiers.org/wormbase/"]="WormBase"
    # note in curie_map WormBase is
    # 'https://www.wormbase.org/get?name='

    # various, transient?
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/gene/")]="NCBIGene"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/genome/")]="NCBIGenome"
    prefix[tokenize("http://www.ncbi.nlm.nih.gov/assembly?term=")]="NCBIAssembly"
    # kegg
    prefix[tokenize("http://www.genome.jp/kegg/pathway/map/")]="KEGG-img"
    # in EOM
    prefix[tokenize("https://elementsofmorphology.nih.gov/images/terms/")]="EOM_IMG"
    prefix[tokenize("http://elementsofmorphology.nih.gov/index.cgi?tid=")]="EOM"  # w/o httpS

{'fraction_non_alphanumeric': 0.1683295194508009, 'fraction_numerical': 0.005034324942791762, 'mean_word_length': 3.2763209393346377, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 12, 'lorem ipsum': 0, 'www.': 20, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '886473', 'n_tokens_mistral': 3709, 'n_tokens_neox': 3523, 'n_words': 1019}