<commit_before>#!/usr/bin/env python
"""Run FreeSurfer on a T1 image and capture all output as NIDM
"""
import os

import urllib
import urlparse
from StringIO import StringIO

from nipype.interfaces.base import CommandLine
CommandLine.set_default_terminal_output('allatonce')

from nipype import config
config.enable_provenance()

import rdflib

def run_freesurfer(subject_id, T1_images, subjects_dir, T2_image=None):
    """Run freesurfer, convert to nidm and extract stats
    """
    from nipype import freesurfer as fs
    from nipype import Node
    from fs_dir_to_graph import to_graph
    from query_convert_fs_stats import get_collections, process_collection

    recon = Node(fs.ReconAll(), name='recon')
    recon.inputs.T1_files = T1_images
    recon.inputs.subject_id = subject_id
    recon.inputs.subjects_dir = subjects_dir
    recon.inputs.openmp = 4
    if T2_image:
        recon.inputs.T2_file = T2_image
    recon.base_dir = os.path.abspath(os.path.join('working', subject_id))

    results = recon.run()
    provgraph = results.provenance
    newgraph = to_graph(os.path.join(results.outputs.subjects_dir,
                                     results.outputs.subject_id))
    provgraph.add_bundle(newgraph)
    provgraph.rdf().serialize('test1.ttl', format='turtle')
    results = get_collections(provgraph.rdf())
    collections = []
    for row in results:
        collections.append(str(row[0]))
    if len(collections) > 1:
        raise ValueError('More than one freesurfer directory collection found')
    provgraph, termsrdf = process_collection(provgraph, collections.pop())
    rdfgraph = provgraph.rdf() + termsrdf
    return provgraph, rdfgraph


"""
Construct input graph as an example. this code will be removed from the
production version, but shows an example of what the graph of the required
input should look like from a provenance model standpoint
"""
from uuid import uuid1
import hashlib
from socket import getfqdn
from utils import hash_infile, foaf, niiri, nif, crypto, get_id
from utils import prov as pm

T1s = [os.path.abspath('SAD_024.nii.gz')]
subject = 'SAD_024'

ingraph = pm.ProvBundle(identifier=get_id())
ingraph.add_namespace(foaf)
ingraph.add_namespace(niiri)
ingraph.add_namespace(nif)
ingraph.add_namespace(crypto)

agent = ingraph.agent(get_id(),
                      {pm.PROV["type"]: pm.PROV["Person"],
                       foaf["name"]: subject}
                      )
t1_collection = ingraph.collection(get_id())
ingraph.wasAttributedTo(t1_collection, agent)
for t1path in T1s:
    file_md5_hash = hash_infile(t1path, crypto=hashlib.md5)
    file_sha512_hash = hash_infile(t1path, crypto=hashlib.sha512)
    url = "file://%s%s" % (getfqdn(), t1path)
    url_get = pm.URIRef(url)
    obj_attr = [(pm.PROV["location"], url_get),
                (crypto["md5"], "%s" % file_md5_hash),
                (crypto["sha"], "%s" % file_sha512_hash),
                (pm.PROV["type"], nif["nlx_inv_20090243"])
                ]
    t1entity = ingraph.entity(get_id(), obj_attr)
    ingraph.hadMember(t1_collection, t1entity)

rdfingraph = ingraph.rdf()
print rdfingraph.serialize(format='turtle')

"""
The input graph as generated by the above code
"""

'''
@prefix crypto: <http://www.w3.org/2000/10/swap/crypto#> .
@prefix foaf: <http://xmlns.com/foaf/0.1/> .
@prefix nif: <http://neurolex.org/wiki/> .
@prefix niiri: <http://iri.nidash.org/> .
@prefix prov: <http://www.w3.org/ns/prov#> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
@prefix xsd1: <http://www.w3.org/2001/XMLSchema-datatypes#> .

niiri:e04910f5498211e3863b14109fcf6ae7 a prov:Collection,
        prov:Entity ;
    prov:hadMember niiri:e04fcb6b498211e3abc614109fcf6ae7 ;
    prov:wasAttributedTo niiri:e0490cc7498211e3a26f14109fcf6ae7 .

niiri:e0490cc7498211e3a26f14109fcf6ae7 a prov:Agent,
        prov:Person ;
    foaf:name "SAD_024" .

niiri:e04fcb6b498211e3abc614109fcf6ae7 a nif:nlx_inv_20090243,
        prov:Entity ;
    crypto:md5 "d79a4aaba19e45c0d0bbf83cd4a8e5e7" ;
    crypto:sha "c9ecc6e18cebb58303ef1f27278d5a19db4430644c67cd59c2f3a56fb53e4b94e1114e4886c8be29087bdfb5f0026201853f24be103cf95fffe9370d908cf196" ;
    prov:location "file://warpspeed.local/software/temp/nipype-tutorial/temp/fsscript/SAD_024.nii.gz" .
'''

"""
Parse the input graph to get subject id and a list of T1 files

"""
subject_query = """
    PREFIX prov: <http://www.w3.org/ns/prov#>
    PREFIX foaf: <http://xmlns.com/foaf/0.1/>
    select ?subject_id where
    {?id a prov:Agent;
        foaf:name ?subject_id .
     ?c a prov:Collection;
        prov:wasAttributedTo ?id .
    }
"""
sq_result = rdfingraph.query(subject_query)
try:
    subject_id = str(sq_result.bindings[0]['?subject_id'])
except:
    subject_id = 'anon' + uuid1().hex

t1_query = """
    PREFIX prov: <http://www.w3.org/ns/prov#>
    PREFIX nif: <http://neurolex.org/wiki/>
    PREFIX crypto: <http://www.w3.org/2000/10/swap/crypto#>
    select ?t1path ?sha where
    {?c a prov:Collection;
        prov:hadMember ?e .
     ?e a nif:nlx_inv_20090243;
        crypto:sha ?sha;
        prov:location ?t1path .
    }
"""
t1_result = rdfingraph.query(t1_query)

"""
Retrieve all the files into cwd
"""
out_T1_files = []
for idx, info in enumerate(t1_result.bindings):
    o = urlparse.urlparse(info['?t1path'])
    if o.scheme.startswith('file'):
        uri = 'file://' + o.path
    else:
        uri = info['?t1path']
    filename = os.path.join(os.getcwd(),
                            'T1_%d_' % idx + os.path.split(o.path)[-1])
    urllib.urlretrieve(uri, filename)
    if hash_infile(filename, crypto=hashlib.sha512) != str(info['?sha']):
        raise IOError("Hash of file doesn't match remote hash")
    out_T1_files.append(filename)


"""
Run freesurfer and convert to rdf
"""
subject_dir = os.path.abspath('subjects')
if not os.path.exists(subject_dir):
    os.mkdir(subject_dir)
provgraph, rdfgraph = run_freesurfer(subject_id,
                                     out_T1_files,
                                     subject_dir)

# TODO: Need to reconcile rdfingraph and rdfgraph based on file hashes

newgraph = rdflib.Graph().parse(StringIO(rdfgraph.serialize()))
newgraph.serialize('outfile.ttl', format='turtle')

context = {('@%s' % k): v for k, v in newgraph.namespaces()}
newgraph.serialize('outfile.json', context=context, format='json-ld')
<commit_msg>enh: updating fs script<commit_after>#!/usr/bin/env python
"""Run FreeSurfer on a T1 image and capture all output as NIDM
"""
import os

import urllib
import urlparse
from StringIO import StringIO

from nipype.interfaces.base import CommandLine
CommandLine.set_default_terminal_output('allatonce')

from nipype import config
config.enable_provenance()

import rdflib

def run_freesurfer(subject_id, T1_images, subjects_dir, T2_image=None):
    """Run freesurfer, convert to nidm and extract stats
    """
    from nipype import freesurfer as fs
    from nipype import Node
    from fs_dir_to_graph import to_graph
    from query_convert_fs_stats import get_collections, process_collection

    recon = Node(fs.ReconAll(), name='recon')
    recon.inputs.T1_files = T1_images
    recon.inputs.subject_id = subject_id
    recon.inputs.subjects_dir = subjects_dir
    recon.inputs.openmp = 4
    if T2_image:
        recon.inputs.T2_file = T2_image
    recon.base_dir = os.path.abspath(os.path.join('working', subject_id))

    results = recon.run()
    provgraph = results.provenance
    newgraph = to_graph(os.path.join(results.outputs.subjects_dir,
                                     results.outputs.subject_id))
    provgraph.add_bundle(newgraph)
    provgraph.rdf().serialize('test1.ttl', format='turtle')
    results = get_collections(provgraph.rdf())
    collections = []
    for row in results:
        collections.append(str(row[0]))
    if len(collections) > 1:
        raise ValueError('More than one freesurfer directory collection found')
    provgraph, termsrdf = process_collection(provgraph, collections.pop())
    rdfgraph = provgraph.rdf() + termsrdf
    return provgraph, rdfgraph


"""
Construct input graph as an example. this code will be removed from the
production version, but shows an example of what the graph of the required
input should look like from a provenance model standpoint
"""
from uuid import uuid1
import hashlib
from socket import getfqdn
from utils import hash_infile, foaf, niiri, nif, crypto, get_id
from utils import prov as pm

T1s = [os.path.abspath('SAD_024.nii.gz')]
subject = 'SAD_024'

ingraph = pm.ProvBundle(identifier=get_id())
ingraph.add_namespace(foaf)
ingraph.add_namespace(niiri)
ingraph.add_namespace(nif)
ingraph.add_namespace(crypto)

agent = ingraph.agent(get_id(),
                      {pm.PROV["type"]: pm.PROV["Person"],
                       foaf["name"]: subject}
                      )
t1_collection = ingraph.collection(get_id())
ingraph.wasAttributedTo(t1_collection, agent)
for t1path in T1s:
    file_md5_hash = hash_infile(t1path, crypto=hashlib.md5)
    file_sha512_hash = hash_infile(t1path, crypto=hashlib.sha512)
    url = "file://%s%s" % (getfqdn(), t1path)
    url_get = pm.URIRef(url)
    obj_attr = [(pm.PROV["location"], url_get),
                (crypto["md5"], "%s" % file_md5_hash),
                (crypto["sha"], "%s" % file_sha512_hash),
                (pm.PROV["type"], nif["nlx_inv_20090243"])
                ]
    t1entity = ingraph.entity(get_id(), obj_attr)
    ingraph.hadMember(t1_collection, t1entity)

rdfingraph = ingraph.rdf()
print rdfingraph.serialize(format='turtle')

"""
The input graph as generated by the above code
"""

'''
@prefix crypto: <http://www.w3.org/2000/10/swap/crypto#> .
@prefix foaf: <http://xmlns.com/foaf/0.1/> .
@prefix nif: <http://neurolex.org/wiki/> .
@prefix niiri: <http://iri.nidash.org/> .
@prefix prov: <http://www.w3.org/ns/prov#> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
@prefix xsd1: <http://www.w3.org/2001/XMLSchema-datatypes#> .

niiri:e04910f5498211e3863b14109fcf6ae7 a prov:Collection,
        prov:Entity ;
    prov:hadMember niiri:e04fcb6b498211e3abc614109fcf6ae7 ;
    prov:wasAttributedTo niiri:e0490cc7498211e3a26f14109fcf6ae7 .

niiri:e0490cc7498211e3a26f14109fcf6ae7 a prov:Agent,
        prov:Person ;
    foaf:name "SAD_024" .

niiri:e04fcb6b498211e3abc614109fcf6ae7 a nif:nlx_inv_20090243,
        prov:Entity ;
    crypto:md5 "d79a4aaba19e45c0d0bbf83cd4a8e5e7" ;
    crypto:sha "c9ecc6e18cebb58303ef1f27278d5a19db4430644c67cd59c2f3a56fb53e4b94e1114e4886c8be29087bdfb5f0026201853f24be103cf95fffe9370d908cf196" ;
    prov:location "file://warpspeed.local/software/temp/nipype-tutorial/temp/fsscript/SAD_024.nii.gz" .
'''


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(prog='fs_dir_to_graph.py',
                                     description=__doc__)
    parser.add_argument('-i', '--in_graph', type=str, required=True,
                        help='Path to input graph turtle file')
    parser.add_argument('-e', '--endpoint', type=str,
                        help='SPARQL endpoint to use for update')
    parser.add_argument('-g', '--graph_iri', type=str,
                        help='Graph IRI to store the triples')
    parser.add_argument('-o', '--output_dir', type=str, required=True,
                        help='Output directory')

    args = parser.parse_args()

    rdfingraph = rdflib.Graph().parse(args.in_graph, format='turtle')

    """
    Parse the input graph to get subject id and a list of T1 files

    """
    subject_query = """
        PREFIX prov: <http://www.w3.org/ns/prov#>
        PREFIX foaf: <http://xmlns.com/foaf/0.1/>
        select ?subject_id where
        {?id a prov:Agent;
            foaf:name ?subject_id .
         ?c a prov:Collection;
            prov:wasAttributedTo ?id .
        }
    """
    sq_result = rdfingraph.query(subject_query)
    try:
        subject_id = str(sq_result.bindings[0]['?subject_id'])
    except:
        subject_id = 'anon' + uuid1().hex

    t1_query = """
        PREFIX prov: <http://www.w3.org/ns/prov#>
        PREFIX nif: <http://neurolex.org/wiki/>
        PREFIX crypto: <http://www.w3.org/2000/10/swap/crypto#>
        select ?t1path ?sha where
        {?c a prov:Collection;
            prov:hadMember ?e .
         ?e a nif:nlx_inv_20090243;
            crypto:sha ?sha;
            prov:location ?t1path .
        }
    """
    t1_result = rdfingraph.query(t1_query)

    """
    Retrieve all the files into cwd
    """
    out_T1_files = []
    for idx, info in enumerate(t1_result.bindings):
        o = urlparse.urlparse(info['?t1path'])
        if o.scheme.startswith('file'):
            uri = 'file://' + o.path
        else:
            uri = info['?t1path']
        filename = os.path.join(os.getcwd(),
                                'T1_%d_' % idx + os.path.split(o.path)[-1])
        urllib.urlretrieve(uri, filename)
        if hash_infile(filename, crypto=hashlib.sha512) != str(info['?sha']):
            raise IOError("Hash of file doesn't match remote hash")
        out_T1_files.append(filename)


    """
    Run freesurfer and convert to rdf
    """
    subject_dir = os.path.abspath('subjects')
    if not os.path.exists(subject_dir):
        os.mkdir(subject_dir)
    provgraph, rdfgraph = run_freesurfer(subject_id,
                                         out_T1_files,
                                         subject_dir)

    # TODO: Need to reconcile rdfingraph and rdfgraph based on file hashes

    newgraph = rdflib.Graph().parse(StringIO(rdfgraph.serialize()))
    newgraph.serialize('outfile.ttl', format='turtle')

    context = {('@%s' % k): v for k, v in newgraph.namespaces()}
    newgraph.serialize('outfile.json', context=context, format='json-ld')
{'fraction_non_alphanumeric': 0.12672254581616707, 'fraction_numerical': 0.04858644693848558, 'mean_word_length': 3.1641526175687664, 'pattern_counts': {'":': 1, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 20, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4590042', 'n_tokens_mistral': 5395, 'n_tokens_neox': 5023, 'n_words': 972}