<commit_before>package gngs
/*
 *  Groovy NGS Utils - Some simple utilites for processing Next Generation Sequencing data.
 *
 *  Copyright (C) 2014 <NAME>, <EMAIL>in<at>gmail.com
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2.1 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 */

import static RelationshipType.*

import graxxia.TSV

class Pedigrees {

    /**
     * Lookup table to find family by pedigree Id
     */
    Map<String,Pedigree> families = [:]
    
    /**
     * Look up table to find pedigree by subject Ids
     */
    Map<String, Pedigree> subjects = [:]
    
    Pedigrees add(Pedigrees other) {
        new Pedigrees(families: families + other.families, subjects: subjects + other.subjects)
    }
    
    /**
     * Add the pedigree to this Pedigrees object
     * 
     * @param ped
     */
    void add(Pedigree ped) {
        ped.individuals.each { Subject s ->
            subjects[s.id] = ped
        }
        this.families[ped.id] = ped
    }
    
    /**
     * Find the largest possible set of samples that are unrelated to 
     * the given sample. If null is passed, the maximal set of unrelated
     * samples from the entire set will be computed.
     * 
     * @return
     */
    List<Subject> findMaximalUnrelatedSet(String sampleId=null) {
        
        
        List<Pedigree> unrelatedFamilies = null
        
        if(sampleId != null) {
            // Get all the families that are not those of the sample in question
            Pedigree samplePedigree = families[sampleId]
            if(samplePedigree == null) 
                throw new IllegalArgumentException("Sample " + sampleId + " is not a known sample")
                
            unrelatedFamilies = families*.value.grep { it.id != samplePedigree.id}
        }
        else {
            unrelatedFamilies = families*.value
        }
            
        // Each family is independent: choosing the maximal subset of 
        // members from each family will result in the overall maximal number of samples
        List results = []
        for(Pedigree pedigree in unrelatedFamilies) {
            List familyBestSet = pedigree.findMaximalUnrelatedSet()
            results.addAll(familyBestSet)
        }
        return results
    }
    
    /**
     * Parse a PED file to create a set of pedigrees from it
     * 
     * @param pedFileName
     * @return
     */
    static Pedigrees parse(String pedFileName) {
        new File(pedFileName).withReader { r ->
            parse(r)
        }
    }
    
    final static Set<String> NULL_VALUES = [".","0"]
    
    static Pedigrees parse(Reader r) {
        Map<String,Pedigree> families = [:]
        Map<String,Pedigree> subjectsToFamilies = [:]
        List<Subject> subjects = new TSV(r,columnNames:['familyId','id', 'paternalId', 'maternalId', 'sex', 'phenotype']).collect { line ->
            
            if(!families.containsKey(line.familyId))
                families[line.familyId] = new Pedigree(id:line.familyId)
                
            Pedigree p = families[line.familyId]
            def sex = line.sex ?: "other"
            Subject s = new Subject(id: line.id, sex: Sex.decode(sex), phenoTypes:[line.phenotype])
            
            RelationshipType childType = s.sex == Sex.FEMALE ? DAUGHTER : SON
            if(line.paternalId && !NULL_VALUES.contains(line.paternalId))
                s.relationships.add(new Relationship(type:childType,from:s.id, to: line.paternalId))
                
            if(line.maternalId && !NULL_VALUES.contains(line.maternalId))
                s.relationships.add(new Relationship(type:childType,from:s.id, to: line.maternalId)) 
            
            p.individuals.add(s)
            p.phenoTypes.add(line.phenotype)
            
            subjectsToFamilies[s.id] = p
            return s
        }
        
        // The relationships at this point are one-way. We would like to have them two way, so we 
        // go through each subject and if they have a parent, fill them in as a child in the parent's<commit_msg>support filtering of samples from pedigree in loading<commit_after>package gngs
/*
 *  Groovy NGS Utils - Some simple utilites for processing Next Generation Sequencing data.
 *
 *  Copyright (C) 2014 <NAME>, <EMAIL><at><EMAIL>
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2.1 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 */

import static RelationshipType.*

import graxxia.TSV
import static Sex.*

class Pedigrees {
    
    final static List PED_COLUMNS = ['familyId','id', 'paternalId', 'maternalId', 'sex', 'phenotype']

    /**
     * Lookup table to find family by pedigree Id
     */
    Map<String,Pedigree> families = [:]
    
    /**
     * Look up table to find pedigree by subject Ids
     */
    Map<String, Pedigree> subjects = [:]
    
    Pedigrees add(Pedigrees other) {
        new Pedigrees(families: families + other.families, subjects: subjects + other.subjects)
    }
    
    /**
     * Add the pedigree to this Pedigrees object
     * 
     * @param ped
     */
    void add(Pedigree ped) {
        ped.individuals.each { Subject s ->
            subjects[s.id] = ped
        }
        this.families[ped.id] = ped
    }
    
    /**
     * Find the largest possible set of samples that are unrelated to 
     * the given sample. If null is passed, the maximal set of unrelated
     * samples from the entire set will be computed.
     * 
     * @return
     */
    List<Subject> findMaximalUnrelatedSet(String sampleId=null) {
        
        
        List<Pedigree> unrelatedFamilies = null
        
        if(sampleId != null) {
            // Get all the families that are not those of the sample in question
            Pedigree samplePedigree = families[sampleId]
            if(samplePedigree == null) 
                throw new IllegalArgumentException("Sample " + sampleId + " is not a known sample")
                
            unrelatedFamilies = families*.value.grep { it.id != samplePedigree.id}
        }
        else {
            unrelatedFamilies = families*.value
        }
            
        // Each family is independent: choosing the maximal subset of 
        // members from each family will result in the overall maximal number of samples
        List results = []
        for(Pedigree pedigree in unrelatedFamilies) {
            List familyBestSet = pedigree.findMaximalUnrelatedSet()
            results.addAll(familyBestSet)
        }
        return results
    }
    
    /**
     * Parse a PED file to create a set of pedigrees from it
     * 
     * See {@link #parse(Reader,Closure)}
     * 
     * @param pedFileName   path to file
     * @param c             filtering closure
     * 
     * @return
     */
    static Pedigrees parse(String pedFileName, Closure c = null) {
        new File(pedFileName).withReader { r ->
            parse(r,c)
        }
    }
    
    final static Set<String> NULL_VALUES = [".","0"]
    
    /**
     * Parse a PED file from the given reader, with optional filtering via the provided closure
     * 
     * @param r the reader to read the PED file from
     * @param c an optional Closure that will be called for each line (see {@link #PED_COLUMNS})
     *          which will cause samples to be skipped from parsing if it returns false
     * 
     * @return  a Pedigrees object representing the families parsed from the PED file
     */
    static Pedigrees parse(Reader r, Closure c = null) {
        Map<String,Pedigree> families = [:]
        Map<String,Pedigree> subjectsToFamilies = [:]
        Map<String,Subject> validSubjects = [:]
        List<Subject> subjects = new TSV(r,columnNames:PED_COLUMNS, columnTypes:[String,String,String,String,Integer,Integer]).findResults { line ->
            
            if(c != null) {
                if(c(line) == false)
                    return
            }
            
            if(line.id in validSubjects)
                throw new IllegalStateException("Sample $line.id is present multiple times in PED file")
                
            
            if(!families.containsKey(line.familyId))
                families[line.familyId] = new Pedigree(id:line.familyId)
                
            Pedigree p = families[line.familyId]
            def sex = line.sex ?: "other"
            Subject s = new Subject(id: line.id, sex: Sex.decode(sex), phenoTypes:[line.phenotype])
            
            RelationshipType childType = s.sex == Sex.FEMALE ? DAUGHTER : SON
            if(line.paternalId && !NULL_VALUES.contains(line.paternalId))
                s.relationships.add(new Relationship(type:childType,from:s.id, to: line.paternalId))
                
            if(line.maternalId && !NULL_VALUES.contains(line.maternalId))
                s.relationships.add(new Relationship(type:childType,from:s.id, to: line.maternalId)) 
            
            p.individuals.add(s)
            p.phenoTypes.add(line.phenotype)
            
            subjectsToFamilies[s.id] = p
            return s
        }
        
        // The relationships at this point are one-way. We would like to have them two way, so we 
        // go through each subject and if they have a parent, fill them in as a child in the parent's
{'fraction_non_alphanumeric': 0.08450433108758422, 'fraction_numerical': 0.001347449470644851, 'mean_word_length': 2.0806403794841386, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '394950', 'n_tokens_mistral': 2997, 'n_tokens_neox': 2826, 'n_words': 945}