<commit_before>    [

        // ====================
        // FEDERATION SERVICES
        // ====================
        // --OMIM--
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nlx_151835-1.json?exportType=data&q=HP:0001337",
            desc : "DIRECT Query on OmimDiseasePhenotype using an HPO ID (Tremor)",
            maxTimeMilliseconds : 5000,
            expects : {
                format : "json",
                min_results : 0,
                must_contain : {
                    disorder_id : "OMIM:270500" // Ataxia, Spastic, Childhood-Onset, Autosomal Recessive, With Optic Atrophy And Mental Retardation
                }
            }
        },

        // --OMIM closure--
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nlx_151835-1.json?exportType=data&q=HP:0001337&includeSubclasses=true&count=1000",
            desc : "INFERRED Query OmimDiseasePhenotype using an HPO ID (Tremor) - include subclasses - we expect diseases directly annotated and annotated to subclasses",
            maxTimeMilliseconds : 5000,
            expects : {
                format : "json",
                min_results : 20,
                must_contain : 
                [
                    {
                        aspect_text : "organ abnormality",
                        disorder_id : "OMIM:300619" // Cataract, Ataxia, Short Stature, And Mental Retardation
                    },
                    {  
                        aspect_text : "organ abnormality",
                        phenotype_id : "HP:0001337" // always include annotations to direct
                    },
                    {  
                        aspect_text : "organ abnormality",
                        phenotype_id : "HP:0002378" // annotation to hand tremor - indirect
                    },
                    {  
                        aspect_text : "organ abnormality",
                        phenotype_id : "HP:0002599", // Head titubation
                        "disorder_id": "OMIM:275550", // Trichorrhexis Nodosa Syndrome
                    },
                    
                ]
            }
        },

        // --MGI closure: high level term--
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nif-0000-00096-6.json?exportType=data&count=100&q=MP:0000428&includeSubclasses=true&",
            desc : "INFERRED MGI Geno Pheno using an MP ID (craniofacial) - include subclasses - we expect genotypes directly annotated and annotated to subclasses",
            maxTimeMilliseconds : 5000,
            expects : {
                format : "json",
                min_results : 80,
            }
        },

        // --MGI closure: high level term--
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nif-0000-00096-6.json?exportType=data&count=1000&q=MP:0003677&includeSubclasses=true&",
            desc : "INFERRED MGI Geno Pheno using an MP ID (craniofacial) - include subclasses - we expect genotypes directly annotated and annotated to subclasses",
            maxTimeMilliseconds : 5000,
            expects : {
                format : "json",
                min_results : 2,
                must_contain : 
                [
                    {
                        effective_genotype_id: "MGI:3686776",
                        genomic_background_label: "involves: 129S7/SvEvBrd * C57BL/6 * CBA",
                        phenotype_id: "MP:0003679",
                        phenotype_label: "ear lobe hypoplasia",
                        publication_id: "PMID:16914493",
                        
                    }
                ]
            }
        },

        // --OMIM variant search--
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nif-0000-03216-9.json?exportType=data&count=1000&q=Huntington",
            desc : "Query OmimDiseaseVariants using string matching (expect Hungtindon's disease)",
            maxTimeMilliseconds : 5000,
            expects : {
                format : "json",
                min_results : 3,
                must_contain : {
                    phenotype_id : "OMIM:143100" // ID for Huntington's disease (note in this view, phenotype subsumes phenotype and disease)
                }
            }
        },

        // --OMIM variant--
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nif-0000-03216-9.json?exportType=data&count=1000&q=OMIM:214290",
            desc : "Query OmimDiseaseVariants using OMIM ID (Cervical Vertebrae, Agenesis Of)",
            maxTimeMilliseconds : 3000,
            expects : {
                format : "json",
                max_results : 1,
                must_contain : {
                    phenotype_id : "OMIM:214290" // Cervical Vertebrae, Agenesis Of
                },
                must_not_contain : {
                    phenotype_id : "OMIM:103400" // non-queried for ID
                }
            }
        },

        // ORPHANET
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nif-0000-21306-2.json?exportType=data&count=50&q=Huntington",
            desc : "Query ORPHANET using string matching (expect Hungtindon's disease)",
            maxTimeMilliseconds : 5000,
            expects : {
                format : "json",
                min_results : 3,
                must_contain : {
                    disease_id: "ORPHANET:399", // ID for Huntington's disease
                }
            }
        },


        // ====================
        // VOCABULARY SERVICES
        // ====================
        {
            component : "vocabulary",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/vocabulary.json?prefix=small+adrenal+gland&vocabulary=monarch",
            desc : "Autocomplete query form 'small adrenal...'",
            maxTimeMilliseconds : 400,
            expects : {
                format : "json",
                min_results : 2,
                must_contain : {
                    id : "MP_0002768"
                },
            }
        },

        // ====================
        // ONTOQUEST QUERIES
        // ====================
        {
            component : "ontoquest",
            subcomponent : "concepts",
            priority : 1,
            url : "http://nif-services-stage.neuinfo.org/ontoquest-lamhdi/concepts/term/HP_0003325",
            desc : "Concept details for 'limb girdle muscle weakness''",
            notes : "Currently just checks for string matches in returned XML",
            maxTimeMilliseconds : 2000,
            expects : {
                format : "xml",
                raw_contains : 
                ["Limb-girdle muscle weakness",
                 "HP:0008971", // alt ID
                 "Weakness of the limb-girdle muscles"     // definition field
                ]
            }
        },

        {
            component : "ontoquest",
            subcomponent : "rel",
            priority : 1,
            url : "http://nif-services-stage.neuinfo.org/ontoquest-lamhdi/rel/all/MP_0002789?level=1",
            desc : "Relationships for male pseudohermaphroditism",
            notes : "Currently just checks for string matches in returned XML",
            maxTimeMilliseconds : 2000,
            expects : {
                format : "xml",
                raw_contains : [
                    "HP_0000037", // equivalent HP class
                    "MP_0002787", // parent class pseudohermaphroditism
                ],
            }
        },

        // ====================
        // MONARCH QUERIES
        // ====================
        // --html--
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu",
            desc : "Top level monarch portal page",
            notes : "Currently just checks for string matches in returned HTML",
            expects : {
                format : "html",
                raw_contains : "Welcome to Monarch",
            }
        },

        // --html--
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/disease/DOID_14330",
            desc : "A DO disease page, Parkinsons",
            expects : {
                format : "html",
                raw_contains : 
                [
                    "DOID_14330",
                    "Parkinsonism", // HP term associated
                    "PINK1", // 
                    "Parkinson Disease 14", // OMIM subtype
                ]
            }
        },

        // --html--
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/phenotype/HP:0001337",
            desc : "A HPO phenotype disease page, Tremor",
            expects : {
                format : "html",
                raw_contains : 
                [
                    "Phenotype: Tremor",
                    "Cataract, Ataxia, Short Stature, And Mental Retardation", // associated with a subtype of tremor
                    "Grid2", // MGI genotype, for titubation
                ]
            }
        },

        // --json--
        // monarch API calls can return deeply nested/structured objects, these are
        // checked recursively
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/phenotype/NCBIGene:388552.json",
            desc : "JSON for a gene",
            expects : {
                format : "json",
                must_contain : [
                    {
                        tax_xref: {
                            "id": "NCBITaxon:9606"
                        }
                    }
                ]
            }
        },
            

        // --json--
        // monarch API calls can return deeply nested/structured objects, these are
        // checked recursively
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/phenotype/HP:0001337.json",
            desc : "JSON for A HPO phenotype disease page, Tremor (and subtypes)",
            expects : {
                format : "json",
                must_contain : [
                    {
                        disease_associations : {
                            disease: {
                                label: "Nipah Virus Disease"
                            }
                        }
                    },
                    {
                        disease_associations : {
                            type: "Association",
                            disease: {
                                id: "OMIM:253550",
                                label: "Spinal Muscular Atrophy, Type Ii"
                            },
                            phenotype: {
                                id: "HP:0002378",
                                label: "Hand tremor"
                            },
                            source: {
                                id: "nlx_151835-1",
                                label: "HPO"
                            }
                        }
                    },
                    {
                        genotype_associations : {
                            has_genotype : {
                                id: "MGI:3844321",
                                label: "Grid2<Lc-J>/Grid2<+> [BALB/cByJ]",
                                type: "effective_genotype",
                                has_part: {
                                    id: "MGI:3844321",
                                    label: "Grid2<Lc-J>/Grid2<+> [BALB/cByJ]",
                                    type: "intrinsic_genotype",
                                    has_part: {
                                        type: "genomic_variation_complement",
                                        has_part : {
                                            type: "variant_single_locus_complement",
                                            has_variant_part : {
                                                has_part : {
                                                    type: "variant_locus",
                                                    has_part : {
                                                        type: "sequence_alteration"
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                ],
                raw_contains : 
                [
                    "Tremor",
                    "Cataract, Ataxia, Short Stature, And Mental Retardation", // associated with a subtype of tremor
                    "Grid2", // MGI genotype, for titubation
                ]
            }
        },

        // --simsearch json--
        {
            component : "monarch",
            subcomponent : "simsearch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/simsearch/phenotype/?input_items=HP:0002360,HP:0002015,HP:0002063,HP:0002322&target_species=10090",
            desc : "Simsearch (layered on owlsim)",
            expects : {
                format : "json",
                raw_contains : 
                [
                    "MGI",
                    "HP:0002360",
                ]
            }
        },

        // --simsearch html--
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/analyze/phenotypes",
            desc : "Top level monarch portal page page for phenotype profile search",
            notes : "Currently just checks for string matches in returned HTML",
            expects : {
                format : "html",
                raw_contains : "Monarch Analysis: phenotypes",
            }
        },


        // --analyze html--
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/analyze/phenotypes/?target_species=10090&input_items=MP%3A0000788%0D%0AMP%3A0000822%0D%0AMP%3A0000914%0D%0AMP%3A0000929%0D%0AMP%3A0000930%0D%0AMP%3A0001286%0D%0AMP%3A0001393%0D%0AMP%3A0001688%0D%0AMP%3A0001698%0D%0AMP%3A0001732%0D%0AMP%3A0001787%0D%0AMP%3A0002064%0D%0AMP%3A0002083%0D%0AMP%3A0002151%0D%0AMP%3A0002950%0D%0AMP%3A0003012%0D%0AMP%3A0003424%0D%0AMP%3A0003651%0D%0AMP%3A0004948%0D%0AMP%3A0005657&limit=100",
            desc : "Results for phenotype profile search",
            notes : "Currently just checks for string matches in returned HTML",
            expects : {
                format : "html",
                raw_contains : "Cfl1",
            }
        },

        // --redirects--
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/reference/PMID:16516152",
            desc : "Pubmed query, which should redirect",
            expects : {
                redirect: true
            }
        },


            
    ]
<commit_msg>using a different example for direct queries<commit_after>    [

        // ====================
        // FEDERATION SERVICES
        // ====================
        // --OMIM--
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nlx_151835-1.json?exportType=data&q=HP:0200085",
            desc : "DIRECT Query on OmimDiseasePhenotype using an HPO ID (Limb Tremor)",
            maxTimeMilliseconds : 5000,
            expects : {
                format : "json",
                min_results : 2,
                must_contain : {
                    disorder_id : "OMIM:105830" // Angelman
                }
            }
        },

        // --OMIM closure--
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nlx_151835-1.json?exportType=data&q=HP:0001337&includeSubclasses=true&count=1000",
            desc : "INFERRED Query OmimDiseasePhenotype using an HPO ID (Tremor) - include subclasses - we expect diseases directly annotated and annotated to subclasses",
            maxTimeMilliseconds : 5000,
            expects : {
                format : "json",
                min_results : 20,
                must_contain : 
                [
                    {
                        aspect_text : "organ abnormality",
                        disorder_id : "OMIM:300619" // Cataract, Ataxia, Short Stature, And Mental Retardation
                    },
                    {  
                        aspect_text : "organ abnormality",
                        phenotype_id : "HP:0001337" // always include annotations to direct
                    },
                    {  
                        aspect_text : "organ abnormality",
                        phenotype_id : "HP:0002378" // annotation to hand tremor - indirect
                    },
                    {  
                        aspect_text : "organ abnormality",
                        phenotype_id : "HP:0002599", // Head titubation
                        "disorder_id": "OMIM:275550", // Trichorrhexis Nodosa Syndrome
                    },
                    
                ]
            }
        },

        // --MGI closure: high level term--
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nif-0000-00096-6.json?exportType=data&count=100&q=MP:0000428&includeSubclasses=true&",
            desc : "INFERRED MGI Geno Pheno using an MP ID (craniofacial) - include subclasses - we expect genotypes directly annotated and annotated to subclasses",
            maxTimeMilliseconds : 5000,
            expects : {
                format : "json",
                min_results : 80,
            }
        },

        // --MGI closure: high level term--
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nif-0000-00096-6.json?exportType=data&count=1000&q=MP:0003677&includeSubclasses=true&",
            desc : "INFERRED MGI Geno Pheno using an MP ID (craniofacial) - include subclasses - we expect genotypes directly annotated and annotated to subclasses",
            maxTimeMilliseconds : 5000,
            expects : {
                format : "json",
                min_results : 2,
                must_contain : 
                [
                    {
                        effective_genotype_id: "MGI:3686776",
                        genomic_background_label: "involves: 129S7/SvEvBrd * C57BL/6 * CBA",
                        phenotype_id: "MP:0003679",
                        phenotype_label: "ear lobe hypoplasia",
                        publication_id: "PMID:16914493",
                        
                    }
                ]
            }
        },

        // --OMIM variant search--
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nif-0000-03216-9.json?exportType=data&count=1000&q=Huntington",
            desc : "Query OmimDiseaseVariants using string matching (expect Hungtindon's disease)",
            maxTimeMilliseconds : 5000,
            expects : {
                format : "json",
                min_results : 3,
                must_contain : {
                    phenotype_id : "OMIM:143100" // ID for Huntington's disease (note in this view, phenotype subsumes phenotype and disease)
                }
            }
        },

        // --OMIM variant--
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nif-0000-03216-9.json?exportType=data&count=1000&q=OMIM:214290",
            desc : "Query OmimDiseaseVariants using OMIM ID (Cervical Vertebrae, Agenesis Of)",
            maxTimeMilliseconds : 3000,
            expects : {
                format : "json",
                max_results : 1,
                must_contain : {
                    phenotype_id : "OMIM:214290" // Cervical Vertebrae, Agenesis Of
                },
                must_not_contain : {
                    phenotype_id : "OMIM:103400" // non-queried for ID
                }
            }
        },

        // ORPHANET
        {
            component : "federation",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/federation/data/nif-0000-21306-2.json?exportType=data&count=50&q=Huntington",
            desc : "Query ORPHANET using string matching (expect Hungtindon's disease)",
            maxTimeMilliseconds : 5000,
            expects : {
                format : "json",
                min_results : 3,
                must_contain : {
                    disease_id: "ORPHANET:399", // ID for Huntington's disease
                }
            }
        },


        // ====================
        // VOCABULARY SERVICES
        // ====================
        {
            component : "vocabulary",
            priority : 1,
            url : "http://beta.neuinfo.org/services/v1/vocabulary.json?prefix=small+adrenal+gland&vocabulary=monarch",
            desc : "Autocomplete query form 'small adrenal...'",
            maxTimeMilliseconds : 400,
            expects : {
                format : "json",
                min_results : 2,
                must_contain : {
                    id : "MP_0002768"
                },
            }
        },

        // ====================
        // ONTOQUEST QUERIES
        // ====================
        {
            component : "ontoquest",
            subcomponent : "concepts",
            priority : 1,
            url : "http://nif-services-stage.neuinfo.org/ontoquest-lamhdi/concepts/term/HP_0003325",
            desc : "Concept details for 'limb girdle muscle weakness''",
            notes : "Currently just checks for string matches in returned XML",
            maxTimeMilliseconds : 2000,
            expects : {
                format : "xml",
                raw_contains : 
                ["Limb-girdle muscle weakness",
                 "HP:0008971", // alt ID
                 "Weakness of the limb-girdle muscles"     // definition field
                ]
            }
        },

        {
            component : "ontoquest",
            subcomponent : "rel",
            priority : 1,
            url : "http://nif-services-stage.neuinfo.org/ontoquest-lamhdi/rel/all/MP_0002789?level=1",
            desc : "Relationships for male pseudohermaphroditism",
            notes : "Currently just checks for string matches in returned XML",
            maxTimeMilliseconds : 2000,
            expects : {
                format : "xml",
                raw_contains : [
                    "HP_0000037", // equivalent HP class
                    "MP_0002787", // parent class pseudohermaphroditism
                ],
            }
        },

        // ====================
        // MONARCH QUERIES
        // ====================
        // --html--
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu",
            desc : "Top level monarch portal page",
            notes : "Currently just checks for string matches in returned HTML",
            expects : {
                format : "html",
                raw_contains : "Welcome to Monarch",
            }
        },

        // --html--
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/disease/DOID_14330",
            desc : "A DO disease page, Parkinsons",
            expects : {
                format : "html",
                raw_contains : 
                [
                    "DOID_14330",
                    "Parkinsonism", // HP term associated
                    "PINK1", // 
                    "Parkinson Disease 14", // OMIM subtype
                ]
            }
        },

        // --html--
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/phenotype/HP:0001337",
            desc : "A HPO phenotype disease page, Tremor",
            expects : {
                format : "html",
                raw_contains : 
                [
                    "Phenotype: Tremor",
                    "Cataract, Ataxia, Short Stature, And Mental Retardation", // associated with a subtype of tremor
                    "Grid2", // MGI genotype, for titubation
                ]
            }
        },

        // --json--
        // monarch API calls can return deeply nested/structured objects, these are
        // checked recursively
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/phenotype/NCBIGene:388552.json",
            desc : "JSON for a gene",
            expects : {
                format : "json",
                must_contain : [
                    {
                        tax_xref: {
                            "id": "NCBITaxon:9606"
                        }
                    }
                ]
            }
        },
            

        // --json--
        // monarch API calls can return deeply nested/structured objects, these are
        // checked recursively
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/phenotype/HP:0001337.json",
            desc : "JSON for A HPO phenotype disease page, Tremor (and subtypes)",
            expects : {
                format : "json",
                must_contain : [
                    {
                        disease_associations : {
                            disease: {
                                label: "Nipah Virus Disease"
                            }
                        }
                    },
                    {
                        disease_associations : {
                            type: "Association",
                            disease: {
                                id: "OMIM:253550",
                                label: "Spinal Muscular Atrophy, Type Ii"
                            },
                            phenotype: {
                                id: "HP:0002378",
                                label: "Hand tremor"
                            },
                            source: {
                                id: "nlx_151835-1",
                                label: "HPO"
                            }
                        }
                    },
                    {
                        genotype_associations : {
                            has_genotype : {
                                id: "MGI:3844321",
                                label: "Grid2<Lc-J>/Grid2<+> [BALB/cByJ]",
                                type: "effective_genotype",
                                has_part: {
                                    id: "MGI:3844321",
                                    label: "Grid2<Lc-J>/Grid2<+> [BALB/cByJ]",
                                    type: "intrinsic_genotype",
                                    has_part: {
                                        type: "genomic_variation_complement",
                                        has_part : {
                                            type: "variant_single_locus_complement",
                                            has_variant_part : {
                                                has_part : {
                                                    type: "variant_locus",
                                                    has_part : {
                                                        type: "sequence_alteration"
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                ],
                raw_contains : 
                [
                    "Tremor",
                    "Cataract, Ataxia, Short Stature, And Mental Retardation", // associated with a subtype of tremor
                    "Grid2", // MGI genotype, for titubation
                ]
            }
        },

        // --simsearch json--
        {
            component : "monarch",
            subcomponent : "simsearch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/simsearch/phenotype/?input_items=HP:0002360,HP:0002015,HP:0002063,HP:0002322&target_species=10090",
            desc : "Simsearch (layered on owlsim)",
            expects : {
                format : "json",
                raw_contains : 
                [
                    "MGI",
                    "HP:0002360",
                ]
            }
        },

        // --simsearch html--
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/analyze/phenotypes",
            desc : "Top level monarch portal page page for phenotype profile search",
            notes : "Currently just checks for string matches in returned HTML",
            expects : {
                format : "html",
                raw_contains : "Monarch Analysis: phenotypes",
            }
        },


        // --analyze html--
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/analyze/phenotypes/?target_species=10090&input_items=MP%3A0000788%0D%0AMP%3A0000822%0D%0AMP%3A0000914%0D%0AMP%3A0000929%0D%0AMP%3A0000930%0D%0AMP%3A0001286%0D%0AMP%3A0001393%0D%0AMP%3A0001688%0D%0AMP%3A0001698%0D%0AMP%3A0001732%0D%0AMP%3A0001787%0D%0AMP%3A0002064%0D%0AMP%3A0002083%0D%0AMP%3A0002151%0D%0AMP%3A0002950%0D%0AMP%3A0003012%0D%0AMP%3A0003424%0D%0AMP%3A0003651%0D%0AMP%3A0004948%0D%0AMP%3A0005657&limit=100",
            desc : "Results for phenotype profile search",
            notes : "Currently just checks for string matches in returned HTML",
            expects : {
                format : "html",
                raw_contains : "Cfl1",
            }
        },

        // --redirects--
        {
            component : "monarch",
            priority : 1,
            url : "http://tartini.crbs.ucsd.edu/reference/PMID:16516152",
            desc : "Pubmed query, which should redirect",
            expects : {
                redirect: true
            }
        },


            
    ]

{'fraction_non_alphanumeric': 0.10408344429649356, 'fraction_numerical': 0.04178555576691396, 'mean_word_length': 1.047316154994483, 'pattern_counts': {'":': 4, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 60, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7351368', 'n_tokens_mistral': 9260, 'n_tokens_neox': 7901, 'n_words': 1549}