<commit_before>#!/usr/bin/python
import sys

import requests
import json
import collections
import ntpath

def main(argv):
    print "running solr main method"
    solr=Solr("http://ves-ebi-d0.ebi.ac.uk:8090/mi/impc/dev/solr/")
    solr.getAllPhenCenterPipelinesAndProceduresAndParameters()

class Solr:
    def __init__(self, solrRootUrl):
        self.numberOfFilesInSolr=0
        self.solrRootUrl=solrRootUrl
        self.solrExperimentRootUrl=solrRootUrl+"experiment/"
        print "initialising solr python object with solr root url="+solrRootUrl
        print "initialising solr python object with solrExperimentRootUrl="+self.solrExperimentRootUrl
        #pipelines= http://ves-ebi-d0.ebi.ac.uk:8090/mi/impc/dev/solr/experiment/select?q=observation_type:image_record&fq=download_file_path:(download_file_path:*mousephenotype.org*%20AND%20!download_file_path:*.pdf%20!download_file_path:*.mov)&facet.mincount=1&facet=true&facet.field=pipeline_stable_id&wt=json&indent=on&rows=0
        self.phenotyping_centers=[]
        self.pipelines=[]
        self.procedures=[]
        self.parameters=[]
        self.standardFilter='download_file_path:*mousephenotype.org*%20AND%20!download_file_path:*.mov%20AND%20!download_file_path:*.fcs%20AND%20!download_file_path:*.bz2'

        #self.getParametersForProcedureAndPipeline("TCP_001", "IMPC_XRY_001")

    #procedures for pipeline=http://ves-ebi-d0.ebi.ac.uk:8090/mi/impc/dev/solr/experiment/select?q=observation_type:image_record&fq=download_file_path:(download_file_path:*mousephenotype.org*%20AND%20!download_file_path:*.pdf%20!download_file_path:*.mov%20AND%20pipeline_stable_id:TCP_001)&facet.mincount=1&facet=true&facet.field=procedure_stable_id&wt=json&indent=on&rows=0
    #parameters for pipline, procedure http://ves-ebi-d0.ebi.ac.uk:8090/mi/impc/dev/solr/experiment/select?q=observation_type:image_record&fq=download_file_path:(download_file_path:*mousephenotype.org*%20AND%20!download_file_path:*.pdf%20!download_file_path:*.mov%20AND%20pipeline_stable_id:TCP_001%20AND%20procedure_stable_id:IMPC_XRY_001)&facet.mincount=1&facet=true&facet.field=parameter_stable_id&wt=json&indent=on&rows=0
    #get first 100 image_record names for pipeline, procedure, parameter
    #http://ves-ebi-d0.ebi.ac.uk:8090/mi/impc/dev/solr/experiment/select?q=observation_type:image_record&fq=download_file_path:(download_file_path:*mousephenotype.org*%20AND%20!download_file_path:*.pdf%20!download_file_path:*.mov%20AND%20pipeline_stable_id:TCP_001%20AND%20procedure_stable_id:IMPC_XRY_001%20AND%20parameter_stable_id:IMPC_XRY_048_001)&facet.mincount=1&facet=true&facet.field=parameter_stable_id&wt=json&indent=on&rows=100

    #get a map of directories paths to list of filenames

    def getAllPhenCenterPipelinesAndProceduresAndParameters(self):
        self.phenotyping_centers=self.getPhenotypingCenters()
        self.directory_map = collections.OrderedDict()
        for phenCenter in self.phenotyping_centers:
            #print "phenCenter="+phenCenter
            self.pipelines=self.getPipelines(phenCenter)
            for pipe in self.pipelines:
                self.procedures=self.getProceduresForPipeline(phenCenter, pipe)
                #print "pipe="+pipe
                for proc in self.procedures:<commit_msg>Add OR clause standard filter to allow pulling of 3i images<commit_after>#!/usr/bin/python
import sys

import requests
import json
import collections
import ntpath

def main(argv):
    print "running solr main method"
    solr=Solr("http://ves-ebi-d0.ebi.ac.uk:8090/mi/impc/dev/solr/")
    solr.getAllPhenCenterPipelinesAndProceduresAndParameters()

class Solr:
    def __init__(self, solrRootUrl):
        self.numberOfFilesInSolr=0
        self.solrRootUrl=solrRootUrl
        self.solrExperimentRootUrl=solrRootUrl+"experiment/"
        print "initialising solr python object with solr root url="+solrRootUrl
        print "initialising solr python object with solrExperimentRootUrl="+self.solrExperimentRootUrl
        #pipelines= http://ves-ebi-d0.ebi.ac.uk:8090/mi/impc/dev/solr/experiment/select?q=observation_type:image_record&fq=download_file_path:(download_file_path:*mousephenotype.org*%20AND%20!download_file_path:*.pdf%20!download_file_path:*.mov)&facet.mincount=1&facet=true&facet.field=pipeline_stable_id&wt=json&indent=on&rows=0
        self.phenotyping_centers=[]
        self.pipelines=[]
        self.procedures=[]
        self.parameters=[]
        # KB 21/11/2017 Added OR part to allow 3i images to be pulled as well
        self.standardFilter='download_file_path:(*mousephenotype.org*%20OR%20*images/3i*)%20AND%20!download_file_path:*.mov%20AND%20!download_file_path:*.fcs%20AND%20!download_file_path:*.bz2'

        #self.getParametersForProcedureAndPipeline("TCP_001", "IMPC_XRY_001")

    #procedures for pipeline=http://ves-ebi-d0.ebi.ac.uk:8090/mi/impc/dev/solr/experiment/select?q=observation_type:image_record&fq=download_file_path:(download_file_path:*mousephenotype.org*%20AND%20!download_file_path:*.pdf%20!download_file_path:*.mov%20AND%20pipeline_stable_id:TCP_001)&facet.mincount=1&facet=true&facet.field=procedure_stable_id&wt=json&indent=on&rows=0
    #parameters for pipline, procedure http://ves-ebi-d0.ebi.ac.uk:8090/mi/impc/dev/solr/experiment/select?q=observation_type:image_record&fq=download_file_path:(download_file_path:*mousephenotype.org*%20AND%20!download_file_path:*.pdf%20!download_file_path:*.mov%20AND%20pipeline_stable_id:TCP_001%20AND%20procedure_stable_id:IMPC_XRY_001)&facet.mincount=1&facet=true&facet.field=parameter_stable_id&wt=json&indent=on&rows=0
    #get first 100 image_record names for pipeline, procedure, parameter
    #http://ves-ebi-d0.ebi.ac.uk:8090/mi/impc/dev/solr/experiment/select?q=observation_type:image_record&fq=download_file_path:(download_file_path:*mousephenotype.org*%20AND%20!download_file_path:*.pdf%20!download_file_path:*.mov%20AND%20pipeline_stable_id:TCP_001%20AND%20procedure_stable_id:IMPC_XRY_001%20AND%20parameter_stable_id:IMPC_XRY_048_001)&facet.mincount=1&facet=true&facet.field=parameter_stable_id&wt=json&indent=on&rows=100

    #get a map of directories paths to list of filenames

    def getAllPhenCenterPipelinesAndProceduresAndParameters(self):
        self.phenotyping_centers=self.getPhenotypingCenters()
        self.directory_map = collections.OrderedDict()
        for phenCenter in self.phenotyping_centers:
            #print "phenCenter="+phenCenter
            self.pipelines=self.getPipelines(phenCenter)
            for pipe in self.pipelines:
                self.procedures=self.getProceduresForPipeline(phenCenter, pipe)
                #print "pipe="+pipe
                for proc in self.procedures:
{'fraction_non_alphanumeric': 0.1170673435866806, 'fraction_numerical': 0.040167239062266685, 'mean_word_length': 5.85568065506653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2970371', 'n_tokens_mistral': 2584, 'n_tokens_neox': 2371, 'n_words': 233}