<commit_before>/*
 * The Gemma project
 * 
 * Copyright (c) 2006 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.loader.expression.arrayDesign;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.core.context.SecurityContext;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;

import ubic.gemma.analysis.report.ArrayDesignReportService;
import ubic.gemma.analysis.sequence.ProbeMapper;
import ubic.gemma.analysis.sequence.ProbeMapperConfig;
import ubic.gemma.externalDb.GoldenPathSequenceAnalysis;
import ubic.gemma.model.common.description.ExternalDatabase;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
import ubic.gemma.model.expression.arrayDesign.ArrayDesignService;
import ubic.gemma.model.expression.designElement.CompositeSequence;
import ubic.gemma.model.expression.designElement.CompositeSequenceService;
import ubic.gemma.model.genome.Gene;
import ubic.gemma.model.genome.Taxon;
import ubic.gemma.model.genome.biosequence.BioSequence;
import ubic.gemma.model.genome.biosequence.BioSequenceService;
import ubic.gemma.model.genome.biosequence.SequenceType;
import ubic.gemma.model.genome.gene.GeneProduct;
import ubic.gemma.model.genome.gene.GeneService;
import ubic.gemma.model.genome.sequenceAnalysis.AnnotationAssociation;
import ubic.gemma.model.genome.sequenceAnalysis.AnnotationAssociationService;
import ubic.gemma.model.genome.sequenceAnalysis.BlatAssociation;
import ubic.gemma.model.genome.sequenceAnalysis.BlatResult;
import ubic.gemma.model.genome.sequenceAnalysis.BlatResultService;
import ubic.gemma.persistence.PersisterHelper;

/**
 * For an array design, generate gene product mappings for the sequences.
 * 
 * @author pavlidis
 * @version $Id$
 */
@Service
public class ArrayDesignProbeMapperService {

    private static Log log = LogFactory.getLog( ArrayDesignProbeMapperService.class.getName() );

    private static final int QUEUE_SIZE = 20000;

    @Autowired
    private AnnotationAssociationService annotationAssociationService;

    @Autowired
    private ArrayDesignService arrayDesignService;

    @Autowired
    private BioSequenceService bioSequenceService;

    @Autowired
    private BlatResultService blatResultService;

    @Autowired
    private CompositeSequenceService compositeSequenceService;

    @Autowired
    private GeneService geneService;

    @Autowired
    ArrayDesignReportService arrayDesignReportService;

    @Autowired
    private PersisterHelper persisterHelper;

    @Autowired
    private ProbeMapper probeMapper;

    /**
     * Do probe mapping, writing the results to the database and using default settings.
     * 
     * @param arrayDesign
     */
    public void processArrayDesign( ArrayDesign arrayDesign ) {
        this.processArrayDesign( arrayDesign, new ProbeMapperConfig(), true );
    }

    /**
     * @param arrayDesign
     * @param config
     * @param useDB if false, the results will not be written to the database, but printed to stdout instead.
     */
    public void processArrayDesign( ArrayDesign arrayDesign, ProbeMapperConfig config, boolean useDB ) {

        Collection<Taxon> taxa = arrayDesignService.getTaxa( arrayDesign.getId() );

        Taxon taxon = arrayDesign.getPrimaryTaxon();
        if ( taxa.size() > 1 && taxon == null ) {
            throw new IllegalArgumentException(
                    "Array design has sequence from multiple taxa and has no primary taxon set: " + arrayDesign );
        }

        GoldenPathSequenceAnalysis goldenPathDb = new GoldenPathSequenceAnalysis( taxon );

        BlockingQueue<BlatAssociation> persistingQueue = new ArrayBlockingQueue<BlatAssociation>( QUEUE_SIZE );
        AtomicBoolean generatorDone = new AtomicBoolean( false );
        AtomicBoolean loaderDone = new AtomicBoolean( false );

        load( persistingQueue, generatorDone, loaderDone );

        if ( useDB ) {
            log.info( "Removing any old associations" );
            arrayDesignService.deleteGeneProductAssociations( arrayDesign );
        }

        int count = 0;
        int hits = 0;
        log.info( "Start processing " + arrayDesign.getCompositeSequences().size() + " probes ..." );
        for ( CompositeSequence compositeSequence : arrayDesign.getCompositeSequences() ) {
            BioSequence bs = compositeSequence.getBiologicalCharacteristic();

            if ( bs == null ) continue;

            /*
             * It isn't 100% clear what the right thing to do is. But this seems at least _reasonable_.
             */
            if ( !bs.getTaxon().equals( taxon ) ) {
                continue;
            }

            final Collection<BlatResult> blatResults = blatResultService.findByBioSequence( bs );

            if ( blatResults == null || blatResults.isEmpty() ) continue;

            Map<String, Collection<BlatAssociation>> results = probeMapper.processBlatResults( goldenPathDb,
                    blatResults, config );

            if ( log.isDebugEnabled() )
                log.debug( "Found " + results.size() + " mappings for " + compositeSequence + " (" + blatResults.size()
                        + " BLAT results)" );

            for ( Collection<BlatAssociation> col : results.values() ) {
                for ( BlatAssociation association : col ) {
                    if ( log.isDebugEnabled() ) log.debug( association );
                }

                if ( useDB ) {
                    // persisting is done in a separate thread.
                    persistingQueue.addAll( col );
                } else {
                    printResult( compositeSequence, col );
                }
                ++hits;
            }

            if ( ++count % 100 == 0 ) {
                log.info( "Processed " + count + " composite sequences" + " with blat results; " + hits
                        + " mappings found." );
            }
        }
<commit_msg>make logging less frequent, since it's so much faster than it used to be<commit_after>/*
 * The Gemma project
 * 
 * Copyright (c) 2006 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.loader.expression.arrayDesign;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.core.context.SecurityContext;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;

import ubic.gemma.analysis.report.ArrayDesignReportService;
import ubic.gemma.analysis.sequence.ProbeMapper;
import ubic.gemma.analysis.sequence.ProbeMapperConfig;
import ubic.gemma.externalDb.GoldenPathSequenceAnalysis;
import ubic.gemma.model.common.description.ExternalDatabase;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
import ubic.gemma.model.expression.arrayDesign.ArrayDesignService;
import ubic.gemma.model.expression.designElement.CompositeSequence;
import ubic.gemma.model.expression.designElement.CompositeSequenceService;
import ubic.gemma.model.genome.Gene;
import ubic.gemma.model.genome.Taxon;
import ubic.gemma.model.genome.biosequence.BioSequence;
import ubic.gemma.model.genome.biosequence.BioSequenceService;
import ubic.gemma.model.genome.biosequence.SequenceType;
import ubic.gemma.model.genome.gene.GeneProduct;
import ubic.gemma.model.genome.gene.GeneService;
import ubic.gemma.model.genome.sequenceAnalysis.AnnotationAssociation;
import ubic.gemma.model.genome.sequenceAnalysis.AnnotationAssociationService;
import ubic.gemma.model.genome.sequenceAnalysis.BlatAssociation;
import ubic.gemma.model.genome.sequenceAnalysis.BlatResult;
import ubic.gemma.model.genome.sequenceAnalysis.BlatResultService;
import ubic.gemma.persistence.PersisterHelper;

/**
 * For an array design, generate gene product mappings for the sequences.
 * 
 * @author pavlidis
 * @version $Id$
 */
@Service
public class ArrayDesignProbeMapperService {

    private static Log log = LogFactory.getLog( ArrayDesignProbeMapperService.class.getName() );

    private static final int QUEUE_SIZE = 20000;

    @Autowired
    private AnnotationAssociationService annotationAssociationService;

    @Autowired
    private ArrayDesignService arrayDesignService;

    @Autowired
    private BioSequenceService bioSequenceService;

    @Autowired
    private BlatResultService blatResultService;

    @Autowired
    private CompositeSequenceService compositeSequenceService;

    @Autowired
    private GeneService geneService;

    @Autowired
    ArrayDesignReportService arrayDesignReportService;

    @Autowired
    private PersisterHelper persisterHelper;

    @Autowired
    private ProbeMapper probeMapper;

    /**
     * Do probe mapping, writing the results to the database and using default settings.
     * 
     * @param arrayDesign
     */
    public void processArrayDesign( ArrayDesign arrayDesign ) {
        this.processArrayDesign( arrayDesign, new ProbeMapperConfig(), true );
    }

    /**
     * @param arrayDesign
     * @param config
     * @param useDB if false, the results will not be written to the database, but printed to stdout instead.
     */
    public void processArrayDesign( ArrayDesign arrayDesign, ProbeMapperConfig config, boolean useDB ) {

        Collection<Taxon> taxa = arrayDesignService.getTaxa( arrayDesign.getId() );

        Taxon taxon = arrayDesign.getPrimaryTaxon();
        if ( taxa.size() > 1 && taxon == null ) {
            throw new IllegalArgumentException(
                    "Array design has sequence from multiple taxa and has no primary taxon set: " + arrayDesign );
        }

        GoldenPathSequenceAnalysis goldenPathDb = new GoldenPathSequenceAnalysis( taxon );

        BlockingQueue<BlatAssociation> persistingQueue = new ArrayBlockingQueue<BlatAssociation>( QUEUE_SIZE );
        AtomicBoolean generatorDone = new AtomicBoolean( false );
        AtomicBoolean loaderDone = new AtomicBoolean( false );

        load( persistingQueue, generatorDone, loaderDone );

        if ( useDB ) {
            log.info( "Removing any old associations" );
            arrayDesignService.deleteGeneProductAssociations( arrayDesign );
        }

        int count = 0;
        int hits = 0;
        log.info( "Start processing " + arrayDesign.getCompositeSequences().size() + " probes ..." );
        for ( CompositeSequence compositeSequence : arrayDesign.getCompositeSequences() ) {
            BioSequence bs = compositeSequence.getBiologicalCharacteristic();

            if ( bs == null ) continue;

            /*
             * It isn't 100% clear what the right thing to do is. But this seems at least _reasonable_.
             */
            if ( !bs.getTaxon().equals( taxon ) ) {
                continue;
            }

            final Collection<BlatResult> blatResults = blatResultService.findByBioSequence( bs );

            if ( blatResults == null || blatResults.isEmpty() ) continue;

            Map<String, Collection<BlatAssociation>> results = probeMapper.processBlatResults( goldenPathDb,
                    blatResults, config );

            if ( log.isDebugEnabled() )
                log.debug( "Found " + results.size() + " mappings for " + compositeSequence + " (" + blatResults.size()
                        + " BLAT results)" );

            for ( Collection<BlatAssociation> col : results.values() ) {
                for ( BlatAssociation association : col ) {
                    if ( log.isDebugEnabled() ) log.debug( association );
                }

                if ( useDB ) {
                    // persisting is done in a separate thread.
                    persistingQueue.addAll( col );
                } else {
                    printResult( compositeSequence, col );
                }
                ++hits;
            }

            if ( ++count % 200 == 0 ) {
                log.info( "Processed " + count + " composite sequences" + " with blat results; " + hits
                        + " mappings found." );
            }
        }

{'fraction_non_alphanumeric': 0.08601998157726919, 'fraction_numerical': 0.003259406221214483, 'mean_word_length': 3.08155002891845, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1571448', 'n_tokens_mistral': 3785, 'n_tokens_neox': 3621, 'n_words': 951}