<commit_before>/*
 * Copyright (C) 2014 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
package com.google.cloud.genomics.dataflow.utils;

import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.List;
import java.util.logging.Logger;

import com.google.api.client.repackaged.com.google.common.base.Strings;
import com.google.api.services.genomics.Genomics;
import com.google.api.services.genomics.model.SearchVariantsRequest;
import com.google.cloud.dataflow.sdk.options.Default;
import com.google.cloud.dataflow.sdk.options.Description;
import com.google.cloud.dataflow.sdk.util.gcsfs.GcsPath;
import com.google.cloud.genomics.utils.Contig;
import com.google.cloud.genomics.utils.Contig.SexChromosomeFilter;
import com.google.cloud.genomics.utils.GenomicsFactory;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;

/**
 * A common options class for all pipelines that operate over a single dataset and write their
 * analysis to a file.
 */
public interface GenomicsDatasetOptions extends GenomicsOptions {
  public static class Methods {

    private static final Logger LOG = Logger.getLogger(GenomicsDatasetOptions.class.getName());

    // TODO: If needed, add getReadRequests method
    public static List<SearchVariantsRequest> getVariantRequests(GenomicsDatasetOptions options,
        GenomicsFactory.OfflineAuth auth, SexChromosomeFilter sexChromosomeFilter) throws IOException,
        GeneralSecurityException {
      String datasetId = options.getDatasetId();
      Genomics genomics = auth.getGenomics(auth.getDefaultFactory());

      Iterable<Contig> contigs =
          options.isAllReferences() ? Contig.getContigsInVariantSet(genomics, datasetId, sexChromosomeFilter)
              : Contig.parseContigsFromCommandLine(options.getReferences());

      List<SearchVariantsRequest> requests = Lists.newArrayList();
      for (Contig contig : contigs) {
        for (Contig shard : contig.getShards(options.getBasesPerShard())) {
          LOG.info("Adding request with " + shard.referenceName + " " + shard.start + " to "
              + shard.end);
          SearchVariantsRequest request = shard.getVariantsRequest(datasetId);
          if (options.getPageSize() > 0) {
            request.setPageSize(options.getPageSize());
          }
          if (options.getMaxCalls() > 0) {
            request.setMaxCalls(options.getMaxCalls());
          }
          requests.add(request);
        }
      }
      return requests;
    }

    public static void validateOptions(GenomicsDatasetOptions options) {
      Preconditions.checkArgument(0 < options.getBinSize(), "binSize must be greater than zero");
      if (null != options.getOutput()) {
        try {
          // Check that we can parse the path.
          GcsPath valid = GcsPath.fromUri(options.getOutput());
          // GcsPath allows for empty bucket or filename, but that doesn't make for a good output file.
          Preconditions.checkArgument(!Strings.isNullOrEmpty(valid.getBucket()), "Bucket must be specified");
          Preconditions.checkArgument(!Strings.isNullOrEmpty(valid.getObject()), "Filename prefix must be specified");
        } catch (Exception x) {
          Preconditions.checkState(false, "output must be a valid Google Cloud Storage URL (starting with gs://)");
        }
      }
      GenomicsOptions.Methods.validateOptions(options);
    }
  }

  @Description("The ID of the Google Genomics dataset this pipeline is accessing. "
      + "Defaults to 1000 Genomes.")
  @Default.String("10473108253681171589")
  String getDatasetId();
  void setDatasetId(String datasetId);

  @Description("The IDs of the Google Genomics call sets this pipeline is working with, comma "
      + "delimited.Defaults to 1000 Genomes HG00261.")
  @Default.String("10473108253681171589-0")
  String getCallSetIds();
  void setCallSetIds(String callSetIds);

  // TODO refactor this option so that its only present and validated for pipelines that do write output to GCS.
  @Description("Google Cloud Storage path prefix of the files to which to write pipeline output, if applicable.")
  String getOutput();
  void setOutput(String output);

  @Description("The IDs of the Google Genomics transcript sets this pipeline is working with, "
      + "comma delimited. Defaults to UCSC refGene (hg19).")
  @Default.String("CIjfoPXj9LqPlAEQ6Mm91Ya458eqAQ")
  String getTranscriptSetIds();
  void setTranscriptSetIds(String transcriptSetIds);

  @Description("The IDs of the Google Genomics variant annotation sets this pipeline is working "
      + "with, comma delimited. Defaults to ClinVar (GRCh37).")
  @Default.String("CILSqfjtlY6tHxC0nNH-4cu-xlQ")
  String getVariantAnnotationSetIds();
  void setVariantAnnotationSetIds(String variantAnnotationSetIds);

  @Description("By default, variants analyses will be run on BRCA1.  Pass this flag to run on all "
      + "references present in the dataset.  Note that certain jobs such as PCA and IBS "
      + "will automatically exclude X and Y chromosomes when this option is true.")
  boolean isAllReferences();
  void setAllReferences(boolean allReferences);

  @Description("Comma separated tuples of reference:start:end,... Defaults to " + Contig.BRCA1)
  @Default.String(Contig.BRCA1)
  String getReferences();
  void setReferences(String references);

  @Description("The maximum number of bases per shard.")
  @Default.Long(Contig.DEFAULT_NUMBER_OF_BASES_PER_SHARD)
  long getBasesPerShard();
  void setBasesPerShard(long basesPerShard);

  @Description("The maximum number of calls to return. However, at least one variant will always "
      + "be returned, even if it has more calls than this limit.  If unspecified, the default "
      + "number of calls will be returned.")<commit_msg>Update UCSC transcript set to reimported version<commit_after>/*
 * Copyright (C) 2014 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
package com.google.cloud.genomics.dataflow.utils;

import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.List;
import java.util.logging.Logger;

import com.google.api.client.repackaged.com.google.common.base.Strings;
import com.google.api.services.genomics.Genomics;
import com.google.api.services.genomics.model.SearchVariantsRequest;
import com.google.cloud.dataflow.sdk.options.Default;
import com.google.cloud.dataflow.sdk.options.Description;
import com.google.cloud.dataflow.sdk.util.gcsfs.GcsPath;
import com.google.cloud.genomics.utils.Contig;
import com.google.cloud.genomics.utils.Contig.SexChromosomeFilter;
import com.google.cloud.genomics.utils.GenomicsFactory;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;

/**
 * A common options class for all pipelines that operate over a single dataset and write their
 * analysis to a file.
 */
public interface GenomicsDatasetOptions extends GenomicsOptions {
  public static class Methods {

    private static final Logger LOG = Logger.getLogger(GenomicsDatasetOptions.class.getName());

    // TODO: If needed, add getReadRequests method
    public static List<SearchVariantsRequest> getVariantRequests(GenomicsDatasetOptions options,
        GenomicsFactory.OfflineAuth auth, SexChromosomeFilter sexChromosomeFilter) throws IOException,
        GeneralSecurityException {
      String datasetId = options.getDatasetId();
      Genomics genomics = auth.getGenomics(auth.getDefaultFactory());

      Iterable<Contig> contigs =
          options.isAllReferences() ? Contig.getContigsInVariantSet(genomics, datasetId, sexChromosomeFilter)
              : Contig.parseContigsFromCommandLine(options.getReferences());

      List<SearchVariantsRequest> requests = Lists.newArrayList();
      for (Contig contig : contigs) {
        for (Contig shard : contig.getShards(options.getBasesPerShard())) {
          LOG.info("Adding request with " + shard.referenceName + " " + shard.start + " to "
              + shard.end);
          SearchVariantsRequest request = shard.getVariantsRequest(datasetId);
          if (options.getPageSize() > 0) {
            request.setPageSize(options.getPageSize());
          }
          if (options.getMaxCalls() > 0) {
            request.setMaxCalls(options.getMaxCalls());
          }
          requests.add(request);
        }
      }
      return requests;
    }

    public static void validateOptions(GenomicsDatasetOptions options) {
      Preconditions.checkArgument(0 < options.getBinSize(), "binSize must be greater than zero");
      if (null != options.getOutput()) {
        try {
          // Check that we can parse the path.
          GcsPath valid = GcsPath.fromUri(options.getOutput());
          // GcsPath allows for empty bucket or filename, but that doesn't make for a good output file.
          Preconditions.checkArgument(!Strings.isNullOrEmpty(valid.getBucket()), "Bucket must be specified");
          Preconditions.checkArgument(!Strings.isNullOrEmpty(valid.getObject()), "Filename prefix must be specified");
        } catch (Exception x) {
          Preconditions.checkState(false, "output must be a valid Google Cloud Storage URL (starting with gs://)");
        }
      }
      GenomicsOptions.Methods.validateOptions(options);
    }
  }

  @Description("The ID of the Google Genomics dataset this pipeline is accessing. "
      + "Defaults to 1000 Genomes.")
  @Default.String("10473108253681171589")
  String getDatasetId();
  void setDatasetId(String datasetId);

  @Description("The IDs of the Google Genomics call sets this pipeline is working with, comma "
      + "delimited.Defaults to 1000 Genomes HG00261.")
  @Default.String("10473108253681171589-0")
  String getCallSetIds();
  void setCallSetIds(String callSetIds);

  // TODO refactor this option so that its only present and validated for pipelines that do write output to GCS.
  @Description("Google Cloud Storage path prefix of the files to which to write pipeline output, if applicable.")
  String getOutput();
  void setOutput(String output);

  @Description("The IDs of the Google Genomics transcript sets this pipeline is working with, "
      + "comma delimited. Defaults to UCSC refGene (hg19).")
  @Default.String("CIjfoPXj9LqPlAEQ5vnql4KewYuSAQ")
  String getTranscriptSetIds();
  void setTranscriptSetIds(String transcriptSetIds);

  @Description("The IDs of the Google Genomics variant annotation sets this pipeline is working "
      + "with, comma delimited. Defaults to ClinVar (GRCh37).")
  @Default.String("CILSqfjtlY6tHxC0nNH-4cu-xlQ")
  String getVariantAnnotationSetIds();
  void setVariantAnnotationSetIds(String variantAnnotationSetIds);

  @Description("By default, variants analyses will be run on BRCA1.  Pass this flag to run on all "
      + "references present in the dataset.  Note that certain jobs such as PCA and IBS "
      + "will automatically exclude X and Y chromosomes when this option is true.")
  boolean isAllReferences();
  void setAllReferences(boolean allReferences);

  @Description("Comma separated tuples of reference:start:end,... Defaults to " + Contig.BRCA1)
  @Default.String(Contig.BRCA1)
  String getReferences();
  void setReferences(String references);

  @Description("The maximum number of bases per shard.")
  @Default.Long(Contig.DEFAULT_NUMBER_OF_BASES_PER_SHARD)
  long getBasesPerShard();
  void setBasesPerShard(long basesPerShard);

  @Description("The maximum number of calls to return. However, at least one variant will always "
      + "be returned, even if it has more calls than this limit.  If unspecified, the default "
      + "number of calls will be returned.")
{'fraction_non_alphanumeric': 0.09004028117842193, 'fraction_numerical': 0.012637232446094306, 'mean_word_length': 3.9211037699183833, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7663890', 'n_tokens_mistral': 3674, 'n_tokens_neox': 3425, 'n_words': 1069}