<commit_before>/*
 * Copyright (c) 2007-2012 The Broad Institute, Inc.
 * SOFTWARE COPYRIGHT NOTICE
 * This software and its documentation are the copyright of the Broad Institute, Inc. All rights are reserved.
 *
 * This software is supplied without any warranty or guaranteed support whatsoever. The Broad Institute is not responsible for its use, misuse, or functionality.
 *
 * This software is licensed under the terms of the GNU Lesser General Public License (LGPL),
 * Version 2.1 which is available at http://www.opensource.org/licenses/lgpl-2.1.php.
 */

/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package org.broad.igv.feature.genome;

import org.apache.log4j.Logger;
import org.broad.igv.Globals;
import org.broad.igv.util.FileUtils;
import org.broad.igv.util.HttpUtils;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
 * /**
 *
 * @author jrobinso
 */
public class GenomeImporter {
    public static final int MAX_CONTIGS = 1500000;

    static Logger log = Logger.getLogger(GenomeImporter.class);
    public static final Pattern SEQUENCE_NAME_SPLITTER = Pattern.compile("\\s+");


    /**
     * Create a zip containing all the information and data required to load a
     * genome. All file/directory validation is assume to have been done by validation
     * outside of this method.
     *
     * @param genomeFile
     * @param genomeId          Id of the genome.
     * @param genomeDisplayName The genome name that is user-friendly.
     * @param fastaFile         The location of a fasta file, or directory of fasta files
     * @param refFlatFile       RefFlat file.
     * @param cytobandFile      Cytoband file.
     * @return The newly created genome archive file.
     */
    public File createGenomeArchive(File genomeFile,
                                    String genomeId,
                                    String genomeDisplayName,
                                    String fastaFile,
                                    File refFlatFile,
                                    File cytobandFile,
                                    File chrAliasFile) throws IOException {

        if ((genomeFile == null) || (genomeId == null) || (genomeDisplayName == null)) {

            log.error("Invalid input for genome creation: ");
            log.error("\tGenome file=" + genomeFile.getAbsolutePath());
            log.error("\tGenome Id=" + genomeId);
            log.error("\tGenome Name" + genomeDisplayName);
            return null;
        }


        File propertyFile = null;

        File archive = null;
        FileWriter propertyFileWriter = null;
        try {
            boolean fastaDirectory = false;
            List<String> fastaFileNames = new ArrayList<String>();

            if (!FileUtils.resourceExists(fastaFile)) {
                String msg = "File not found: " + fastaFile;
                throw new GenomeException(msg);
            }
            if (fastaFile.toLowerCase().endsWith(Globals.ZIP_EXTENSION)) {
                String msg = "Error.  Zip archives are not supported.  Please select a fasta file.";
                throw new GenomeException(msg);
            }
            if (fastaFile.toLowerCase().endsWith(Globals.GZIP_FILE_EXTENSION)) {
                String msg = "Error.  GZipped files are not supported.  Please select a non-gzipped fasta file.";
                throw new GenomeException(msg);
            }

            List<String> fastaIndexPathList = new ArrayList<String>();
            String fastaIndexPath = fastaFile + ".fai";

            File sequenceInputFile = new File(fastaFile);
            if (sequenceInputFile.exists()) {
                // Local file
                if (sequenceInputFile.isDirectory()) {
                    fastaDirectory = true;
                    List<File> files = getSequenceFiles(sequenceInputFile);
                    for (File file : files) {
                        if (file.getName().toLowerCase().endsWith(Globals.GZIP_FILE_EXTENSION)) {
                            String msg = "<html>Error.  One or more fasta files are gzipped: " + file.getName() +
                                    "<br>All fasta files must be gunzipped prior to importing.";
                            throw new GenomeException(msg);
                        }

                        File indexFile = new File(fastaIndexPath);
                        if (!indexFile.exists()) {
                            FastaUtils.createIndexFile(fastaFile, fastaIndexPath);
                        }
                        fastaIndexPathList.add(fastaIndexPath);
                        fastaFileNames.add(file.getName());
                    }
                } else {
                    // Index if neccessary
                    File indexFile = new File(fastaIndexPath);
                    if (!indexFile.exists()) {
                        FastaUtils.createIndexFile(fastaFile, fastaIndexPath);
                    }
                    fastaIndexPathList.add(fastaIndexPath);
                }
            } else {
                if (!FileUtils.resourceExists(fastaIndexPath)) {
                    String msg = "<html>Index file " + fastaIndexPath + " Not found. " +
                            "<br>Remote fasta files must be indexed prior to importing.";
                    throw new GenomeException(msg);
                }
            }


            fastaFile = FileUtils.getRelativePath(genomeFile.getParent(), fastaFile);

            // Create "in memory" property file
            byte[] propertyBytes = createGenomePropertyFile(genomeId, genomeDisplayName, fastaFile, refFlatFile,
                    cytobandFile, chrAliasFile, fastaDirectory, fastaFileNames);<commit_msg>Fix to import genome from a folder<commit_after>/*
 * Copyright (c) 2007-2012 The Broad Institute, Inc.
 * SOFTWARE COPYRIGHT NOTICE
 * This software and its documentation are the copyright of the Broad Institute, Inc. All rights are reserved.
 *
 * This software is supplied without any warranty or guaranteed support whatsoever. The Broad Institute is not responsible for its use, misuse, or functionality.
 *
 * This software is licensed under the terms of the GNU Lesser General Public License (LGPL),
 * Version 2.1 which is available at http://www.opensource.org/licenses/lgpl-2.1.php.
 */

/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package org.broad.igv.feature.genome;

import org.apache.log4j.Logger;
import org.broad.igv.Globals;
import org.broad.igv.util.FileUtils;
import org.broad.igv.util.HttpUtils;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
 * /**
 *
 * @author jrobinso
 */
public class GenomeImporter {
    public static final int MAX_CONTIGS = 1500000;

    static Logger log = Logger.getLogger(GenomeImporter.class);
    public static final Pattern SEQUENCE_NAME_SPLITTER = Pattern.compile("\\s+");


    /**
     * Create a zip containing all the information and data required to load a
     * genome. All file/directory validation is assume to have been done by validation
     * outside of this method.
     *
     * @param genomeFile
     * @param genomeId          Id of the genome.
     * @param genomeDisplayName The genome name that is user-friendly.
     * @param fastaFile         The location of a fasta file, or directory of fasta files
     * @param refFlatFile       RefFlat file.
     * @param cytobandFile      Cytoband file.
     * @return The newly created genome archive file.
     */
    public File createGenomeArchive(File genomeFile,
                                    String genomeId,
                                    String genomeDisplayName,
                                    String fastaFile,
                                    File refFlatFile,
                                    File cytobandFile,
                                    File chrAliasFile) throws IOException {

        if ((genomeFile == null) || (genomeId == null) || (genomeDisplayName == null)) {

            log.error("Invalid input for genome creation: ");
            log.error("\tGenome file=" + genomeFile.getAbsolutePath());
            log.error("\tGenome Id=" + genomeId);
            log.error("\tGenome Name" + genomeDisplayName);
            return null;
        }


        File propertyFile = null;

        File archive = null;
        FileWriter propertyFileWriter = null;
        try {
            boolean fastaDirectory = false;
            List<String> fastaFileNames = new ArrayList<String>();

            if (!FileUtils.resourceExists(fastaFile)) {
                String msg = "File not found: " + fastaFile;
                throw new GenomeException(msg);
            }
            if (fastaFile.toLowerCase().endsWith(Globals.ZIP_EXTENSION)) {
                String msg = "Error.  Zip archives are not supported.  Please select a fasta file.";
                throw new GenomeException(msg);
            }
            if (fastaFile.toLowerCase().endsWith(Globals.GZIP_FILE_EXTENSION)) {
                String msg = "Error.  GZipped files are not supported.  Please select a non-gzipped fasta file.";
                throw new GenomeException(msg);
            }

            List<String> fastaIndexPathList = new ArrayList<String>();
            String fastaIndexPath = fastaFile + ".fai";

            File sequenceInputFile = new File(fastaFile);
            if (sequenceInputFile.exists()) {
                // Local file
                if (sequenceInputFile.isDirectory()) {
                    fastaDirectory = true;
                    List<File> files = getSequenceFiles(sequenceInputFile);
                    for (File file : files) {
                        if (file.getName().toLowerCase().endsWith(Globals.GZIP_FILE_EXTENSION)) {
                            String msg = "<html>Error.  One or more fasta files are gzipped: " + file.getName() +
                                    "<br>All fasta files must be gunzipped prior to importing.";
                            throw new GenomeException(msg);
                        }

                        File indexFile = new File(sequenceInputFile, file.getName() + ".fai");
                        if (!indexFile.exists()) {
                            FastaUtils.createIndexFile(file.getAbsolutePath(), indexFile.getAbsolutePath());
                        }
                        fastaIndexPathList.add(fastaIndexPath);
                        fastaFileNames.add(file.getName());
                    }
                } else {
                    // Index if neccessary
                    File indexFile = new File(fastaIndexPath);
                    if (!indexFile.exists()) {
                        FastaUtils.createIndexFile(fastaFile, fastaIndexPath);
                    }
                    fastaIndexPathList.add(fastaIndexPath);
                }
            } else {
                if (!FileUtils.resourceExists(fastaIndexPath)) {
                    String msg = "<html>Index file " + fastaIndexPath + " Not found. " +
                            "<br>Remote fasta files must be indexed prior to importing.";
                    throw new GenomeException(msg);
                }
            }


            fastaFile = FileUtils.getRelativePath(genomeFile.getParent(), fastaFile);

            // Create "in memory" property file
            byte[] propertyBytes = createGenomePropertyFile(genomeId, genomeDisplayName, fastaFile, refFlatFile,
                    cytobandFile, chrAliasFile, fastaDirectory, fastaFileNames);
{'fraction_non_alphanumeric': 0.08479309469408479, 'fraction_numerical': 0.003384953880003385, 'mean_word_length': 1.7978219696969697, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3627347', 'n_tokens_mistral': 3074, 'n_tokens_neox': 2876, 'n_words': 855}