<commit_before>/*
 * The baseCode project
 * 
 * Copyright (c) 2006 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.basecode.util;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.Arrays;
import java.util.Collection;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.StringTokenizer;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * @author keshav
 * @author Pavlidis
 * @author <NAME>
 * @version $Id$
 */
public class FileTools {
    private static Log log = LogFactory.getLog( FileTools.class.getName() );
    protected final static String PNG_EXTENSION = ".png";
    protected final static String GIF_EXTENSION = ".gif";
    protected final static String TXT_EXTENSION = ".txt";
    protected final static String[] XML_EXTENSIONS = { ".XML", ".RDF-XML", ".rdf-xml.gz", ".rdf-xml.zip", ".xml.zip",
            ".xml.gz" };

    protected final static String[] IMAGE_EXTENSIONS = { PNG_EXTENSION, GIF_EXTENSION, "PNG", "GIF", "JPEG", "JPG" };
    protected final static String[] DATA_EXTENSIONS = { TXT_EXTENSION, ".TXT", "txt.gz", "txt.zip", "txt.gzip" };
    // default values
    public final static String DEFAULT_DATA_EXTENSION = TXT_EXTENSION;
    public final static String DEFAULT_IMAGE_EXTENSION = PNG_EXTENSION;
    public final static String DEFAULT_XML_EXTENSION = ".xml";

    /**
     * @param file
     * @throws IOException
     */
    public static void checkPathIsReadableFile( String file ) throws IOException {
        File infile = new File( file );
        if ( !infile.exists() || !infile.canRead() ) {
            throw new IOException( "Could not find file: " + file );
        }
    }

    /**
     * Returns the extension of a file.
     * 
     * @param filename
     * @return
     * @return
     */
    public static String getExtension( String filename ) {
        String extension = null;
        int i = filename.lastIndexOf( '.' );

        if ( i > 0 && i < filename.length() - 1 ) {
            extension = filename.substring( i + 1 ).toLowerCase();
        }
        return extension;
    } // end getExtension

    /**
     * @param filename
     * @return
     */
    public static String chompExtension( String filename ) {
        int j = filename.lastIndexOf( '.' );
        if ( j > 1 ) {
            return filename.substring( 0, filename.lastIndexOf( '.' ) );
        }
        return filename;
    }

    /**
     * @param filename
     * @param newExtension
     * @return the new filename with the changed extension, but does not modify the <code>filename</code> parameter.
     */
    public static String changeExtension( String filename, String newExtension ) {

        String filenameWithoutExtension = chompExtension( filename );
        return ( filenameWithoutExtension + "." + newExtension );
    } // end getWithChangedExtension

    /**
     * @param filename
     * @return
     */
    public static boolean hasImageExtension( String filename ) {
        for ( int i = 0; i < FileTools.IMAGE_EXTENSIONS.length; i++ ) {
            if ( filename.toUpperCase().endsWith( FileTools.IMAGE_EXTENSIONS[i].toUpperCase() ) ) {
                return true;
            }
        }
        return false;

    } // end hasImageExtension

    /**
     * @param filename
     * @return
     */
    public static boolean hasXMLExtension( String filename ) {
        for ( int i = 0; i < FileTools.XML_EXTENSIONS.length; i++ ) {
            if ( filename.toUpperCase().endsWith( FileTools.XML_EXTENSIONS[i].toUpperCase() ) ) {
                return true;
            }
        }
        return false;
    }

    /**
     * @param filename
     * @return the new filename with the added extension, but does not modify the <code>filename</code> parameter.
     */
    public static String addImageExtension( String filename ) {
        return ( filename + ( FileTools.DEFAULT_IMAGE_EXTENSION.startsWith( "." ) ? "" : "." ) + FileTools.DEFAULT_IMAGE_EXTENSION );
    }

    /**
     * @param filename
     * @return the new filename with the added extension, but does not modify the <code>filename</code> parameter.
     */
    public static String addDataExtension( String filename ) {
        return ( filename + ( FileTools.DEFAULT_DATA_EXTENSION.startsWith( "." ) ? "" : "." ) + FileTools.DEFAULT_DATA_EXTENSION );
    }

    /**
     * @param dirname directory name
     * @return
     */
    public static boolean testDir( String dirname ) {
        if ( dirname != null && dirname.length() > 0 ) {
            File f = new File( dirname );
            if ( f.isDirectory() && f.canRead() ) {
                return true;
            }
        }
        return false;
    }

    /**
     * @param filename
     * @return
     */
    public static boolean testFile( String filename ) {
        if ( filename != null && filename.length() > 0 ) {
            File f = new File( filename );
            if ( f.isFile() && f.canRead() ) {
                return true;
            }
        }
        return false;
    }

    /**
     * Test whether a File is writeable.
     * 
     * @param file
     * @return
     */
    public static boolean testFile( File file ) {
        if ( file != null ) {
            if ( file.isFile() && file.canRead() ) {
                return true;
            }
        }
        return false;
    }

    /**
     * @param filename
     * @return
     */
    public static boolean isZipped( String filename ) {
        String capfileName = filename.toUpperCase();
        if ( capfileName.endsWith( ".ZIP" ) ) {
            return true;
        }
        return false;
    }
<commit_msg>utility method<commit_after>/*
 * The baseCode project
 * 
 * Copyright (c) 2006 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.basecode.util;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Arrays;
import java.util.Collection;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.StringTokenizer;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * @author keshav
 * @author Pavlidis
 * @author <NAME>
 * @version $Id$
 */
public class FileTools {
    private static Log log = LogFactory.getLog( FileTools.class.getName() );
    protected final static String PNG_EXTENSION = ".png";
    protected final static String GIF_EXTENSION = ".gif";
    protected final static String TXT_EXTENSION = ".txt";
    protected final static String[] XML_EXTENSIONS = { ".XML", ".RDF-XML", ".rdf-xml.gz", ".rdf-xml.zip", ".xml.zip",
            ".xml.gz" };

    protected final static String[] IMAGE_EXTENSIONS = { PNG_EXTENSION, GIF_EXTENSION, "PNG", "GIF", "JPEG", "JPG" };
    protected final static String[] DATA_EXTENSIONS = { TXT_EXTENSION, ".TXT", "txt.gz", "txt.zip", "txt.gzip" };
    // default values
    public final static String DEFAULT_DATA_EXTENSION = TXT_EXTENSION;
    public final static String DEFAULT_IMAGE_EXTENSION = PNG_EXTENSION;
    public final static String DEFAULT_XML_EXTENSION = ".xml";

    /**
     * @param file
     * @throws IOException
     */
    public static void checkPathIsReadableFile( String file ) throws IOException {
        File infile = new File( file );
        if ( !infile.exists() || !infile.canRead() ) {
            throw new IOException( "Could not find file: " + file );
        }
    }

    /**
     * Returns the extension of a file.
     * 
     * @param filename
     * @return
     * @return
     */
    public static String getExtension( String filename ) {
        String extension = null;
        int i = filename.lastIndexOf( '.' );

        if ( i > 0 && i < filename.length() - 1 ) {
            extension = filename.substring( i + 1 ).toLowerCase();
        }
        return extension;
    } // end getExtension

    /**
     * @param filename
     * @return
     */
    public static String chompExtension( String filename ) {
        int j = filename.lastIndexOf( '.' );
        if ( j > 1 ) {
            return filename.substring( 0, filename.lastIndexOf( '.' ) );
        }
        return filename;
    }

    /**
     * @param filename
     * @param newExtension
     * @return the new filename with the changed extension, but does not modify the <code>filename</code> parameter.
     */
    public static String changeExtension( String filename, String newExtension ) {

        String filenameWithoutExtension = chompExtension( filename );
        return ( filenameWithoutExtension + "." + newExtension );
    } // end getWithChangedExtension

    /**
     * @param filename
     * @return
     */
    public static boolean hasImageExtension( String filename ) {
        for ( int i = 0; i < FileTools.IMAGE_EXTENSIONS.length; i++ ) {
            if ( filename.toUpperCase().endsWith( FileTools.IMAGE_EXTENSIONS[i].toUpperCase() ) ) {
                return true;
            }
        }
        return false;

    } // end hasImageExtension

    /**
     * @param filename
     * @return
     */
    public static boolean hasXMLExtension( String filename ) {
        for ( int i = 0; i < FileTools.XML_EXTENSIONS.length; i++ ) {
            if ( filename.toUpperCase().endsWith( FileTools.XML_EXTENSIONS[i].toUpperCase() ) ) {
                return true;
            }
        }
        return false;
    }

    /**
     * @param filename
     * @return the new filename with the added extension, but does not modify the <code>filename</code> parameter.
     */
    public static String addImageExtension( String filename ) {
        return ( filename + ( FileTools.DEFAULT_IMAGE_EXTENSION.startsWith( "." ) ? "" : "." ) + FileTools.DEFAULT_IMAGE_EXTENSION );
    }

    /**
     * @param filename
     * @return the new filename with the added extension, but does not modify the <code>filename</code> parameter.
     */
    public static String addDataExtension( String filename ) {
        return ( filename + ( FileTools.DEFAULT_DATA_EXTENSION.startsWith( "." ) ? "" : "." ) + FileTools.DEFAULT_DATA_EXTENSION );
    }

    /**
     * @param dirname directory name
     * @return
     */
    public static boolean testDir( String dirname ) {
        if ( dirname != null && dirname.length() > 0 ) {
            File f = new File( dirname );
            if ( f.isDirectory() && f.canRead() ) {
                return true;
            }
        }
        return false;
    }

    /**
     * @param filename
     * @return
     */
    public static boolean testFile( String filename ) {
        if ( filename != null && filename.length() > 0 ) {
            File f = new File( filename );
            if ( f.isFile() && f.canRead() ) {
                return true;
            }
        }
        return false;

    }

    /**
     * @param resourcePath
     * @return
     * @throws URISyntaxException
     */
    public static String resourceToPath( String resourcePath ) throws URISyntaxException {
        if ( StringUtils.isBlank( resourcePath ) ) throw new IllegalArgumentException();
        URL resource = FileTools.class.getResource( resourcePath );
        if ( resource == null ) throw new IllegalArgumentException( "Could not get URL for resource=" + resourcePath );
        return new File( resource.toURI() ).getAbsolutePath();
    }

    /**
     * Test whether a File is writeable.
     * 
     * @param file
     * @return
     */
    public static boolean testFile( File file ) {
        if ( file != null ) {
            if ( file.isFile() && file.canRead() ) {
                return true;
            }
        }
        return false;
    }

    /**
     * @param filename
     * @return
     */
    public static boolean isZipped( String filename ) {
        String capfileName = filename.toUpperCase();
        if ( capfileName.endsWith( ".ZIP" ) ) {
            return true;
        }
        return false;
    }

{'fraction_non_alphanumeric': 0.11167367895600086, 'fraction_numerical': 0.0024245881765670683, 'mean_word_length': 2.355023923444976, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1543439', 'n_tokens_mistral': 4010, 'n_tokens_neox': 3907, 'n_words': 1148}