<commit_before>/*
 * Copyright (c) 2003-2004, <NAME>, All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without 
 * modification, are permitted provided that the following conditions are met:
 * 
 * - Redistributions of source code must retain the above copyright notice, 
 *   this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright notice, 
 *   this list of conditions and the following disclaimer in the documentation 
 *   and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 */
package classycle;

import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import classycle.classfile.ClassConstant;
import classycle.classfile.Constant;
import classycle.classfile.UTF8Constant;
import classycle.graph.AtomicVertex;
import classycle.graph.NameAttributes;

/**
 *  Utility methods for parsing class files and creating directed graphs.
 *  The nodes of the graph are classes. The initial vertex of an edge is the
 *  class which uses the class specified by the terminal vertex.
 *
 *  @author <NAME>
 */
public class Parser {
  private static final int ACC_INTERFACE = 0x200, ACC_ABSTRACT = 0x400;

  private static class UnresolvedNode implements Comparable {
    NameAttributes attributes;
    ArrayList nodes = new ArrayList();

    public UnresolvedNode() {}

    public int compareTo(Object obj) {
      return attributes.getName().compareTo(
                ((UnresolvedNode) obj).attributes.getName());
    }
  }

  /** Private constructor to prohibit instanciation. */
  private Parser() {
  }

  /**
   *  Reads the specified class files and creates a directed graph where each
   *  vertex represents a class. The head vertex of an arc is a class which is
   *  used by the tail vertex of the arc.
   *  The elements of <tt>classFiles</tt> are file names (relative to the
   *  working directory) which are interpreted depending on its file type as
   *  <ul>
   *    <li>name of a class file (file type <tt>.class</tt>)
   *    <li>name of a folder containing class files
   *    <li>name of a zip file containing class file (file type <tt>.zip</tt>)
   *    <li>name of a jar file (file type <tt>.jar</tt>)
   *  </ul>
   *  Folders and zip/jar files are searched recursively for class files.
   *  @param classFiles Array of file names.
   *  @return directed graph.
   */
  public static AtomicVertex[] readClassFiles(String[] classFiles)
                                                          throws IOException {
    ArrayList unresolvedNodes = new ArrayList();
    for (int i = 0; i < classFiles.length; i++) {
      File file = new File(classFiles[i]);
      if (file.isDirectory() || file.getName().endsWith(".class")) {
        analyseClassFile(file, unresolvedNodes);
      } else if (file.getName().endsWith(".zip") 
                 || file.getName().endsWith(".jar")) {
        analyseClassFiles(new ZipFile(file.getAbsoluteFile()), 
                          unresolvedNodes);
      } else {
        throw new IOException(classFiles[i] + " is an invalid file.");
      }
    }
    UnresolvedNode[] nodes = new UnresolvedNode[unresolvedNodes.size()];
    nodes = (UnresolvedNode[]) unresolvedNodes.toArray(nodes);
    Arrays.sort(nodes);
    return createGraph(nodes);
  }

  private static void analyseClassFile(File file, ArrayList unresolvedNodes)
                                                          throws IOException {
    if (file.isDirectory()) {
      String[] files = file.list();
      for (int i = 0; i < files.length; i++) {
        File child = new File(file, files[i]);
        if (child.isDirectory() || files[i].endsWith(".class")) {
          analyseClassFile(child, unresolvedNodes);
        }
      }
    } else {
      unresolvedNodes.add(extractNode(file));
    }
  }

  private static UnresolvedNode extractNode(File file) throws IOException {
    InputStream stream = null;
    UnresolvedNode result = null;
    try {
      stream = new FileInputStream(file);
      result = Parser.createNode(stream, (int) file.length());
    } catch (IOException e) {
      throw e;
    } finally {
      try {
        stream.close();<commit_msg>change:  - parses also war and ear files  - filters classes with StringPattern<commit_after>/*
 * Copyright (c) 2003-2004, <NAME>, All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without 
 * modification, are permitted provided that the following conditions are met:
 * 
 * - Redistributions of source code must retain the above copyright notice, 
 *   this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright notice, 
 *   this list of conditions and the following disclaimer in the documentation 
 *   and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 */
package classycle;

import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import classycle.classfile.ClassConstant;
import classycle.classfile.Constant;
import classycle.classfile.UTF8Constant;
import classycle.graph.AtomicVertex;
import classycle.graph.NameAttributes;
import classycle.util.StringPattern;
import classycle.util.TrueStringPattern;

/**
 *  Utility methods for parsing class files and creating directed graphs.
 *  The nodes of the graph are classes. The initial vertex of an edge is the
 *  class which uses the class specified by the terminal vertex.
 *
 *  @author <NAME>
 */
public class Parser {
  private static final int ACC_INTERFACE = 0x200, ACC_ABSTRACT = 0x400;
  private static final String[] ZIP_FILE_TYPES 
      = new String[] {".zip", ".jar", ".war", ".ear"}; 

  private static class UnresolvedNode implements Comparable {
    NameAttributes attributes;
    ArrayList nodes = new ArrayList();

    public UnresolvedNode() {}

    public int compareTo(Object obj) {
      return attributes.getName().compareTo(
                ((UnresolvedNode) obj).attributes.getName());
    }
    
    public boolean isMatchedBy(StringPattern pattern)
    {
      return pattern.matches(attributes.getName());
    }
  }

  /** Private constructor to prohibit instanciation. */
  private Parser() {
  }
  
  /**
   * Reads and parses class files and creates a direct graph. Short-cut of 
   * <tt>readClassFiles(classFiles, new {@link TrueStringPattern}());</tt>
   */
  public static AtomicVertex[] readClassFiles(String[] classFiles)
                               throws IOException 
  {
    return readClassFiles(classFiles, new TrueStringPattern());
  }

  /**
   *  Reads the specified class files and creates a directed graph where each
   *  vertex represents a class. The head vertex of an arc is a class which is
   *  used by the tail vertex of the arc.
   *  The elements of <tt>classFiles</tt> are file names (relative to the
   *  working directory) which are interpreted depending on its file type as
   *  <ul>
   *    <li>name of a class file (file type <tt>.class</tt>)
   *    <li>name of a folder containing class files
   *    <li>name of a file of type <code>.zip</code>, <code>.jar</code>,
   *        <code>.war</code>, or <code>.ear</code>
   *        containing class file
   *  </ul>
   *  Folders and zip/jar/war/ear files are searched recursively 
   *  for class files.
   *  @param classFiles Array of file names.
   *  @param pattern Pattern fully qualified class names have to match in order
   *                 to be added to the graph. Otherwise they count as
   *                 'external'.
   *  @return directed graph.
   */
  public static AtomicVertex[] readClassFiles(String[] classFiles, 
                                              StringPattern pattern)
                                                          throws IOException {
    ArrayList unresolvedNodes = new ArrayList();
    for (int i = 0; i < classFiles.length; i++) {
      File file = new File(classFiles[i]);
      if (file.isDirectory() || file.getName().endsWith(".class")) {
        analyseClassFile(file, unresolvedNodes);
      } else if (isZipFile(file)) {
        analyseClassFiles(new ZipFile(file.getAbsoluteFile()), 
                          unresolvedNodes);
      } else {
        throw new IOException(classFiles[i] + " is an invalid file.");
      }
    }
    ArrayList filteredNodes = new ArrayList();
    for (int i = 0, n = unresolvedNodes.size(); i < n; i++)
    {
      UnresolvedNode node = (UnresolvedNode) unresolvedNodes.get(i);
      if (node.isMatchedBy(pattern))
      {
        filteredNodes.add(node);
      }
    }
    UnresolvedNode[] nodes = new UnresolvedNode[filteredNodes.size()];
    nodes = (UnresolvedNode[]) filteredNodes.toArray(nodes);
    Arrays.sort(nodes);
    return createGraph(nodes);
  }

  private static boolean isZipFile(File file) 
  {
    boolean result = false;
    String name = file.getName();
    for (int i = 0; i < ZIP_FILE_TYPES.length; i++) 
    {
      if (name.endsWith(ZIP_FILE_TYPES[i]))
      {
        result = true;
        break;
      }
    }
    return result;
  }

  private static void analyseClassFile(File file, ArrayList unresolvedNodes)
                                                          throws IOException {
    if (file.isDirectory()) {
      String[] files = file.list();
      for (int i = 0; i < files.length; i++) {
        File child = new File(file, files[i]);
        if (child.isDirectory() || files[i].endsWith(".class")) {
          analyseClassFile(child, unresolvedNodes);
        }
      }
    } else {
      unresolvedNodes.add(extractNode(file));
    }
  }

  private static UnresolvedNode extractNode(File file) throws IOException {
    InputStream stream = null;
    UnresolvedNode result = null;
    try {
      stream = new FileInputStream(file);
      result = Parser.createNode(stream, (int) file.length());
    } catch (IOException e) {
      throw e;
    } finally {
      try {
        stream.close();
{'fraction_non_alphanumeric': 0.09230769230769231, 'fraction_numerical': 0.0033264033264033266, 'mean_word_length': 2.9185402411208865, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5982730', 'n_tokens_mistral': 3349, 'n_tokens_neox': 3129, 'n_words': 1175}