<commit_before>package edu.uci.ics.sourcerer.extractor.ast;

import static edu.uci.ics.sourcerer.util.io.Logging.logger;

import java.util.Collection;
import java.util.List;
import java.util.logging.Level;

import org.eclipse.core.resources.IFile;
import org.eclipse.jdt.core.IClassFile;
import org.eclipse.jdt.core.ICompilationUnit;
import org.eclipse.jdt.core.JavaCore;
import org.eclipse.jdt.core.compiler.IProblem;
import org.eclipse.jdt.core.dom.AST;
import org.eclipse.jdt.core.dom.ASTParser;
import org.eclipse.jdt.core.dom.CompilationUnit;
import org.eclipse.jdt.core.dom.IBinding;
import org.eclipse.jdt.core.dom.ImportDeclaration;

//import ca.mcgill.cs.swevo.ppa.PPAOptions;
//import ca.mcgill.cs.swevo.ppa.ui.PPAUtil;
import edu.uci.ics.sourcerer.extractor.io.WriterBundle;
import edu.uci.ics.sourcerer.util.Helper;
import edu.uci.ics.sourcerer.util.io.PropertyManager;

public final class FeatureExtractor {
  private ASTParser parser;
  private WriterBundle bundle;
  private boolean doPPA;
  
  public FeatureExtractor() {
    parser = ASTParser.newParser(AST.JLS3);
  }
  
  public void setOutput(PropertyManager properties) {
    if (bundle != null) {
      bundle.close();
    }
    bundle = new WriterBundle(properties);
  }
  
  public void setPPA(boolean doPPA) {
    this.doPPA = doPPA;
  }
  
  public void close() {
    bundle.close();
  }
  
  public void extractClassFiles(Collection<IClassFile> classFiles) {
    ClassFileExtractor extractor = new ClassFileExtractor(bundle);
    for (IClassFile classFile : classFiles) {
      try {
        extractor.extractClassFile(classFile);
      } catch (NullPointerException e) {
        logger.log(Level.SEVERE, "Unable to extract " + classFile.getElementName(), e);
      }
    }
  }
   
  public void extractSourceFiles(Collection<IFile> sourceFiles) {
    ReferenceExtractorVisitor visitor = new ReferenceExtractorVisitor(bundle);
    
    int total = 0;
    
    for (IFile source : sourceFiles) {
      if (doPPA) {
//        PPAUtil.getCU(source, new PPAOptions()).accept(visitor);
      } else {
        ICompilationUnit icu = JavaCore.createCompilationUnitFrom(source);
        
        parser.setStatementsRecovery(true);
        parser.setResolveBindings(true);
        parser.setBindingsRecovery(true);
        parser.setSource(icu);
        
        try {
          CompilationUnit unit;
          do {
            unit = (CompilationUnit)parser.createAST(null);
          } while (reattemptCompilation(unit));
          unit.accept(visitor);
        } catch (NullPointerException e) {
          logger.log(Level.SEVERE, "Unable to create AST for " + icu.getResource().getLocation().toString(), e);
          total--;
        }
      }
      
      if (++total % 1000 == 0) {
        logger.info(total + " files extracted");
      }
    }
    
    logger.info(total + " files extracted");
  }
  
  @SuppressWarnings("unchecked")
  private boolean reattemptCompilation(CompilationUnit unit) {
    IProblem[] problems = unit.getProblems();
    if (problems.length == 0) {
      return false;
    } else {
      boolean foundIndirectRef = false;
      Collection<String> missingPrefix = Helper.newHashSet();
      Collection<String> missingRefs = Helper.newHashSet();
      // Check problems for missing indirect references
      for (IProblem problem : problems) {
        if (problem.isError()) {
          String message = problem.getMessage();
          if (message.startsWith("The type") && message.endsWith("is indirectly referenced from required .class files")) {<commit_msg>Remove log info -> " # files extracted"<commit_after>package edu.uci.ics.sourcerer.extractor.ast;

import static edu.uci.ics.sourcerer.util.io.Logging.logger;

import java.util.Collection;
import java.util.List;
import java.util.logging.Level;

import org.eclipse.core.resources.IFile;
import org.eclipse.jdt.core.IClassFile;
import org.eclipse.jdt.core.ICompilationUnit;
import org.eclipse.jdt.core.JavaCore;
import org.eclipse.jdt.core.compiler.IProblem;
import org.eclipse.jdt.core.dom.AST;
import org.eclipse.jdt.core.dom.ASTParser;
import org.eclipse.jdt.core.dom.CompilationUnit;
import org.eclipse.jdt.core.dom.IBinding;
import org.eclipse.jdt.core.dom.ImportDeclaration;

//import ca.mcgill.cs.swevo.ppa.PPAOptions;
//import ca.mcgill.cs.swevo.ppa.ui.PPAUtil;
import edu.uci.ics.sourcerer.extractor.io.WriterBundle;
import edu.uci.ics.sourcerer.util.Helper;
import edu.uci.ics.sourcerer.util.io.PropertyManager;

public final class FeatureExtractor {
  private ASTParser parser;
  private WriterBundle bundle;
  private boolean doPPA;
  
  public FeatureExtractor() {
    parser = ASTParser.newParser(AST.JLS3);
  }
  
  public void setOutput(PropertyManager properties) {
    if (bundle != null) {
      bundle.close();
    }
    bundle = new WriterBundle(properties);
  }
  
  public void setPPA(boolean doPPA) {
    this.doPPA = doPPA;
  }
  
  public void close() {
    bundle.close();
  }
  
  public void extractClassFiles(Collection<IClassFile> classFiles) {
    ClassFileExtractor extractor = new ClassFileExtractor(bundle);
    for (IClassFile classFile : classFiles) {
      try {
        extractor.extractClassFile(classFile);
      } catch (NullPointerException e) {
        logger.log(Level.SEVERE, "Unable to extract " + classFile.getElementName(), e);
      }
    }
  }
   
  public void extractSourceFiles(Collection<IFile> sourceFiles) {
    ReferenceExtractorVisitor visitor = new ReferenceExtractorVisitor(bundle);
    
    int total = 0;
    
    for (IFile source : sourceFiles) {
      if (doPPA) {
//        PPAUtil.getCU(source, new PPAOptions()).accept(visitor);
      } else {
        ICompilationUnit icu = JavaCore.createCompilationUnitFrom(source);
        
        parser.setStatementsRecovery(true);
        parser.setResolveBindings(true);
        parser.setBindingsRecovery(true);
        parser.setSource(icu);
        
        try {
          CompilationUnit unit;
          do {
            unit = (CompilationUnit)parser.createAST(null);
          } while (reattemptCompilation(unit));
          unit.accept(visitor);
        } catch (NullPointerException e) {
          logger.log(Level.SEVERE, "Unable to create AST for " + icu.getResource().getLocation().toString(), e);
          total--;
        }
      }
      
      if (++total % 1000 == 0) {
//        logger.info(total + " files extracted");
      }
    }
    
//    logger.info(total + " files extracted");
  }
  
  @SuppressWarnings("unchecked")
  private boolean reattemptCompilation(CompilationUnit unit) {
    IProblem[] problems = unit.getProblems();
    if (problems.length == 0) {
      return false;
    } else {
      boolean foundIndirectRef = false;
      Collection<String> missingPrefix = Helper.newHashSet();
      Collection<String> missingRefs = Helper.newHashSet();
      // Check problems for missing indirect references
      for (IProblem problem : problems) {
        if (problem.isError()) {
          String message = problem.getMessage();
          if (message.startsWith("The type") && message.endsWith("is indirectly referenced from required .class files")) {
{'fraction_non_alphanumeric': 0.1142337807606264, 'fraction_numerical': 0.0022371364653243847, 'mean_word_length': 3.0711439954467843, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3957812', 'n_tokens_mistral': 2255, 'n_tokens_neox': 2198, 'n_words': 422}