<commit_before>package me.tomassetti.turin.compiler;

import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.google.common.collect.ImmutableList;
import me.tomassetti.turin.parser.analysis.resolvers.ComposedResolver;
import me.tomassetti.turin.parser.analysis.resolvers.InFileResolver;
import me.tomassetti.turin.parser.analysis.resolvers.Resolver;
import me.tomassetti.turin.parser.analysis.resolvers.SrcResolver;
import me.tomassetti.turin.parser.ast.*;

import java.io.*;
import java.util.ArrayList;
import java.util.List;

import me.tomassetti.turin.parser.Parser;

public class Compiler {

    private static String VERSION = "Alpha 000";

    private Resolver resolver;
    private Options options;

    public Compiler(Resolver resolver, Options options) {
        this.resolver = resolver;
        this.options = options;
    }

    public List<ClassFileDefinition> compile(TurinFile turinFile) {
        return new Compilation(resolver).compile(turinFile);
    }

    public static class Options {

        @Parameter(names = {"-o", "--output"})
        private String destinationDir = "turin_classes";

        @Parameter(names = {"-cp", "--classpath"}, variableArity = true)
        private List<String> classPathElements = new ArrayList<>();

        @Parameter(names = {"-v", "--verbose"})
        private boolean verbose = false;

        @Parameter(names = {"-d", "--debug"})
        private boolean debug = false;

        @Parameter(names = {"-h", "--help"})
        private boolean help = false;

        @Parameter(description = "Files or directories to compile")
        private List<String> sources = new ArrayList<>();
    }

    private static Resolver getResolver(List<String> sources, List<String> classPathElements, List<TurinFile> turinFiles) {
        // TODO use all the elements for resolving
        return new ComposedResolver(ImmutableList.of(new InFileResolver(), new SrcResolver(turinFiles)));
    }

    private void compile(File file) throws IOException {
        if (file.isDirectory()) {
            compileDir(file);
        } else {
            compileFile(file);
        }
    }

    private void compileFile(File file) throws IOException {
        TurinFile turinFile = new Parser().parse(new FileInputStream(file));

        for (ClassFileDefinition classFileDefinition : compile(turinFile)) {
            if (options.verbose) {
                System.out.println(" Writing [" + classFileDefinition.getName() + "]");
            }
            File classFile = new File(options.destinationDir +"/" + classFileDefinition.getName().replaceAll("\\.", "/") + ".class");
            classFile.getParentFile().mkdirs();
            FileOutputStream fos = new FileOutputStream(classFile);
            fos.write(classFileDefinition.getBytecode());
            fos.close();
        }
    }

    private void compileDir(File file) throws IOException {
        for (File child : file.listFiles()) {
            compile(child);
        }
    }

    public static void main(String[] args) throws IOException {
        System.out.println("-------------------------------------");
        System.out.println(" Turin Compiler - version " + VERSION);
        System.out.println("-------------------------------------\n");

        Options options = new Options();
        JCommander commander = null;
        try {
            commander = new JCommander(options, args);
        } catch (Throwable t) {
            System.err.println("Problem parsing options: " + t.getMessage());
            System.exit(1);
            return;
        }

        if (options.help) {
            System.out.println("Help demanded - printing usage");
            commander.usage();
            return;
        }

        if (options.sources.isEmpty()) {
            System.err.println("No sources specified");
            commander.usage();
            return;
        }

        Parser parser = new Parser();

        // First we collect all TurinFiles and we pass it to the resolver
        List<TurinFile> turinFiles = new ArrayList<>();
        for (String source : options.sources) {
            try {
                turinFiles.addAll(parser.parseAllIn(new File(source)));
            } catch (FileNotFoundException e){
                System.err.println("Error: " + e.getMessage());
                System.exit(1);
                return;
            }
        }
        Resolver resolver = getResolver(options.sources, options.classPathElements, turinFiles);

        // Then we compile all files
        // TODO consider classpath
        Compiler instance = new Compiler(resolver, options);
        for (TurinFile turinFile : turinFiles) {
            for (ClassFileDefinition classFileDefinition : instance.compile(turinFile)) {
                saveClassFile(classFileDefinition, options);
            }
        }
    }

    private static void saveClassFile(ClassFileDefinition classFileDefinition, Options options) {
        File output = null;
        try {
            output = new File(new File(options.destinationDir).getAbsolutePath() + "/" + classFileDefinition.getName().replaceAll("\\.", "/") + ".class");
            if (options.verbose) {
                System.out.println(" [saving "+output.getPath()+"]");
            }
            output.getParentFile().mkdirs();
            FileOutputStream fos = new FileOutputStream(output);
            fos.write(classFileDefinition.getBytecode());
        } catch (IOException e) {
            System.err.println("Problem writing file "+output+": "+ e.getMessage());
            System.exit(3);
        }
    }

}
<commit_msg>update version<commit_after>package me.tomassetti.turin.compiler;

import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.google.common.collect.ImmutableList;
import me.tomassetti.turin.parser.analysis.resolvers.ComposedResolver;
import me.tomassetti.turin.parser.analysis.resolvers.InFileResolver;
import me.tomassetti.turin.parser.analysis.resolvers.Resolver;
import me.tomassetti.turin.parser.analysis.resolvers.SrcResolver;
import me.tomassetti.turin.parser.ast.*;

import java.io.*;
import java.util.ArrayList;
import java.util.List;

import me.tomassetti.turin.parser.Parser;

public class Compiler {

    private static String VERSION = "0.1 (Crocetta)";

    private Resolver resolver;
    private Options options;

    public Compiler(Resolver resolver, Options options) {
        this.resolver = resolver;
        this.options = options;
    }

    public List<ClassFileDefinition> compile(TurinFile turinFile) {
        return new Compilation(resolver).compile(turinFile);
    }

    public static class Options {

        @Parameter(names = {"-o", "--output"})
        private String destinationDir = "turin_classes";

        @Parameter(names = {"-cp", "--classpath"}, variableArity = true)
        private List<String> classPathElements = new ArrayList<>();

        @Parameter(names = {"-v", "--verbose"})
        private boolean verbose = false;

        @Parameter(names = {"-d", "--debug"})
        private boolean debug = false;

        @Parameter(names = {"-h", "--help"})
        private boolean help = false;

        @Parameter(description = "Files or directories to compile")
        private List<String> sources = new ArrayList<>();
    }

    private static Resolver getResolver(List<String> sources, List<String> classPathElements, List<TurinFile> turinFiles) {
        // TODO use all the elements for resolving
        return new ComposedResolver(ImmutableList.of(new InFileResolver(), new SrcResolver(turinFiles)));
    }

    private void compile(File file) throws IOException {
        if (file.isDirectory()) {
            compileDir(file);
        } else {
            compileFile(file);
        }
    }

    private void compileFile(File file) throws IOException {
        TurinFile turinFile = new Parser().parse(new FileInputStream(file));

        for (ClassFileDefinition classFileDefinition : compile(turinFile)) {
            if (options.verbose) {
                System.out.println(" Writing [" + classFileDefinition.getName() + "]");
            }
            File classFile = new File(options.destinationDir +"/" + classFileDefinition.getName().replaceAll("\\.", "/") + ".class");
            classFile.getParentFile().mkdirs();
            FileOutputStream fos = new FileOutputStream(classFile);
            fos.write(classFileDefinition.getBytecode());
            fos.close();
        }
    }

    private void compileDir(File file) throws IOException {
        for (File child : file.listFiles()) {
            compile(child);
        }
    }

    public static void main(String[] args) throws IOException {
        System.out.println("-------------------------------------");
        System.out.println(" Turin Compiler - version " + VERSION);
        System.out.println("-------------------------------------\n");

        Options options = new Options();
        JCommander commander = null;
        try {
            commander = new JCommander(options, args);
        } catch (Throwable t) {
            System.err.println("Problem parsing options: " + t.getMessage());
            System.exit(1);
            return;
        }

        if (options.help) {
            System.out.println("Help demanded - printing usage");
            commander.usage();
            return;
        }

        if (options.sources.isEmpty()) {
            System.err.println("No sources specified");
            commander.usage();
            return;
        }

        Parser parser = new Parser();

        // First we collect all TurinFiles and we pass it to the resolver
        List<TurinFile> turinFiles = new ArrayList<>();
        for (String source : options.sources) {
            try {
                turinFiles.addAll(parser.parseAllIn(new File(source)));
            } catch (FileNotFoundException e){
                System.err.println("Error: " + e.getMessage());
                System.exit(1);
                return;
            }
        }
        Resolver resolver = getResolver(options.sources, options.classPathElements, turinFiles);

        // Then we compile all files
        // TODO consider classpath
        Compiler instance = new Compiler(resolver, options);
        for (TurinFile turinFile : turinFiles) {
            for (ClassFileDefinition classFileDefinition : instance.compile(turinFile)) {
                saveClassFile(classFileDefinition, options);
            }
        }
    }

    private static void saveClassFile(ClassFileDefinition classFileDefinition, Options options) {
        File output = null;
        try {
            output = new File(new File(options.destinationDir).getAbsolutePath() + "/" + classFileDefinition.getName().replaceAll("\\.", "/") + ".class");
            if (options.verbose) {
                System.out.println(" [saving "+output.getPath()+"]");
            }
            output.getParentFile().mkdirs();
            FileOutputStream fos = new FileOutputStream(output);
            fos.write(classFileDefinition.getBytecode());
        } catch (IOException e) {
            System.err.println("Problem writing file "+output+": "+ e.getMessage());
            System.exit(3);
        }
    }

}

{'fraction_non_alphanumeric': 0.12903511883646684, 'fraction_numerical': 0.0009755232351897836, 'mean_word_length': 2.2867968522296707, 'pattern_counts': {'":': 2, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5147433', 'n_tokens_mistral': 2979, 'n_tokens_neox': 2885, 'n_words': 673}