<commit_before>package org.cafebabepy.runtime;

import org.cafebabepy.annotation.DefineCafeBabePyModule;
import org.cafebabepy.annotation.DefineCafeBabePyType;
import org.cafebabepy.runtime.module.builtins.PyIntType;
import org.cafebabepy.runtime.module.builtins.PyStrType;
import org.cafebabepy.runtime.object.*;
import org.cafebabepy.util.BinaryConsumer;
import org.cafebabepy.util.ModuleOrClassSplitter;
import org.cafebabepy.util.ReflectionUtils;

import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;

import static org.cafebabepy.util.ProtocolNames.__iter__;
import static org.cafebabepy.util.ProtocolNames.__next__;

/**
 * Created by yotchang4s on 2017/05/12.
 */
public final class Python {

    public static final String MAIN_MODULE_NAME = "__main__";

    public static final String BUILTINS_MODULE_NAME = "builtins";

    public static final String TYPES_MODULE_NAME = "types";

    // FIXME sys.modulesに持って行きたい
    private Map<String, PyObject> moduleMap;

    private PyObject objectObject;

    private PyObject noneObject;

    private PyBoolObject trueObject;

    private PyBoolObject falseObject;

    private PyObject notImplementedTypeObject;

    private Python() {
        this.moduleMap = new ConcurrentHashMap<>();
    }

    public static Python createRuntime() {
        Python runtime = new Python();
        runtime.initialize();

        return runtime;
    }

    private void initialize() {
        initializeBuiltins("org.cafebabepy.runtime.module.builtins");
        initializeBuiltins("org.cafebabepy.runtime.module.types");

        initializeBuiltins("org.cafebabepy.runtime.module");

        initializeObjects();
    }

    @SuppressWarnings("unchecked")
    public void initializeBuiltins(String packageName) {
        Set<Class<?>> builtinsClasses;

        // FIXME 本当は form builtins import * の形にしたい
        try {
            builtinsClasses = ReflectionUtils.getClasses(packageName);

        } catch (IOException e) {
            throw new CafeBabePyException("Fail initialize package '" + packageName + "'");
        }

        PyObject module = null;
        for (Class<?> c : builtinsClasses) {
            DefineCafeBabePyModule defineCafeBabePyModule = c.getAnnotation(DefineCafeBabePyModule.class);
            if (defineCafeBabePyModule == null || !PyObject.class.isAssignableFrom(c)) {
                continue;
            }

            Class<PyObject> clazz = (Class<PyObject>) c;

            // Check duplicate module
            if (module != null) {
                throw new CafeBabePyException(
                        "Duplicate module '"
                                + clazz.getName()
                                + "' and '"
                                + module.getClass().getName()
                                + "'");

            }

            module = createType(clazz, defineCafeBabePyModule.name());
            module.preInitialize();
        }

        if (module == null) {
            throw new CafeBabePyException("'" + packageName + "' module not found");
        }

        Set<PyObject> types = new HashSet<>();

        for (Class<?> c : builtinsClasses) {
            DefineCafeBabePyType defineCafeBabePyType = c.getAnnotation(DefineCafeBabePyType.class);
            if (defineCafeBabePyType == null || !PyObject.class.isAssignableFrom(c)) {
                continue;
            }

            Class<PyObject> clazz = (Class<PyObject>) c;

            PyObject type = createType(clazz, defineCafeBabePyType.name());
            type.preInitialize();

            types.add(type);
        }

        module.postInitialize();

        for (PyObject type : types) {
            type.postInitialize();
        }
    }

    private PyObject createType(Class<PyObject> clazz, String typeFullName) {
        try {<commit_msg>Add check duplicate type<commit_after>package org.cafebabepy.runtime;

import org.cafebabepy.annotation.DefineCafeBabePyModule;
import org.cafebabepy.annotation.DefineCafeBabePyType;
import org.cafebabepy.runtime.module.builtins.PyIntType;
import org.cafebabepy.runtime.module.builtins.PyStrType;
import org.cafebabepy.runtime.object.*;
import org.cafebabepy.util.BinaryConsumer;
import org.cafebabepy.util.ModuleOrClassSplitter;
import org.cafebabepy.util.ReflectionUtils;

import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;

import static org.cafebabepy.util.ProtocolNames.__iter__;
import static org.cafebabepy.util.ProtocolNames.__next__;

/**
 * Created by yotchang4s on 2017/05/12.
 */
public final class Python {

    public static final String MAIN_MODULE_NAME = "__main__";

    public static final String BUILTINS_MODULE_NAME = "builtins";

    public static final String TYPES_MODULE_NAME = "types";

    // FIXME sys.modulesに持って行きたい
    private Map<String, PyObject> moduleMap;

    private PyObject objectObject;

    private PyObject noneObject;

    private PyBoolObject trueObject;

    private PyBoolObject falseObject;

    private PyObject notImplementedTypeObject;

    private Python() {
        this.moduleMap = new ConcurrentHashMap<>();
    }

    public static Python createRuntime() {
        Python runtime = new Python();
        runtime.initialize();

        return runtime;
    }

    private void initialize() {
        initializeBuiltins("org.cafebabepy.runtime.module.builtins");
        initializeBuiltins("org.cafebabepy.runtime.module.types");

        initializeBuiltins("org.cafebabepy.runtime.module");

        initializeObjects();
    }

    @SuppressWarnings("unchecked")
    public void initializeBuiltins(String packageName) {
        Set<Class<?>> builtinsClasses;

        // FIXME 本当は form builtins import * の形にしたい
        try {
            builtinsClasses = ReflectionUtils.getClasses(packageName);

        } catch (IOException e) {
            throw new CafeBabePyException("Fail initialize package '" + packageName + "'");
        }

        PyObject module = null;
        for (Class<?> c : builtinsClasses) {
            DefineCafeBabePyModule defineCafeBabePyModule = c.getAnnotation(DefineCafeBabePyModule.class);
            if (defineCafeBabePyModule == null || !PyObject.class.isAssignableFrom(c)) {
                continue;
            }

            Class<PyObject> clazz = (Class<PyObject>) c;

            // Check duplicate module
            if (module != null) {
                throw new CafeBabePyException(
                        "Duplicate module '"
                                + clazz.getName()
                                + "' and '"
                                + module.getClass().getName()
                                + "'");

            }

            module = createType(clazz, defineCafeBabePyModule.name());
            module.preInitialize();
        }

        if (module == null) {
            throw new CafeBabePyException("'" + packageName + "' module not found");
        }

        Set<PyObject> types = new HashSet<>();
        Set<String> checkDuplicateTypes = new HashSet<>();

        for (Class<?> c : builtinsClasses) {
            DefineCafeBabePyType defineCafeBabePyType = c.getAnnotation(DefineCafeBabePyType.class);
            if (defineCafeBabePyType == null || !PyObject.class.isAssignableFrom(c)) {
                continue;
            }

            Class<PyObject> clazz = (Class<PyObject>) c;

            PyObject type = createType(clazz, defineCafeBabePyType.name());
            type.preInitialize();

            if (checkDuplicateTypes.contains(defineCafeBabePyType.name())) {
                throw new CafeBabePyException("Duplicate type '" + defineCafeBabePyType.name() + "'");
            }
            types.add(type);
            checkDuplicateTypes.add(defineCafeBabePyType.name());
        }

        module.postInitialize();

        for (PyObject type : types) {
            type.postInitialize();
        }
    }

    private PyObject createType(Class<PyObject> clazz, String typeFullName) {
        try {
{'fraction_non_alphanumeric': 0.09848119575699132, 'fraction_numerical': 0.0021697203471552555, 'mean_word_length': 2.4802852348993287, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3219215', 'n_tokens_mistral': 2402, 'n_tokens_neox': 2298, 'n_words': 451}