<commit_before>/*
 * Copyright 2017 - 2021 <NAME> (i-net software)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package de.inetsoftware.jwebassembly.module;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.logging.Level;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import de.inetsoftware.classparser.ClassFile;
import de.inetsoftware.classparser.Code;
import de.inetsoftware.classparser.ConstantClass;
import de.inetsoftware.classparser.MethodInfo;
import de.inetsoftware.jwebassembly.JWebAssembly;
import de.inetsoftware.jwebassembly.WasmException;
import de.inetsoftware.jwebassembly.javascript.JavaScriptWriter;
import de.inetsoftware.jwebassembly.module.TypeManager.StructType;
import de.inetsoftware.jwebassembly.wasm.AnyType;
import de.inetsoftware.jwebassembly.wasm.FunctionType;
import de.inetsoftware.jwebassembly.wasm.ValueType;
import de.inetsoftware.jwebassembly.wasm.ValueTypeParser;
import de.inetsoftware.jwebassembly.watparser.WatParser;

/**
 * Generate the WebAssembly output.
 * 
 * @author <NAME>
 */
public class ModuleGenerator {

    private final ModuleWriter              writer;

    private final JavaScriptWriter          javaScript;

    private final ClassFileLoader           classFileLoader;

    private final JavaMethodWasmCodeBuilder javaCodeBuilder;

    private final WatParser                 watParser;

    private String                          sourceFile;

    private String                          className;

    private String                          methodName;

    private final FunctionManager           functions;

    private final TypeManager               types;

    private final StringManager             strings;

    private final CodeOptimizer             optimizer;

    /**
     * Create a new generator.
     * 
     * @param writer
     *            the target writer
     * @param target
     *            the target for the module data
     * @param libraries
     *            libraries 
     */
    public ModuleGenerator( @Nonnull ModuleWriter writer, WasmTarget target, @Nonnull List<URL> libraries ) {
        this.javaCodeBuilder = new JavaMethodWasmCodeBuilder();
        this.watParser = new WatParser();
        this.writer = writer;
        this.javaScript = new JavaScriptWriter( target );
        this.classFileLoader = new ClassFileLoader( new URLClassLoader( libraries.toArray( new URL[libraries.size()] ) ) );
        WasmOptions options = writer.options;
        functions = options.functions;
        types = options.types;
        strings = options.strings;
        optimizer = options.optimizer;
        javaCodeBuilder.init( options, classFileLoader );
        ((WasmCodeBuilder)watParser).init( options, classFileLoader );
        types.init( classFileLoader );

        scanLibraries( libraries );
    }

    /**
     * Scan the libraries for annotated methods
     * 
     * @param libraries
     *            libraries
     */
    private void scanLibraries( @Nonnull List<URL> libraries ) {
        // search for replacement methods in the libraries
        for( URL url : libraries ) {
            try {
                File file = new File(url.toURI());
                if( file.isDirectory() ) {
                    for( Iterator<Path> iterator = Files.walk( file.toPath() ).iterator(); iterator.hasNext(); ) {
                        Path path = iterator.next();
                        if( path.toString().endsWith( ".class" ) ) {
                            ClassFile classFile = new ClassFile( new BufferedInputStream( Files.newInputStream( path ) ) );
                            prepare( classFile );
                        }
                    }
                }
            } catch( Exception e ) {
                e.printStackTrace();
            }

            try (ZipInputStream input = new ZipInputStream( url.openStream() )) {
                do {
                    ZipEntry entry = input.getNextEntry();
                    if( entry == null ) {
                        break;
                    }
                    if( entry.getName().endsWith( ".class" ) ) {
                        try {
                            ClassFile classFile = new ClassFile( new BufferedInputStream( input ) {
                                @Override
                                public void close() {
                                } // does not close the zip stream
                            } );
                            prepare( classFile );
                        } catch( Throwable th ) {
                            JWebAssembly.LOGGER.log( Level.SEVERE, "Parsing error with " + entry.getName() + " in " + url, th );
                        }
                    }
                } while( true );
            } catch( IOException e ) {
                e.printStackTrace();
            }
        }
    }

    /**
     * Prepare the content of the class.
     * 
     * @param classFile
     *            the class file
     * @throws WasmException
     *             if some Java code can't converted
     * @throws IOException
     *             if any I/O error occur
     */
    public void prepare( @Nonnull ClassFile classFile ) throws IOException {
        classFileLoader.cache( classFile );
        // check if this class replace another class
        Map<String,Object> annotationValues;
        if( (annotationValues = classFile.getAnnotation( JWebAssembly.REPLACE_ANNOTATION )) != null ) {
            String signatureName = (String)annotationValues.get( "value" );
            if( signatureName != null ) {
                classFileLoader.replace( signatureName, classFile );
            }
        }

        // check if this class extends another class with partial code
        if( (annotationValues = classFile.getAnnotation( JWebAssembly.PARTIAL_ANNOTATION )) != null ) {
            String signatureName = (String)annotationValues.get( "value" );
            if( signatureName != null ) {
                classFileLoader.partial( signatureName, classFile );
            }
        }

        iterateMethods( classFile, m -> prepareMethod( m ) );
    }

    /**
     * Scan all needed methods/functions in a loop. If the scan find more needed content then the loop continue.
     * 
     * @throws IOException
     *             if any I/O error occur
     */
    private void scanFunctions() throws IOException {
        FunctionName next;
        NEXT:
        while( (next = functions.nextScannLater()) != null ) {
            className = next.className;
            methodName = next.methodName;
            JWebAssembly.LOGGER.fine( "scan " + next.signatureName );
            if( next instanceof SyntheticFunctionName ) {
                SyntheticFunctionName synth = (SyntheticFunctionName)next;
                if( synth.hasWasmCode() ) {
                    synth.getCodeBuilder( watParser );
                } else {
                    functions.markAsImport( synth, synth.getAnnotation() );
                }
                functions.markAsScanned( next );
                continue;
            }

            MethodInfo method = null;
            ClassFile classFile = classFileLoader.get( next.className );
            if( classFile != null ) {
                sourceFile = classFile.getSourceFile();
                className = classFile.getThisClass().getName();
                method = classFile.getMethod( next.methodName, next.signature );
            }
            if( method == null ) {
                method = functions.replace( next, null );
            }
            if( method != null ) {
                createInstructions( functions.replace( next, method ) );
                functions.markAsScanned( next );
                if( functions.needThisParameter( next ) ) {
                    types.valueOf( next.className ); // for the case that the type unknown yet
                }
                continue;<commit_msg>Handle call of Object methods from arrays<commit_after>/*
 * Copyright 2017 - 2021 <NAME> (i-net software)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package de.inetsoftware.jwebassembly.module;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.logging.Level;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import de.inetsoftware.classparser.ClassFile;
import de.inetsoftware.classparser.Code;
import de.inetsoftware.classparser.ConstantClass;
import de.inetsoftware.classparser.MethodInfo;
import de.inetsoftware.jwebassembly.JWebAssembly;
import de.inetsoftware.jwebassembly.WasmException;
import de.inetsoftware.jwebassembly.javascript.JavaScriptWriter;
import de.inetsoftware.jwebassembly.module.TypeManager.StructType;
import de.inetsoftware.jwebassembly.wasm.AnyType;
import de.inetsoftware.jwebassembly.wasm.FunctionType;
import de.inetsoftware.jwebassembly.wasm.ValueType;
import de.inetsoftware.jwebassembly.wasm.ValueTypeParser;
import de.inetsoftware.jwebassembly.watparser.WatParser;

/**
 * Generate the WebAssembly output.
 * 
 * @author <NAME>
 */
public class ModuleGenerator {

    private final ModuleWriter              writer;

    private final JavaScriptWriter          javaScript;

    private final ClassFileLoader           classFileLoader;

    private final JavaMethodWasmCodeBuilder javaCodeBuilder;

    private final WatParser                 watParser;

    private String                          sourceFile;

    private String                          className;

    private String                          methodName;

    private final FunctionManager           functions;

    private final TypeManager               types;

    private final StringManager             strings;

    private final CodeOptimizer             optimizer;

    /**
     * Create a new generator.
     * 
     * @param writer
     *            the target writer
     * @param target
     *            the target for the module data
     * @param libraries
     *            libraries 
     */
    public ModuleGenerator( @Nonnull ModuleWriter writer, WasmTarget target, @Nonnull List<URL> libraries ) {
        this.javaCodeBuilder = new JavaMethodWasmCodeBuilder();
        this.watParser = new WatParser();
        this.writer = writer;
        this.javaScript = new JavaScriptWriter( target );
        this.classFileLoader = new ClassFileLoader( new URLClassLoader( libraries.toArray( new URL[libraries.size()] ) ) );
        WasmOptions options = writer.options;
        functions = options.functions;
        types = options.types;
        strings = options.strings;
        optimizer = options.optimizer;
        javaCodeBuilder.init( options, classFileLoader );
        ((WasmCodeBuilder)watParser).init( options, classFileLoader );
        types.init( classFileLoader );

        scanLibraries( libraries );
    }

    /**
     * Scan the libraries for annotated methods
     * 
     * @param libraries
     *            libraries
     */
    private void scanLibraries( @Nonnull List<URL> libraries ) {
        // search for replacement methods in the libraries
        for( URL url : libraries ) {
            try {
                File file = new File(url.toURI());
                if( file.isDirectory() ) {
                    for( Iterator<Path> iterator = Files.walk( file.toPath() ).iterator(); iterator.hasNext(); ) {
                        Path path = iterator.next();
                        if( path.toString().endsWith( ".class" ) ) {
                            ClassFile classFile = new ClassFile( new BufferedInputStream( Files.newInputStream( path ) ) );
                            prepare( classFile );
                        }
                    }
                }
            } catch( Exception e ) {
                e.printStackTrace();
            }

            try (ZipInputStream input = new ZipInputStream( url.openStream() )) {
                do {
                    ZipEntry entry = input.getNextEntry();
                    if( entry == null ) {
                        break;
                    }
                    if( entry.getName().endsWith( ".class" ) ) {
                        try {
                            ClassFile classFile = new ClassFile( new BufferedInputStream( input ) {
                                @Override
                                public void close() {
                                } // does not close the zip stream
                            } );
                            prepare( classFile );
                        } catch( Throwable th ) {
                            JWebAssembly.LOGGER.log( Level.SEVERE, "Parsing error with " + entry.getName() + " in " + url, th );
                        }
                    }
                } while( true );
            } catch( IOException e ) {
                e.printStackTrace();
            }
        }
    }

    /**
     * Prepare the content of the class.
     * 
     * @param classFile
     *            the class file
     * @throws WasmException
     *             if some Java code can't converted
     * @throws IOException
     *             if any I/O error occur
     */
    public void prepare( @Nonnull ClassFile classFile ) throws IOException {
        classFileLoader.cache( classFile );
        // check if this class replace another class
        Map<String,Object> annotationValues;
        if( (annotationValues = classFile.getAnnotation( JWebAssembly.REPLACE_ANNOTATION )) != null ) {
            String signatureName = (String)annotationValues.get( "value" );
            if( signatureName != null ) {
                classFileLoader.replace( signatureName, classFile );
            }
        }

        // check if this class extends another class with partial code
        if( (annotationValues = classFile.getAnnotation( JWebAssembly.PARTIAL_ANNOTATION )) != null ) {
            String signatureName = (String)annotationValues.get( "value" );
            if( signatureName != null ) {
                classFileLoader.partial( signatureName, classFile );
            }
        }

        iterateMethods( classFile, m -> prepareMethod( m ) );
    }

    /**
     * Scan all needed methods/functions in a loop. If the scan find more needed content then the loop continue.
     * 
     * @throws IOException
     *             if any I/O error occur
     */
    private void scanFunctions() throws IOException {
        FunctionName next;
        NEXT:
        while( (next = functions.nextScannLater()) != null ) {
            className = next.className;
            methodName = next.methodName;
            JWebAssembly.LOGGER.fine( "scan " + next.signatureName );
            if( next instanceof SyntheticFunctionName ) {
                SyntheticFunctionName synth = (SyntheticFunctionName)next;
                if( synth.hasWasmCode() ) {
                    synth.getCodeBuilder( watParser );
                } else {
                    functions.markAsImport( synth, synth.getAnnotation() );
                }
                functions.markAsScanned( next );
                continue;
            }

            MethodInfo method = null;
            ClassFile classFile = classFileLoader.get( next.className );
            if( classFile != null ) {
                sourceFile = classFile.getSourceFile();
                className = classFile.getThisClass().getName();
                method = classFile.getMethod( next.methodName, next.signature );
            }
            if( method == null ) {
                method = functions.replace( next, null );
            }
            if( method != null ) {
                createInstructions( functions.replace( next, method ) );
                functions.markAsScanned( next );
                if( functions.needThisParameter( next ) ) {
                    types.valueOf( next.className ); // for the case that the type unknown yet
                }
                continue;
            }

            if( classFile == null ) {
                // all extends from object, also arrays
                classFile = classFileLoader.get( "java/lang/Object" );
{'fraction_non_alphanumeric': 0.08513027852650494, 'fraction_numerical': 0.0013477088948787063, 'mean_word_length': 1.8250317258883249, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1086071', 'n_tokens_mistral': 4458, 'n_tokens_neox': 4219, 'n_words': 1191}