<commit_before>package scripts

import org.apache.commons.lang.StringUtils
import org.reflections.Reflections
import org.zstack.core.Platform
import org.zstack.header.exception.CloudRuntimeException
import org.zstack.header.query.APIQueryReply
import org.zstack.header.rest.APINoSee
import org.zstack.header.rest.RestResponse
import org.zstack.header.rest.SDK
import org.zstack.rest.sdk.SdkTemplate
import org.zstack.rest.sdk.SdkFile
import org.zstack.utils.FieldUtils
import org.zstack.utils.Utils
import org.zstack.utils.logging.CLogger

import java.lang.reflect.Field

/**
 * Created by xing5 on 2016/12/11.
 */
class SdkDataStructureGenerator implements SdkTemplate {
    CLogger logger = Utils.getLogger(SdkDataStructureGenerator.class)

    Set<Class> responseClasses
    Map<Class, SdkFile> sdkFileMap = [:]
    Set<Class> laterResolvedClasses = []

    Map<String, String> sourceClassMap = [:]

    Reflections reflections = Platform.reflections

    SdkDataStructureGenerator() {
        Reflections reflections = Platform.getReflections()
        responseClasses = reflections.getTypesAnnotatedWith(RestResponse.class)
    }

    @Override
    List<SdkFile> generate() {
        responseClasses.each { c ->
            try {
                generateResponseClass(c)
            } catch (Throwable t) {
                throw new CloudRuntimeException("failed to generate SDK for the class[${c.name}]", t)
            }
        }
        resolveAllClasses()
        generateSourceDestClassMap()

        def ret = sdkFileMap.values() as List
        ret.add(generateSourceDestClassMap())

        return ret
    }

    def generateSourceDestClassMap() {
        def srcToDst = []
        def dstToSrc = []

        sourceClassMap.each { k, v ->
            srcToDst.add("""\t\t\tput("${k}", "${v}");""")
            dstToSrc.add("""\t\t\tput("${v}", "${k}");""")
        }

        SdkFile f = new SdkFile()
        f.fileName = "SourceClassMap.java"
        f.content = """package org.zstack.sdk;

import java.util.HashMap;

public class SourceClassMap {
    final static HashMap<String, String> srcToDstMapping = new HashMap() {
        {
${srcToDst.join("\n")}
        }
    };

    final static HashMap<String, String> dstToSrcMapping = new HashMap() {
        {
${dstToSrc.join("\n")}
        }
    };
}
"""
        return f
    }

    def resolveAllClasses() {
        if (laterResolvedClasses.isEmpty()) {
            return
        }

        Set<Class> toResolve = []
        toResolve.addAll(laterResolvedClasses)

        toResolve.each { Class clz ->
            try {
                resolveClass(clz)
                laterResolvedClasses.remove(clz)
            } catch (Throwable t) {
                throw new CloudRuntimeException("failed to generate SDK for the class[${clz.name}]", t)
            }
        }

        resolveAllClasses()
    }

    def getTargetClassName(Class clz) {
        SDK at = clz.getAnnotation(SDK.class)
        if (at == null || at.sdkClassName().isEmpty()) {
            return clz.simpleName
        }

        return at.sdkClassName()
    }

    def resolveClass(Class clz) {
        if (sdkFileMap.containsKey(clz)) {
            return
        }

        if (!Object.class.isAssignableFrom(clz.superclass)) {
            addToLaterResolvedClassesIfNeed(clz.superclass)
        }

        def output = []

        if (!Enum.class.isAssignableFrom(clz)) {
            for (Field f : clz.getDeclaredFields()) {
                if (f.isAnnotationPresent(APINoSee.class)) {
                    continue
                }

                output.add(makeFieldText(f.name, f))
            }
        } else {
            for (Enum e : clz.getEnumConstants()) {
                output.add("\t${e.name()},")
            }
        }

        SdkFile file = new SdkFile()
        file.fileName = "${getTargetClassName(clz)}.java"
        if (!Enum.class.isAssignableFrom(clz)) {
            file.content = """package org.zstack.sdk;

public class ${getTargetClassName(clz)} ${Object.class == clz.superclass ? "" : "extends " + clz.superclass.simpleName} {

${output.join("\n")}
}
"""
        } else {
            file.content = """package org.zstack.sdk;

public enum ${getTargetClassName(clz)} {
${output.join("\n")}
}
"""
        }

        sourceClassMap[clz.name] = "org.zstack.sdk.${getTargetClassName(clz)}"
        sdkFileMap.put(clz, file)
    }

    def isZStackClass(Class clz) {
        if (clz.name.startsWith("java.") || int.class == clz || long.class == clz
                || short.class == clz || char.class == clz || boolean.class == clz || float.class == clz
                || double.class == clz) {
            return false
        } else if (clz.canonicalName.startsWith("org.zstack")) {
            return true
        } else {
            throw new CloudRuntimeException("${clz.name} is neither JRE class nor ZStack class")
        }
    }

    def addToLaterResolvedClassesIfNeed(Class clz) {
        if (!sdkFileMap.containsKey(clz)) {
            laterResolvedClasses.add(clz)
        }
    }

    def generateResponseClass(Class responseClass) {
        logger.debug("generating class: ${responseClass.name}")

        RestResponse at = responseClass.getAnnotation(RestResponse.class)

        def fields = [:]

        def addToFields = { String fname, Field f ->
            if (isZStackClass(f.type)) {
                addToLaterResolvedClassesIfNeed(f.type)
                fields[fname] = f
            } else {
                fields[fname] = f
            }
        }<commit_msg>fix sdk generator: generate derived classes<commit_after>package scripts

import org.apache.commons.lang.StringUtils
import org.reflections.Reflections
import org.zstack.core.Platform
import org.zstack.header.exception.CloudRuntimeException
import org.zstack.header.query.APIQueryReply
import org.zstack.header.rest.APINoSee
import org.zstack.header.rest.RestResponse
import org.zstack.header.rest.SDK
import org.zstack.rest.sdk.SdkFile
import org.zstack.rest.sdk.SdkTemplate
import org.zstack.utils.FieldUtils
import org.zstack.utils.Utils
import org.zstack.utils.logging.CLogger

import java.lang.reflect.Field

/**
 * Created by xing5 on 2016/12/11.
 */
class SdkDataStructureGenerator implements SdkTemplate {
    CLogger logger = Utils.getLogger(SdkDataStructureGenerator.class)

    Set<Class> responseClasses
    Map<Class, SdkFile> sdkFileMap = [:]
    Set<Class> laterResolvedClasses = []

    Map<String, String> sourceClassMap = [:]

    Reflections reflections = Platform.reflections

    SdkDataStructureGenerator() {
        Reflections reflections = Platform.getReflections()
        responseClasses = reflections.getTypesAnnotatedWith(RestResponse.class)
    }

    @Override
    List<SdkFile> generate() {
        responseClasses.each { c ->
            try {
                generateResponseClass(c)
            } catch (Throwable t) {
                throw new CloudRuntimeException("failed to generate SDK for the class[${c.name}]", t)
            }
        }

        resolveAllClasses()

        def ret = sdkFileMap.values() as List
        ret.add(generateSourceDestClassMap())

        return ret
    }

    def generateSourceDestClassMap() {
        def srcToDst = []
        def dstToSrc = []

        sourceClassMap.each { k, v ->
            srcToDst.add("""\t\t\tput("${k}", "${v}");""")
            dstToSrc.add("""\t\t\tput("${v}", "${k}");""")
        }

        srcToDst.sort()
        dstToSrc.sort()

        SdkFile f = new SdkFile()
        f.fileName = "SourceClassMap.java"
        f.content = """package org.zstack.sdk;

import java.util.HashMap;

public class SourceClassMap {
    final static HashMap<String, String> srcToDstMapping = new HashMap() {
        {
${srcToDst.join("\n")}
        }
    };

    final static HashMap<String, String> dstToSrcMapping = new HashMap() {
        {
${dstToSrc.join("\n")}
        }
    };
}
"""
        return f
    }

    def resolveAllClasses() {
        if (laterResolvedClasses.isEmpty()) {
            return
        }

        Set<Class> toResolve = []
        toResolve.addAll(laterResolvedClasses)

        toResolve.each { Class clz ->
            try {
                resolveClass(clz)
                laterResolvedClasses.remove(clz)
            } catch (Throwable t) {
                throw new CloudRuntimeException("failed to generate SDK for the class[${clz.name}]", t)
            }
        }

        resolveAllClasses()
    }

    def getTargetClassName(Class clz) {
        SDK at = clz.getAnnotation(SDK.class)
        if (at == null || at.sdkClassName().isEmpty()) {
            return clz.simpleName
        }

        return at.sdkClassName()
    }

    def resolveClass(Class clz) {
        if (sdkFileMap.containsKey(clz)) {
            return
        }

        if (!Object.class.isAssignableFrom(clz.superclass)) {
            addToLaterResolvedClassesIfNeed(clz.superclass)
        }

        def output = []

        if (!Enum.class.isAssignableFrom(clz)) {
            for (Field f : clz.getDeclaredFields()) {
                if (f.isAnnotationPresent(APINoSee.class)) {
                    continue
                }

                output.add(makeFieldText(f.name, f))
            }
        } else {
            for (Enum e : clz.getEnumConstants()) {
                output.add("\t${e.name()},")
            }
        }

        SdkFile file = new SdkFile()
        file.fileName = "${getTargetClassName(clz)}.java"
        if (!Enum.class.isAssignableFrom(clz)) {
            file.content = """package org.zstack.sdk;

public class ${getTargetClassName(clz)} ${Object.class == clz.superclass ? "" : "extends " + clz.superclass.simpleName} {

${output.join("\n")}
}
"""
        } else {
            file.content = """package org.zstack.sdk;

public enum ${getTargetClassName(clz)} {
${output.join("\n")}
}
"""
        }

        sourceClassMap[clz.name] = "org.zstack.sdk.${getTargetClassName(clz)}"
        sdkFileMap.put(clz, file)
    }

    def isZStackClass(Class clz) {
        if (clz.name.startsWith("java.") || int.class == clz || long.class == clz
                || short.class == clz || char.class == clz || boolean.class == clz || float.class == clz
                || double.class == clz) {
            return false
        } else if (clz.canonicalName.startsWith("org.zstack")) {
            return true
        } else {
            throw new CloudRuntimeException("${clz.name} is neither JRE class nor ZStack class")
        }
    }

    def addToLaterResolvedClassesIfNeed(Class clz) {
        if (!sdkFileMap.containsKey(clz)) {
            laterResolvedClasses.add(clz)
        }
        Platform.reflections.getSubTypesOf(clz).forEach({ i ->
            if (!sdkFileMap.containsKey(i)) {
                laterResolvedClasses.add(i)
            }
        })
    }

    def generateResponseClass(Class responseClass) {
        logger.debug("generating class: ${responseClass.name}")

        RestResponse at = responseClass.getAnnotation(RestResponse.class)

        def fields = [:]

        def addToFields = { String fname, Field f ->
            if (isZStackClass(f.type)) {
                addToLaterResolvedClassesIfNeed(f.type)
                fields[fname] = f
            } else {
                fields[fname] = f
            }
        }
{'fraction_non_alphanumeric': 0.12245437714890241, 'fraction_numerical': 0.0015868817773075905, 'mean_word_length': 2.1130625686059274, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5106819', 'n_tokens_mistral': 3460, 'n_tokens_neox': 3341, 'n_words': 621}