<commit_before>package com.github.agiledon.sisyphus.sis;

import com.github.agiledon.sisyphus.exception.FailedDeserializationException;
import com.github.agiledon.sisyphus.exception.FailedSerializationException;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;

import static com.github.agiledon.sisyphus.sis.rule.ParsingRule.sisClassTree;
import static com.github.agiledon.sisyphus.sis.rule.ParsingRule.createRootClass;
import static com.github.agiledon.sisyphus.sis.util.Reflection.*;
import static com.google.common.base.Preconditions.checkArgument;

public class SyntaxParser {
    private final Logger logger = LoggerFactory.getLogger(SyntaxParser.class);
    private static final String LINE_BREAK = "\n";
    private boolean ignoreNullField = true;

    public SyntaxParser() {
    }

    public SyntaxParser(boolean ignoreNullField) {
        this.ignoreNullField = ignoreNullField;
    }

    public SisClass parseClassFromResource(String resource) {
        SisClass rootClass;

        try {
            List<String> lines = splitLines(resource);
            checkArgument(lines != null && lines.size() >= 1, "data file is error");
            rootClass = createRootClass(lines.get(0));
            for (int i = 1; i < lines.size(); i++) {
                rootClass = sisClassTree(rootClass, lines.get(i));
            }
            return rootClass;
        } catch (IllegalArgumentException ex) {
            return logAndRethrowException(ex);
        } catch (Exception ex) {
            return logAndRethrowException(ex);
        }
    }

    public <T> SisClass parseClassFromObject(T sourceObject) {
        return parseClass(sourceObject, null, 0);
    }

    private <T> SisClass parseClass(T sourceObject, String fieldName, int level) {
        if (sourceObject == null) {
            throw new FailedSerializationException("The target is null");
        }
        if (isArray(sourceObject)) {
            return createArrayClass(sourceObject, fieldName, level);
        }
        if (isList(sourceObject)) {
            return createListClass(sourceObject, fieldName, level);
        }

        return createNormalClass(sourceObject, fieldName, level);
    }

    private <T> SisClass createArrayClass(T sourceObject, String fieldName, int level) {
        SisArrayClass sisArrayClass = new SisArrayClass(fieldName);
        sisArrayClass.setCurrentLevel(level);

        Object[] array = (Object[]) sourceObject;
        if (ignoreNullField && (array == null || array.length == 0)) {
            return sisArrayClass;
        }

        try {
            Class<?> elementType = getElementTypeForArray(sourceObject.getClass());
            if (isPrimitiveType(elementType)) {
                if (array == null || array.length == 0) {
                    sisArrayClass.addBasicElement(new BasicElement(""));
                    return sisArrayClass;
                }
                for (Object element : array) {
                    sisArrayClass.addBasicElement(new BasicElement(element.toString()));
                }
                return sisArrayClass;
            }
            if (array == null || array.length == 0) {
                sisArrayClass.addChildClass(parseClass(getFieldValue(elementType, ""), null, level + 1));
                return sisArrayClass;
            }
            for (Object element : array) {
                sisArrayClass.addChildClass(parseClass(element, null, level + 1));
            }
            return sisArrayClass;
        } catch (ClassNotFoundException e) {
            logger.warn(e.getMessage());
            return sisArrayClass;
        }
    }

    private <T> SisClass createListClass(T sourceObject, String fieldName, int level) {
        SisListClass sisListClass = new SisListClass(fieldName);
        sisListClass.setCurrentLevel(level);

        ArrayList list = (ArrayList) sourceObject;
        if (list == null || list.size() == 0) {
            return sisListClass;
        }

        Object firstElement = list.get(0);
        if (isPrimitiveType(firstElement.getClass())) {
            for (Object element : list) {
                sisListClass.addBasicElement(new BasicElement(element.toString()));
            }
        } else {
            for (Object element : list) {
                sisListClass.addChildClass(parseClass(element, null, level + 1));
            }
        }

        return sisListClass;
    }

    private <T> SisNormalClass createNormalClass(T sourceObject, String fieldName, int level) {
        SisNormalClass sisNormalClass = new SisNormalClass(fieldName);
        sisNormalClass.setCurrentLevel(level);

        Field[] declaredFields = sourceObject.getClass().getDeclaredFields();
        for (Field declaredField : declaredFields) {
            setOrIgnoreFieldValue(sourceObject, sisNormalClass, declaredField, level);
        }
        return sisNormalClass;
    }

    private <T> void setOrIgnoreFieldValue(T sourceObject, SisNormalClass sisNormalClass, Field declaredField, int level) {
        try {
            declaredField.setAccessible(true);
            Object fieldValue = declaredField.get(sourceObject);
            if (fieldValue == null && ignoreNullField) {
                return;
            }
            if (isPrimitiveType(declaredField.getType())) {
                if (fieldValue == null) {
                    fieldValue = getFieldValue(declaredField.getType(), "");
                }
                sisNormalClass.addBasicField(new BasicField(declaredField.getName(), fieldValue.toString()));
            } else {
                if (fieldValue == null) {
                    fieldValue = createInstance(declaredField.getType());
                }
                sisNormalClass.addChildClass(parseClass(fieldValue, declaredField.getName(), level + 1));
            }
        } catch (IllegalAccessException e) {
            logAndRethrowException(e);
        } catch (InstantiationException e) {
            logAndRethrowException(e);
        } catch (InvocationTargetException e) {
            logAndRethrowException(e);
        } catch (Exception e) {
            logAndRethrowException(e);
        }
    }

    private <T> boolean isList(T sourceObject) {
        Class<?> sourceClass = sourceObject.getClass();
        return sourceClass.getSuperclass() != null &&
                (sourceClass.getSuperclass().getSimpleName().equals("ArrayList") ||
                        sourceClass.getSuperclass().getSimpleName().equals("List") ||
                        sourceClass.getSimpleName().equals("ArrayList"));
    }

    private <T> boolean isArray(T sourceObject) {
        return sourceObject.getClass().isArray();
    }

    private SisClass logAndRethrowException(Exception ex) {
        logger.error(ex.getMessage());
        throw new FailedDeserializationException(ex);
    }

    private List<String> splitLines(String content) {
        Iterable<String> lines = Splitter.on(LINE_BREAK)
                .omitEmptyStrings()
                .split(content);

        return Lists.newArrayList(lines);
    }
}
<commit_msg>extract methods for array<commit_after>package com.github.agiledon.sisyphus.sis;

import com.github.agiledon.sisyphus.exception.FailedDeserializationException;
import com.github.agiledon.sisyphus.exception.FailedSerializationException;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;

import static com.github.agiledon.sisyphus.sis.rule.ParsingRule.sisClassTree;
import static com.github.agiledon.sisyphus.sis.rule.ParsingRule.createRootClass;
import static com.github.agiledon.sisyphus.sis.util.Reflection.*;
import static com.google.common.base.Preconditions.checkArgument;

public class SyntaxParser {
    private final Logger logger = LoggerFactory.getLogger(SyntaxParser.class);
    private static final String LINE_BREAK = "\n";
    private boolean ignoreNullField = true;

    public SyntaxParser() {
    }

    public SyntaxParser(boolean ignoreNullField) {
        this.ignoreNullField = ignoreNullField;
    }

    public SisClass parseClassFromResource(String resource) {
        SisClass rootClass;

        try {
            List<String> lines = splitLines(resource);
            checkArgument(lines != null && lines.size() >= 1, "data file is error");
            rootClass = createRootClass(lines.get(0));
            for (int i = 1; i < lines.size(); i++) {
                rootClass = sisClassTree(rootClass, lines.get(i));
            }
            return rootClass;
        } catch (IllegalArgumentException ex) {
            return logAndRethrowException(ex);
        } catch (Exception ex) {
            return logAndRethrowException(ex);
        }
    }

    public <T> SisClass parseClassFromObject(T sourceObject) {
        return parseClass(sourceObject, null, 0);
    }

    private <T> SisClass parseClass(T sourceObject, String fieldName, int level) {
        if (sourceObject == null) {
            throw new FailedSerializationException("The target is null");
        }
        if (isArray(sourceObject)) {
            return createArrayClass(sourceObject, fieldName, level);
        }
        if (isList(sourceObject)) {
            return createListClass(sourceObject, fieldName, level);
        }

        return createNormalClass(sourceObject, fieldName, level);
    }

    private <T> SisClass createArrayClass(T sourceObject, String fieldName, int level) {
        SisArrayClass sisArrayClass = new SisArrayClass(fieldName);
        sisArrayClass.setCurrentLevel(level);

        Object[] array = (Object[]) sourceObject;
        if (ignoreNullField && (array == null || array.length == 0)) {
            return sisArrayClass;
        }

        try {
            Class<?> elementType = getElementTypeForArray(sourceObject.getClass());
            if (isPrimitiveType(elementType)) {
                addElements(sisArrayClass, array);
            } else {
                addChildClasses(level, sisArrayClass, array, elementType);
            }
            return sisArrayClass;
        } catch (ClassNotFoundException e) {
            logger.warn(e.getMessage());
            return sisArrayClass;
        }
    }

    private void addChildClasses(int level, SisArrayClass sisArrayClass, Object[] array, Class<?> elementType) {
        if (array == null || array.length == 0) {
            sisArrayClass.addChildClass(parseClass(getFieldValue(elementType, ""), null, level + 1));
        } else {
            for (Object element : array) {
                sisArrayClass.addChildClass(parseClass(element, null, level + 1));
            }
        }
    }

    private void addElements(SisArrayClass sisArrayClass, Object[] array) {
        if (array == null || array.length == 0) {
            sisArrayClass.addBasicElement(new BasicElement(""));
        } else {
            for (Object element : array) {
                sisArrayClass.addBasicElement(new BasicElement(element.toString()));
            }
        }
    }

    private <T> SisClass createListClass(T sourceObject, String fieldName, int level) {
        SisListClass sisListClass = new SisListClass(fieldName);
        sisListClass.setCurrentLevel(level);

        ArrayList list = (ArrayList) sourceObject;
        if (list == null || list.size() == 0) {
            return sisListClass;
        }

        Object firstElement = list.get(0);
        if (isPrimitiveType(firstElement.getClass())) {
            for (Object element : list) {
                sisListClass.addBasicElement(new BasicElement(element.toString()));
            }
        } else {
            for (Object element : list) {
                sisListClass.addChildClass(parseClass(element, null, level + 1));
            }
        }

        return sisListClass;
    }

    private <T> SisNormalClass createNormalClass(T sourceObject, String fieldName, int level) {
        SisNormalClass sisNormalClass = new SisNormalClass(fieldName);
        sisNormalClass.setCurrentLevel(level);

        Field[] declaredFields = sourceObject.getClass().getDeclaredFields();
        for (Field declaredField : declaredFields) {
            setOrIgnoreFieldValue(sourceObject, sisNormalClass, declaredField, level);
        }
        return sisNormalClass;
    }

    private <T> void setOrIgnoreFieldValue(T sourceObject, SisNormalClass sisNormalClass, Field declaredField, int level) {
        try {
            declaredField.setAccessible(true);
            Object fieldValue = declaredField.get(sourceObject);
            if (fieldValue == null && ignoreNullField) {
                return;
            }
            if (isPrimitiveType(declaredField.getType())) {
                if (fieldValue == null) {
                    fieldValue = getFieldValue(declaredField.getType(), "");
                }
                sisNormalClass.addBasicField(new BasicField(declaredField.getName(), fieldValue.toString()));
            } else {
                if (fieldValue == null) {
                    fieldValue = createInstance(declaredField.getType());
                }
                sisNormalClass.addChildClass(parseClass(fieldValue, declaredField.getName(), level + 1));
            }
        } catch (IllegalAccessException e) {
            logAndRethrowException(e);
        } catch (InstantiationException e) {
            logAndRethrowException(e);
        } catch (InvocationTargetException e) {
            logAndRethrowException(e);
        } catch (Exception e) {
            logAndRethrowException(e);
        }
    }

    private <T> boolean isList(T sourceObject) {
        Class<?> sourceClass = sourceObject.getClass();
        return sourceClass.getSuperclass() != null &&
                (sourceClass.getSuperclass().getSimpleName().equals("ArrayList") ||
                        sourceClass.getSuperclass().getSimpleName().equals("List") ||
                        sourceClass.getSimpleName().equals("ArrayList"));
    }

    private <T> boolean isArray(T sourceObject) {
        return sourceObject.getClass().isArray();
    }

    private SisClass logAndRethrowException(Exception ex) {
        logger.error(ex.getMessage());
        throw new FailedDeserializationException(ex);
    }

    private List<String> splitLines(String content) {
        Iterable<String> lines = Splitter.on(LINE_BREAK)
                .omitEmptyStrings()
                .split(content);

        return Lists.newArrayList(lines);
    }
}

{'fraction_non_alphanumeric': 0.10040485829959514, 'fraction_numerical': 0.0020242914979757085, 'mean_word_length': 2.420493884144934, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1564362', 'n_tokens_mistral': 3978, 'n_tokens_neox': 3863, 'n_words': 804}