<commit_before>package org.dhis2.mobile.processors;

import android.content.Context;
import android.support.annotation.NonNull;

import org.dhis2.mobile.io.holders.DataElementOperand;
import org.dhis2.mobile.io.holders.DataSetCategoryOptions;
import org.dhis2.mobile.io.models.CategoryCombo;
import org.dhis2.mobile.io.models.Field;
import org.dhis2.mobile.io.models.Form;
import org.dhis2.mobile.io.models.Group;
import org.dhis2.mobile.network.HTTPClient;
import org.dhis2.mobile.network.NetworkException;
import org.dhis2.mobile.network.Response;
import org.dhis2.mobile.network.URLConstants;
import org.dhis2.mobile.utils.PrefUtils;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

public class DataSetMetaData {

    public static String download(Context context,
            String formId) throws NetworkException {
        String url = buildUrl(context, formId);
        String credentials = PrefUtils.getCredentials(context);
        Response response = HTTPClient.get(url, credentials);
        if (response.getCode() >= 200 && response.getCode() < 300) {
            return response.getBody();
        }else{
            throw new NetworkException(response.getCode());
        }
    }

    private static String buildUrl(Context context, String formId) {
        String server = PrefUtils.getServerURL(context);
        String url = server
                + URLConstants.DATASET_VALUES_URL + "/" + formId + "?"
                + URLConstants.DATA_SET_DATA_ELEMENTS_META_DATA_PARAM;

        return url;
    }


    public static void addCompulsoryDataElements(List<DataElementOperand> compulsoryUIds,
            Form form) {
        if (form == null && compulsoryUIds == null) {
            return;
        }
        for (DataElementOperand dataElementOperand : compulsoryUIds) {
            for (Group group : form.getGroups()) {
                addCompulsoryDataElements(dataElementOperand, group);
            }
        }
    }

    private static void addCompulsoryDataElements(DataElementOperand dataElementOperand,
            Group group) {
        for (Field field : group.getFields()) {
            if (field.getDataElement().equals(dataElementOperand.getDataElementUid())) {
                if (field.getCategoryOptionCombo().equals(
                        dataElementOperand.getCategoryOptionComboUid())) {
                    field.setCompulsory(true);
                }
            }
        }
    }


    public static Form removeFieldsWithInvalidCategoryOptionRelation(Form form,
            DataSetCategoryOptions dataSetCategoryOptions) {
        for (Group group : form.getGroups()) {
            boolean defaultSection=false;
            if(form.getGroups().size()==1 && form.getGroups().get(0).getLabel().equals("default")){
                defaultSection=true;
            }
            ArrayList<Field> validFields = getValidatedFieldList(
                    group, dataSetCategoryOptions, defaultSection);
            group.setFields(validFields);
        }
        return form;
    }

    @NonNull
    private static ArrayList<Field> getValidatedFieldList(
            Group group, DataSetCategoryOptions dataSetCategoryOptions, boolean defaultSection) {
        ArrayList<Field> validFields = new ArrayList<>();
        for (int i = 0; i < group.getFields().size(); i++) {
            Field field = group.getFields().get(i);
            checkIfAFieldIsValid(validFields, field, group.getLabel(), dataSetCategoryOptions, defaultSection);
        }
        return validFields;
    }

    private static void checkIfAFieldIsValid(
            ArrayList<Field> validFields, Field field, String section,
            DataSetCategoryOptions dataSetCategoryOptions, boolean defaultSection) {
        HashMap<String, List<CategoryCombo>> categoryComboByDataElement =
                dataSetCategoryOptions.getCategoryComboByDataElement();
        HashMap<String, List<String>> categoryOptionComboBySection =
                dataSetCategoryOptions.getCategoryComboDataElementBySection();

        if (categoryComboByDataElement.containsKey(field.getDataElement())) {
            if (isValidField(field, section, dataSetCategoryOptions, categoryComboByDataElement,
                    categoryOptionComboBySection, defaultSection)) {
                validFields.add(field);
            }
        }
    }

    private static boolean isValidField(Field field, String section,
            DataSetCategoryOptions dataSetCategoryOptions,
            HashMap<String, List<CategoryCombo>> categoryComboByDataElement,
            HashMap<String, List<String>> categoryOptionComboBySection, boolean defaultSection) {
        if (categoryComboByDataElement.get(field.getDataElement()) == null) {
            if (isValidDefaultField(field, dataSetCategoryOptions)) return true;
            if (defaultSection || isValidSectionField(field, section, categoryOptionComboBySection)) return true;
        } else if (isAValidCategoryOptionCombo(categoryComboByDataElement, field)) {
            return true;
        }
        return false;
    }

    private static boolean isValidDefaultField(Field field,
            DataSetCategoryOptions dataSetCategoryOptions) {
        if (dataSetCategoryOptions.getDefaultCategoryCombo().getCategoryOptionComboUIdList()
                .contains(
                        field.getCategoryOptionCombo())) {
            return true;
        }
        return false;
    }

    private static boolean isValidSectionField(Field field, String section,
            HashMap<String, List<String>> categoryOptionComboBySection) {
        if (categoryOptionComboBySection != null && categoryOptionComboBySection.containsKey(
                section)) {
            for (String validCategoryOptionUId : categoryOptionComboBySection.get(section)) {
                if (field.getCategoryOptionCombo().equals(validCategoryOptionUId)) {<commit_msg>removes section bad fix, the data sets withoutsections need be filtered like the others<commit_after>package org.dhis2.mobile.processors;

import android.content.Context;
import android.support.annotation.NonNull;

import org.dhis2.mobile.io.holders.DataElementOperand;
import org.dhis2.mobile.io.holders.DataSetCategoryOptions;
import org.dhis2.mobile.io.models.CategoryCombo;
import org.dhis2.mobile.io.models.Field;
import org.dhis2.mobile.io.models.Form;
import org.dhis2.mobile.io.models.Group;
import org.dhis2.mobile.network.HTTPClient;
import org.dhis2.mobile.network.NetworkException;
import org.dhis2.mobile.network.Response;
import org.dhis2.mobile.network.URLConstants;
import org.dhis2.mobile.utils.PrefUtils;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

public class DataSetMetaData {

    public static String download(Context context,
            String formId) throws NetworkException {
        String url = buildUrl(context, formId);
        String credentials = PrefUtils.getCredentials(context);
        Response response = HTTPClient.get(url, credentials);
        if (response.getCode() >= 200 && response.getCode() < 300) {
            return response.getBody();
        }else{
            throw new NetworkException(response.getCode());
        }
    }

    private static String buildUrl(Context context, String formId) {
        String server = PrefUtils.getServerURL(context);
        String url = server
                + URLConstants.DATASET_VALUES_URL + "/" + formId + "?"
                + URLConstants.DATA_SET_DATA_ELEMENTS_META_DATA_PARAM;

        return url;
    }


    public static void addCompulsoryDataElements(List<DataElementOperand> compulsoryUIds,
            Form form) {
        if (form == null && compulsoryUIds == null) {
            return;
        }
        for (DataElementOperand dataElementOperand : compulsoryUIds) {
            for (Group group : form.getGroups()) {
                addCompulsoryDataElements(dataElementOperand, group);
            }
        }
    }

    private static void addCompulsoryDataElements(DataElementOperand dataElementOperand,
            Group group) {
        for (Field field : group.getFields()) {
            if (field.getDataElement().equals(dataElementOperand.getDataElementUid())) {
                if (field.getCategoryOptionCombo().equals(
                        dataElementOperand.getCategoryOptionComboUid())) {
                    field.setCompulsory(true);
                }
            }
        }
    }


    public static Form removeFieldsWithInvalidCategoryOptionRelation(Form form,
            DataSetCategoryOptions dataSetCategoryOptions) {
        for (Group group : form.getGroups()) {
            ArrayList<Field> validFields = getValidatedFieldList(
                    group, dataSetCategoryOptions);
            group.setFields(validFields);
        }
        return form;
    }

    @NonNull
    private static ArrayList<Field> getValidatedFieldList(
            Group group, DataSetCategoryOptions dataSetCategoryOptions) {
        ArrayList<Field> validFields = new ArrayList<>();
        for (int i = 0; i < group.getFields().size(); i++) {
            Field field = group.getFields().get(i);
            checkIfAFieldIsValid(validFields, field, group.getLabel(), dataSetCategoryOptions);
        }
        return validFields;
    }

    private static void checkIfAFieldIsValid(
            ArrayList<Field> validFields, Field field, String section,
            DataSetCategoryOptions dataSetCategoryOptions) {
        HashMap<String, List<CategoryCombo>> categoryComboByDataElement =
                dataSetCategoryOptions.getCategoryComboByDataElement();
        HashMap<String, List<String>> categoryOptionComboBySection =
                dataSetCategoryOptions.getCategoryComboDataElementBySection();

        if (categoryComboByDataElement.containsKey(field.getDataElement())) {
            if (isValidField(field, section, dataSetCategoryOptions, categoryComboByDataElement,
                    categoryOptionComboBySection)) {
                validFields.add(field);
            }
        }
    }

    private static boolean isValidField(Field field, String section,
            DataSetCategoryOptions dataSetCategoryOptions,
            HashMap<String, List<CategoryCombo>> categoryComboByDataElement,
            HashMap<String, List<String>> categoryOptionComboBySection) {
        if (categoryComboByDataElement.get(field.getDataElement()) == null) {
            if (isValidDefaultField(field, dataSetCategoryOptions)) return true;
            if (isValidSectionField(field, section, categoryOptionComboBySection)) return true;
        } else if (isAValidCategoryOptionCombo(categoryComboByDataElement, field)) {
            return true;
        }
        return false;
    }

    private static boolean isValidDefaultField(Field field,
            DataSetCategoryOptions dataSetCategoryOptions) {
        if (dataSetCategoryOptions.getDefaultCategoryCombo().getCategoryOptionComboUIdList()
                .contains(
                        field.getCategoryOptionCombo())) {
            return true;
        }
        return false;
    }

    private static boolean isValidSectionField(Field field, String section,
            HashMap<String, List<String>> categoryOptionComboBySection) {
        if (categoryOptionComboBySection != null && categoryOptionComboBySection.containsKey(
                section)) {
            for (String validCategoryOptionUId : categoryOptionComboBySection.get(section)) {
                if (field.getCategoryOptionCombo().equals(validCategoryOptionUId)) {
{'fraction_non_alphanumeric': 0.08583282911228282, 'fraction_numerical': 0.003457515774915723, 'mean_word_length': 2.7984241628365067, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1922018', 'n_tokens_mistral': 2920, 'n_tokens_neox': 2935, 'n_words': 580}