<commit_before>/*
 * Copyright (c) 2010-2016 Evolveum
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.evolveum.polygon.connector.sap;

import com.sap.conn.jco.*;
import org.identityconnectors.common.Base64;
import org.identityconnectors.common.StringUtil;
import org.identityconnectors.common.logging.Log;
import org.identityconnectors.common.security.GuardedString;
import org.identityconnectors.framework.common.FrameworkUtil;
import org.identityconnectors.framework.common.exceptions.*;
import org.identityconnectors.framework.common.objects.*;
import org.identityconnectors.framework.common.objects.filter.FilterTranslator;
import org.identityconnectors.framework.spi.Configuration;
import org.identityconnectors.framework.spi.ConnectorClass;
import org.identityconnectors.framework.spi.PoolableConnector;
import org.identityconnectors.framework.spi.operations.*;

import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;

@ConnectorClass(displayNameKey = "sap.connector.display", configurationClass = SapConfiguration.class)
public class SapConnector implements PoolableConnector, TestOp, SchemaOp, SearchOp<SapFilter>, CreateOp, DeleteOp, UpdateOp,
        SyncOp, ScriptOnConnectorOp {

    private static final Log LOG = Log.getLog(SapConnector.class);

    private static final String SEPARATOR = "."; // between structure and his attributes, for example ADDRESS.FIRSTNAME
    // used BAPI functions in connector
    private static final String[] BAPI_FUNCTION_LIST = {"BAPI_USER_GETLIST", "BAPI_USER_GET_DETAIL", "BAPI_USER_CREATE1",
            "BAPI_TRANSACTION_COMMIT", "BAPI_TRANSACTION_ROLLBACK", "BAPI_USER_DELETE",
            "BAPI_USER_CHANGE", "BAPI_USER_LOCK", "BAPI_USER_UNLOCK", "BAPI_USER_ACTGROUPS_ASSIGN",
            "RFC_GET_TABLE_ENTRIES", "SUSR_USER_CHANGE_PASSWORD_RFC", "SUSR_GENERATE_PASSWORD",
            "BAPI_USER_PROFILES_ASSIGN", "BAPI_HELPVALUES_GET",
            "SUSR_LOGIN_CHECK_RFC", "PASSWORD_FORMAL_CHECK",
            "SUSR_GET_ADMIN_USER_LOGIN_INFO"
//            , "SUSR_BAPI_USER_UNLOCK"

            /*"BAPI_ADDRESSORG_GETDETAIL", "BAPI_ORGUNITEXT_DATA_GET", */ /* BAPI to Read Organization Addresses, Get data on organizational unit  */

            /*"BAPI_OUTEMPLOYEE_GETLIST", "BAPI_EMPLOYEE_GETDATA", */ /* List of employees in a payroll area, Find Personnel Numbers for Specified Search Criteria  - not found in SAP :((( */

            /*"BAPI_USER_LOCPROFILES_ASSIGN", "BAPI_USER_LOCPROFILES_READ", "BAPI_USER_LOCPROFILES_DELETE" */ /* CUA landscape*/
            /*"BAPI_USER_LOCACTGROUPS_READ", "BAPI_USER_LOCACTGROUPS_DELETE", "BAPI_USER_LOCACTGROUPS_ASSIGN" */ /* CUA landscape*/

            /*"BAPI_USER_EXISTENCE_CHECK", */ /* handled over Exception and parameter RETURN */
            /*"BAPI_USER_ACTGROUPS_DELETE", */ /* replaced with BAPI_USER_ACTGROUPS_ASSIGN */
            /*"BAPI_USER_PROFILES_DELETE" */ /* replaced with BAPI_USER_PROFILES_ASSIGN, NON CUA landscape */
            /*"BAPI_USER_DISPLAY",*/ /*Don't need, using BAPI_USER_GET_DETAIL */
            /*"SUSR_BAPI_USER_LOCK", "BAPI_USER_CREATE" */ /*DO NOT USE !*/
    };

    // SAP roles, groups, profiles
    private static final String AGR_NAME = "AGR_NAME";
    public static final String ACTIVITYGROUPS = "ACTIVITYGROUPS";
    public static final String ACTIVITYGROUPS__ARG_NAME = ACTIVITYGROUPS + SEPARATOR + AGR_NAME;

    private static final String BAPIPROF = "BAPIPROF";
    public static final String PROFILES = "PROFILES";
    public static final String PROFILES_BAPIPROF = PROFILES + SEPARATOR + BAPIPROF;
    public static final String PROFILE_NAME = "PROFILE"; // also as objectClass name

    private static final String USERGROUP = "USERGROUP";
    public static final String GROUPS = "GROUPS";
    public static final String GROUPS_USERGROUP = GROUPS + SEPARATOR + USERGROUP;

    // see for example http://www.sapdatasheet.org/abap/func/BAPI_USER_GET_DETAIL.html
    // these "Paremeter name"-s we can read and write in Type "Exporting"
    private static final String[] READ_WRITE_PARAMETERS = {"ADDRESS", "DEFAULTS", "UCLASS", "LOGONDATA", "ALIAS", "COMPANY", "REF_USER"};
    // these "Paremeter name"-s we can only read (don't have appropirade parameters in BAPI_USER_CHANGE)
    static final String[] READ_ONLY_PARAMETERS = {"ISLOCKED", "LASTMODIFIED", "SNC", "ADMINDATA", "IDENTITY"};
    // variable version of READ_ONLY_PARAMETERS that could be set through connector configuration
    private String[] readOnlyParams;
    // these attributes in "ADDRESS" parameter name we can't update, only set, because "ADDRESSX" in BAPI_USER_CHANGE don't contains these fields
    private static final String[] CREATE_ONLY_ATTRIBUTES = {"ADDRESS" + SEPARATOR + "COUNTY_CODE", "ADDRESS" + SEPARATOR + "COUNTY",
            "ADDRESS" + SEPARATOR + "TOWNSHIP_CODE", "ADDRESS" + SEPARATOR + "TOWNSHIP", "DEFAULTS" + SEPARATOR + "CATTKENNZ"};


    // supported tables reading & writing in tables parameter type
    // "UCLASSSYS", "EXTIDHEAD", "EXTIDPART", "SYSTEMS" not supported yet
    public static final String[] TABLETYPE_PARAMETER_LIST = {"PARAMETER", PROFILES, ACTIVITYGROUPS,
            "RETURN", "ADDTEL", "ADDFAX", "ADDTTX", "ADDTLX", "ADDSMTP", "ADDRML", "ADDX400", "ADDRFC", "ADDPRT", "ADDSSF",
            "ADDURI", "ADDPAG", "ADDCOMREM", "PARAMETER1", GROUPS/*, "UCLASSSYS", "EXTIDHEAD", "EXTIDPART", "SYSTEMS"*/};

    // these tables on update use "change flag" explicitly as is
    private static final List<String> TABLES_WITH_CHANGE_FLAG = Arrays.asList(GROUPS, "PARAMETER", "PARAMETER1");
    // these communication data tables on update use "change flag" in "ADDCOMX"
    private static final List<String> COMMUNICATION_DATA_TABLES_WITH_CHANGE_FLAG = Arrays.asList("ADDTEL",
            "ADDFAX", "ADDTTX", "ADDTLX", "ADDSMTP", "ADDRML", "ADDX400", "ADDRFC", "ADDPRT", "ADDSSF", "ADDURI", "ADDPAG", "ADDCOMREM");

    // with these tables we can manipulate as XML line with all his attributes (for example ACTIVITYGROUPS)
    // and also as only string keys (for example ACTIVITYGROUPS.AGR_NAME) - used in midPoint associations
    public static final Map<String, String> TABLETYPE_PARAMETER_KEYS = new HashMap<>();

    static {
        TABLETYPE_PARAMETER_KEYS.put(ACTIVITYGROUPS, AGR_NAME);
        TABLETYPE_PARAMETER_KEYS.put(PROFILES, BAPIPROF);
        TABLETYPE_PARAMETER_KEYS.put(GROUPS, USERGROUP);
    }

    // LOGONDATA
    private static final String GLTGV = "GLTGV";        //User valid from, AttributeInfo ENABLE_DATE, "LOGONDATA.GLTGV"
    private static final String GLTGB = "GLTGB";        //User valid to, AttributeInfo DISABLE_DATE; "LOGONDATA.GLTGB"

    //ISLOCKED
    private static final String LOCAL_LOCK = "LOCAL_LOCK";     // Local_Lock - Logon generally locked for the local system
    private static final String WRNG_LOGON = "WRNG_LOGON";     // WRNG_LOGON - Password logon locked by incorrect logon attempts

    public static final String USERNAME = "USERNAME";

    // PASSWORD
    private static final String BAPIPWD = "<PASSWORD>";

    //USER_LOGIN_INFO prefix
    private static final String USER_LOGIN_INFO = "USER_LOGIN_INFO";

    public static final SimpleDateFormat SAP_DF = new SimpleDateFormat("yyyy-MM-dd");
    public static final SimpleDateFormat DATE_TIME = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

    private static final String SELECT = "X";

    private SapConfiguration configuration;
    private JCoDestination destination;
    private Map<String, Integer> sapAttributesLength = new HashMap<String, Integer>();
    private Map<String, String> sapAttributesType = new HashMap<String, String>();

    @Override
    public Configuration getConfiguration() {
        return configuration;
    }

    @Override
    public void init(Configuration configuration) {
        this.configuration = (SapConfiguration) configuration;
        LOG.info("Initialization start, configuration: {0}", this.configuration.toString());

        CustomDestinationDataProvider myProvider = CustomDestinationDataProvider.getInstance();

        //register the provider with the JCo environment;
        //catch IllegalStateException if an instance is already registered
        try {
            if (!com.sap.conn.jco.ext.Environment.isDestinationDataProviderRegistered()) {
                com.sap.conn.jco.ext.Environment.registerDestinationDataProvider(myProvider);
            }
        } catch (IllegalStateException providerAlreadyRegisteredException) {
            //somebody else registered its implementation,
            //stop the execution
            throw new ConnectorIOException(providerAlreadyRegisteredException.getMessage(), providerAlreadyRegisteredException);
        }

        //properties for destination from config
        Properties props = this.configuration.getDestinationProperties();

        String destinationName = this.configuration.getFinalDestinationName();

        Properties destProps = myProvider.getDestinationProperties(destinationName);
        if (destProps == null || !destProps.equals(props)){
            myProvider.setDestinationProperties(destinationName, props);
        }
        // set read only parameters from gui connector configuration
        readOnlyParams =  this.configuration.getReadOnlyParams();

        if (this.configuration.SNC_MODE_ON.equals(this.configuration.getSncMode())) {
            createDestinationDataFile(destinationName, props);
        }

        // create destination & ping it
        try {
            this.destination = JCoDestinationManager.getDestination(destinationName);
            this.destination.ping();
        } catch (JCoException e) {
            throw new ConnectorIOException(e.getMessage(), e);
        }

        // validate & initialize tables
        configuration.validate();
        // read schema
        schema();

        LOG.info("Initialization finished");
    }


    @Override
    public void checkAlive() {
        if (this.destination == null || !this.destination.isValid()) {
            LOG.ok("check alive: FAILED");
            throw new ConnectorException("Connection check failed");
        }
        try {
            this.destination.ping();
        } catch (JCoException e) {
            LOG.ok("connection ping FAILED: "+e);
            throw new ConnectorException("Connection ping failed", e);
        }
        LOG.ok("check alive: OK");
    }


    private void createDestinationDataFile(String destinationName, Properties connectProperties)
    {
        String fileName = destinationName+".jcoDestination";
        try
        {
            File destCfg = new File(fileName);
            destCfg.deleteOnExit(); // only temporary files
            FileOutputStream fos = new FileOutputStream(destCfg,false);
            connectProperties.store(fos, "for tests only !");
            fos.close();
            LOG.ok("destination file was created: "+destCfg.getName()+", "+destCfg.getAbsolutePath());
        }
        catch (Exception e)
        {
            throw new RuntimeException("Unable to create the destination file: "+fileName, e);
        }
    }

    private void deleteDestinationDataFile(String destinationName)
    {
        String fileName = destinationName+".jcoDestination";
        try
        {
            File destCfg = new File(fileName);
            boolean deleted = destCfg.delete();
            if (deleted) {
                LOG.ok("destination file was deleted: " + destCfg.getName() + ", " + destCfg.getAbsolutePath());
            }
        }
        catch (Exception e)
        {
            LOG.warn("Unable to delete the destination file: "+fileName, e);
        }
    }

    @Override
    public void dispose() {
        this.configuration = null;
        if ((this.destination != null) && (JCoContext.isStateful(this.destination))) {
            if (this.configuration.SNC_MODE_ON.equals(this.configuration.getSncMode())) {
                deleteDestinationDataFile(this.destination.getDestinationName());
            }
            try {
                JCoContext.end(this.destination);
            } catch (JCoException jcoe) {
                throw new ConnectorIOException(jcoe.getMessage(), jcoe);
            }
        }
    }

    @Override
    public void test() {
        try {
            this.destination.ping();
            if (configuration.getTestBapiFunctionPermission()) {
                List<String> notFoundFunctions = new LinkedList<String>();
                for (String function : BAPI_FUNCTION_LIST) {
                    if (!configuration.getUseTransaction() && function.contains("_TRANSACTION_")) {
                        continue;
                    }
                    JCoFunction jcoFunc = this.destination.getRepository().getFunction(function);
                    if (jcoFunc == null)
                        notFoundFunctions.add(function);
                }
                if (notFoundFunctions.size() > 0) {
                    throw new ConfigurationException("these BAPI functions are not accessible: " + notFoundFunctions);
                }
                // testing creation of transaction
                if (configuration.getUseTransaction()) {
                    JCoContext.begin(destination);
                    JCoContext.end(destination);
                }
            }
        } catch (JCoException e) {
            throw new ConfigurationException(e.getMessage(), e);
        }
    }

    @Override
    public Schema schema() {
        SchemaBuilder builder = new SchemaBuilder(SapConnector.class);

        buildAccountObjectClass(builder);

        buildTableObjectClasses(builder);

        buildProfileObjectClass(builder);

        return builder.build();
    }

    private void buildAccountObjectClass(SchemaBuilder builder) {
        ObjectClassInfoBuilder objClassBuilder = new ObjectClassInfoBuilder();

        try {
            String function = "BAPI_USER_GET_DETAIL";
            getSchemaFromBapiFunction(function, READ_WRITE_PARAMETERS, objClassBuilder, false);
            getSchemaFromBapiFunction(function, readOnlyParams, objClassBuilder, true);
        } catch (Exception e) {
            throw new ConnectorIOException("Error when parse user schema from SAP: " + e, e);
        }
        // __NAME__ and __UID__ is default and was renamed to USERNAME in schema if it's needed
        AttributeInfoBuilder uidAib = new AttributeInfoBuilder(Uid.NAME);
        uidAib.setRequired(true);
        if (this.configuration.getUseNativeNames()) {
            uidAib.setNativeName(USERNAME);
        }
        objClassBuilder.addAttributeInfo(uidAib.build());

        AttributeInfoBuilder nameAib = new AttributeInfoBuilder(Name.NAME);
        if (this.configuration.getUseNativeNames()) {
            nameAib.setNativeName(USERNAME);
        }
        objClassBuilder.addAttributeInfo(nameAib.build());

        sapAttributesType.put(USERNAME, "java.lang.String");
        sapAttributesLength.put(USERNAME, 12);

        objClassBuilder.addAttributeInfo(OperationalAttributeInfos.ENABLE);     // enable / disable - ISLOCKED.LOCAL_LOCK
        objClassBuilder.addAttributeInfo(OperationalAttributeInfos.LOCK_OUT);   // unlock - ISLOCKED.WRNG_LOGON
        objClassBuilder.addAttributeInfo(OperationalAttributeInfos.ENABLE_DATE); // LOGONDATA.GLTGV
        objClassBuilder.addAttributeInfo(OperationalAttributeInfos.DISABLE_DATE); //LOGONDATA.GLTGB

        AttributeInfoBuilder passwordAIB = new AttributeInfoBuilder(OperationalAttributes.PASSWORD_NAME, GuardedString.class);
        passwordAIB.setReadable(false); // write only
        passwordAIB.setReturnedByDefault(false);
        objClassBuilder.addAttributeInfo(passwordAIB.build());
        sapAttributesType.put(BAPIPWD, "java.lang.String");
        sapAttributesLength.put(BAPIPWD, 40);

        // tables
        for (String table : configuration.getTableParameterNames()) {
            AttributeInfoBuilder attributeActivityGroup = new AttributeInfoBuilder(table);
            attributeActivityGroup.setMultiValued(true);
            objClassBuilder.addAttributeInfo(attributeActivityGroup.build());
            // id's from tables
            if (TABLETYPE_PARAMETER_KEYS.containsKey(table)) {
                AttributeInfoBuilder attributeActivityGroupIds = new AttributeInfoBuilder(table + SEPARATOR + TABLETYPE_PARAMETER_KEYS.get(table));
                attributeActivityGroupIds.setMultiValued(true);
                objClassBuilder.addAttributeInfo(attributeActivityGroupIds.build());
            }
        }
        // user_login_infos
        if (this.configuration.getAlsoReadLoginInfo()) {
            boolean readOnly = true;
            objClassBuilder.addAttributeInfo(createAttributeInfo(null, USER_LOGIN_INFO + SEPARATOR + "LAST_LOGON_DATE", Long.class, readOnly));
            objClassBuilder.addAttributeInfo(createAttributeInfo(null, USER_LOGIN_INFO + SEPARATOR + "LOCK_STATUS", String.class, readOnly));
            objClassBuilder.addAttributeInfo(createAttributeInfo(null, USER_LOGIN_INFO + SEPARATOR + "PASSWORD_STATUS", String.class, readOnly));
        }

        builder.defineObjectClass(objClassBuilder.build());
    }

    private void buildTableObjectClasses(SchemaBuilder builder) {
        for (Map.Entry<String, Map<String, Integer>> table : configuration.getTableMetadatas().entrySet()) {
            String tableName = table.getKey();
            Map<String, Integer> columnsMetadata = table.getValue();

            ObjectClassInfoBuilder objClassBuilder = new ObjectClassInfoBuilder();
            objClassBuilder.setType(configuration.getTableAliases().get(tableName));

            for (Map.Entry<String, Integer> column : columnsMetadata.entrySet()) {
                String columnName = column.getKey();
                AttributeInfoBuilder attributeInfoBuilder = new AttributeInfoBuilder(columnName);
                attributeInfoBuilder.setCreateable(false);
                attributeInfoBuilder.setUpdateable(false);
                objClassBuilder.addAttributeInfo(attributeInfoBuilder.build());
            }

            builder.defineObjectClass(objClassBuilder.build());
        }
    }

    private void buildProfileObjectClass(SchemaBuilder builder) {
        ObjectClassInfoBuilder objClassBuilder = new ObjectClassInfoBuilder();
        objClassBuilder.setType(PROFILE_NAME);

        AttributeInfoBuilder uidAib = new AttributeInfoBuilder(Uid.NAME);
        uidAib.setRequired(true);
        uidAib.setCreateable(false);
        uidAib.setUpdateable(false);
        if (this.configuration.getUseNativeNames()) {
            uidAib.setNativeName(PROFILE_NAME);
        }
        objClassBuilder.addAttributeInfo(uidAib.build());

        AttributeInfoBuilder nameAib = new AttributeInfoBuilder(Name.NAME);
        nameAib.setCreateable(false);
        nameAib.setUpdateable(false);
        if (this.configuration.getUseNativeNames()) {
            nameAib.setNativeName(PROFILE_NAME);
        }
        objClassBuilder.addAttributeInfo(nameAib.build());

        builder.defineObjectClass(objClassBuilder.build());
    }

    private List<String> getAllStructureParameters(JCoParameterList epl) {
        List<String> parameters = new LinkedList<String>();
        JCoListMetaData lmd = epl.getListMetaData();
        for (int i = 0; i < lmd.getFieldCount(); i++) {
            if (lmd.isStructure(i)) {
                parameters.add(lmd.getName(i));
            }
        }

        return parameters;
    }

    private void getSchemaFromBapiFunction(String bapiFunction, String[] parameterList, ObjectClassInfoBuilder objClassBuilder, boolean readOnly) throws JCoException, ClassNotFoundException {
        JCoFunction function = destination.getRepository().getFunction(bapiFunction);
        if (function == null)
            throw new RuntimeException(bapiFunction + " not found in SAP.");

        List<String> parameters = null;
        JCoParameterList epl = function.getExportParameterList();
        if (parameterList == null) {
            LOG.ok("reading all structure parameters from ExportParameterList");
            parameters = getAllStructureParameters(epl);
        } else {
            parameters = Arrays.asList(parameterList);
        }

        for (String param : parameters) {
            JCoStructure structure = epl.getStructure(param);
            JCoRecordMetaData rmd = structure.getRecordMetaData();
//            LOG.ok("STRUCTURE for " + param + ":");
            for (int r = 0; r < rmd.getFieldCount(); r++) {
                String name = rmd.getName(r);
                Integer length = rmd.getLength(r);
                String attrName = param + SEPARATOR + name;
                String className = rmd.getClassNameOfField(r);
                Class classs = null;
                try {
                    classs = Class.forName(className);
                } catch (ClassNotFoundException cnfe) {
                    if (!"byte[]".equals(className)) {
                        LOG.warn("Not supported class type: " + attrName + "\t" + length + "\t" + className + "\t" + rmd.getRecordTypeName(r) + "\t" + rmd.getTypeAsString(r) + ", ex: " + cnfe);
                    }
                }
                if (classs != null && FrameworkUtil.isSupportedAttributeType(classs)) {
                    objClassBuilder.addAttributeInfo(createAttributeInfo(param, attrName, classs, readOnly));
                } else if ("java.util.Date".equals(className)) {
                    objClassBuilder.addAttributeInfo(createAttributeInfo(param, attrName, Long.class, readOnly));
                    LOG.ok(className + " simulated as java.lang.Long over connector for: " + attrName);
                } else if ("byte[]".equals(className)) {
                    objClassBuilder.addAttributeInfo(createAttributeInfo(param, attrName, byte[].class, readOnly));
                } else {
                    objClassBuilder.addAttributeInfo(createAttributeInfo(param, attrName, String.class, readOnly));
                    LOG.warn("TODO: implement better support for " + className + ", attribute " + attrName + " if you need it, I'm using java.lang.String");
                }
                this.sapAttributesLength.put(attrName, length);
                this.sapAttributesType.put(attrName, className);

//                LOG.ok(attrName + "\t" + length + "\t" + className + "\t" + rmd.getRecordTypeName(r) + "\t" + rmd.getTypeAsString(r));
            }
        }
    }


    private AttributeInfo createAttributeInfo(String structure, String attrName, Class classs, boolean readOnly) {
        AttributeInfoBuilder attributeInfoBuilder = new AttributeInfoBuilder(attrName, classs);
        if (readOnly) {
            attributeInfoBuilder.setCreateable(false);
            attributeInfoBuilder.setUpdateable(false);
        } else if (contains(CREATE_ONLY_ATTRIBUTES, structure + SEPARATOR + attrName)) {
            attributeInfoBuilder.setCreateable(true);
            attributeInfoBuilder.setUpdateable(false);
        }

        return attributeInfoBuilder.build();
    }

    @Override
    public FilterTranslator<SapFilter> createFilterTranslator(ObjectClass objectClass, OperationOptions operationOptions) {
        if (objectClass.is(ObjectClass.ACCOUNT_NAME)) {
            return new SapAccountFilterTranslator();
        }
        else {
            return new SapBasicFilterTranslator();
        }
    }

    @Override
    public void executeQuery(ObjectClass objectClass, SapFilter query, ResultsHandler handler, OperationOptions options) {
        LOG.info("executeQuery: {0}, options: {1}, objectClass: {2}", query, options, objectClass);

        if (objectClass.is(ObjectClass.ACCOUNT_NAME)) {

            executeAccountQuery(query, handler, options);

        } else if (objectClass.is(PROFILE_NAME)) {

            executeProfileQuery(query, handler);

        } else {
            String found = null;
            for (String tableName : configuration.getTableAliases().keySet()) {
                String tableAlias = configuration.getTableAliases().get(tableName);
                if (objectClass.is(tableAlias)) {
                    found = tableName;
                }
            }

            if (found == null) {
                throw new UnsupportedOperationException("Unsupported object class " + objectClass + ", expected: " + configuration.getTableMetadatas());
            }

            executeTableQuery(found, query, handler);
        }

    }

    private void executeProfileQuery(SapFilter query, ResultsHandler handler) {
        try {
            // find all or find by key

            JCoFunction function = destination.getRepository().getFunction("BAPI_HELPVALUES_GET");
            if (function == null)
                throw new RuntimeException("BAPI_HELPVALUES_GET not found in SAP.");

            function.getImportParameterList().setValue("OBJNAME", "USER");
            function.getImportParameterList().setValue("METHOD", "ProfilesAssign");
            function.getImportParameterList().setValue("PARAMETER", "Profiles");
            function.getImportParameterList().setValue("FIELD", BAPIPROF);

            function.execute(destination);

            JCoTable entries = function.getTableParameterList().getTable("VALUES_FOR_FIELD");


            entries.firstRow();
            LOG.ok("Number of entries in input: {0}, filter: {1}", entries.getNumRows(), query != null ? query.getBasicByNameEquals() : "(empty)");
            if (entries.getNumRows() > 0) {
                do {
                    String profile = entries.getString("VALUES");

                    if (query != null && query.getBasicByNameEquals() != null && !profile.equalsIgnoreCase(query.getBasicByNameEquals())) {
                        // TODO: case sensitive or not?
                        // not matched, ignore this
                        continue;
                    }

                    ConnectorObjectBuilder builder = new ConnectorObjectBuilder();
                    builder.setUid(profile);
                    builder.setName(profile);

                    ObjectClass objectClass = new ObjectClass(PROFILE_NAME);
                    builder.setObjectClass(objectClass);

                    ConnectorObject build = builder.build();
                    LOG.ok("ConnectorObject: {0}", build);
                    handler.handle(build);
                } while (entries.nextRow());
            }
        } catch (JCoException e) {
            throw new ConnectorIOException(e.getMessage(), e);
        }
    }

    private void executeTableQuery(String tableName, SapFilter query, ResultsHandler handler) {
        try {
            // find all or find by key

            JCoFunction function = destination.getRepository().getFunction("RFC_GET_TABLE_ENTRIES");
            if (function == null)
                throw new RuntimeException("RFC_GET_TABLE_ENTRIES not found in SAP.");

            function.getImportParameterList().setValue("TABLE_NAME", tableName);
            // find by key
            if (query != null && query.getBasicByNameEquals() != null) {
                function.getImportParameterList().setValue("GEN_KEY", query.getBasicByNameEquals());
                LOG.ok("query by Key: " + query.getBasicByNameEquals() + " on table: " + tableName);
            }
            // else find all

            function.execute(destination);

            JCoTable entries = function.getTableParameterList().getTable("ENTRIES");

            LOG.info("Entries: " + function.getExportParameterList().getValue("NUMBER_OF_ENTRIES"));

            entries.firstRow();
            if (entries.getNumRows() > 0) {
                do {
                    String value = entries.getString("WA");
                    int index = 0;

                    ConnectorObjectBuilder builder = new ConnectorObjectBuilder();
                    List<String> keys = new LinkedList<String>();

                    for (Map.Entry<String, Integer> entry : configuration.getTableMetadatas().get(tableName).entrySet()) {
                        String column = entry.getKey();
                        Integer length = entry.getValue();

                        String columnValue = value.substring(index, index + length).trim();
                        // ignore columns, what is selected as :IGNORE
                        if (!configuration.getTableIgnores().get(tableName).contains(column)) {
                            addAttr(builder, column, columnValue);
                        }
                        if (configuration.getTableKeys().get(tableName).contains(column)) {
                            keys.add(columnValue);
                        }

                        index += length;
                    }

                    StringBuilder concatenatedKey = new StringBuilder();
                    for (String key : keys) {
                        if (concatenatedKey.length() != 0) {
                            concatenatedKey.append(":");
                        }
                        concatenatedKey.append(key);
                    }
                    if (StringUtil.isEmpty(concatenatedKey.toString())) {
                        LOG.warn("ignoring empty key: " + concatenatedKey);
                        continue;
                    }

                    builder.setUid(concatenatedKey.toString());
                    builder.setName(concatenatedKey.toString());

                    ObjectClass objectClass = new ObjectClass(configuration.getTableAliases().get(tableName));
                    builder.setObjectClass(objectClass);

                    ConnectorObject build = builder.build();
                    LOG.ok("ConnectorObject: {0}", build);
                    handler.handle(build);

                } while (entries.nextRow());
            }
        } catch (JCoException e) {
            throw new ConnectorIOException(e.getMessage(), e);
        }
    }

    private void executeAccountQuery(SapFilter query, ResultsHandler handler, OperationOptions options) {
        try {
            // find by NAME (or UID - same as name)
            if (query != null && query.byNameEquals() != null) {

                JCoFunction function = destination.getRepository().getFunction("BAPI_USER_GET_DETAIL");
                function.getImportParameterList().setValue(USERNAME, query.byNameEquals());
                executeFunction(function);

                JCoFunction userLoginInfoFunc = runUserLoginInfoFunction(query.byNameEquals());

                ConnectorObject connectorObject = convertUserToConnectorObject(function, userLoginInfoFunc);
                handler.handle(connectorObject);

            } // find by name contains
            else if (query != null && query.byNameContains() != null) {
                JCoFunction function = destination.getRepository().getFunction("BAPI_USER_GETLIST");
                JCoTable exp = function.getTableParameterList().getTable("SELECTION_EXP");
                exp.appendRow();
                exp.setValue("PARAMETER", USERNAME);
                exp.setValue("OPTION", "CP");
                exp.setValue("LOW", query.byNameContains());

                executeFunction(function);
<commit_msg>Fix "TABLE_EMPTY" (MID-7753)<commit_after>/*
 * Copyright (c) 2010-2016 Evolveum
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.evolveum.polygon.connector.sap;

import com.sap.conn.jco.*;
import org.identityconnectors.common.Base64;
import org.identityconnectors.common.StringUtil;
import org.identityconnectors.common.logging.Log;
import org.identityconnectors.common.security.GuardedString;
import org.identityconnectors.framework.common.FrameworkUtil;
import org.identityconnectors.framework.common.exceptions.*;
import org.identityconnectors.framework.common.objects.*;
import org.identityconnectors.framework.common.objects.filter.FilterTranslator;
import org.identityconnectors.framework.spi.Configuration;
import org.identityconnectors.framework.spi.ConnectorClass;
import org.identityconnectors.framework.spi.PoolableConnector;
import org.identityconnectors.framework.spi.operations.*;

import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;

@ConnectorClass(displayNameKey = "sap.connector.display", configurationClass = SapConfiguration.class)
public class SapConnector implements PoolableConnector, TestOp, SchemaOp, SearchOp<SapFilter>, CreateOp, DeleteOp, UpdateOp,
        SyncOp, ScriptOnConnectorOp {

    private static final Log LOG = Log.getLog(SapConnector.class);

    private static final String SEPARATOR = "."; // between structure and his attributes, for example ADDRESS.FIRSTNAME
    // used BAPI functions in connector
    private static final String[] BAPI_FUNCTION_LIST = {"BAPI_USER_GETLIST", "BAPI_USER_GET_DETAIL", "BAPI_USER_CREATE1",
            "BAPI_TRANSACTION_COMMIT", "BAPI_TRANSACTION_ROLLBACK", "BAPI_USER_DELETE",
            "BAPI_USER_CHANGE", "BAPI_USER_LOCK", "BAPI_USER_UNLOCK", "BAPI_USER_ACTGROUPS_ASSIGN",
            "RFC_GET_TABLE_ENTRIES", "SUSR_USER_CHANGE_PASSWORD_RFC", "SUSR_GENERATE_PASSWORD",
            "BAPI_USER_PROFILES_ASSIGN", "BAPI_HELPVALUES_GET",
            "SUSR_LOGIN_CHECK_RFC", "PASSWORD_FORMAL_CHECK",
            "SUSR_GET_ADMIN_USER_LOGIN_INFO"
//            , "SUSR_BAPI_USER_UNLOCK"

            /*"BAPI_ADDRESSORG_GETDETAIL", "BAPI_ORGUNITEXT_DATA_GET", */ /* BAPI to Read Organization Addresses, Get data on organizational unit  */

            /*"BAPI_OUTEMPLOYEE_GETLIST", "BAPI_EMPLOYEE_GETDATA", */ /* List of employees in a payroll area, Find Personnel Numbers for Specified Search Criteria  - not found in SAP :((( */

            /*"BAPI_USER_LOCPROFILES_ASSIGN", "BAPI_USER_LOCPROFILES_READ", "BAPI_USER_LOCPROFILES_DELETE" */ /* CUA landscape*/
            /*"BAPI_USER_LOCACTGROUPS_READ", "BAPI_USER_LOCACTGROUPS_DELETE", "BAPI_USER_LOCACTGROUPS_ASSIGN" */ /* CUA landscape*/

            /*"BAPI_USER_EXISTENCE_CHECK", */ /* handled over Exception and parameter RETURN */
            /*"BAPI_USER_ACTGROUPS_DELETE", */ /* replaced with BAPI_USER_ACTGROUPS_ASSIGN */
            /*"BAPI_USER_PROFILES_DELETE" */ /* replaced with BAPI_USER_PROFILES_ASSIGN, NON CUA landscape */
            /*"BAPI_USER_DISPLAY",*/ /*Don't need, using BAPI_USER_GET_DETAIL */
            /*"SUSR_BAPI_USER_LOCK", "BAPI_USER_CREATE" */ /*DO NOT USE !*/
    };

    // SAP roles, groups, profiles
    private static final String AGR_NAME = "AGR_NAME";
    public static final String ACTIVITYGROUPS = "ACTIVITYGROUPS";
    public static final String ACTIVITYGROUPS__ARG_NAME = ACTIVITYGROUPS + SEPARATOR + AGR_NAME;

    private static final String BAPIPROF = "BAPIPROF";
    public static final String PROFILES = "PROFILES";
    public static final String PROFILES_BAPIPROF = PROFILES + SEPARATOR + BAPIPROF;
    public static final String PROFILE_NAME = "PROFILE"; // also as objectClass name

    private static final String USERGROUP = "USERGROUP";
    public static final String GROUPS = "GROUPS";
    public static final String GROUPS_USERGROUP = GROUPS + SEPARATOR + USERGROUP;

    // see for example http://www.sapdatasheet.org/abap/func/BAPI_USER_GET_DETAIL.html
    // these "Paremeter name"-s we can read and write in Type "Exporting"
    private static final String[] READ_WRITE_PARAMETERS = {"ADDRESS", "DEFAULTS", "UCLASS", "LOGONDATA", "ALIAS", "COMPANY", "REF_USER"};
    // these "Paremeter name"-s we can only read (don't have appropirade parameters in BAPI_USER_CHANGE)
    static final String[] READ_ONLY_PARAMETERS = {"ISLOCKED", "LASTMODIFIED", "SNC", "ADMINDATA", "IDENTITY"};
    // variable version of READ_ONLY_PARAMETERS that could be set through connector configuration
    private String[] readOnlyParams;
    // these attributes in "ADDRESS" parameter name we can't update, only set, because "ADDRESSX" in BAPI_USER_CHANGE don't contains these fields
    private static final String[] CREATE_ONLY_ATTRIBUTES = {"ADDRESS" + SEPARATOR + "COUNTY_CODE", "ADDRESS" + SEPARATOR + "COUNTY",
            "ADDRESS" + SEPARATOR + "TOWNSHIP_CODE", "ADDRESS" + SEPARATOR + "TOWNSHIP", "DEFAULTS" + SEPARATOR + "CATTKENNZ"};


    // supported tables reading & writing in tables parameter type
    // "UCLASSSYS", "EXTIDHEAD", "EXTIDPART", "SYSTEMS" not supported yet
    public static final String[] TABLETYPE_PARAMETER_LIST = {"PARAMETER", PROFILES, ACTIVITYGROUPS,
            "RETURN", "ADDTEL", "ADDFAX", "ADDTTX", "ADDTLX", "ADDSMTP", "ADDRML", "ADDX400", "ADDRFC", "ADDPRT", "ADDSSF",
            "ADDURI", "ADDPAG", "ADDCOMREM", "PARAMETER1", GROUPS/*, "UCLASSSYS", "EXTIDHEAD", "EXTIDPART", "SYSTEMS"*/};

    // these tables on update use "change flag" explicitly as is
    private static final List<String> TABLES_WITH_CHANGE_FLAG = Arrays.asList(GROUPS, "PARAMETER", "PARAMETER1");
    // these communication data tables on update use "change flag" in "ADDCOMX"
    private static final List<String> COMMUNICATION_DATA_TABLES_WITH_CHANGE_FLAG = Arrays.asList("ADDTEL",
            "ADDFAX", "ADDTTX", "ADDTLX", "ADDSMTP", "ADDRML", "ADDX400", "ADDRFC", "ADDPRT", "ADDSSF", "ADDURI", "ADDPAG", "ADDCOMREM");

    // with these tables we can manipulate as XML line with all his attributes (for example ACTIVITYGROUPS)
    // and also as only string keys (for example ACTIVITYGROUPS.AGR_NAME) - used in midPoint associations
    public static final Map<String, String> TABLETYPE_PARAMETER_KEYS = new HashMap<>();

    static {
        TABLETYPE_PARAMETER_KEYS.put(ACTIVITYGROUPS, AGR_NAME);
        TABLETYPE_PARAMETER_KEYS.put(PROFILES, BAPIPROF);
        TABLETYPE_PARAMETER_KEYS.put(GROUPS, USERGROUP);
    }

    // LOGONDATA
    private static final String GLTGV = "GLTGV";        //User valid from, AttributeInfo ENABLE_DATE, "LOGONDATA.GLTGV"
    private static final String GLTGB = "GLTGB";        //User valid to, AttributeInfo DISABLE_DATE; "LOGONDATA.GLTGB"

    //ISLOCKED
    private static final String LOCAL_LOCK = "LOCAL_LOCK";     // Local_Lock - Logon generally locked for the local system
    private static final String WRNG_LOGON = "WRNG_LOGON";     // WRNG_LOGON - Password logon locked by incorrect logon attempts

    public static final String USERNAME = "USERNAME";

    // PASSWORD
    private static final String BAPIPWD = "<PASSWORD>";

    //USER_LOGIN_INFO prefix
    private static final String USER_LOGIN_INFO = "USER_LOGIN_INFO";

    public static final SimpleDateFormat SAP_DF = new SimpleDateFormat("yyyy-MM-dd");
    public static final SimpleDateFormat DATE_TIME = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

    private static final String SELECT = "X";

    private SapConfiguration configuration;
    private JCoDestination destination;
    private Map<String, Integer> sapAttributesLength = new HashMap<String, Integer>();
    private Map<String, String> sapAttributesType = new HashMap<String, String>();

    @Override
    public Configuration getConfiguration() {
        return configuration;
    }

    @Override
    public void init(Configuration configuration) {
        this.configuration = (SapConfiguration) configuration;
        LOG.info("Initialization start, configuration: {0}", this.configuration.toString());

        CustomDestinationDataProvider myProvider = CustomDestinationDataProvider.getInstance();

        //register the provider with the JCo environment;
        //catch IllegalStateException if an instance is already registered
        try {
            if (!com.sap.conn.jco.ext.Environment.isDestinationDataProviderRegistered()) {
                com.sap.conn.jco.ext.Environment.registerDestinationDataProvider(myProvider);
            }
        } catch (IllegalStateException providerAlreadyRegisteredException) {
            //somebody else registered its implementation,
            //stop the execution
            throw new ConnectorIOException(providerAlreadyRegisteredException.getMessage(), providerAlreadyRegisteredException);
        }

        //properties for destination from config
        Properties props = this.configuration.getDestinationProperties();

        String destinationName = this.configuration.getFinalDestinationName();

        Properties destProps = myProvider.getDestinationProperties(destinationName);
        if (destProps == null || !destProps.equals(props)){
            myProvider.setDestinationProperties(destinationName, props);
        }
        // set read only parameters from gui connector configuration
        readOnlyParams =  this.configuration.getReadOnlyParams();

        if (this.configuration.SNC_MODE_ON.equals(this.configuration.getSncMode())) {
            createDestinationDataFile(destinationName, props);
        }

        // create destination & ping it
        try {
            this.destination = JCoDestinationManager.getDestination(destinationName);
            this.destination.ping();
        } catch (JCoException e) {
            throw new ConnectorIOException(e.getMessage(), e);
        }

        // validate & initialize tables
        configuration.validate();
        // read schema
        schema();

        LOG.info("Initialization finished");
    }


    @Override
    public void checkAlive() {
        if (this.destination == null || !this.destination.isValid()) {
            LOG.ok("check alive: FAILED");
            throw new ConnectorException("Connection check failed");
        }
        try {
            this.destination.ping();
        } catch (JCoException e) {
            LOG.ok("connection ping FAILED: "+e);
            throw new ConnectorException("Connection ping failed", e);
        }
        LOG.ok("check alive: OK");
    }


    private void createDestinationDataFile(String destinationName, Properties connectProperties)
    {
        String fileName = destinationName+".jcoDestination";
        try
        {
            File destCfg = new File(fileName);
            destCfg.deleteOnExit(); // only temporary files
            FileOutputStream fos = new FileOutputStream(destCfg,false);
            connectProperties.store(fos, "for tests only !");
            fos.close();
            LOG.ok("destination file was created: "+destCfg.getName()+", "+destCfg.getAbsolutePath());
        }
        catch (Exception e)
        {
            throw new RuntimeException("Unable to create the destination file: "+fileName, e);
        }
    }

    private void deleteDestinationDataFile(String destinationName)
    {
        String fileName = destinationName+".jcoDestination";
        try
        {
            File destCfg = new File(fileName);
            boolean deleted = destCfg.delete();
            if (deleted) {
                LOG.ok("destination file was deleted: " + destCfg.getName() + ", " + destCfg.getAbsolutePath());
            }
        }
        catch (Exception e)
        {
            LOG.warn("Unable to delete the destination file: "+fileName, e);
        }
    }

    @Override
    public void dispose() {
        this.configuration = null;
        if ((this.destination != null) && (JCoContext.isStateful(this.destination))) {
            if (this.configuration.SNC_MODE_ON.equals(this.configuration.getSncMode())) {
                deleteDestinationDataFile(this.destination.getDestinationName());
            }
            try {
                JCoContext.end(this.destination);
            } catch (JCoException jcoe) {
                throw new ConnectorIOException(jcoe.getMessage(), jcoe);
            }
        }
    }

    @Override
    public void test() {
        try {
            this.destination.ping();
            if (configuration.getTestBapiFunctionPermission()) {
                List<String> notFoundFunctions = new LinkedList<String>();
                for (String function : BAPI_FUNCTION_LIST) {
                    if (!configuration.getUseTransaction() && function.contains("_TRANSACTION_")) {
                        continue;
                    }
                    JCoFunction jcoFunc = this.destination.getRepository().getFunction(function);
                    if (jcoFunc == null)
                        notFoundFunctions.add(function);
                }
                if (notFoundFunctions.size() > 0) {
                    throw new ConfigurationException("these BAPI functions are not accessible: " + notFoundFunctions);
                }
                // testing creation of transaction
                if (configuration.getUseTransaction()) {
                    JCoContext.begin(destination);
                    JCoContext.end(destination);
                }
            }
        } catch (JCoException e) {
            throw new ConfigurationException(e.getMessage(), e);
        }
    }

    @Override
    public Schema schema() {
        SchemaBuilder builder = new SchemaBuilder(SapConnector.class);

        buildAccountObjectClass(builder);

        buildTableObjectClasses(builder);

        buildProfileObjectClass(builder);

        return builder.build();
    }

    private void buildAccountObjectClass(SchemaBuilder builder) {
        ObjectClassInfoBuilder objClassBuilder = new ObjectClassInfoBuilder();

        try {
            String function = "BAPI_USER_GET_DETAIL";
            getSchemaFromBapiFunction(function, READ_WRITE_PARAMETERS, objClassBuilder, false);
            getSchemaFromBapiFunction(function, readOnlyParams, objClassBuilder, true);
        } catch (Exception e) {
            throw new ConnectorIOException("Error when parse user schema from SAP: " + e, e);
        }
        // __NAME__ and __UID__ is default and was renamed to USERNAME in schema if it's needed
        AttributeInfoBuilder uidAib = new AttributeInfoBuilder(Uid.NAME);
        uidAib.setRequired(true);
        if (this.configuration.getUseNativeNames()) {
            uidAib.setNativeName(USERNAME);
        }
        objClassBuilder.addAttributeInfo(uidAib.build());

        AttributeInfoBuilder nameAib = new AttributeInfoBuilder(Name.NAME);
        if (this.configuration.getUseNativeNames()) {
            nameAib.setNativeName(USERNAME);
        }
        objClassBuilder.addAttributeInfo(nameAib.build());

        sapAttributesType.put(USERNAME, "java.lang.String");
        sapAttributesLength.put(USERNAME, 12);

        objClassBuilder.addAttributeInfo(OperationalAttributeInfos.ENABLE);     // enable / disable - ISLOCKED.LOCAL_LOCK
        objClassBuilder.addAttributeInfo(OperationalAttributeInfos.LOCK_OUT);   // unlock - ISLOCKED.WRNG_LOGON
        objClassBuilder.addAttributeInfo(OperationalAttributeInfos.ENABLE_DATE); // LOGONDATA.GLTGV
        objClassBuilder.addAttributeInfo(OperationalAttributeInfos.DISABLE_DATE); //LOGONDATA.GLTGB

        AttributeInfoBuilder passwordAIB = new AttributeInfoBuilder(OperationalAttributes.PASSWORD_NAME, GuardedString.class);
        passwordAIB.setReadable(false); // write only
        passwordAIB.setReturnedByDefault(false);
        objClassBuilder.addAttributeInfo(passwordAIB.build());
        sapAttributesType.put(BAPIPWD, "java.lang.String");
        sapAttributesLength.put(BAPIPWD, 40);

        // tables
        for (String table : configuration.getTableParameterNames()) {
            AttributeInfoBuilder attributeActivityGroup = new AttributeInfoBuilder(table);
            attributeActivityGroup.setMultiValued(true);
            objClassBuilder.addAttributeInfo(attributeActivityGroup.build());
            // id's from tables
            if (TABLETYPE_PARAMETER_KEYS.containsKey(table)) {
                AttributeInfoBuilder attributeActivityGroupIds = new AttributeInfoBuilder(table + SEPARATOR + TABLETYPE_PARAMETER_KEYS.get(table));
                attributeActivityGroupIds.setMultiValued(true);
                objClassBuilder.addAttributeInfo(attributeActivityGroupIds.build());
            }
        }
        // user_login_infos
        if (this.configuration.getAlsoReadLoginInfo()) {
            boolean readOnly = true;
            objClassBuilder.addAttributeInfo(createAttributeInfo(null, USER_LOGIN_INFO + SEPARATOR + "LAST_LOGON_DATE", Long.class, readOnly));
            objClassBuilder.addAttributeInfo(createAttributeInfo(null, USER_LOGIN_INFO + SEPARATOR + "LOCK_STATUS", String.class, readOnly));
            objClassBuilder.addAttributeInfo(createAttributeInfo(null, USER_LOGIN_INFO + SEPARATOR + "PASSWORD_STATUS", String.class, readOnly));
        }

        builder.defineObjectClass(objClassBuilder.build());
    }

    private void buildTableObjectClasses(SchemaBuilder builder) {
        for (Map.Entry<String, Map<String, Integer>> table : configuration.getTableMetadatas().entrySet()) {
            String tableName = table.getKey();
            Map<String, Integer> columnsMetadata = table.getValue();

            ObjectClassInfoBuilder objClassBuilder = new ObjectClassInfoBuilder();
            objClassBuilder.setType(configuration.getTableAliases().get(tableName));

            for (Map.Entry<String, Integer> column : columnsMetadata.entrySet()) {
                String columnName = column.getKey();
                AttributeInfoBuilder attributeInfoBuilder = new AttributeInfoBuilder(columnName);
                attributeInfoBuilder.setCreateable(false);
                attributeInfoBuilder.setUpdateable(false);
                objClassBuilder.addAttributeInfo(attributeInfoBuilder.build());
            }

            builder.defineObjectClass(objClassBuilder.build());
        }
    }

    private void buildProfileObjectClass(SchemaBuilder builder) {
        ObjectClassInfoBuilder objClassBuilder = new ObjectClassInfoBuilder();
        objClassBuilder.setType(PROFILE_NAME);

        AttributeInfoBuilder uidAib = new AttributeInfoBuilder(Uid.NAME);
        uidAib.setRequired(true);
        uidAib.setCreateable(false);
        uidAib.setUpdateable(false);
        if (this.configuration.getUseNativeNames()) {
            uidAib.setNativeName(PROFILE_NAME);
        }
        objClassBuilder.addAttributeInfo(uidAib.build());

        AttributeInfoBuilder nameAib = new AttributeInfoBuilder(Name.NAME);
        nameAib.setCreateable(false);
        nameAib.setUpdateable(false);
        if (this.configuration.getUseNativeNames()) {
            nameAib.setNativeName(PROFILE_NAME);
        }
        objClassBuilder.addAttributeInfo(nameAib.build());

        builder.defineObjectClass(objClassBuilder.build());
    }

    private List<String> getAllStructureParameters(JCoParameterList epl) {
        List<String> parameters = new LinkedList<String>();
        JCoListMetaData lmd = epl.getListMetaData();
        for (int i = 0; i < lmd.getFieldCount(); i++) {
            if (lmd.isStructure(i)) {
                parameters.add(lmd.getName(i));
            }
        }

        return parameters;
    }

    private void getSchemaFromBapiFunction(String bapiFunction, String[] parameterList, ObjectClassInfoBuilder objClassBuilder, boolean readOnly) throws JCoException, ClassNotFoundException {
        JCoFunction function = destination.getRepository().getFunction(bapiFunction);
        if (function == null)
            throw new RuntimeException(bapiFunction + " not found in SAP.");

        List<String> parameters = null;
        JCoParameterList epl = function.getExportParameterList();
        if (parameterList == null) {
            LOG.ok("reading all structure parameters from ExportParameterList");
            parameters = getAllStructureParameters(epl);
        } else {
            parameters = Arrays.asList(parameterList);
        }

        for (String param : parameters) {
            JCoStructure structure = epl.getStructure(param);
            JCoRecordMetaData rmd = structure.getRecordMetaData();
//            LOG.ok("STRUCTURE for " + param + ":");
            for (int r = 0; r < rmd.getFieldCount(); r++) {
                String name = rmd.getName(r);
                Integer length = rmd.getLength(r);
                String attrName = param + SEPARATOR + name;
                String className = rmd.getClassNameOfField(r);
                Class classs = null;
                try {
                    classs = Class.forName(className);
                } catch (ClassNotFoundException cnfe) {
                    if (!"byte[]".equals(className)) {
                        LOG.warn("Not supported class type: " + attrName + "\t" + length + "\t" + className + "\t" + rmd.getRecordTypeName(r) + "\t" + rmd.getTypeAsString(r) + ", ex: " + cnfe);
                    }
                }
                if (classs != null && FrameworkUtil.isSupportedAttributeType(classs)) {
                    objClassBuilder.addAttributeInfo(createAttributeInfo(param, attrName, classs, readOnly));
                } else if ("java.util.Date".equals(className)) {
                    objClassBuilder.addAttributeInfo(createAttributeInfo(param, attrName, Long.class, readOnly));
                    LOG.ok(className + " simulated as java.lang.Long over connector for: " + attrName);
                } else if ("byte[]".equals(className)) {
                    objClassBuilder.addAttributeInfo(createAttributeInfo(param, attrName, byte[].class, readOnly));
                } else {
                    objClassBuilder.addAttributeInfo(createAttributeInfo(param, attrName, String.class, readOnly));
                    LOG.warn("TODO: implement better support for " + className + ", attribute " + attrName + " if you need it, I'm using java.lang.String");
                }
                this.sapAttributesLength.put(attrName, length);
                this.sapAttributesType.put(attrName, className);

//                LOG.ok(attrName + "\t" + length + "\t" + className + "\t" + rmd.getRecordTypeName(r) + "\t" + rmd.getTypeAsString(r));
            }
        }
    }


    private AttributeInfo createAttributeInfo(String structure, String attrName, Class classs, boolean readOnly) {
        AttributeInfoBuilder attributeInfoBuilder = new AttributeInfoBuilder(attrName, classs);
        if (readOnly) {
            attributeInfoBuilder.setCreateable(false);
            attributeInfoBuilder.setUpdateable(false);
        } else if (contains(CREATE_ONLY_ATTRIBUTES, structure + SEPARATOR + attrName)) {
            attributeInfoBuilder.setCreateable(true);
            attributeInfoBuilder.setUpdateable(false);
        }

        return attributeInfoBuilder.build();
    }

    @Override
    public FilterTranslator<SapFilter> createFilterTranslator(ObjectClass objectClass, OperationOptions operationOptions) {
        if (objectClass.is(ObjectClass.ACCOUNT_NAME)) {
            return new SapAccountFilterTranslator();
        }
        else {
            return new SapBasicFilterTranslator();
        }
    }

    @Override
    public void executeQuery(ObjectClass objectClass, SapFilter query, ResultsHandler handler, OperationOptions options) {
        LOG.info("executeQuery: {0}, options: {1}, objectClass: {2}", query, options, objectClass);

        if (objectClass.is(ObjectClass.ACCOUNT_NAME)) {

            executeAccountQuery(query, handler, options);

        } else if (objectClass.is(PROFILE_NAME)) {

            executeProfileQuery(query, handler);

        } else {
            String found = null;
            for (String tableName : configuration.getTableAliases().keySet()) {
                String tableAlias = configuration.getTableAliases().get(tableName);
                if (objectClass.is(tableAlias)) {
                    found = tableName;
                }
            }

            if (found == null) {
                throw new UnsupportedOperationException("Unsupported object class " + objectClass + ", expected: " + configuration.getTableMetadatas());
            }

            executeTableQuery(found, query, handler);
        }

    }

    private void executeProfileQuery(SapFilter query, ResultsHandler handler) {
        try {
            // find all or find by key

            JCoFunction function = destination.getRepository().getFunction("BAPI_HELPVALUES_GET");
            if (function == null)
                throw new RuntimeException("BAPI_HELPVALUES_GET not found in SAP.");

            function.getImportParameterList().setValue("OBJNAME", "USER");
            function.getImportParameterList().setValue("METHOD", "ProfilesAssign");
            function.getImportParameterList().setValue("PARAMETER", "Profiles");
            function.getImportParameterList().setValue("FIELD", BAPIPROF);

            function.execute(destination);

            JCoTable entries = function.getTableParameterList().getTable("VALUES_FOR_FIELD");


            entries.firstRow();
            LOG.ok("Number of entries in input: {0}, filter: {1}", entries.getNumRows(), query != null ? query.getBasicByNameEquals() : "(empty)");
            if (entries.getNumRows() > 0) {
                do {
                    String profile = entries.getString("VALUES");

                    if (query != null && query.getBasicByNameEquals() != null && !profile.equalsIgnoreCase(query.getBasicByNameEquals())) {
                        // TODO: case sensitive or not?
                        // not matched, ignore this
                        continue;
                    }

                    ConnectorObjectBuilder builder = new ConnectorObjectBuilder();
                    builder.setUid(profile);
                    builder.setName(profile);

                    ObjectClass objectClass = new ObjectClass(PROFILE_NAME);
                    builder.setObjectClass(objectClass);

                    ConnectorObject build = builder.build();
                    LOG.ok("ConnectorObject: {0}", build);
                    handler.handle(build);
                } while (entries.nextRow());
            }
        } catch (JCoException e) {
            throw new ConnectorIOException(e.getMessage(), e);
        }
    }

    private void executeTableQuery(String tableName, SapFilter query, ResultsHandler handler) {
        try {
            // find all or find by key

            JCoFunction function = destination.getRepository().getFunction("RFC_GET_TABLE_ENTRIES");
            if (function == null)
                throw new RuntimeException("RFC_GET_TABLE_ENTRIES not found in SAP.");

            function.getImportParameterList().setValue("TABLE_NAME", tableName);
            // find by key
            if (query != null && query.getBasicByNameEquals() != null) {
                function.getImportParameterList().setValue("GEN_KEY", query.getBasicByNameEquals());
                LOG.ok("query by Key: " + query.getBasicByNameEquals() + " on table: " + tableName);
            }
            // else find all

            function.execute(destination);

            JCoTable entries = function.getTableParameterList().getTable("ENTRIES");

            LOG.info("Entries: " + function.getExportParameterList().getValue("NUMBER_OF_ENTRIES"));

            entries.firstRow();
            if (entries.getNumRows() > 0) {
                do {
                    String value = entries.getString("WA");
                    int index = 0;

                    ConnectorObjectBuilder builder = new ConnectorObjectBuilder();
                    List<String> keys = new LinkedList<String>();

                    for (Map.Entry<String, Integer> entry : configuration.getTableMetadatas().get(tableName).entrySet()) {
                        String column = entry.getKey();
                        Integer length = entry.getValue();

                        String columnValue = value.substring(index, index + length).trim();
                        // ignore columns, what is selected as :IGNORE
                        if (!configuration.getTableIgnores().get(tableName).contains(column)) {
                            addAttr(builder, column, columnValue);
                        }
                        if (configuration.getTableKeys().get(tableName).contains(column)) {
                            keys.add(columnValue);
                        }

                        index += length;
                    }

                    StringBuilder concatenatedKey = new StringBuilder();
                    for (String key : keys) {
                        if (concatenatedKey.length() != 0) {
                            concatenatedKey.append(":");
                        }
                        concatenatedKey.append(key);
                    }
                    if (StringUtil.isEmpty(concatenatedKey.toString())) {
                        LOG.warn("ignoring empty key: " + concatenatedKey);
                        continue;
                    }

                    builder.setUid(concatenatedKey.toString());
                    builder.setName(concatenatedKey.toString());

                    ObjectClass objectClass = new ObjectClass(configuration.getTableAliases().get(tableName));
                    builder.setObjectClass(objectClass);

                    ConnectorObject build = builder.build();
                    LOG.ok("ConnectorObject: {0}", build);
                    handler.handle(build);

                } while (entries.nextRow());
            }
        } catch (JCoException e) {
            //there is no other way of checking this
            if("TABLE_EMPTY".equals(e.getKey()))
                return;
            throw new ConnectorIOException(e.getMessage(), e);
        }
    }

    private void executeAccountQuery(SapFilter query, ResultsHandler handler, OperationOptions options) {
        try {
            // find by NAME (or UID - same as name)
            if (query != null && query.byNameEquals() != null) {

                JCoFunction function = destination.getRepository().getFunction("BAPI_USER_GET_DETAIL");
                function.getImportParameterList().setValue(USERNAME, query.byNameEquals());
                executeFunction(function);

                JCoFunction userLoginInfoFunc = runUserLoginInfoFunction(query.byNameEquals());

                ConnectorObject connectorObject = convertUserToConnectorObject(function, userLoginInfoFunc);
                handler.handle(connectorObject);

            } // find by name contains
            else if (query != null && query.byNameContains() != null) {
                JCoFunction function = destination.getRepository().getFunction("BAPI_USER_GETLIST");
                JCoTable exp = function.getTableParameterList().getTable("SELECTION_EXP");
                exp.appendRow();
                exp.setValue("PARAMETER", USERNAME);
                exp.setValue("OPTION", "CP");
                exp.setValue("LOW", query.byNameContains());

                executeFunction(function);

{'fraction_non_alphanumeric': 0.0989208352204401, 'fraction_numerical': 0.001374334306820134, 'mean_word_length': 2.5728155339805827, 'pattern_counts': {'":': 4, '<': 55, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2143429', 'n_tokens_mistral': 16829, 'n_tokens_neox': 16417, 'n_words': 3666}