<commit_before>/*
 *   Copyright 2014 OSBI Ltd
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

package org.saiku.service.datasource;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.SystemUtils;
import org.saiku.database.dto.MondrianSchema;
import org.saiku.datasources.connection.IConnectionManager;
import org.saiku.datasources.connection.RepositoryFile;
import org.saiku.datasources.datasource.SaikuDatasource;
import org.saiku.repository.*;
import org.saiku.service.importer.JujuSource;
import org.saiku.service.importer.LegacyImporter;
import org.saiku.service.importer.LegacyImporterImpl;
import org.saiku.service.user.UserService;
import org.saiku.service.util.exception.SaikuServiceException;

import org.saiku.service.util.security.authentication.PasswordProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.core.session.SessionRegistry;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;

import javax.jcr.PathNotFoundException;
import javax.jcr.RepositoryException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpSession;

/**
 * A Datasource Manager for the Saiku Repository API layer.
 */
public class RepositoryDatasourceManager implements IDatasourceManager {
    private static final String ORBIS_WORKSPACE_DIR = "workspace";
    private final Map<String, SaikuDatasource> datasources =
            Collections.synchronizedMap(new HashMap<String, SaikuDatasource>());


    public IConnectionManager connectionManager;
    private ScopedRepo sessionRegistry;
    private boolean workspaces;

    public void setConnectionManager(IConnectionManager connectionManager) {
        this.connectionManager = connectionManager;
    }

    private UserService userService;
    private static final Logger log = LoggerFactory.getLogger(RepositoryDatasourceManager.class);
    private String configurationpath;
    private String datadir;
    private IRepositoryManager irm;
    private String foodmartdir;
    private String foodmartschema;
    private String foodmarturl;
    private PasswordProvider repopasswordprovider;
    private String oldpassword;
    private String earthquakeurl;
    private String earthquakedir;
    private String earthquakeschema;
    private String defaultRole;
    private String externalparameters;
    private String type;
    private String separator = "/";
    private String host;
    private String port;
    private String username;
    private String password;
    private String database;

    public void load() {
        Properties ext = checkForExternalDataSourceProperties();

        if (type.equals("marklogic")) {
            irm = MarkLogicRepositoryManager.getMarkLogicRepositoryManager(host, Integer.parseInt(port), username, password, database, cleanse(datadir));
        } else if (type.equals("classpath")) {
            separator = "/";
            log.debug("init datadir= "+datadir);
            irm = ClassPathRepositoryManager.getClassPathRepositoryManager(cleanse(datadir), defaultRole, sessionRegistry, workspaces);
            log.debug("2nd init datadir= "+datadir);
        } else {
            irm = JackRabbitRepositoryManager.getJackRabbitRepositoryManager(configurationpath, datadir, repopasswordprovider.getPassword(),
                    oldpassword, defaultRole);
        }

        try {
            irm.start(userService);
            this.saveInternalFile("/etc" + separator + ".repo_version", "d20f0bea-681a-11e5-9d70-feff819cdc9f", null);
        } catch (RepositoryException e) {
            log.error("Could not start repo", e);
        }

        datasources.clear();


            List<DataSource> exporteddatasources = null;
            try {
                exporteddatasources = irm.getAllDataSources();
            } catch (RepositoryException e1) {
                log.error("Could not export data sources", e1);
            }

            if (exporteddatasources != null) {
                int i = 0;
                while(i < exporteddatasources.size()){
                    DataSource file = exporteddatasources.get(i);
                    try {
                        if (file.getName() != null && file.getType() != null) {
                            Properties props = new Properties();
                            if (file.getDriver() != null) {
                                props.put("driver", file.getDriver());
                            } else if (file.getPropertyKey() != null && ext.containsKey("datasource." + file.getPropertyKey() + ".driver")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".driver");
                                props.put("driver", p);
                            }
                            if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".location")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".location");
                                if (ext.containsKey("datasource." + file.getPropertyKey() + ".schemaoverride")) {
                                    String[] spl = p.split(";");
                                    spl[1] = "Catalog=mondrian://" + file.getSchema();
                                    StringBuilder sb = new StringBuilder();
                                    for (String str : spl) {
                                        sb.append(str + ";");
                                    }
                                    props.put("location", sb.toString());
                                } else {
                                    props.put("location", p);
                                }
                            } else if (file.getLocation() != null) {
                                props.put("location", file.getLocation());
                            }
                            if (file.getUsername() != null && file.getPropertyKey() == null) {
                                props.put("username", file.getUsername());
                            } else if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".username")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".username");
                                props.put("username", p);
                            }
                            if (file.getPassword() != null && file.getPropertyKey() == null) {
                                props.put("password", file.getPassword());
                            } else if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".password")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".password");
                                props.put("password", p);
                            }
                            if (file.getPath() != null) {
                                props.put("path", file.getPath());
                            } else if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".path")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".path");
                                props.put("path", p);
                            }
                            if (file.getId() != null) {
                                props.put("id", file.getId());
                            }
                            if (file.getSecurityenabled() != null) {
                                props.put("security.enabled", file.getSecurityenabled());
                            } else if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".security.enabled")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".security.enabled");
                                props.put("security.enabled", p);
                            }
                            if (file.getSecuritytype() != null) {
                                props.put("security.type", file.getSecuritytype());
                            } else if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".security.type")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".security.type");
                                props.put("security.type", p);
                            }
                            if (file.getSecuritymapping() != null) {
                                props.put("security.mapping", file.getSecuritymapping());
                            } else if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".security.mapping")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".security.mapping");
                                props.put("security.mapping", p);
                            }
                            if (file.getAdvanced() != null) {
                                props.put("advanced", file.getAdvanced());
                            }
                            if (file.getCsv() != null) {
                                props.put("csv", file.getCsv());
                            }
                            if (file.getEnabled() != null) {
                                props.put("enabled", file.getEnabled());
                            }
                            if (file.getPropertyKey() != null) {
                                props.put("propertykey", file.getPropertyKey());
                            }
                            SaikuDatasource.Type t = SaikuDatasource.Type.valueOf(file.getType().toUpperCase());
                            SaikuDatasource ds = new SaikuDatasource(file.getName(), t, props);
                            datasources.put(file.getName(), ds);
                        }
                    }
                        catch(Exception e){
                            //throw new SaikuServiceException("Failed to add datasource", e);
                            log.error("Failed to add datasource", e);
                        }
                    i++;


                }
            }



    }

    public Properties checkForExternalDataSourceProperties() {
        Properties p = new Properties();
        InputStream input;

        try {
            input = new FileInputStream(externalparameters);
            p.load(input);
        } catch (IOException e) {
            log.debug("file did not exist");
        }

        return p;

    }

    public String[] getAvailablePropertiesKeys() {
        Properties p = new Properties();
        InputStream input;

        try {
            input = new FileInputStream(externalparameters);
            p.load(input);
        } catch (IOException e) {
            log.debug("file did not exist");
        }

        String[] arr = p.keySet().toArray(new String[p.keySet().size()]);

        ArrayList<String> newlist = new ArrayList<>();
        for (String str : arr) {
            String[] s = str.split("\\.");
            newlist.add(s[1]);
        }
        Set<String> unique = new HashSet<>(newlist);

        return unique.toArray(new String[unique.size()]);
    }

    public void unload() {
        irm.shutdown();
    }

    public SaikuDatasource addDatasource(SaikuDatasource datasource) throws Exception {
        DataSource ds = new DataSource(datasource);

        if (ds.getCsv() != null && ds.getCsv().equals("true")) {
            if(this.workspaces) {
                String s = this.getworkspacedir();
                if (s.endsWith("/")) {
                    s = s.substring(0, s.length() - 1);
                }
                if (ds.getName().startsWith(s)) {
                    ds.setName(ds.getName().replace(s + "_", ""));
                }
            }
            String split[] = ds.getLocation().split("=");
            String loc = split[2];
            if(split[2].startsWith("mondrian:")){
                split[2] = "mondrian:/"+getDatadir() + "/datasources/" + ds.getName() + "-csv.json;Catalog";
            }
            else {
                split[2] = getDatadir() + "/datasources/" + ds.getName() + "-csv.json;Catalog";
            }
            for (int i = 0; i < split.length - 1; i++) {
                split[i] = split[i] + "=";
            }
            ds.setLocation(StringUtils.join(split));

            log.debug("LOC IS: " + loc);
            String path = loc.substring(0, loc.lastIndexOf(";"));

            log.debug("PATH IS: " + path);
            path = path.replace("\\", "/");

            log.debug("Trimmed path is: " + path);
            if(!datadir.equals("${CLASSPATH_REPO_PATH_UNPARSED}")) {
                path = path.replaceFirst(datadir, "");
            }
            boolean f = true;
            if (new File(getDatadir() + separator + path).exists() && new File(getDatadir() + separator + path).isDirectory()) {
                f = false;
            }
            if(!path.startsWith("mondrian:")) {
                irm.saveInternalFile(this.getCSVJson(f, ds.getName(), getDatadir() + path),
                    separator + "datasources" + separator + ds.getName() + "-csv.json", null);

            }
            else{
                irm.saveInternalFile(this.getCSVJson(f, ds.getName(), path),
                    separator + "datasources" + separator + ds.getName() + "-csv.json", null);

            }
            irm.saveDataSource(ds, separator + "datasources" + separator + ds.getName() + ".sds", "fixme");
            
            String name = ds.getName();
            
            if (workspaces) {
                name = getworkspacedir().substring(0, getworkspacedir().length() - 1) + "_" + ds.getName();
            }

            // In a workspace environment it is necessary to prefix the datasource name with the workspace name
            connectionManager.refreshConnection(name);
        } else {
            irm.saveDataSource(ds, separator + "datasources" + separator + ds.getName() + ".sds", "fixme");

        }

        String name = ds.getName();
        
        if (workspaces) {
            name = getworkspacedir().substring(0, getworkspacedir().length() - 1) + "_" + ds.getName();
        }
        
        SaikuDatasource sds = new SaikuDatasource(name, SaikuDatasource.Type.OLAP, datasource.getProperties());
        
        // It stores the datasource name prefixed with the workspace name
        datasources.put(name, sds);

        return datasource;
    }

    public SaikuDatasource setDatasource(SaikuDatasource datasource) throws Exception {
        return addDatasource(datasource);
    }

    public List<SaikuDatasource> addDatasources(List<SaikuDatasource> dsources) {<commit_msg>Storing the datasource before refreshing it<commit_after>/*
 *   Copyright 2014 OSBI Ltd
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

package org.saiku.service.datasource;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.SystemUtils;
import org.saiku.database.dto.MondrianSchema;
import org.saiku.datasources.connection.IConnectionManager;
import org.saiku.datasources.connection.RepositoryFile;
import org.saiku.datasources.datasource.SaikuDatasource;
import org.saiku.repository.*;
import org.saiku.service.importer.JujuSource;
import org.saiku.service.importer.LegacyImporter;
import org.saiku.service.importer.LegacyImporterImpl;
import org.saiku.service.user.UserService;
import org.saiku.service.util.exception.SaikuServiceException;

import org.saiku.service.util.security.authentication.PasswordProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.core.session.SessionRegistry;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;

import javax.jcr.PathNotFoundException;
import javax.jcr.RepositoryException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpSession;

/**
 * A Datasource Manager for the Saiku Repository API layer.
 */
public class RepositoryDatasourceManager implements IDatasourceManager {
    private static final String ORBIS_WORKSPACE_DIR = "workspace";
    private final Map<String, SaikuDatasource> datasources =
            Collections.synchronizedMap(new HashMap<String, SaikuDatasource>());


    public IConnectionManager connectionManager;
    private ScopedRepo sessionRegistry;
    private boolean workspaces;

    public void setConnectionManager(IConnectionManager connectionManager) {
        this.connectionManager = connectionManager;
    }

    private UserService userService;
    private static final Logger log = LoggerFactory.getLogger(RepositoryDatasourceManager.class);
    private String configurationpath;
    private String datadir;
    private IRepositoryManager irm;
    private String foodmartdir;
    private String foodmartschema;
    private String foodmarturl;
    private PasswordProvider repopasswordprovider;
    private String oldpassword;
    private String earthquakeurl;
    private String earthquakedir;
    private String earthquakeschema;
    private String defaultRole;
    private String externalparameters;
    private String type;
    private String separator = "/";
    private String host;
    private String port;
    private String username;
    private String password;
    private String database;

    public void load() {
        Properties ext = checkForExternalDataSourceProperties();

        if (type.equals("marklogic")) {
            irm = MarkLogicRepositoryManager.getMarkLogicRepositoryManager(host, Integer.parseInt(port), username, password, database, cleanse(datadir));
        } else if (type.equals("classpath")) {
            separator = "/";
            log.debug("init datadir= "+datadir);
            irm = ClassPathRepositoryManager.getClassPathRepositoryManager(cleanse(datadir), defaultRole, sessionRegistry, workspaces);
            log.debug("2nd init datadir= "+datadir);
        } else {
            irm = JackRabbitRepositoryManager.getJackRabbitRepositoryManager(configurationpath, datadir, repopasswordprovider.getPassword(),
                    oldpassword, defaultRole);
        }

        try {
            irm.start(userService);
            this.saveInternalFile("/etc" + separator + ".repo_version", "d20f0bea-681a-11e5-9d70-feff819cdc9f", null);
        } catch (RepositoryException e) {
            log.error("Could not start repo", e);
        }

        datasources.clear();


            List<DataSource> exporteddatasources = null;
            try {
                exporteddatasources = irm.getAllDataSources();
            } catch (RepositoryException e1) {
                log.error("Could not export data sources", e1);
            }

            if (exporteddatasources != null) {
                int i = 0;
                while(i < exporteddatasources.size()){
                    DataSource file = exporteddatasources.get(i);
                    try {
                        if (file.getName() != null && file.getType() != null) {
                            Properties props = new Properties();
                            if (file.getDriver() != null) {
                                props.put("driver", file.getDriver());
                            } else if (file.getPropertyKey() != null && ext.containsKey("datasource." + file.getPropertyKey() + ".driver")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".driver");
                                props.put("driver", p);
                            }
                            if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".location")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".location");
                                if (ext.containsKey("datasource." + file.getPropertyKey() + ".schemaoverride")) {
                                    String[] spl = p.split(";");
                                    spl[1] = "Catalog=mondrian://" + file.getSchema();
                                    StringBuilder sb = new StringBuilder();
                                    for (String str : spl) {
                                        sb.append(str + ";");
                                    }
                                    props.put("location", sb.toString());
                                } else {
                                    props.put("location", p);
                                }
                            } else if (file.getLocation() != null) {
                                props.put("location", file.getLocation());
                            }
                            if (file.getUsername() != null && file.getPropertyKey() == null) {
                                props.put("username", file.getUsername());
                            } else if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".username")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".username");
                                props.put("username", p);
                            }
                            if (file.getPassword() != null && file.getPropertyKey() == null) {
                                props.put("password", file.<PASSWORD>());
                            } else if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".password")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".password");
                                props.put("password", p);
                            }
                            if (file.getPath() != null) {
                                props.put("path", file.getPath());
                            } else if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".path")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".path");
                                props.put("path", p);
                            }
                            if (file.getId() != null) {
                                props.put("id", file.getId());
                            }
                            if (file.getSecurityenabled() != null) {
                                props.put("security.enabled", file.getSecurityenabled());
                            } else if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".security.enabled")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".security.enabled");
                                props.put("security.enabled", p);
                            }
                            if (file.getSecuritytype() != null) {
                                props.put("security.type", file.getSecuritytype());
                            } else if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".security.type")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".security.type");
                                props.put("security.type", p);
                            }
                            if (file.getSecuritymapping() != null) {
                                props.put("security.mapping", file.getSecuritymapping());
                            } else if (file.getPropertyKey() != null &&
                                    ext.containsKey("datasource." + file.getPropertyKey() + ".security.mapping")) {
                                String p = ext.getProperty("datasource." + file.getPropertyKey() + ".security.mapping");
                                props.put("security.mapping", p);
                            }
                            if (file.getAdvanced() != null) {
                                props.put("advanced", file.getAdvanced());
                            }
                            if (file.getCsv() != null) {
                                props.put("csv", file.getCsv());
                            }
                            if (file.getEnabled() != null) {
                                props.put("enabled", file.getEnabled());
                            }
                            if (file.getPropertyKey() != null) {
                                props.put("propertykey", file.getPropertyKey());
                            }
                            SaikuDatasource.Type t = SaikuDatasource.Type.valueOf(file.getType().toUpperCase());
                            SaikuDatasource ds = new SaikuDatasource(file.getName(), t, props);
                            datasources.put(file.getName(), ds);
                        }
                    }
                        catch(Exception e){
                            //throw new SaikuServiceException("Failed to add datasource", e);
                            log.error("Failed to add datasource", e);
                        }
                    i++;


                }
            }



    }

    public Properties checkForExternalDataSourceProperties() {
        Properties p = new Properties();
        InputStream input;

        try {
            input = new FileInputStream(externalparameters);
            p.load(input);
        } catch (IOException e) {
            log.debug("file did not exist");
        }

        return p;

    }

    public String[] getAvailablePropertiesKeys() {
        Properties p = new Properties();
        InputStream input;

        try {
            input = new FileInputStream(externalparameters);
            p.load(input);
        } catch (IOException e) {
            log.debug("file did not exist");
        }

        String[] arr = p.keySet().toArray(new String[p.keySet().size()]);

        ArrayList<String> newlist = new ArrayList<>();
        for (String str : arr) {
            String[] s = str.split("\\.");
            newlist.add(s[1]);
        }
        Set<String> unique = new HashSet<>(newlist);

        return unique.toArray(new String[unique.size()]);
    }

    public void unload() {
        irm.shutdown();
    }

    public SaikuDatasource addDatasource(SaikuDatasource datasource) throws Exception {
        DataSource ds = new DataSource(datasource);

        if (ds.getCsv() != null && ds.getCsv().equals("true")) {
            if(this.workspaces) {
                String s = this.getworkspacedir();
                if (s.endsWith("/")) {
                    s = s.substring(0, s.length() - 1);
                }
                if (ds.getName().startsWith(s)) {
                    ds.setName(ds.getName().replace(s + "_", ""));
                }
            }
            String split[] = ds.getLocation().split("=");
            String loc = split[2];
            if(split[2].startsWith("mondrian:")){
                split[2] = "mondrian:/"+getDatadir() + "/datasources/" + ds.getName() + "-csv.json;Catalog";
            }
            else {
                split[2] = getDatadir() + "/datasources/" + ds.getName() + "-csv.json;Catalog";
            }
            for (int i = 0; i < split.length - 1; i++) {
                split[i] = split[i] + "=";
            }
            ds.setLocation(StringUtils.join(split));

            log.debug("LOC IS: " + loc);
            String path = loc.substring(0, loc.lastIndexOf(";"));

            log.debug("PATH IS: " + path);
            path = path.replace("\\", "/");

            log.debug("Trimmed path is: " + path);
            if(!datadir.equals("${CLASSPATH_REPO_PATH_UNPARSED}")) {
                path = path.replaceFirst(datadir, "");
            }
            boolean f = true;
            if (new File(getDatadir() + separator + path).exists() && new File(getDatadir() + separator + path).isDirectory()) {
                f = false;
            }
            if(!path.startsWith("mondrian:")) {
                irm.saveInternalFile(this.getCSVJson(f, ds.getName(), getDatadir() + path),
                    separator + "datasources" + separator + ds.getName() + "-csv.json", null);

            }
            else{
                irm.saveInternalFile(this.getCSVJson(f, ds.getName(), path),
                    separator + "datasources" + separator + ds.getName() + "-csv.json", null);

            }
            irm.saveDataSource(ds, separator + "datasources" + separator + ds.getName() + ".sds", "fixme");
            
            String name = ds.getName();
            
            if (workspaces) {
                name = getworkspacedir().substring(0, getworkspacedir().length() - 1) + "_" + ds.getName();
            }
            
            // Adding the connection before refreshing it
            SaikuDatasource sds = new SaikuDatasource(name, SaikuDatasource.Type.OLAP, datasource.getProperties());
            datasources.put(name, sds);

            // In a workspace environment it is necessary to prefix the datasource name with the workspace name
            connectionManager.refreshConnection(name);
        } else {
            irm.saveDataSource(ds, separator + "datasources" + separator + ds.getName() + ".sds", "fixme");

        }

        String name = ds.getName();
        
        if (workspaces) {
            name = getworkspacedir().substring(0, getworkspacedir().length() - 1) + "_" + ds.getName();
        }
        
        SaikuDatasource sds = new SaikuDatasource(name, SaikuDatasource.Type.OLAP, datasource.getProperties());
        
        // It stores the datasource name prefixed with the workspace name
        datasources.put(name, sds);

        return datasource;
    }

    public SaikuDatasource setDatasource(SaikuDatasource datasource) throws Exception {
        return addDatasource(datasource);
    }

    public List<SaikuDatasource> addDatasources(List<SaikuDatasource> dsources) {
{'fraction_non_alphanumeric': 0.11483165396903416, 'fraction_numerical': 0.0027648070779061193, 'mean_word_length': 1.6455099553027224, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5112287', 'n_tokens_mistral': 8172, 'n_tokens_neox': 7859, 'n_words': 1701}