<commit_before>/*
 * DBeaver - Universal Database Manager
 * Copyright (C) 2010-2019 <NAME> (<EMAIL>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/*
 * Created on Jul 13, 2004
 */
package org.jkiss.dbeaver.ext.erd.model;

import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IProject;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.draw2d.AbsoluteBendpoint;
import org.eclipse.draw2d.Bendpoint;
import org.eclipse.draw2d.RelativeBendpoint;
import org.eclipse.draw2d.geometry.Point;
import org.eclipse.draw2d.geometry.Rectangle;
import org.eclipse.jface.resource.StringConverter;
import org.eclipse.swt.graphics.Color;
import org.eclipse.swt.widgets.Display;
import org.jkiss.code.Nullable;
import org.jkiss.dbeaver.DBException;
import org.jkiss.dbeaver.Log;
import org.jkiss.dbeaver.ext.erd.ERDConstants;
import org.jkiss.dbeaver.ext.erd.editor.ERDAttributeVisibility;
import org.jkiss.dbeaver.ext.erd.part.*;
import org.jkiss.dbeaver.model.*;
import org.jkiss.dbeaver.model.app.DBPProject;
import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor;
import org.jkiss.dbeaver.model.runtime.VoidProgressMonitor;
import org.jkiss.dbeaver.model.struct.*;
import org.jkiss.dbeaver.runtime.DBWorkbench;
import org.jkiss.dbeaver.ui.SharedFonts;
import org.jkiss.dbeaver.ui.UIUtils;
import org.jkiss.dbeaver.utils.GeneralUtils;
import org.jkiss.dbeaver.utils.RuntimeUtils;
import org.jkiss.utils.CommonUtils;
import org.jkiss.utils.xml.XMLBuilder;
import org.jkiss.utils.xml.XMLException;
import org.jkiss.utils.xml.XMLUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;

import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.util.*;

/**
 * Entity diagram loader/saver
 * @author <NAME>
 */
public class DiagramLoader
{
    private static final Log log = Log.getLog(DiagramLoader.class);

    private static final String TAG_DIAGRAM = "diagram";
    private static final String TAG_ENTITIES = "entities";
    private static final String TAG_DATA_SOURCE = "data-source";
    private static final String TAG_ENTITY = "entity";
    private static final String TAG_PATH = "path";
    private static final String TAG_RELATIONS = "relations";
    private static final String TAG_RELATION = "relation";
    private static final String TAG_BEND = "bend";

    private static final String ATTR_VERSION = "version";
    private static final String ATTR_NAME = "name";
    private static final String ATTR_TIME = "time";
    private static final String ATTR_ALIAS = "alias";
    private static final String ATTR_ID = "id";
    private static final String ATTR_ORDER = "order";
    private static final String ATTR_TRANSPARENT = "transparent";
    private static final String ATTR_COLOR_BG = "color-bg";
    private static final String ATTR_COLOR_FG = "color-fg";
    private static final String ATTR_FONT = "font";
    private static final String ATTR_BORDER_WIDTH = "border-width";
    private static final String ATTR_FQ_NAME = "fq-name";
    private static final String ATTR_REF_NAME = "ref-name";
    private static final String ATTR_TYPE = "type";
    private static final String ATTR_PK_REF = "pk-ref";
    private static final String ATTR_FK_REF = "fk-ref";
    private static final String ATTR_ATTRIBUTE_VISIBILITY = "showAttrs";
    private static final String TAG_COLUMN = "column";
    private static final String ATTR_X = "x";
    private static final String ATTR_Y = "y";
    private static final String ATTR_W = "w";
    private static final String ATTR_H = "h";

    private static final int ERD_VERSION_1 = 1;
    private static final String BEND_ABSOLUTE = "abs";
    private static final String BEND_RELATIVE = "rel";
    private static final String TAG_NOTES = "notes";
    private static final String TAG_NOTE = "note";

    private static class ElementSaveInfo {
        final ERDElement element;
        final NodePart nodePart;
        final int objectId;

        private ElementSaveInfo(ERDElement element, NodePart nodePart, int objectId)
        {
            this.element = element;
            this.nodePart = nodePart;
            this.objectId = objectId;
        }
    }

    private static class ElementLoadInfo {
        final String objectId;
        final DBSEntity entity;
        final ERDNote note;
        final EntityDiagram.NodeVisualInfo visualInfo;

        private ElementLoadInfo(String objectId, DBSEntity entity, ERDNote note, EntityDiagram.NodeVisualInfo visualInfo)
        {
            this.objectId = objectId;
            this.entity = entity;
            this.note = note;
            this.visualInfo = visualInfo;
        }
    }

    private static class RelationLoadInfo {
        final String name;
        final String type;
        final ElementLoadInfo pkTable;
        final ElementLoadInfo fkTable;
        final Map<String, String> columns = new LinkedHashMap<>();
        final List<Point> bends = new ArrayList<>();

        private RelationLoadInfo(String name, String type, ElementLoadInfo pkTable, ElementLoadInfo fkTable)
        {
            this.name = name;
            this.type = type;
            this.pkTable = pkTable;
            this.fkTable = fkTable;
        }
    }

    private static class DataSourceObjects {
        List<ERDEntity> entities = new ArrayList<>();
    }

    public static List<DBPDataSourceContainer> extractContainers(IFile resource)
        throws IOException, XMLException, DBException
    {
        List<DBPDataSourceContainer> containers = new ArrayList<>();

        DBPProject projectMeta = DBWorkbench.getPlatform().getWorkspace().getProject(resource.getProject());
        if (projectMeta == null) {
            return containers;
        }
        try (InputStream is = resource.getContents()) {
            final Document document = XMLUtils.parseDocument(is);
            final Element diagramElem = document.getDocumentElement();

            final Element entitiesElem = XMLUtils.getChildElement(diagramElem, TAG_ENTITIES);
            if (entitiesElem != null) {
                // Parse data source
                for (Element dsElem : XMLUtils.getChildElementList(entitiesElem, TAG_DATA_SOURCE)) {
                    String dsId = dsElem.getAttribute(ATTR_ID);
                    if (!CommonUtils.isEmpty(dsId)) {
                        // Get connected datasource
                        final DBPDataSourceContainer dataSourceContainer = projectMeta.getDataSourceRegistry().getDataSource(dsId);
                        if (dataSourceContainer != null) {
                            containers.add(dataSourceContainer);
                        }
                    }
                }
            }
        } catch (CoreException e) {
            throw new DBException("Error reading resource contents", e);
        }
        return containers;
    }

    public static void load(DBRProgressMonitor monitor, IProject project, DiagramPart diagramPart, InputStream in)
        throws XMLException, DBException
    {
        monitor.beginTask("Parse diagram", 1);
        final EntityDiagram diagram = diagramPart.getDiagram();

        DBPProject projectMeta = DBWorkbench.getPlatform().getWorkspace().getProject(project);
        if (projectMeta == null) {
            throw new DBException("Cannot find datasource registry for project '" + project.getName() + "'");
        }

        final Document document = XMLUtils.parseDocument(in);
        monitor.done();

        loadDiagram(monitor, document, projectMeta, diagram);
    }

    public static void loadDiagram(DBRProgressMonitor monitor, Document document, DBPProject projectMeta, EntityDiagram diagram) throws DBException {
        final Element diagramElem = document.getDocumentElement();

        // Check version
        final String diagramVersion = diagramElem.getAttribute(ATTR_VERSION);
        if (CommonUtils.isEmpty(diagramVersion)) {
            throw new DBException("Diagram version not found");
        }
        if (!diagramVersion.equals(String.valueOf(ERD_VERSION_1))) {
            throw new DBException("Unsupported diagram version: " + diagramVersion);
        }

        List<ElementLoadInfo> tableInfos = new ArrayList<>();
        List<RelationLoadInfo> relInfos = new ArrayList<>();
        Map<String, ElementLoadInfo> elementMap = new HashMap<>();

        final Element entitiesElem = XMLUtils.getChildElement(diagramElem, TAG_ENTITIES);
        if (entitiesElem != null) {
            // Parse data source
            for (Element dsElem : XMLUtils.getChildElementList(entitiesElem, TAG_DATA_SOURCE)) {
                String dsId = dsElem.getAttribute(ATTR_ID);
                if (CommonUtils.isEmpty(dsId)) {
                    log.warn("Missing datasource ID");
                    continue;
                }
                // Get connected datasource
                final DBPDataSourceContainer dataSourceContainer = projectMeta.getDataSourceRegistry().getDataSource(dsId);
                if (dataSourceContainer == null) {
                    log.warn("Datasource '" + dsId + "' not found");
                    continue;
                }
                if (!dataSourceContainer.isConnected()) {
                    monitor.subTask("Connect to '" + dataSourceContainer.getName() + "'");
                    try {
                        dataSourceContainer.connect(monitor, true, true);
                    } catch (DBException e) {
                        diagram.addErrorMessage("Can't connect to '" + dataSourceContainer.getName() + "': " + e.getMessage());
                        continue;
                    }
                }
                final DBPDataSource dataSource = dataSourceContainer.getDataSource();
                if (!(dataSource instanceof DBSObjectContainer)) {
                    diagram.addErrorMessage("Datasource '" + dataSourceContainer.getName() + "' entities cannot be loaded - no entity container found");
                    continue;
                }
                DBSObjectContainer rootContainer = (DBSObjectContainer)dataSource;
                // Parse entities
                Collection<Element> entityElemList = XMLUtils.getChildElementList(dsElem, TAG_ENTITY);
                monitor.beginTask("Parse entities", entityElemList.size());
                for (Element entityElem : entityElemList) {
                    String tableId = entityElem.getAttribute(ATTR_ID);
                    String tableName = entityElem.getAttribute(ATTR_NAME);
                    monitor.subTask("Load " + tableName);
                    List<String> path = new ArrayList<>();
                    for (Element pathElem : XMLUtils.getChildElementList(entityElem, TAG_PATH)) {
                        path.add(0, pathElem.getAttribute(ATTR_NAME));
                    }
                    DBSObjectContainer container = rootContainer;
                    for (String conName : path) {
                        final DBSObject child = container.getChild(monitor, conName);
                        if (child == null) {
                            diagram.addErrorMessage("Container '" + conName + "' not found within '" + container.getName() + "'. Skip table '" + tableName + "'.");
                            container = null;
                            break;
                        }
                        if (child instanceof DBSObjectContainer) {
                            container = (DBSObjectContainer) child;
                        } else {
                            diagram.addErrorMessage("Object '" + child.getName() + "' is not a container");
                            container = null;
                            break;
                        }
                    }
                    if (container == null) {
                        continue;
                    }
                    final DBSObject child = container.getChild(monitor, tableName);
                    if (!(child instanceof DBSEntity)) {
                        diagram.addErrorMessage("Cannot find table '" + tableName + "' in '" + container.getName() + "'");
                        continue;
                    }
                    String locX = entityElem.getAttribute(ATTR_X);<commit_msg>#7295 ERD load errors: replace "cannotfind table" with debug warning.<commit_after>/*
 * DBeaver - Universal Database Manager
 * Copyright (C) 2010-2019 <NAME> (<EMAIL>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/*
 * Created on Jul 13, 2004
 */
package org.jkiss.dbeaver.ext.erd.model;

import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IProject;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.draw2d.AbsoluteBendpoint;
import org.eclipse.draw2d.Bendpoint;
import org.eclipse.draw2d.RelativeBendpoint;
import org.eclipse.draw2d.geometry.Point;
import org.eclipse.draw2d.geometry.Rectangle;
import org.eclipse.jface.resource.StringConverter;
import org.eclipse.swt.graphics.Color;
import org.eclipse.swt.widgets.Display;
import org.jkiss.code.Nullable;
import org.jkiss.dbeaver.DBException;
import org.jkiss.dbeaver.Log;
import org.jkiss.dbeaver.ext.erd.ERDConstants;
import org.jkiss.dbeaver.ext.erd.editor.ERDAttributeVisibility;
import org.jkiss.dbeaver.ext.erd.part.*;
import org.jkiss.dbeaver.model.*;
import org.jkiss.dbeaver.model.app.DBPProject;
import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor;
import org.jkiss.dbeaver.model.runtime.VoidProgressMonitor;
import org.jkiss.dbeaver.model.struct.*;
import org.jkiss.dbeaver.runtime.DBWorkbench;
import org.jkiss.dbeaver.ui.SharedFonts;
import org.jkiss.dbeaver.ui.UIUtils;
import org.jkiss.dbeaver.utils.GeneralUtils;
import org.jkiss.dbeaver.utils.RuntimeUtils;
import org.jkiss.utils.CommonUtils;
import org.jkiss.utils.xml.XMLBuilder;
import org.jkiss.utils.xml.XMLException;
import org.jkiss.utils.xml.XMLUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;

import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.util.*;

/**
 * Entity diagram loader/saver
 * @author <NAME>
 */
public class DiagramLoader
{
    private static final Log log = Log.getLog(DiagramLoader.class);

    private static final String TAG_DIAGRAM = "diagram";
    private static final String TAG_ENTITIES = "entities";
    private static final String TAG_DATA_SOURCE = "data-source";
    private static final String TAG_ENTITY = "entity";
    private static final String TAG_PATH = "path";
    private static final String TAG_RELATIONS = "relations";
    private static final String TAG_RELATION = "relation";
    private static final String TAG_BEND = "bend";

    private static final String ATTR_VERSION = "version";
    private static final String ATTR_NAME = "name";
    private static final String ATTR_TIME = "time";
    private static final String ATTR_ALIAS = "alias";
    private static final String ATTR_ID = "id";
    private static final String ATTR_ORDER = "order";
    private static final String ATTR_TRANSPARENT = "transparent";
    private static final String ATTR_COLOR_BG = "color-bg";
    private static final String ATTR_COLOR_FG = "color-fg";
    private static final String ATTR_FONT = "font";
    private static final String ATTR_BORDER_WIDTH = "border-width";
    private static final String ATTR_FQ_NAME = "fq-name";
    private static final String ATTR_REF_NAME = "ref-name";
    private static final String ATTR_TYPE = "type";
    private static final String ATTR_PK_REF = "pk-ref";
    private static final String ATTR_FK_REF = "fk-ref";
    private static final String ATTR_ATTRIBUTE_VISIBILITY = "showAttrs";
    private static final String TAG_COLUMN = "column";
    private static final String ATTR_X = "x";
    private static final String ATTR_Y = "y";
    private static final String ATTR_W = "w";
    private static final String ATTR_H = "h";

    private static final int ERD_VERSION_1 = 1;
    private static final String BEND_ABSOLUTE = "abs";
    private static final String BEND_RELATIVE = "rel";
    private static final String TAG_NOTES = "notes";
    private static final String TAG_NOTE = "note";

    private static class ElementSaveInfo {
        final ERDElement element;
        final NodePart nodePart;
        final int objectId;

        private ElementSaveInfo(ERDElement element, NodePart nodePart, int objectId)
        {
            this.element = element;
            this.nodePart = nodePart;
            this.objectId = objectId;
        }
    }

    private static class ElementLoadInfo {
        final String objectId;
        final DBSEntity entity;
        final ERDNote note;
        final EntityDiagram.NodeVisualInfo visualInfo;

        private ElementLoadInfo(String objectId, DBSEntity entity, ERDNote note, EntityDiagram.NodeVisualInfo visualInfo)
        {
            this.objectId = objectId;
            this.entity = entity;
            this.note = note;
            this.visualInfo = visualInfo;
        }
    }

    private static class RelationLoadInfo {
        final String name;
        final String type;
        final ElementLoadInfo pkTable;
        final ElementLoadInfo fkTable;
        final Map<String, String> columns = new LinkedHashMap<>();
        final List<Point> bends = new ArrayList<>();

        private RelationLoadInfo(String name, String type, ElementLoadInfo pkTable, ElementLoadInfo fkTable)
        {
            this.name = name;
            this.type = type;
            this.pkTable = pkTable;
            this.fkTable = fkTable;
        }
    }

    private static class DataSourceObjects {
        List<ERDEntity> entities = new ArrayList<>();
    }

    public static List<DBPDataSourceContainer> extractContainers(IFile resource)
        throws IOException, XMLException, DBException
    {
        List<DBPDataSourceContainer> containers = new ArrayList<>();

        DBPProject projectMeta = DBWorkbench.getPlatform().getWorkspace().getProject(resource.getProject());
        if (projectMeta == null) {
            return containers;
        }
        try (InputStream is = resource.getContents()) {
            final Document document = XMLUtils.parseDocument(is);
            final Element diagramElem = document.getDocumentElement();

            final Element entitiesElem = XMLUtils.getChildElement(diagramElem, TAG_ENTITIES);
            if (entitiesElem != null) {
                // Parse data source
                for (Element dsElem : XMLUtils.getChildElementList(entitiesElem, TAG_DATA_SOURCE)) {
                    String dsId = dsElem.getAttribute(ATTR_ID);
                    if (!CommonUtils.isEmpty(dsId)) {
                        // Get connected datasource
                        final DBPDataSourceContainer dataSourceContainer = projectMeta.getDataSourceRegistry().getDataSource(dsId);
                        if (dataSourceContainer != null) {
                            containers.add(dataSourceContainer);
                        }
                    }
                }
            }
        } catch (CoreException e) {
            throw new DBException("Error reading resource contents", e);
        }
        return containers;
    }

    public static void load(DBRProgressMonitor monitor, IProject project, DiagramPart diagramPart, InputStream in)
        throws XMLException, DBException
    {
        monitor.beginTask("Parse diagram", 1);
        final EntityDiagram diagram = diagramPart.getDiagram();

        DBPProject projectMeta = DBWorkbench.getPlatform().getWorkspace().getProject(project);
        if (projectMeta == null) {
            throw new DBException("Cannot find datasource registry for project '" + project.getName() + "'");
        }

        final Document document = XMLUtils.parseDocument(in);
        monitor.done();

        loadDiagram(monitor, document, projectMeta, diagram);
    }

    public static void loadDiagram(DBRProgressMonitor monitor, Document document, DBPProject projectMeta, EntityDiagram diagram) throws DBException {
        final Element diagramElem = document.getDocumentElement();

        // Check version
        final String diagramVersion = diagramElem.getAttribute(ATTR_VERSION);
        if (CommonUtils.isEmpty(diagramVersion)) {
            throw new DBException("Diagram version not found");
        }
        if (!diagramVersion.equals(String.valueOf(ERD_VERSION_1))) {
            throw new DBException("Unsupported diagram version: " + diagramVersion);
        }

        List<ElementLoadInfo> tableInfos = new ArrayList<>();
        List<RelationLoadInfo> relInfos = new ArrayList<>();
        Map<String, ElementLoadInfo> elementMap = new HashMap<>();

        final Element entitiesElem = XMLUtils.getChildElement(diagramElem, TAG_ENTITIES);
        if (entitiesElem != null) {
            // Parse data source
            for (Element dsElem : XMLUtils.getChildElementList(entitiesElem, TAG_DATA_SOURCE)) {
                String dsId = dsElem.getAttribute(ATTR_ID);
                if (CommonUtils.isEmpty(dsId)) {
                    log.warn("Missing datasource ID");
                    continue;
                }
                // Get connected datasource
                final DBPDataSourceContainer dataSourceContainer = projectMeta.getDataSourceRegistry().getDataSource(dsId);
                if (dataSourceContainer == null) {
                    log.warn("Datasource '" + dsId + "' not found");
                    continue;
                }
                if (!dataSourceContainer.isConnected()) {
                    monitor.subTask("Connect to '" + dataSourceContainer.getName() + "'");
                    try {
                        dataSourceContainer.connect(monitor, true, true);
                    } catch (DBException e) {
                        diagram.addErrorMessage("Can't connect to '" + dataSourceContainer.getName() + "': " + e.getMessage());
                        continue;
                    }
                }
                final DBPDataSource dataSource = dataSourceContainer.getDataSource();
                if (!(dataSource instanceof DBSObjectContainer)) {
                    diagram.addErrorMessage("Datasource '" + dataSourceContainer.getName() + "' entities cannot be loaded - no entity container found");
                    continue;
                }
                DBSObjectContainer rootContainer = (DBSObjectContainer)dataSource;
                // Parse entities
                Collection<Element> entityElemList = XMLUtils.getChildElementList(dsElem, TAG_ENTITY);
                monitor.beginTask("Parse entities", entityElemList.size());
                for (Element entityElem : entityElemList) {
                    String tableId = entityElem.getAttribute(ATTR_ID);
                    String tableName = entityElem.getAttribute(ATTR_NAME);
                    monitor.subTask("Load " + tableName);
                    List<String> path = new ArrayList<>();
                    for (Element pathElem : XMLUtils.getChildElementList(entityElem, TAG_PATH)) {
                        path.add(0, pathElem.getAttribute(ATTR_NAME));
                    }
                    DBSObjectContainer container = rootContainer;
                    for (String conName : path) {
                        final DBSObject child = container.getChild(monitor, conName);
                        if (child == null) {
                            diagram.addErrorMessage("Container '" + conName + "' not found within '" + container.getName() + "'. Skip table '" + tableName + "'.");
                            container = null;
                            break;
                        }
                        if (child instanceof DBSObjectContainer) {
                            container = (DBSObjectContainer) child;
                        } else {
                            diagram.addErrorMessage("Object '" + child.getName() + "' is not a container");
                            container = null;
                            break;
                        }
                    }
                    if (container == null) {
                        continue;
                    }
                    final DBSObject child = container.getChild(monitor, tableName);
                    if (!(child instanceof DBSEntity)) {
                        log.debug("Cannot find table '" + tableName + "' in '" + container.getName() + "'");
                        continue;
                    }
                    String locX = entityElem.getAttribute(ATTR_X);
{'fraction_non_alphanumeric': 0.08816387278709072, 'fraction_numerical': 0.002506658311138963, 'mean_word_length': 2.327642382379773, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6038941', 'n_tokens_mistral': 6604, 'n_tokens_neox': 6379, 'n_words': 1724}