<commit_before>/*
 *  eXist Open Source Native XML Database
 *  Copyright (C) 2001-2010 The eXist Project
 *  http://exist-db.org
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public License
 *  as published by the Free Software Foundation; either version 2
 *  of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *  $Id$
 */
package org.exist.backup;

import java.io.*;
import java.util.*;
import org.apache.log4j.Logger;

import org.exist.security.PermissionDeniedException;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.helpers.NamespaceSupport;

import org.exist.Namespaces;
import org.exist.collections.Collection;
import org.exist.dom.*;
import org.exist.management.Agent;
import org.exist.management.AgentFactory;
import org.exist.security.internal.AccountImpl;
import org.exist.stax.EmbeddedXMLStreamReader;
import org.exist.storage.DBBroker;
import org.exist.storage.NativeBroker;
import org.exist.storage.ProcessMonitor;
import org.exist.storage.btree.BTreeCallback;
import org.exist.storage.btree.Value;
import org.exist.storage.index.CollectionStore;
import org.exist.storage.io.VariableByteInput;
import org.exist.storage.serializers.EXistOutputKeys;
import org.exist.util.UTF8;
import org.exist.util.serializer.AttrList;
import org.exist.util.serializer.Receiver;
import org.exist.util.serializer.SAXSerializer;
import org.exist.util.serializer.SerializerPool;
import org.exist.xmldb.XmldbURI;
import org.exist.xquery.TerminatedException;
import org.exist.xquery.XPathException;
import org.exist.xquery.util.URIUtils;
import org.exist.xquery.value.DateTimeValue;


import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.OutputKeys;
import org.exist.storage.DataBackup;


/**
 * Embedded database export tool class. Tries to export as much data as possible, even if parts of the collection hierarchy are corrupted or documents
 * are no longer readable. Features:
 *
 * <ul>
 * <li>Descendant collections will be exported properly even if their ancestor collection is corrupted.</li>
 * <li>Documents which are intact but belong to a destroyed collection will be stored into a special collection /db/__lost_and_found__.</li>
 * <li>Damaged documents are detected by ConsistencyCheck and are removed from the backup.</li>
 * <li>The format of the exported data is compatible with backups generated via the standard backup tool (Java admin client).</li>
 * </ul>
 *
 * <p>The class should be used in combination with {@link ConsistencyCheck}. The error lists returned by ConsistencyCheck can be passed to {@link
 * #export(org.exist.collections.Collection, BackupWriter, java.util.Date, BackupDescriptor, java.util.List, org.exist.dom.MutableDocumentSet)}.</p>
 */
public class SystemExport
{
    private final static Logger     LOG                     = Logger.getLogger( SystemExport.class );

    private static final XmldbURI   TEMP_COLLECTION         = XmldbURI.createInternal( XmldbURI.TEMP_COLLECTION );
    private static final XmldbURI   CONTENTS_URI            = XmldbURI.createInternal( "__contents__.xml" );
    private static final XmldbURI   LOST_URI                = XmldbURI.createInternal( "__lost_and_found__" );

    private static final int        currVersion             = 1;

    private int                     collectionCount         = -1;

    public Properties               defaultOutputProperties = new Properties();

    public Properties               contentsOutputProps     = new Properties();

    private DBBroker                broker;
    private StatusCallback          callback                = null;
    private boolean                 directAccess            = false;
    private ProcessMonitor.Monitor  monitor                 = null;

    {
        defaultOutputProperties.setProperty( OutputKeys.INDENT, "no" );
        defaultOutputProperties.setProperty( OutputKeys.ENCODING, "UTF-8" );
        defaultOutputProperties.setProperty( OutputKeys.OMIT_XML_DECLARATION, "no" );
        defaultOutputProperties.setProperty( EXistOutputKeys.EXPAND_XINCLUDES, "no" );
        defaultOutputProperties.setProperty( EXistOutputKeys.PROCESS_XSL_PI, "no" );
    }

    {
        contentsOutputProps.setProperty( OutputKeys.INDENT, "yes" );
    }

    public SystemExport( DBBroker broker, StatusCallback callback, ProcessMonitor.Monitor monitor, boolean direct )
    {
        this.broker       = broker;
        this.callback     = callback;
        this.monitor      = monitor;
        this.directAccess = direct;
    }

    public File export( String targetDir, boolean incremental, boolean zip, List<ErrorReport> errorList )
    {
        return( export( targetDir, incremental, -1, zip, errorList ) );
    }


    /**
     * Export the contents of the database, trying to preserve as much data as possible. To be effective, this method should be used in combination
     * with class {@link ConsistencyCheck}.
     *
     * @param   targetDir    the output directory or file to which data will be written. Output will be written to a zip file if target ends with
     *                       .zip.
     * @param   incremental  DOCUMENT ME!
     * @param   maxInc       DOCUMENT ME!
     * @param   zip          DOCUMENT ME!
     * @param   errorList    a list of {@link ErrorReport} objects as returned by methods in {@link ConsistencyCheck}.
     *
     * @return  DOCUMENT ME!
     */
    public File export( String targetDir, boolean incremental, int maxInc, boolean zip, List<ErrorReport> errorList )
    {
        File backupFile = null;

        try {
            BackupDirectory  directory  = new BackupDirectory( targetDir );
            BackupDescriptor prevBackup = null;

            if( incremental ) {
                prevBackup = directory.lastBackupFile();
                LOG.info( "Creating incremental backup. Prev backup: " + ( ( prevBackup == null ) ? "none" : prevBackup.getSymbolicPath() ) );
            }

            Properties properties = new Properties();
            int        seqNr      = 1;

            if( incremental ) {
                properties.setProperty( BackupDescriptor.PREVIOUS_PROP_NAME, ( prevBackup == null ) ? "" : prevBackup.getName() );

                if( prevBackup != null ) {
                    Properties prevProp = prevBackup.getProperties();

                    if( prevProp != null ) {
                        String seqNrStr = prevProp.getProperty( BackupDescriptor.NUMBER_IN_SEQUENCE_PROP_NAME, "1" );

                        try {
                            seqNr = Integer.parseInt( seqNrStr );

                            if( seqNr == maxInc ) {
                                seqNr       = 1;
                                incremental = false;
                                prevBackup  = null;
                            } else {
                                ++seqNr;
                            }
                        }
                        catch( NumberFormatException e ) {
                            LOG.warn( "Bad sequence number in backup descriptor: " + prevBackup.getName() );
                        }
                    }
                }
            }
            properties.setProperty( BackupDescriptor.NUMBER_IN_SEQUENCE_PROP_NAME, Integer.toString( seqNr ) );
            properties.setProperty( BackupDescriptor.INCREMENTAL_PROP_NAME, incremental ? "yes" : "no" );

            try {
                properties.setProperty( BackupDescriptor.DATE_PROP_NAME, new DateTimeValue( new Date() ).getStringValue() );
            }
            catch( XPathException e ) {
            }

            backupFile = directory.createBackup( incremental && ( prevBackup != null ), zip );
            BackupWriter output;

            if( zip ) {
                output = new ZipWriter( backupFile, XmldbURI.ROOT_COLLECTION );
            } else {
                output = new FileSystemWriter( backupFile );
            }
            output.setProperties( properties );

            Date               date = ( prevBackup == null ) ? null : prevBackup.getDate();
            CollectionCallback cb   = new CollectionCallback( output, date, prevBackup, errorList, true );
            broker.getCollectionsFailsafe( cb );

            exportOrphans( output, cb.getDocs(), errorList );

            output.close();
            return( backupFile );
        }
        catch( IOException e ) {
            reportError( "A write error occurred while exporting data: '" + e.getMessage() + "'. Aborting export.", e );
            return( null );
        }
        catch( TerminatedException e ) {

            if( backupFile != null ) {
                backupFile.delete();
            }
            return( null );
        }
    }


    private void reportError( String message, Throwable e )
    {
        if( callback != null ) {
            callback.error( "EXPORT: " + message, e );
        }
    }


    private static boolean isDamaged( DocumentImpl doc, List<ErrorReport> errorList )
    {
        if( errorList == null ) {
            return( false );
        }

        for( org.exist.backup.ErrorReport report : errorList ) {

            if( ( report.getErrcode() == org.exist.backup.ErrorReport.RESOURCE_ACCESS_FAILED ) && ( ( (ErrorReport.ResourceError)report ).getDocumentId() == doc.getDocId() ) ) {
                return( true );
            }
        }
        return( false );
    }


    @SuppressWarnings( "unused" )
    private static boolean isDamaged( Collection collection, List<ErrorReport> errorList )
    {
        if( errorList == null ) {
            return( false );
        }

        for( ErrorReport report : errorList ) {

            if( ( report.getErrcode() == org.exist.backup.ErrorReport.CHILD_COLLECTION ) && ( ( (ErrorReport.CollectionError)report ).getCollectionId() == collection.getId() ) ) {
                return( true );
            }
        }
        return( false );
    }


    private static boolean isDamagedChild( XmldbURI uri, List<ErrorReport> errorList )
    {
        if( errorList == null ) {
            return( false );
        }

        for( org.exist.backup.ErrorReport report : errorList ) {

            if( ( report.getErrcode() == org.exist.backup.ErrorReport.CHILD_COLLECTION ) && ( (org.exist.backup.ErrorReport.CollectionError)report ).getCollectionURI().equalsInternal( uri ) ) {
                return( true );
            }
        }
        return( false );
    }


    /**
     * Scan all document records in collections.dbx and try to find orphaned documents whose parent collection got destroyed or is damaged.
     *
     * @param  output     the backup writer
     * @param  docs       a document set containing all the documents which were exported regularily. the method will ignore those.
     * @param  errorList  a list of {@link org.exist.backup.ErrorReport} objects as returned by methods in {@link ConsistencyCheck}
     */
    private void exportOrphans( BackupWriter output, DocumentSet docs, List<ErrorReport> errorList )
    {
        output.newCollection( "/db/__lost_and_found__" );

        try {
            Writer        contents   = output.newContents();

            // serializer writes to __contents__.xml
            SAXSerializer serializer = (SAXSerializer)SerializerPool.getInstance().borrowObject( SAXSerializer.class );
            serializer.setOutput( contents, contentsOutputProps );

            serializer.startDocument();
            serializer.startPrefixMapping( "", Namespaces.EXIST_NS );
            AttributesImpl attr = new AttributesImpl();
            attr.addAttribute( Namespaces.EXIST_NS, "name", "name", "CDATA", "/db/__lost_and_found__" );
            attr.addAttribute( Namespaces.EXIST_NS, "version", "version", "CDATA", String.valueOf( currVersion ) );
            attr.addAttribute( Namespaces.EXIST_NS, "owner", "owner", "CDATA", org.exist.security.SecurityManager.DBA_USER );
            attr.addAttribute( Namespaces.EXIST_NS, "group", "group", "CDATA", org.exist.security.SecurityManager.DBA_GROUP );
            attr.addAttribute( Namespaces.EXIST_NS, "mode", "mode", "CDATA", "0771" );
            serializer.startElement( Namespaces.EXIST_NS, "collection", "collection", attr );

            DocumentCallback docCb = new DocumentCallback( output, serializer, null, null, docs, true );
            broker.getResourcesFailsafe( docCb, directAccess );

            serializer.endElement( Namespaces.EXIST_NS, "collection", "collection" );
            serializer.endPrefixMapping( "" );
            serializer.endDocument();
            output.closeContents();
        }
        catch( Exception e ) {
            e.printStackTrace();

            if( callback != null ) {
                callback.error( e.getMessage(), e );
            }
        }
        finally {
            output.closeCollection();
        }
    }


    /**
     * Export a collection. Write out the collection metadata and save the resources stored in the collection.
     *
     * @param   current     the collection
     * @param   output      the output writer
     * @param   date
     * @param   prevBackup  DOCUMENT ME!
     * @param   errorList   a list of {@link org.exist.backup.ErrorReport} objects as returned by methods in {@link org.exist.backup.ConsistencyCheck}
     * @param   docs        a document set to keep track of all written documents.
     *
     * @throws  IOException
     * @throws  SAXException
     * @throws  TerminatedException  DOCUMENT ME!
     */
    private void export( Collection current, BackupWriter output, Date date, BackupDescriptor prevBackup, List<ErrorReport> errorList, MutableDocumentSet docs ) throws IOException, SAXException, TerminatedException, PermissionDeniedException
    {
        if( callback != null ) {
            callback.startCollection( current.getURI().toString() );
        }

        if( ( monitor != null ) && !monitor.proceed() ) {
            throw( new TerminatedException( "system export terminated by db" ) );
        }

//        if( !current.getURI().equalsInternal( XmldbURI.ROOT_COLLECTION_URI ) ) {
            output.newCollection( Backup.encode( URIUtils.urlDecodeUtf8( current.getURI() ) ) );
//        }

        try {
            Writer        contents   = output.newContents();

            // serializer writes to __contents__.xml
            SAXSerializer serializer = (SAXSerializer)SerializerPool.getInstance().borrowObject( SAXSerializer.class );
            serializer.setOutput( contents, contentsOutputProps );

            serializer.startDocument();
            serializer.startPrefixMapping( "", Namespaces.EXIST_NS );
            XmldbURI       uri  = current.getURI();
            AttributesImpl attr = new AttributesImpl();
            attr.addAttribute( Namespaces.EXIST_NS, "name", "name", "CDATA", uri.toString() );
            attr.addAttribute( Namespaces.EXIST_NS, "version", "version", "CDATA", String.valueOf( currVersion ) );
            attr.addAttribute( Namespaces.EXIST_NS, "owner", "owner", "CDATA", current.getPermissions().getOwner().getName() );
            attr.addAttribute( Namespaces.EXIST_NS, "group", "group", "CDATA", current.getPermissions().getGroup().getName() );
            attr.addAttribute( Namespaces.EXIST_NS, "mode", "mode", "CDATA", Integer.toOctalString( current.getPermissions().getMode() ) );

            try {
                attr.addAttribute( Namespaces.EXIST_NS, "created", "created", "CDATA", new DateTimeValue( new Date( current.getCreationTime() ) ).getStringValue() );
            }
            catch( XPathException e ) {
                e.printStackTrace();
            }
            serializer.startElement( Namespaces.EXIST_NS, "collection", "collection", attr );

            int docsCount = current.getDocumentCountNoLock(broker);
            int count     = 0;

            for( Iterator<DocumentImpl> i = current.iteratorNoLock( broker ); i.hasNext(); count++ ) {
                DocumentImpl doc = i.next();

                if( isDamaged( doc, errorList ) ) {
                    reportError( "Skipping damaged document " + doc.getFileURI(), null );
                    continue;
                }

                if( doc.getFileURI().equalsInternal( CONTENTS_URI ) || doc.getFileURI().equalsInternal( LOST_URI ) ) {
                    continue; // skip __contents__.xml documents
                }
                exportDocument( output, date, prevBackup, serializer, docsCount, count, doc );
                docs.add( doc, false );
            }

            for( Iterator<XmldbURI> i = current.collectionIteratorNoLock(broker); i.hasNext(); ) {
                XmldbURI childUri = i.next();

                if( childUri.equalsInternal( TEMP_COLLECTION ) ) {
                    continue;
                }

                if( isDamagedChild( childUri, errorList ) ) {
                    reportError( "Skipping damaged child collection " + childUri, null );
                    continue;
                }
                attr.clear();
                attr.addAttribute( Namespaces.EXIST_NS, "name", "name", "CDATA", childUri.toString() );
                attr.addAttribute( Namespaces.EXIST_NS, "filename", "filename", "CDATA", Backup.encode( URIUtils.urlDecodeUtf8( childUri.toString() ) ) );
                serializer.startElement( Namespaces.EXIST_NS, "subcollection", "subcollection", attr );
                serializer.endElement( Namespaces.EXIST_NS, "subcollection", "subcollection" );
            }

            if( prevBackup != null ) {

                // Check which collections and resources have been deleted since
                // the
                // last backup
                CheckDeletedHandler check = new CheckDeletedHandler( current, serializer );

                try {
                    prevBackup.parse( check );
                }
                catch( Exception e ) {
                    LOG.error( "Caught exception while trying to parse previous backup descriptor: " + prevBackup.getSymbolicPath(), e );
                }
            }

            // close <collection>
            serializer.endElement( Namespaces.EXIST_NS, "collection", "collection" );
            serializer.endPrefixMapping( "" );
            serializer.endDocument();
            output.closeContents();
        }
        finally {

//            if( !current.getURI().equalsInternal( XmldbURI.ROOT_COLLECTION_URI ) ) {
                output.closeCollection();
//            }
        }
    }


    private void exportDocument( BackupWriter output, Date date, BackupDescriptor prevBackup, SAXSerializer serializer, int docsCount, int count, DocumentImpl doc ) throws IOException, SAXException, TerminatedException
    {
        if( callback != null ) {
            callback.startDocument( doc.getFileURI().toString(), count, docsCount );
        }

        if( ( monitor != null ) && !monitor.proceed() ) {
            throw( new TerminatedException( "system export terminated by db" ) );
        }
        boolean needsBackup = ( prevBackup == null ) || ( date.getTime() < doc.getMetadata().getLastModified() );

        if( needsBackup ) {
            OutputStream os = output.newEntry( Backup.encode( URIUtils.urlDecodeUtf8( doc.getFileURI() ) ) );

            try {

                if( doc.getResourceType() == DocumentImpl.BINARY_FILE ) {
                    broker.readBinaryResource( (BinaryDocument)doc, os );
                } else {
                    BufferedWriter writer            = new BufferedWriter( new OutputStreamWriter( os, "UTF-8" ) );

                    // write resource to contentSerializer
                    SAXSerializer  contentSerializer = (SAXSerializer)SerializerPool.getInstance().borrowObject( SAXSerializer.class );
                    contentSerializer.setOutput( writer, defaultOutputProperties );
                    writeXML( doc, contentSerializer );
                    SerializerPool.getInstance().returnObject( contentSerializer );
                    writer.flush();
                }
            }
            catch( Exception e ) {
                reportError( "A write error occurred while exporting document: '" + doc.getFileURI() + "'. Continuing with next document.", e );
                return;
            }
            finally {
                output.closeEntry();
            }
        }

        // store permissions
        AttributesImpl attr = new AttributesImpl();
        attr.addAttribute( Namespaces.EXIST_NS, "type", "type", "CDATA", ( doc.getResourceType() == DocumentImpl.BINARY_FILE ) ? "BinaryResource" : "XMLResource" );
        attr.addAttribute( Namespaces.EXIST_NS, "name", "name", "CDATA", doc.getFileURI().toString() );
        attr.addAttribute( Namespaces.EXIST_NS, "skip", "skip", "CDATA", ( needsBackup ? "no" : "yes" ) );
        attr.addAttribute( Namespaces.EXIST_NS, "owner", "owner", "CDATA", doc.getPermissions().getOwner().getName() );
        attr.addAttribute( Namespaces.EXIST_NS, "group", "group", "CDATA", doc.getPermissions().getGroup().getName() );
        attr.addAttribute( Namespaces.EXIST_NS, "mode", "mode", "CDATA", Integer.toOctalString( doc.getPermissions().getMode() ) );

        // be careful when accessing document metadata: it is stored in a
        // different place than the
        // main document info and could thus be damaged
        DocumentMetadata metadata = null;

        try {
            metadata = doc.getMetadata();
        }
        catch( Exception e ) {
            // LOG.warn(e.getMessage(), e);
        }

        try {
            String created;
            String modified;

            // metadata could be damaged
            if( metadata != null ) {
                created  = new DateTimeValue( new Date( metadata.getCreated() ) ).getStringValue();
                modified = new DateTimeValue( new Date( metadata.getLastModified() ) ).getStringValue();
            } else {
                created  = new DateTimeValue().getStringValue();
                modified = created;
            }
            attr.addAttribute( Namespaces.EXIST_NS, "created", "created", "CDATA", created );
            attr.addAttribute( Namespaces.EXIST_NS, "modified", "modified", "CDATA", modified );
        }
        catch( XPathException e ) {
            LOG.warn( e.getMessage(), e );
        }

        attr.addAttribute( Namespaces.EXIST_NS, "filename", "filename", "CDATA", Backup.encode( URIUtils.urlDecodeUtf8( doc.getFileURI() ) ) );
        String mimeType = "application/xml";

        if( ( metadata != null ) && ( metadata.getMimeType() != null ) ) {
            mimeType = Backup.encode( metadata.getMimeType() );
        }
        attr.addAttribute( Namespaces.EXIST_NS, "mimetype", "mimetype", "CDATA", mimeType );

        if( ( doc.getResourceType() == DocumentImpl.XML_FILE ) && ( metadata != null ) && ( doc.getDoctype() != null ) ) {

            if( doc.getDoctype().getName() != null ) {
                attr.addAttribute( Namespaces.EXIST_NS, "namedoctype", "namedoctype", "CDATA", doc.getDoctype().getName() );
            }

            if( doc.getDoctype().getPublicId() != null ) {
                attr.addAttribute( Namespaces.EXIST_NS, "publicid", "publicid", "CDATA", doc.getDoctype().getPublicId() );
            }

            if( doc.getDoctype().getSystemId() != null ) {
                attr.addAttribute( Namespaces.EXIST_NS, "systemid", "systemid", "CDATA", doc.getDoctype().getSystemId() );
            }
        }
        serializer.startElement( Namespaces.EXIST_NS, "resource", "resource", attr );
        serializer.endElement( Namespaces.EXIST_NS, "resource", "resource" );
    }


    /**
     * Serialize a document to XML, based on {@link XMLStreamReader}.
     *
     * @param  doc       the document to serialize
     * @param  receiver  the output handler
     */<commit_msg>[bugfix] export ACLPermission during SystemExport<commit_after>/*
 *  eXist Open Source Native XML Database
 *  Copyright (C) 2001-2010 The eXist Project
 *  http://exist-db.org
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public License
 *  as published by the Free Software Foundation; either version 2
 *  of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *  $Id$
 */
package org.exist.backup;

import java.io.*;
import java.util.*;
import org.apache.log4j.Logger;

import org.exist.security.ACLPermission;
import org.exist.security.Permission;
import org.exist.security.PermissionDeniedException;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.helpers.NamespaceSupport;

import org.exist.Namespaces;
import org.exist.collections.Collection;
import org.exist.dom.*;
import org.exist.management.Agent;
import org.exist.management.AgentFactory;
import org.exist.security.internal.AccountImpl;
import org.exist.stax.EmbeddedXMLStreamReader;
import org.exist.storage.DBBroker;
import org.exist.storage.NativeBroker;
import org.exist.storage.ProcessMonitor;
import org.exist.storage.btree.BTreeCallback;
import org.exist.storage.btree.Value;
import org.exist.storage.index.CollectionStore;
import org.exist.storage.io.VariableByteInput;
import org.exist.storage.serializers.EXistOutputKeys;
import org.exist.util.UTF8;
import org.exist.util.serializer.AttrList;
import org.exist.util.serializer.Receiver;
import org.exist.util.serializer.SAXSerializer;
import org.exist.util.serializer.SerializerPool;
import org.exist.xmldb.XmldbURI;
import org.exist.xquery.TerminatedException;
import org.exist.xquery.XPathException;
import org.exist.xquery.util.URIUtils;
import org.exist.xquery.value.DateTimeValue;


import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.OutputKeys;
import org.exist.storage.DataBackup;


/**
 * Embedded database export tool class. Tries to export as much data as possible, even if parts of the collection hierarchy are corrupted or documents
 * are no longer readable. Features:
 *
 * <ul>
 * <li>Descendant collections will be exported properly even if their ancestor collection is corrupted.</li>
 * <li>Documents which are intact but belong to a destroyed collection will be stored into a special collection /db/__lost_and_found__.</li>
 * <li>Damaged documents are detected by ConsistencyCheck and are removed from the backup.</li>
 * <li>The format of the exported data is compatible with backups generated via the standard backup tool (Java admin client).</li>
 * </ul>
 *
 * <p>The class should be used in combination with {@link ConsistencyCheck}. The error lists returned by ConsistencyCheck can be passed to {@link
 * #export(org.exist.collections.Collection, BackupWriter, java.util.Date, BackupDescriptor, java.util.List, org.exist.dom.MutableDocumentSet)}.</p>
 */
public class SystemExport
{
    private final static Logger     LOG                     = Logger.getLogger( SystemExport.class );

    private static final XmldbURI   TEMP_COLLECTION         = XmldbURI.createInternal( XmldbURI.TEMP_COLLECTION );
    private static final XmldbURI   CONTENTS_URI            = XmldbURI.createInternal( "__contents__.xml" );
    private static final XmldbURI   LOST_URI                = XmldbURI.createInternal( "__lost_and_found__" );

    private static final int        currVersion             = 1;

    private int                     collectionCount         = -1;

    public Properties               defaultOutputProperties = new Properties();

    public Properties               contentsOutputProps     = new Properties();

    private DBBroker                broker;
    private StatusCallback          callback                = null;
    private boolean                 directAccess            = false;
    private ProcessMonitor.Monitor  monitor                 = null;

    {
        defaultOutputProperties.setProperty( OutputKeys.INDENT, "no" );
        defaultOutputProperties.setProperty( OutputKeys.ENCODING, "UTF-8" );
        defaultOutputProperties.setProperty( OutputKeys.OMIT_XML_DECLARATION, "no" );
        defaultOutputProperties.setProperty( EXistOutputKeys.EXPAND_XINCLUDES, "no" );
        defaultOutputProperties.setProperty( EXistOutputKeys.PROCESS_XSL_PI, "no" );
    }

    {
        contentsOutputProps.setProperty( OutputKeys.INDENT, "yes" );
    }

    public SystemExport( DBBroker broker, StatusCallback callback, ProcessMonitor.Monitor monitor, boolean direct )
    {
        this.broker       = broker;
        this.callback     = callback;
        this.monitor      = monitor;
        this.directAccess = direct;
    }

    public File export( String targetDir, boolean incremental, boolean zip, List<ErrorReport> errorList )
    {
        return( export( targetDir, incremental, -1, zip, errorList ) );
    }


    /**
     * Export the contents of the database, trying to preserve as much data as possible. To be effective, this method should be used in combination
     * with class {@link ConsistencyCheck}.
     *
     * @param   targetDir    the output directory or file to which data will be written. Output will be written to a zip file if target ends with
     *                       .zip.
     * @param   incremental  DOCUMENT ME!
     * @param   maxInc       DOCUMENT ME!
     * @param   zip          DOCUMENT ME!
     * @param   errorList    a list of {@link ErrorReport} objects as returned by methods in {@link ConsistencyCheck}.
     *
     * @return  DOCUMENT ME!
     */
    public File export( String targetDir, boolean incremental, int maxInc, boolean zip, List<ErrorReport> errorList )
    {
        File backupFile = null;

        try {
            BackupDirectory  directory  = new BackupDirectory( targetDir );
            BackupDescriptor prevBackup = null;

            if( incremental ) {
                prevBackup = directory.lastBackupFile();
                LOG.info( "Creating incremental backup. Prev backup: " + ( ( prevBackup == null ) ? "none" : prevBackup.getSymbolicPath() ) );
            }

            Properties properties = new Properties();
            int        seqNr      = 1;

            if( incremental ) {
                properties.setProperty( BackupDescriptor.PREVIOUS_PROP_NAME, ( prevBackup == null ) ? "" : prevBackup.getName() );

                if( prevBackup != null ) {
                    Properties prevProp = prevBackup.getProperties();

                    if( prevProp != null ) {
                        String seqNrStr = prevProp.getProperty( BackupDescriptor.NUMBER_IN_SEQUENCE_PROP_NAME, "1" );

                        try {
                            seqNr = Integer.parseInt( seqNrStr );

                            if( seqNr == maxInc ) {
                                seqNr       = 1;
                                incremental = false;
                                prevBackup  = null;
                            } else {
                                ++seqNr;
                            }
                        }
                        catch( NumberFormatException e ) {
                            LOG.warn( "Bad sequence number in backup descriptor: " + prevBackup.getName() );
                        }
                    }
                }
            }
            properties.setProperty( BackupDescriptor.NUMBER_IN_SEQUENCE_PROP_NAME, Integer.toString( seqNr ) );
            properties.setProperty( BackupDescriptor.INCREMENTAL_PROP_NAME, incremental ? "yes" : "no" );

            try {
                properties.setProperty( BackupDescriptor.DATE_PROP_NAME, new DateTimeValue( new Date() ).getStringValue() );
            }
            catch( XPathException e ) {
            }

            backupFile = directory.createBackup( incremental && ( prevBackup != null ), zip );
            BackupWriter output;

            if( zip ) {
                output = new ZipWriter( backupFile, XmldbURI.ROOT_COLLECTION );
            } else {
                output = new FileSystemWriter( backupFile );
            }
            output.setProperties( properties );

            Date               date = ( prevBackup == null ) ? null : prevBackup.getDate();
            CollectionCallback cb   = new CollectionCallback( output, date, prevBackup, errorList, true );
            broker.getCollectionsFailsafe( cb );

            exportOrphans( output, cb.getDocs(), errorList );

            output.close();
            return( backupFile );
        }
        catch( IOException e ) {
            reportError( "A write error occurred while exporting data: '" + e.getMessage() + "'. Aborting export.", e );
            return( null );
        }
        catch( TerminatedException e ) {

            if( backupFile != null ) {
                backupFile.delete();
            }
            return( null );
        }
    }


    private void reportError( String message, Throwable e )
    {
        if( callback != null ) {
            callback.error( "EXPORT: " + message, e );
        }
    }


    private static boolean isDamaged( DocumentImpl doc, List<ErrorReport> errorList )
    {
        if( errorList == null ) {
            return( false );
        }

        for( org.exist.backup.ErrorReport report : errorList ) {

            if( ( report.getErrcode() == org.exist.backup.ErrorReport.RESOURCE_ACCESS_FAILED ) && ( ( (ErrorReport.ResourceError)report ).getDocumentId() == doc.getDocId() ) ) {
                return( true );
            }
        }
        return( false );
    }


    @SuppressWarnings( "unused" )
    private static boolean isDamaged( Collection collection, List<ErrorReport> errorList )
    {
        if( errorList == null ) {
            return( false );
        }

        for( ErrorReport report : errorList ) {

            if( ( report.getErrcode() == org.exist.backup.ErrorReport.CHILD_COLLECTION ) && ( ( (ErrorReport.CollectionError)report ).getCollectionId() == collection.getId() ) ) {
                return( true );
            }
        }
        return( false );
    }


    private static boolean isDamagedChild( XmldbURI uri, List<ErrorReport> errorList )
    {
        if( errorList == null ) {
            return( false );
        }

        for( org.exist.backup.ErrorReport report : errorList ) {

            if( ( report.getErrcode() == org.exist.backup.ErrorReport.CHILD_COLLECTION ) && ( (org.exist.backup.ErrorReport.CollectionError)report ).getCollectionURI().equalsInternal( uri ) ) {
                return( true );
            }
        }
        return( false );
    }


    /**
     * Scan all document records in collections.dbx and try to find orphaned documents whose parent collection got destroyed or is damaged.
     *
     * @param  output     the backup writer
     * @param  docs       a document set containing all the documents which were exported regularily. the method will ignore those.
     * @param  errorList  a list of {@link org.exist.backup.ErrorReport} objects as returned by methods in {@link ConsistencyCheck}
     */
    private void exportOrphans( BackupWriter output, DocumentSet docs, List<ErrorReport> errorList )
    {
        output.newCollection( "/db/__lost_and_found__" );

        try {
            Writer        contents   = output.newContents();

            // serializer writes to __contents__.xml
            SAXSerializer serializer = (SAXSerializer)SerializerPool.getInstance().borrowObject( SAXSerializer.class );
            serializer.setOutput( contents, contentsOutputProps );

            serializer.startDocument();
            serializer.startPrefixMapping( "", Namespaces.EXIST_NS );
            AttributesImpl attr = new AttributesImpl();
            attr.addAttribute( Namespaces.EXIST_NS, "name", "name", "CDATA", "/db/__lost_and_found__" );
            attr.addAttribute( Namespaces.EXIST_NS, "version", "version", "CDATA", String.valueOf( currVersion ) );
            attr.addAttribute( Namespaces.EXIST_NS, "owner", "owner", "CDATA", org.exist.security.SecurityManager.DBA_USER );
            attr.addAttribute( Namespaces.EXIST_NS, "group", "group", "CDATA", org.exist.security.SecurityManager.DBA_GROUP );
            attr.addAttribute( Namespaces.EXIST_NS, "mode", "mode", "CDATA", "0771" );
            serializer.startElement( Namespaces.EXIST_NS, "collection", "collection", attr );

            DocumentCallback docCb = new DocumentCallback( output, serializer, null, null, docs, true );
            broker.getResourcesFailsafe( docCb, directAccess );

            serializer.endElement( Namespaces.EXIST_NS, "collection", "collection" );
            serializer.endPrefixMapping( "" );
            serializer.endDocument();
            output.closeContents();
        }
        catch( Exception e ) {
            e.printStackTrace();

            if( callback != null ) {
                callback.error( e.getMessage(), e );
            }
        }
        finally {
            output.closeCollection();
        }
    }


    /**
     * Export a collection. Write out the collection metadata and save the resources stored in the collection.
     *
     * @param   current     the collection
     * @param   output      the output writer
     * @param   date
     * @param   prevBackup  DOCUMENT ME!
     * @param   errorList   a list of {@link org.exist.backup.ErrorReport} objects as returned by methods in {@link org.exist.backup.ConsistencyCheck}
     * @param   docs        a document set to keep track of all written documents.
     *
     * @throws  IOException
     * @throws  SAXException
     * @throws  TerminatedException  DOCUMENT ME!
     */
    private void export( Collection current, BackupWriter output, Date date, BackupDescriptor prevBackup, List<ErrorReport> errorList, MutableDocumentSet docs ) throws IOException, SAXException, TerminatedException, PermissionDeniedException
    {
        if( callback != null ) {
            callback.startCollection( current.getURI().toString() );
        }

        if( ( monitor != null ) && !monitor.proceed() ) {
            throw( new TerminatedException( "system export terminated by db" ) );
        }

//        if( !current.getURI().equalsInternal( XmldbURI.ROOT_COLLECTION_URI ) ) {
            output.newCollection( Backup.encode( URIUtils.urlDecodeUtf8( current.getURI() ) ) );
//        }

        try {
            Writer        contents   = output.newContents();

            // serializer writes to __contents__.xml
            SAXSerializer serializer = (SAXSerializer)SerializerPool.getInstance().borrowObject( SAXSerializer.class );
            serializer.setOutput( contents, contentsOutputProps );

            final Permission perm = current.getPermissions();
            
            serializer.startDocument();
            serializer.startPrefixMapping( "", Namespaces.EXIST_NS );
            XmldbURI       uri  = current.getURI();
            AttributesImpl attr = new AttributesImpl();
            attr.addAttribute( Namespaces.EXIST_NS, "name", "name", "CDATA", uri.toString() );
            attr.addAttribute( Namespaces.EXIST_NS, "version", "version", "CDATA", String.valueOf( currVersion ) );
            Backup.writeUnixStylePermissionAttributes(attr, perm);
            try {
                attr.addAttribute( Namespaces.EXIST_NS, "created", "created", "CDATA", new DateTimeValue( new Date( current.getCreationTime() ) ).getStringValue() );
            }
            catch( XPathException e ) {
                e.printStackTrace();
            }
            serializer.startElement( Namespaces.EXIST_NS, "collection", "collection", attr );

            if(perm instanceof ACLPermission) {
            	Backup.writeACLPermission(serializer, (ACLPermission)perm);
            }

            int docsCount = current.getDocumentCountNoLock(broker);
            int count     = 0;

            for( Iterator<DocumentImpl> i = current.iteratorNoLock( broker ); i.hasNext(); count++ ) {
                DocumentImpl doc = i.next();

                if( isDamaged( doc, errorList ) ) {
                    reportError( "Skipping damaged document " + doc.getFileURI(), null );
                    continue;
                }

                if( doc.getFileURI().equalsInternal( CONTENTS_URI ) || doc.getFileURI().equalsInternal( LOST_URI ) ) {
                    continue; // skip __contents__.xml documents
                }
                exportDocument( output, date, prevBackup, serializer, docsCount, count, doc );
                docs.add( doc, false );
            }

            for( Iterator<XmldbURI> i = current.collectionIteratorNoLock(broker); i.hasNext(); ) {
                XmldbURI childUri = i.next();

                if( childUri.equalsInternal( TEMP_COLLECTION ) ) {
                    continue;
                }

                if( isDamagedChild( childUri, errorList ) ) {
                    reportError( "Skipping damaged child collection " + childUri, null );
                    continue;
                }
                attr.clear();
                attr.addAttribute( Namespaces.EXIST_NS, "name", "name", "CDATA", childUri.toString() );
                attr.addAttribute( Namespaces.EXIST_NS, "filename", "filename", "CDATA", Backup.encode( URIUtils.urlDecodeUtf8( childUri.toString() ) ) );
                serializer.startElement( Namespaces.EXIST_NS, "subcollection", "subcollection", attr );
                serializer.endElement( Namespaces.EXIST_NS, "subcollection", "subcollection" );
            }

            if( prevBackup != null ) {

                // Check which collections and resources have been deleted since
                // the
                // last backup
                CheckDeletedHandler check = new CheckDeletedHandler( current, serializer );

                try {
                    prevBackup.parse( check );
                }
                catch( Exception e ) {
                    LOG.error( "Caught exception while trying to parse previous backup descriptor: " + prevBackup.getSymbolicPath(), e );
                }
            }

            // close <collection>
            serializer.endElement( Namespaces.EXIST_NS, "collection", "collection" );
            serializer.endPrefixMapping( "" );
            serializer.endDocument();
            output.closeContents();
        }
        finally {

//            if( !current.getURI().equalsInternal( XmldbURI.ROOT_COLLECTION_URI ) ) {
                output.closeCollection();
//            }
        }
    }


    private void exportDocument( BackupWriter output, Date date, BackupDescriptor prevBackup, SAXSerializer serializer, int docsCount, int count, DocumentImpl doc ) throws IOException, SAXException, TerminatedException
    {
        if( callback != null ) {
            callback.startDocument( doc.getFileURI().toString(), count, docsCount );
        }

        if( ( monitor != null ) && !monitor.proceed() ) {
            throw( new TerminatedException( "system export terminated by db" ) );
        }
        boolean needsBackup = ( prevBackup == null ) || ( date.getTime() < doc.getMetadata().getLastModified() );

        if( needsBackup ) {
            OutputStream os = output.newEntry( Backup.encode( URIUtils.urlDecodeUtf8( doc.getFileURI() ) ) );

            try {

                if( doc.getResourceType() == DocumentImpl.BINARY_FILE ) {
                    broker.readBinaryResource( (BinaryDocument)doc, os );
                } else {
                    BufferedWriter writer            = new BufferedWriter( new OutputStreamWriter( os, "UTF-8" ) );

                    // write resource to contentSerializer
                    SAXSerializer  contentSerializer = (SAXSerializer)SerializerPool.getInstance().borrowObject( SAXSerializer.class );
                    contentSerializer.setOutput( writer, defaultOutputProperties );
                    writeXML( doc, contentSerializer );
                    SerializerPool.getInstance().returnObject( contentSerializer );
                    writer.flush();
                }
            }
            catch( Exception e ) {
                reportError( "A write error occurred while exporting document: '" + doc.getFileURI() + "'. Continuing with next document.", e );
                return;
            }
            finally {
                output.closeEntry();
            }
        }
        
        Permission perms = doc.getPermissions();

        // store permissions
        AttributesImpl attr = new AttributesImpl();
        attr.addAttribute( Namespaces.EXIST_NS, "type", "type", "CDATA", ( doc.getResourceType() == DocumentImpl.BINARY_FILE ) ? "BinaryResource" : "XMLResource" );
        attr.addAttribute( Namespaces.EXIST_NS, "name", "name", "CDATA", doc.getFileURI().toString() );
        attr.addAttribute( Namespaces.EXIST_NS, "skip", "skip", "CDATA", ( needsBackup ? "no" : "yes" ) );
        Backup.writeUnixStylePermissionAttributes(attr, perms);

        // be careful when accessing document metadata: it is stored in a
        // different place than the
        // main document info and could thus be damaged
        DocumentMetadata metadata = null;

        try {
            metadata = doc.getMetadata();
        }
        catch( Exception e ) {
            // LOG.warn(e.getMessage(), e);
        }

        try {
            String created;
            String modified;

            // metadata could be damaged
            if( metadata != null ) {
                created  = new DateTimeValue( new Date( metadata.getCreated() ) ).getStringValue();
                modified = new DateTimeValue( new Date( metadata.getLastModified() ) ).getStringValue();
            } else {
                created  = new DateTimeValue().getStringValue();
                modified = created;
            }
            attr.addAttribute( Namespaces.EXIST_NS, "created", "created", "CDATA", created );
            attr.addAttribute( Namespaces.EXIST_NS, "modified", "modified", "CDATA", modified );
        }
        catch( XPathException e ) {
            LOG.warn( e.getMessage(), e );
        }

        attr.addAttribute( Namespaces.EXIST_NS, "filename", "filename", "CDATA", Backup.encode( URIUtils.urlDecodeUtf8( doc.getFileURI() ) ) );
        String mimeType = "application/xml";

        if( ( metadata != null ) && ( metadata.getMimeType() != null ) ) {
            mimeType = Backup.encode( metadata.getMimeType() );
        }
        attr.addAttribute( Namespaces.EXIST_NS, "mimetype", "mimetype", "CDATA", mimeType );

        if( ( doc.getResourceType() == DocumentImpl.XML_FILE ) && ( metadata != null ) && ( doc.getDoctype() != null ) ) {

            if( doc.getDoctype().getName() != null ) {
                attr.addAttribute( Namespaces.EXIST_NS, "namedoctype", "namedoctype", "CDATA", doc.getDoctype().getName() );
            }

            if( doc.getDoctype().getPublicId() != null ) {
                attr.addAttribute( Namespaces.EXIST_NS, "publicid", "publicid", "CDATA", doc.getDoctype().getPublicId() );
            }

            if( doc.getDoctype().getSystemId() != null ) {
                attr.addAttribute( Namespaces.EXIST_NS, "systemid", "systemid", "CDATA", doc.getDoctype().getSystemId() );
            }
        }
        serializer.startElement( Namespaces.EXIST_NS, "resource", "resource", attr );
        if(perms instanceof ACLPermission) {
            Backup.writeACLPermission(serializer, (ACLPermission)perms);
        }
        serializer.endElement( Namespaces.EXIST_NS, "resource", "resource" );
    }


    /**
     * Serialize a document to XML, based on {@link XMLStreamReader}.
     *
     * @param  doc       the document to serialize
     * @param  receiver  the output handler
     */
{'fraction_non_alphanumeric': 0.10246888390124465, 'fraction_numerical': 0.0016731279330748827, 'mean_word_length': 2.186049535201196, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 28}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2542502', 'n_tokens_mistral': 13068, 'n_tokens_neox': 12603, 'n_words': 3273}