<commit_before>/**
 * Licensed to Apereo under one or more contributor license
 * agreements. See the NOTICE file distributed with this work
 * for additional information regarding copyright ownership.
 * Apereo licenses this file to you under the Apache License,
 * Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License.  You may obtain a
 * copy of the License at the following location:
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.jasig.portlet.attachment.util;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Collection;
import java.util.List;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import javax.xml.namespace.QName;
import javax.xml.transform.stream.StreamSource;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.jasig.portlet.attachment.model.Attachment;
import org.jasig.portlet.attachment.service.IAttachmentService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.dao.DataAccessException;
import org.springframework.stereotype.Service;

/**
 * Provides import and export capability for Attachments.
 *
 * @author <NAME>, <EMAIL>
 */

@Service
public class ImportExport {
    private static final String FILENAME_SUFFIX = ".attachment.xml";

    Logger log = LoggerFactory.getLogger(this.getClass());

    @Autowired
    IAttachmentService attachmentService;

    @Value("${hibernate.connection.url}")
    private String hibernateURI;

    @Value("${hibernate.attachments.batch.fetch.size:100}")
    private int batchFetchSize = 100;

    public static void main(String[] args) throws Exception {

        if (args.length != 2) {
            usage();
        }

        ApplicationContext context =
                new ClassPathXmlApplicationContext(new String[] {"context/importExportContext.xml"});
        ImportExport importExport = (ImportExport) context.getBean("importExport");

        importExport.notifyUserIfUsingInMemoryDb();

        if ("export".equals(args[0])) {
            importExport.export(args[1]);
        } else if ("import".equalsIgnoreCase(args[0])) {
            importExport.importDir(args[1]);
        } else {
            usage();
        }
    }

    private void notifyUserIfUsingInMemoryDb() {
        if (hibernateURI.startsWith("jdbc:hsqldb:mem")) {
            log.warn("");
            log.warn("NOTICE:");
            log.warn("");
            log.warn("Operation aborted.  Update hibernate.properties. It is currently set to the in-memory DB.");
            log.warn("");
            System.exit(1);
        }
    }

    private void importDir(String directoryName) {
        File dir = checkIfDirExists(directoryName, false);
        Collection<File> files = FileUtils.listFiles(dir, new WildcardFileFilter("*" + FILENAME_SUFFIX), null);
        if (files.size() == 0) {
            log.info("No files of form {} to import in {}", "*" + FILENAME_SUFFIX, directoryName);
        }
        boolean importSucceeded = true;
        for (File file : files) {
            importSucceeded &= importFile(file);
        }
        if (!importSucceeded) {
            System.exit(1);
        }
    }

    private boolean importFile(File filename) {
        try {
            JAXBContext jc = JAXBContext.newInstance(Attachment.class);
            StreamSource xml = new StreamSource(filename);
            Unmarshaller unmarshaller = jc.createUnmarshaller();
            JAXBElement<Attachment> je1 = unmarshaller.unmarshal(xml, Attachment.class);
            Attachment attachment = je1.getValue();

            // An export run after having added SCP v2.0.0+ and uploading some attachments (which would populate
            // column bdata, not data) would
            if (attachment.getData() == null && System.getProperty("importEmpty") == null) {
                log.warn("Attachment with guid {} has no data and will be skipped.  This can occur if you had put"
                        + " SimpleContentPortlet v2.0.0+ on, uploaded attachments which wrote the data to the"
                        + " BDATA column instead of the deprecated DATA column, then went back to a pre 2.0.0"
                        + " version to do the data-export.  Typically you do not want these records imported; they"
                        + " will be correct in the database if you are running import on the same database"
                        + " you ran export on.",
                        attachment.getGuid());
                return false;
            }
            saveOrUpdate(attachment);
        } catch (JAXBException e) {
            log.error("Unable to import attachment {}", filename.getAbsolutePath(), e);
            return false;
        }
        return true;
    }

    private void saveOrUpdate(Attachment attachment) {<commit_msg>CMSPLT-56 documentation<commit_after>/**
 * Licensed to Apereo under one or more contributor license
 * agreements. See the NOTICE file distributed with this work
 * for additional information regarding copyright ownership.
 * Apereo licenses this file to you under the Apache License,
 * Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License.  You may obtain a
 * copy of the License at the following location:
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.jasig.portlet.attachment.util;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Collection;
import java.util.List;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import javax.xml.namespace.QName;
import javax.xml.transform.stream.StreamSource;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.jasig.portlet.attachment.model.Attachment;
import org.jasig.portlet.attachment.service.IAttachmentService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.dao.DataAccessException;
import org.springframework.stereotype.Service;

/**
 * Provides import and export capability for Attachments.
 *
 * @author <NAME>, <EMAIL>
 */

@Service
public class ImportExport {
    private static final String FILENAME_SUFFIX = ".attachment.xml";

    Logger log = LoggerFactory.getLogger(this.getClass());

    @Autowired
    IAttachmentService attachmentService;

    @Value("${hibernate.connection.url}")
    private String hibernateURI;

    @Value("${hibernate.attachments.batch.fetch.size:100}")
    private int batchFetchSize = 100;

    public static void main(String[] args) throws Exception {

        if (args.length != 2) {
            usage();
        }

        ApplicationContext context =
                new ClassPathXmlApplicationContext(new String[] {"context/importExportContext.xml"});
        ImportExport importExport = (ImportExport) context.getBean("importExport");

        importExport.notifyUserIfUsingInMemoryDb();

        if ("export".equals(args[0])) {
            importExport.export(args[1]);
        } else if ("import".equalsIgnoreCase(args[0])) {
            importExport.importDir(args[1]);
        } else {
            usage();
        }
    }

    private void notifyUserIfUsingInMemoryDb() {
        if (hibernateURI.startsWith("jdbc:hsqldb:mem")) {
            log.warn("");
            log.warn("NOTICE:");
            log.warn("");
            log.warn("Operation aborted.  Update hibernate.properties. It is currently set to the in-memory DB.");
            log.warn("");
            System.exit(1);
        }
    }

    private void importDir(String directoryName) {
        File dir = checkIfDirExists(directoryName, false);
        Collection<File> files = FileUtils.listFiles(dir, new WildcardFileFilter("*" + FILENAME_SUFFIX), null);
        if (files.size() == 0) {
            log.info("No files of form {} to import in {}", "*" + FILENAME_SUFFIX, directoryName);
        }
        boolean importSucceeded = true;
        for (File file : files) {
            importSucceeded &= importFile(file);
        }
        if (!importSucceeded) {
            System.exit(1);
        }
    }

    private boolean importFile(File filename) {
        try {
            JAXBContext jc = JAXBContext.newInstance(Attachment.class);
            StreamSource xml = new StreamSource(filename);
            Unmarshaller unmarshaller = jc.createUnmarshaller();
            JAXBElement<Attachment> je1 = unmarshaller.unmarshal(xml, Attachment.class);
            Attachment attachment = je1.getValue();

            // JAXB stores binary data in Base64-encoding, so we can import either an SCP v1.2.x export or an
            // SCP 2.0.0+ export without having to worry about trying to detect the encoding scheme.

            // An export run after having added SCP v2.0.0+ and uploading some attachments (which would populate
            // column bdata, not data) and then backing down to a version of SCP prior to 2.0.0 and doing an
            // export would find no data for the attachments added with v2.0.0.  Notify the user.
            if (attachment.getData() == null && System.getProperty("importEmpty") == null) {
                log.warn("Attachment with guid {} has no data and will be skipped.  This can occur if you had put"
                        + " SimpleContentPortlet v2.0.0+ on, uploaded attachments which wrote the data to the"
                        + " BDATA column instead of the deprecated DATA column, then went back to a pre 2.0.0"
                        + " version to do the data-export.  Typically you do not want these records imported; they"
                        + " will be correct in the database if you are running import on the same database"
                        + " you ran export on.",
                        attachment.getGuid());
                return false;
            }
            saveOrUpdate(attachment);
        } catch (JAXBException e) {
            log.error("Unable to import attachment {}", filename.getAbsolutePath(), e);
            return false;
        }
        return true;
    }

    private void saveOrUpdate(Attachment attachment) {
{'fraction_non_alphanumeric': 0.09658797173875582, 'fraction_numerical': 0.0066344993968636915, 'mean_word_length': 2.7550954383694597, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7150642', 'n_tokens_mistral': 3104, 'n_tokens_neox': 3033, 'n_words': 924}