<commit_before>package xyz.docbleach.module.ooxml;

import org.apache.poi.UnsupportedFileFormatException;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.openxml4j.opc.*;
import org.apache.poi.openxml4j.opc.internal.ContentType;
import org.apache.poi.poifs.filesystem.DocumentFactoryHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import xyz.docbleach.api.BleachSession;
import xyz.docbleach.api.bleach.Bleach;
import xyz.docbleach.api.exception.BleachException;
import xyz.docbleach.api.threat.Threat;
import xyz.docbleach.api.threat.ThreatAction;
import xyz.docbleach.api.threat.ThreatSeverity;
import xyz.docbleach.api.threat.ThreatType;
import xyz.docbleach.api.util.StreamUtils;

import java.io.*;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;

/**
 * Sanitizes Office 2007+ documents (OOXML), in place. This format has two interesting things for
 * us: a list of elements and a list of relationships. It is an Open Format, well documented and we
 * have a list of possible values. Each element/relationship is tied to a mime type, so we just
 * filter at this level: if the mime type is a macro, or an ActiveX object, we remove it.
 */
public class OOXMLBleach implements Bleach {
    private static final Logger LOGGER = LoggerFactory.getLogger(OOXMLBleach.class);
    private static final String SUSPICIOUS_OOXML_FORMAT = "Found and removed suspicious content type: '{}' in '{}' (Size: {})";
    private static final String EXTERNAL_RELATION_FORMAT = "Found an external relationship from '{}' to '{}' (type '{}')";
    private static final String OOXWORD_SCHEME = "ooxWord";

    private static final String[] WHITELISTED_RELATIONS = new String[]{
            // Hyperlinks should be safe enough, right?
            Relations.HYPERLINK
    };

    private static final String[] BLACKLISTED_RELATIONS = new String[]{
            // Macro
            Relations.VBA_PROJECT,
            Relations.VBA_PROJECT_SIGNATURE,
            Relations.WORD_VBA_DATA,

            // OLE Objects
            Relations.OPENXML_OLE_OBJECT,
            Relations.OLE_OBJECT,
            Relations.E1_OBJECT,
            Relations.E2_OBJECT,

            // ActiveX Controls
            Relations.OPENXML_CONTROL,
            Relations.OPENXML_ACTIVEX_CONTROL,
            Relations.OPENXML_ACTIVEX_CONTROL_BIN,
            Relations.ACTIVEX_CONTROL,
            Relations.ACTIVEX_CONTROL_BIN
    };

    private static final String[] BLACKLISTED_CONTENT_TYPES = new String[]{
            // Macro related content types
            ContentTypes.VBA_DATA,
            ContentTypes.VBA_PROJECT,
            ContentTypes.VBA_PROJECT_SIGNATURE,

            // Blacklisting Postscript to prevent 0days
            ContentTypes.POSTSCRIPT,

            // OLE Objects
            ContentTypes.OLE_OBJECT,
            ContentTypes.PACKAGE,

            // ActiveX objects
            ContentTypes.ACTIVEX,
            ContentTypes.OPENXML_ACTIVEX,
            ContentTypes.OPENXML_ACTIVEX_XML,
    };

    private static final Map<String, String> REMAPPED_CONTENT_TYPES = new HashMap<>();

    static {
        // Word
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_DOCM, ContentTypes.MAIN_DOCX);
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_DOTM, ContentTypes.MAIN_DOTX);

        // Excel
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_XLTM, ContentTypes.MAIN_XLTX);
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_XLSM, ContentTypes.MAIN_XLSX);

        // PowerPoint
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_PPSM, ContentTypes.MAIN_PPSX);
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_PPTM, ContentTypes.MAIN_PPTX);
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_POTM, ContentTypes.MAIN_POTX);
    }

    @Override
    public boolean handlesMagic(InputStream stream) {
        try {
            return DocumentFactoryHelper.hasOOXMLHeader(stream);
        } catch (IOException e) {
            LOGGER.warn("An exception occured", e);
            return false;
        }
    }

    @Override
    public String getName() {
        return "Office Bleach";
    }

    @Override
    public void sanitize(InputStream inputStream, OutputStream outputStream, BleachSession session) throws BleachException {
        try {
            inputStream = prepareInputStream(inputStream);
        } catch (IOException e) {
            LOGGER.error("Error in OOXMLBleach", e);
            return;
        }

        try (OPCPackage pkg = OPCPackage.open(inputStream)) {
            sanitize(pkg, session);

            pkg.save(outputStream);

            // Prevent from writing the InputStream, even if this sounds absurd.
            pkg.revert();
        } catch (InvalidFormatException | UnsupportedFileFormatException e) {
            // We can't canitize this file, so we ignore it
            LOGGER.error("Invalid format", e);
            try {
                inputStream.reset();
                StreamUtils.copy(inputStream, outputStream);
            } catch (IOException e2) {
                LOGGER.error("Error in OOXMLBleach", e2);
            }
        } catch (IOException e) {
            throw new BleachException(e);
        }
    }

    private InputStream prepareInputStream(InputStream inputStream) throws IOException {
        // Once opened, the stream can't be recovered. We need to create a "backup" of it, just in case the file is a
        // normal Zip archive and not an OOXML document.
        if (!(inputStream instanceof ByteArrayInputStream)) {
            InputStream oldInputStream = inputStream;
            ByteArrayOutputStream bucket = new ByteArrayOutputStream();
            StreamUtils.copy(oldInputStream, bucket);
            oldInputStream.close();

            inputStream = new ByteArrayInputStream(bucket.toByteArray());
        }

        inputStream.mark(inputStream.available() + 1);<commit_msg>:bug: OOXML: Don't log when file is invalid<commit_after>package xyz.docbleach.module.ooxml;

import org.apache.poi.UnsupportedFileFormatException;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.openxml4j.opc.*;
import org.apache.poi.openxml4j.opc.internal.ContentType;
import org.apache.poi.poifs.filesystem.DocumentFactoryHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import xyz.docbleach.api.BleachSession;
import xyz.docbleach.api.bleach.Bleach;
import xyz.docbleach.api.exception.BleachException;
import xyz.docbleach.api.threat.Threat;
import xyz.docbleach.api.threat.ThreatAction;
import xyz.docbleach.api.threat.ThreatSeverity;
import xyz.docbleach.api.threat.ThreatType;
import xyz.docbleach.api.util.StreamUtils;

import java.io.*;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;

/**
 * Sanitizes Office 2007+ documents (OOXML), in place. This format has two interesting things for
 * us: a list of elements and a list of relationships. It is an Open Format, well documented and we
 * have a list of possible values. Each element/relationship is tied to a mime type, so we just
 * filter at this level: if the mime type is a macro, or an ActiveX object, we remove it.
 */
public class OOXMLBleach implements Bleach {
    private static final Logger LOGGER = LoggerFactory.getLogger(OOXMLBleach.class);
    private static final String SUSPICIOUS_OOXML_FORMAT = "Found and removed suspicious content type: '{}' in '{}' (Size: {})";
    private static final String EXTERNAL_RELATION_FORMAT = "Found an external relationship from '{}' to '{}' (type '{}')";
    private static final String OOXWORD_SCHEME = "ooxWord";

    private static final String[] WHITELISTED_RELATIONS = new String[]{
            // Hyperlinks should be safe enough, right?
            Relations.HYPERLINK
    };

    private static final String[] BLACKLISTED_RELATIONS = new String[]{
            // Macro
            Relations.VBA_PROJECT,
            Relations.VBA_PROJECT_SIGNATURE,
            Relations.WORD_VBA_DATA,

            // OLE Objects
            Relations.OPENXML_OLE_OBJECT,
            Relations.OLE_OBJECT,
            Relations.E1_OBJECT,
            Relations.E2_OBJECT,

            // ActiveX Controls
            Relations.OPENXML_CONTROL,
            Relations.OPENXML_ACTIVEX_CONTROL,
            Relations.OPENXML_ACTIVEX_CONTROL_BIN,
            Relations.ACTIVEX_CONTROL,
            Relations.ACTIVEX_CONTROL_BIN
    };

    private static final String[] BLACKLISTED_CONTENT_TYPES = new String[]{
            // Macro related content types
            ContentTypes.VBA_DATA,
            ContentTypes.VBA_PROJECT,
            ContentTypes.VBA_PROJECT_SIGNATURE,

            // Blacklisting Postscript to prevent 0days
            ContentTypes.POSTSCRIPT,

            // OLE Objects
            ContentTypes.OLE_OBJECT,
            ContentTypes.PACKAGE,

            // ActiveX objects
            ContentTypes.ACTIVEX,
            ContentTypes.OPENXML_ACTIVEX,
            ContentTypes.OPENXML_ACTIVEX_XML,
    };

    private static final Map<String, String> REMAPPED_CONTENT_TYPES = new HashMap<>();

    static {
        // Word
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_DOCM, ContentTypes.MAIN_DOCX);
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_DOTM, ContentTypes.MAIN_DOTX);

        // Excel
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_XLTM, ContentTypes.MAIN_XLTX);
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_XLSM, ContentTypes.MAIN_XLSX);

        // PowerPoint
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_PPSM, ContentTypes.MAIN_PPSX);
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_PPTM, ContentTypes.MAIN_PPTX);
        REMAPPED_CONTENT_TYPES.put(ContentTypes.MAIN_POTM, ContentTypes.MAIN_POTX);
    }

    @Override
    public boolean handlesMagic(InputStream stream) {
        try {
            return DocumentFactoryHelper.hasOOXMLHeader(stream);
        } catch (IOException e) {
            LOGGER.warn("An exception occured", e);
            return false;
        }
    }

    @Override
    public String getName() {
        return "Office Bleach";
    }

    @Override
    public void sanitize(InputStream inputStream, OutputStream outputStream, BleachSession session) throws BleachException {
        try {
            inputStream = prepareInputStream(inputStream);
        } catch (IOException e) {
            LOGGER.error("Error in OOXMLBleach", e);
            return;
        }

        try (OPCPackage pkg = OPCPackage.open(inputStream)) {
            sanitize(pkg, session);

            pkg.save(outputStream);

            // Prevent from writing the InputStream, even if this sounds absurd.
            pkg.revert();
        } catch (InvalidFormatException | UnsupportedFileFormatException e) {
            // We can't canitize this file, so we ignore it
            LOGGER.debug("Invalid format", e);
            try {
                inputStream.reset();
                StreamUtils.copy(inputStream, outputStream);
            } catch (IOException e2) {
                LOGGER.error("Error in OOXMLBleach", e2);
            }
        } catch (IOException e) {
            throw new BleachException(e);
        }
    }

    private InputStream prepareInputStream(InputStream inputStream) throws IOException {
        // Once opened, the stream can't be recovered. We need to create a "backup" of it, just in case the file is a
        // normal Zip archive and not an OOXML document.
        if (!(inputStream instanceof ByteArrayInputStream)) {
            InputStream oldInputStream = inputStream;
            ByteArrayOutputStream bucket = new ByteArrayOutputStream();
            StreamUtils.copy(oldInputStream, bucket);
            oldInputStream.close();

            inputStream = new ByteArrayInputStream(bucket.toByteArray());
        }

        inputStream.mark(inputStream.available() + 1);
{'fraction_non_alphanumeric': 0.08271935699933021, 'fraction_numerical': 0.0025117213663764233, 'mean_word_length': 2.6596200980392157, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1092011', 'n_tokens_mistral': 3526, 'n_tokens_neox': 3372, 'n_words': 796}