<commit_before>module AlaveteliTextMasker
    extend self
    DoNotBinaryMask = [ 'image/tiff',
                        'image/gif',
                        'image/jpeg',
                        'image/png',
                        'image/bmp',
                        'application/zip' ]

    # Replaces all email addresses in (possibly binary) data
    # Also applies custom masks and censor items
    def apply_masks!(text, content_type, options = {})
        # See if content type is one that we mask - things like zip files and
        # images may get broken if we try to. We err on the side of masking too
        # much, as many unknown types will really be text.

        # Special cases for some content types
        case content_type
            when *DoNotBinaryMask
                # do nothing
            when 'text/html'
                apply_text_masks!(text, options)
            when 'application/pdf'
                apply_pdf_masks!(text, options)
            else
                apply_binary_masks!(text, options)
        end
    end

    def apply_pdf_masks!(text, options = {})
        uncompressed_text = nil
        uncompressed_text = AlaveteliExternalCommand.run("pdftk", "-", "output", "-", "uncompress",
                                                         :stdin_string => text)
        # if we managed to uncompress the PDF...
        if !uncompressed_text.blank?
            # then censor stuff (making a copy so can compare again in a bit)
            censored_uncompressed_text = uncompressed_text.dup
            apply_binary_masks!(censored_uncompressed_text, options)
            # if the censor rule removed something...
            if censored_uncompressed_text != uncompressed_text
                # then use the altered file (recompressed)
                recompressed_text = nil
                if AlaveteliConfiguration::use_ghostscript_compression == true
                    command = ["gs", "-sDEVICE=pdfwrite", "-dCompatibilityLevel=1.4", "-dPDFSETTINGS=/screen", "-dNOPAUSE", "-dQUIET", "-dBATCH", "-sOutputFile=-", "-"]
                else
                    command = ["pdftk", "-", "output", "-", "compress"]
                end
                recompressed_text = AlaveteliExternalCommand.run(*(command + [{:stdin_string=>censored_uncompressed_text}]))
                if recompressed_text.blank?
                    # buggy versions of pdftk sometimes fail on
                    # compression, I don't see it's a disaster in
                    # these cases to save an uncompressed version?
                    recompressed_text = censored_uncompressed_text
                    logger.warn "Unable to compress PDF; problem with your pdftk version?"
                end
                if !recompressed_text.blank?
                    text.replace recompressed_text
                end
            end
        end
    end

    private

    # Replace text in place<commit_msg>Call logger explicitly because it's not in scope here<commit_after>module AlaveteliTextMasker
    extend self
    DoNotBinaryMask = [ 'image/tiff',
                        'image/gif',
                        'image/jpeg',
                        'image/png',
                        'image/bmp',
                        'application/zip' ]

    # Replaces all email addresses in (possibly binary) data
    # Also applies custom masks and censor items
    def apply_masks!(text, content_type, options = {})
        # See if content type is one that we mask - things like zip files and
        # images may get broken if we try to. We err on the side of masking too
        # much, as many unknown types will really be text.

        # Special cases for some content types
        case content_type
            when *DoNotBinaryMask
                # do nothing
            when 'text/html'
                apply_text_masks!(text, options)
            when 'application/pdf'
                apply_pdf_masks!(text, options)
            else
                apply_binary_masks!(text, options)
        end
    end

    def apply_pdf_masks!(text, options = {})
        uncompressed_text = nil
        uncompressed_text = AlaveteliExternalCommand.run("pdftk", "-", "output", "-", "uncompress",
                                                         :stdin_string => text)
        # if we managed to uncompress the PDF...
        if !uncompressed_text.blank?
            # then censor stuff (making a copy so can compare again in a bit)
            censored_uncompressed_text = uncompressed_text.dup
            apply_binary_masks!(censored_uncompressed_text, options)
            # if the censor rule removed something...
            if censored_uncompressed_text != uncompressed_text
                # then use the altered file (recompressed)
                recompressed_text = nil
                if AlaveteliConfiguration::use_ghostscript_compression == true
                    command = ["gs", "-sDEVICE=pdfwrite", "-dCompatibilityLevel=1.4", "-dPDFSETTINGS=/screen", "-dNOPAUSE", "-dQUIET", "-dBATCH", "-sOutputFile=-", "-"]
                else
                    command = ["pdftk", "-", "output", "-", "compress"]
                end
                recompressed_text = AlaveteliExternalCommand.run(*(command + [{:stdin_string=>censored_uncompressed_text}]))
                if recompressed_text.blank?
                    # buggy versions of pdftk sometimes fail on
                    # compression, I don't see it's a disaster in
                    # these cases to save an uncompressed version?
                    recompressed_text = censored_uncompressed_text
                    Rails.logger.warn "Unable to compress PDF; problem with your pdftk version?"
                end
                if !recompressed_text.blank?
                    text.replace recompressed_text
                end
            end
        end
    end

    private

    # Replace text in place
{'fraction_non_alphanumeric': 0.07767646065518406, 'fraction_numerical': 0.0006754474839581223, 'mean_word_length': 1.6512981199641898, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2947024', 'n_tokens_mistral': 1546, 'n_tokens_neox': 1462, 'n_words': 445}