<commit_before>##############################################################################
#
#    Copyright (C) 2014-2018 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: <NAME> <<EMAIL>>, <NAME>
#
#    The licence is in the file __manifest__.py
#
##############################################################################
"""
Defines a few functions useful in ../models/import_letters_history.py
"""
import base64
import logging
import os
from io import BytesIO
from time import time

from . import (
    zxing_wrapper,
    zbar_wrapper,
    patternrecognition as pr,
    checkboxreader as cbr,
)

_logger = logging.getLogger(__name__)

try:
    import cv2
    import fitz
    from PyPDF2 import PdfFileWriter, PdfFileReader
except ImportError:
    _logger.warning("Please install PyMuPDF, cv2 and PyPDF2 to use SBC module")

##########################################################################
#                           GENERAL METHODS                              #
##########################################################################


def analyze_attachment(env, file_data, file_name, force_template):
    """
    Analyze attachment (PDF/TIFF) and save everything inside
    import_line_ids.
    The filename is given separately due to the name given by tempfile

    Beware that image is converted to 300DPI, so templates should be defined
    with 300DPI images. Scans should also be with 300 DPI for better results.

    :param env env: Odoo variable env
    :param str file_data: Binary data of the image file to analyze
    :param str file_name: Name of the image file to analyze
    :param correspondence.template force_template: Template

    :returns: Import Line values
    :rtype: list(dict)
    """
    new_dpi = 100.0
    resize_ratio = new_dpi / 300.0

    line_vals = list()
    letter_datas = list()
    _logger.info(f"\tImport file : {file_name}")

    pdf_in_buffer = BytesIO(file_data)
    inputpdf = PdfFileReader(pdf_in_buffer)
    letter_indexes, imgs, lang = _find_qrcodes(env, line_vals, inputpdf, new_dpi)
    _logger.info(f"\t {len(letter_indexes)-1 or 1} letters found!")

    # Construct the data for each detected letter: store as PDF
    if len(letter_indexes) > 1:
        last_index = 0
        for index in letter_indexes[1:]:
            inputpdf = PdfFileReader(pdf_in_buffer)
            output = PdfFileWriter()
            letter_data = BytesIO()
            for i in range(last_index, index):
                output.addPage(inputpdf.getPage(i))
            output.write(letter_data)
            letter_data.seek(0)
            letter_datas.append(letter_data.read())
            last_index = index
    else:
        letter_datas.append(file_data)

    # now try to find the layout for all splitted letters
    for i in range(len(letter_datas)):
        _logger.info(
            f"\tAnalyzing template and language of letter {i + 1}/{len(letter_datas)}"
        )

        letter_vals = line_vals[i]
        file_split = file_name.split(".")
        attach_name = file_split[0] + "-" + str(i) + ".pdf"
        letter_vals["letter_image"] = base64.b64encode(letter_datas[i])
        letter_vals["file_name"] = attach_name
        if force_template:
            letter_vals["template_id"] = force_template.id
        else:
            # use pattern recognition to find the template
            _find_template(env, imgs[i], letter_vals, resize_ratio)
            tic = time()
            if lang:
                letter_vals["letter_language_id"] = lang.id
            _logger.debug(f"\t\tLanguage analysis done in {time() - tic:.3} sec.")

    return line_vals


def _find_qrcodes(env, line_vals, inputpdf, new_dpi):
    """
    Read the image and try to find the QR codes.
    The image should be currently saved as a png with the same name
    than :py:attr:`file_` (except for the extension).
    If QR Code is in wrong orientation, this method will return the given
    file.
    In case of test, the output dictionary contains the image of the QR code
    too.

    :param env env: Odoo variable env
    :param list line_vals: Dictionary that will hold values for import line
    :param inputpdf: PDFReader of the original pdf file
    :returns: binary data of images, numpy arrays of pages to analyze further
    :rtype: list(str), list(np.array)
    """
    # Holds the indexes of the pages where a new letter is detected
    letter_indexes = list()
    page_imgs = list()

    previous_qrcode = ""
    _logger.debug(f"\tThe imported PDF is made of {inputpdf.numPages} pages.")

    for i in range(inputpdf.numPages):
        tic = time()
        output = PdfFileWriter()
        output.addPage(inputpdf.getPage(i))
        page_buffer = BytesIO()
        output.write(page_buffer)
        page_buffer.seek(0)

        # read the qrcode on the current page
        qrcode, img_path = _decode_page(page_buffer.read())
        lang = cbr.find_languages_area(env, img_path)
        if (qrcode and qrcode["data"] != previous_qrcode) or i == 0:
            previous_qrcode = qrcode and qrcode["data"]
            letter_indexes.append(i)
            # we finally resize the image before returning it
            img = cv2.imread(img_path)
            f = new_dpi / 300.0
            try:
                img = cv2.resize(img, (0, 0), fx=f, fy=f, interpolation=cv2.INTER_CUBIC)
            except cv2.error:
                _logger.warning("Error resizing correspondence page", exc_info=True)
            page_imgs.append(img)
            partner_id, child_id = decode_barcode(env, qrcode)
            try:
                page_preview = cv2.imencode(".jpg", img)
                preview_data = base64.b64encode(page_preview[1])<commit_msg>Only look for QRcodes when scanning s2b letters<commit_after>##############################################################################
#
#    Copyright (C) 2014-2018 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: <NAME> <<EMAIL>>, <NAME>
#
#    The licence is in the file __manifest__.py
#
##############################################################################
"""
Defines a few functions useful in ../models/import_letters_history.py
"""
import base64
import logging
import os
from io import BytesIO
from time import time

from . import (
    zxing_wrapper,
    zbar_wrapper,
    patternrecognition as pr,
    checkboxreader as cbr,
)

_logger = logging.getLogger(__name__)

try:
    import cv2
    import fitz
    from PyPDF2 import PdfFileWriter, PdfFileReader
except ImportError:
    _logger.warning("Please install PyMuPDF, cv2 and PyPDF2 to use SBC module")

##########################################################################
#                           GENERAL METHODS                              #
##########################################################################


def analyze_attachment(env, file_data, file_name, force_template):
    """
    Analyze attachment (PDF/TIFF) and save everything inside
    import_line_ids.
    The filename is given separately due to the name given by tempfile

    Beware that image is converted to 300DPI, so templates should be defined
    with 300DPI images. Scans should also be with 300 DPI for better results.

    :param env env: Odoo variable env
    :param str file_data: Binary data of the image file to analyze
    :param str file_name: Name of the image file to analyze
    :param correspondence.template force_template: Template

    :returns: Import Line values
    :rtype: list(dict)
    """
    new_dpi = 100.0
    resize_ratio = new_dpi / 300.0

    line_vals = list()
    letter_datas = list()
    _logger.info(f"\tImport file : {file_name}")

    pdf_in_buffer = BytesIO(file_data)
    inputpdf = PdfFileReader(pdf_in_buffer)
    letter_indexes, imgs, lang = _find_qrcodes(env, line_vals, inputpdf, new_dpi)
    _logger.info(f"\t {len(letter_indexes)-1 or 1} letters found!")

    # Construct the data for each detected letter: store as PDF
    if len(letter_indexes) > 1:
        last_index = 0
        for index in letter_indexes[1:]:
            inputpdf = PdfFileReader(pdf_in_buffer)
            output = PdfFileWriter()
            letter_data = BytesIO()
            for i in range(last_index, index):
                output.addPage(inputpdf.getPage(i))
            output.write(letter_data)
            letter_data.seek(0)
            letter_datas.append(letter_data.read())
            last_index = index
    else:
        letter_datas.append(file_data)

    # now try to find the layout for all splitted letters
    for i in range(len(letter_datas)):
        _logger.info(
            f"\tAnalyzing template and language of letter {i + 1}/{len(letter_datas)}"
        )

        letter_vals = line_vals[i]
        file_split = file_name.split(".")
        attach_name = file_split[0] + "-" + str(i) + ".pdf"
        letter_vals["letter_image"] = base64.b64encode(letter_datas[i])
        letter_vals["file_name"] = attach_name
        if force_template:
            letter_vals["template_id"] = force_template.id
        else:
            # use pattern recognition to find the template
            _find_template(env, imgs[i], letter_vals, resize_ratio)
            tic = time()
            if lang:
                letter_vals["letter_language_id"] = lang.id
            _logger.debug(f"\t\tLanguage analysis done in {time() - tic:.3} sec.")

    return line_vals


def _find_qrcodes(env, line_vals, inputpdf, new_dpi):
    """
    Read the image and try to find the QR codes.
    The image should be currently saved as a png with the same name
    than :py:attr:`file_` (except for the extension).
    If QR Code is in wrong orientation, this method will return the given
    file.
    In case of test, the output dictionary contains the image of the QR code
    too.

    :param env env: Odoo variable env
    :param list line_vals: Dictionary that will hold values for import line
    :param inputpdf: PDFReader of the original pdf file
    :returns: binary data of images, numpy arrays of pages to analyze further
    :rtype: list(str), list(np.array)
    """
    # Holds the indexes of the pages where a new letter is detected
    letter_indexes = list()
    page_imgs = list()

    previous_qrcode = ""
    _logger.debug(f"\tThe imported PDF is made of {inputpdf.numPages} pages.")

    for i in range(inputpdf.numPages):
        tic = time()
        output = PdfFileWriter()
        output.addPage(inputpdf.getPage(i))
        page_buffer = BytesIO()
        output.write(page_buffer)
        page_buffer.seek(0)

        # read the qrcode on the current page
        qrcode, img_path = _decode_page(page_buffer.read())
        lang = cbr.find_languages_area(env, img_path)
        if (qrcode and qrcode["data"] != previous_qrcode and
                qrcode["format"] == "QRCODE") or i == 0:
            previous_qrcode = qrcode["data"]
            letter_indexes.append(i)
            # we finally resize the image before returning it
            img = cv2.imread(img_path)
            f = new_dpi / 300.0
            try:
                img = cv2.resize(img, (0, 0), fx=f, fy=f, interpolation=cv2.INTER_CUBIC)
            except cv2.error:
                _logger.warning("Error resizing correspondence page", exc_info=True)
            page_imgs.append(img)
            partner_id, child_id = decode_barcode(env, qrcode)
            try:
                page_preview = cv2.imencode(".jpg", img)
                preview_data = base64.b64encode(page_preview[1])
{'fraction_non_alphanumeric': 0.13386234259099533, 'fraction_numerical': 0.010781438675176816, 'mean_word_length': 2.718729955099423, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1554247', 'n_tokens_mistral': 3508, 'n_tokens_neox': 3304, 'n_words': 1023}