<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014-2015 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: <NAME>, <NAME> <<EMAIL>>
#
#    The licence is in the file __openerp__.py
#
##############################################################################
"""
This module reads a zip file containing scans of mail and finds the relation
between the database and the mail.
"""
import base64
import logging
import time
import zipfile
from io import BytesIO

from odoo import fields, models, api, _, exceptions
from odoo.tools.config import config
from odoo.addons.sbc_compassion.tools import import_letter_functions as func
from . import translate_connector

logger = logging.getLogger(__name__)

try:
    import pysftp
    from smb.SMBConnection import SMBConnection
    from smb.smb_structs import OperationFailure
    from pyPdf import PdfFileWriter, PdfFileReader
    from pyPdf.pdf import PageObject
except ImportError:
    logger.warning("Please install python dependencies.")


class ImportLettersHistory(models.Model):
    """
    Keep history of imported letters.
    This class add to is parent the possibility to select letters to import
    from a specify config.
    The code is reading QR codes in order to detect child and partner codes
    for every letter, using the zxing library for code detection.
    """
    _name = 'import.letters.history'
    _inherit = ['import.letters.history', 'import.letter.config']

    manual_import = fields.Boolean(
        string=_("Manual import"),
        default=False)

    ##########################################################################
    #                             FIELDS METHODS                             #
    ##########################################################################
    @api.onchange("data", "import_folder_path")
    def _count_nber_letters(self):
        """
        Counts the number of scans. If a zip file is given, the number of
        scans inside is counted.
        """
        if self.manual_import or (
                self.state and self.state != 'draft'):
            super(ImportLettersHistory, self)._count_nber_letters()
        else:
            # files are not selected by user so we find them on NAS
            # folder 'Imports' counter
            tmp = 0

            smb_conn = self._get_smb_connection()
            share_nas = self.env.ref('sbc_switzerland.share_on_nas').value
            imported_letter_path = self.import_folder_path

            if smb_conn and smb_conn.connect(
                SmbConfig.smb_ip, SmbConfig.smb_port) and \
                    imported_letter_path:
                imported_letter_path = self.check_path(imported_letter_path)

                try:
                    listPaths = smb_conn.listPath(
                        share_nas,
                        imported_letter_path)
                except OperationFailure:
                    logger.info('--------------- PATH NO CORRECT -----------')
                    listPaths = []

                for sharedFile in listPaths:
                    if func.check_file(sharedFile.filename) == 1:
                        tmp += 1
                    elif func.isZIP(sharedFile.filename):
                        logger.info(
                            'File to retrieve: {}'.format(
                                imported_letter_path +
                                sharedFile.filename))

                        file_obj = BytesIO()
                        smb_conn.retrieveFile(
                            share_nas,
                            imported_letter_path +
                            sharedFile.filename,
                            file_obj)
                        try:
                            zip_ = zipfile.ZipFile(file_obj, 'r')
                            list_file = zip_.namelist()
                            # loop over all files in zip
                            for tmp_file in list_file:
                                tmp += (func.check_file(
                                    tmp_file) == 1)
                        except zipfile.BadZipfile:
                            raise exceptions.UserError(
                                _('Zip file corrupted (' +
                                  sharedFile.filename + ')'))
                        except zipfile.LargeZipFile:
                            raise exceptions.UserError(
                                _('Zip64 is not supported(' +
                                  sharedFile.filename + ')'))
                smb_conn.close()
            else:
                logger.info("""Failed to list files in imported \
                folder Imports oh the NAS in emplacement: {}""".format(
                    imported_letter_path))

            self.nber_letters = tmp

    ##########################################################################
    #                             VIEW CALLBACKS                             #
    ##########################################################################
    @api.multi
    def button_import(self):
        """
        Analyze the imports in order to create the letter's lines
        """
        if not self.manual_import:
            # when letters are in a folder on NAS redefine method
            for letters_import in self:
                letters_import.state = 'pending'
                if self.env.context.get('async_mode', True):
                    letters_import.with_delay()._run_analyze()
                else:
                    letters_import._run_analyze()
            return True
        else:
            # when letters selected by user, save them on NAS and call
            # super method
            for letters_import in self:
                if letters_import.data and self.env.context.get(
                        'async_mode', True):
                    for attachment in letters_import.data:
                        self._save_imported_letter(attachment)

            return super(ImportLettersHistory, self).button_import()

    @api.multi
    def button_save(self):
        """
        When saving the import_line as correspondences, move pdf file in the
        done folder on the NAS and remove image attachment (Web letter case)
        """
        try:
            for import_letters in self:
                if import_letters.config_id.name == \
                        self.env.ref('sbc_switzerland.web_letter').name:
                    for line in import_letters.import_line_ids:
                        # build part of filename, corresponding to this web
                        # import
                        # letters remove part after '-' caracter (including)
                        # corresponding to the page number
                        part_filename = (line.letter_image.name[:-4]).split(
                            '-')[0]

                        share_nas = self.env.ref(
                            'sbc_switzerland.share_on_nas').value

                        smb_conn = self._get_smb_connection()
                        if smb_conn and smb_conn.connect(
                                SmbConfig.smb_ip, SmbConfig.smb_port):
                            imported_letter_path = self.env.ref(
                                'sbc_switzerland.scan_letter_imported').value
                            listPaths = smb_conn.listPath(
                                share_nas,
                                imported_letter_path)

                            # loop in import folder and find pdf corresponding
                            # to this web letter and eventual attached image
                            image_ext = None
                            file_to_save = False
                            for sharedFile in listPaths:
                                ext = sharedFile.filename[-4:]
                                if part_filename == sharedFile.filename[:-4]:
                                    if ext == '.pdf':
                                        file_to_save = True
                                    else:
                                        image_ext = ext

                            # move web letter on 'Done' folder on the NAS
                            if file_to_save:
                                filename = part_filename + '.pdf'
                                file_obj = BytesIO()
                                smb_conn.retrieveFile(
                                    share_nas, imported_letter_path + filename,
                                    file_obj)
                                file_obj.seek(0)
                                self._copy_imported_to_done_letter(
                                    filename, file_obj, False)
                                # delete files corresponding to web letter in
                                # 'Import'
                                try:
                                    smb_conn.deleteFiles(
                                        share_nas,
                                        imported_letter_path + filename)
                                except Exception as inst:
                                    logger.info(
                                        'Failed to delete pdf web letter'
                                        .format(inst))

                                # image is attached to this letter so we
                                # remove it
                                if image_ext:
                                    try:
                                        smb_conn.deleteFiles(
                                            share_nas,
                                            imported_letter_path +
                                            part_filename + image_ext)
                                    except Exception as inst:
                                        logger.info(
                                            'Failed to delete attached '
                                            'image {}'.format(inst))

                if import_letters.manual_import:
                    self._manage_all_imported_files()

            super(ImportLettersHistory, self).button_save()

        except Exception as e:
            logger.info("Exception during letter import: {}", e)
            # bug during import, so we remove letter sent on translation
            # platform
            for letters in self:
                for corresp in letters.letters_ids:
                    if corresp.state == "Global Partner translation queue":
                        logger.info("LETTER DELETE FROM TRANSLATION PLATFORM")
                        tc = translate_connector.TranslateConnect()
                        tc.remove_translation_with_odoo_id(corresp.id)
            raise e

        return True

    #########################################################################
    #                             PRIVATE METHODS                           #
    #########################################################################
    def _run_analyze(self):
        """
        Analyze each attachment:
        - check for duplicate file names and skip them
        - decompress zip file if necessary<commit_msg>Odoo 8 to Odoo 10 change<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014-2015 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: <NAME>, <NAME> <<EMAIL>>
#
#    The licence is in the file __openerp__.py
#
##############################################################################
"""
This module reads a zip file containing scans of mail and finds the relation
between the database and the mail.
"""
import base64
import logging
import time
import zipfile
from io import BytesIO

from odoo import fields, models, api, _, exceptions
from odoo.tools.config import config
from odoo.addons.sbc_compassion.tools import import_letter_functions as func
from . import translate_connector

logger = logging.getLogger(__name__)

try:
    import pysftp
    from smb.SMBConnection import SMBConnection
    from smb.smb_structs import OperationFailure
    from pyPdf import PdfFileWriter, PdfFileReader
    from pyPdf.pdf import PageObject
    from odoo.addons.queue_job.job import job, related_action
except ImportError:
    logger.warning("Please install python dependencies.")


class ImportLettersHistory(models.Model):
    """
    Keep history of imported letters.
    This class add to is parent the possibility to select letters to import
    from a specify config.
    The code is reading QR codes in order to detect child and partner codes
    for every letter, using the zxing library for code detection.
    """
    _name = 'import.letters.history'
    _inherit = ['import.letters.history', 'import.letter.config']

    manual_import = fields.Boolean(
        string=_("Manual import"),
        default=False)

    ##########################################################################
    #                             FIELDS METHODS                             #
    ##########################################################################
    @api.onchange("data", "import_folder_path")
    def _count_nber_letters(self):
        """
        Counts the number of scans. If a zip file is given, the number of
        scans inside is counted.
        """
        if self.manual_import or (
                self.state and self.state != 'draft'):
            super(ImportLettersHistory, self)._count_nber_letters()
        else:
            # files are not selected by user so we find them on NAS
            # folder 'Imports' counter
            tmp = 0

            smb_conn = self._get_smb_connection()
            share_nas = self.env.ref('sbc_switzerland.share_on_nas').value
            imported_letter_path = self.import_folder_path

            if smb_conn and smb_conn.connect(
                SmbConfig.smb_ip, SmbConfig.smb_port) and \
                    imported_letter_path:
                imported_letter_path = self.check_path(imported_letter_path)

                try:
                    listPaths = smb_conn.listPath(
                        share_nas,
                        imported_letter_path)
                except OperationFailure:
                    logger.info('--------------- PATH NO CORRECT -----------')
                    listPaths = []

                for sharedFile in listPaths:
                    if func.check_file(sharedFile.filename) == 1:
                        tmp += 1
                    elif func.isZIP(sharedFile.filename):
                        logger.info(
                            'File to retrieve: {}'.format(
                                imported_letter_path +
                                sharedFile.filename))

                        file_obj = BytesIO()
                        smb_conn.retrieveFile(
                            share_nas,
                            imported_letter_path +
                            sharedFile.filename,
                            file_obj)
                        try:
                            zip_ = zipfile.ZipFile(file_obj, 'r')
                            list_file = zip_.namelist()
                            # loop over all files in zip
                            for tmp_file in list_file:
                                tmp += (func.check_file(
                                    tmp_file) == 1)
                        except zipfile.BadZipfile:
                            raise exceptions.UserError(
                                _('Zip file corrupted (' +
                                  sharedFile.filename + ')'))
                        except zipfile.LargeZipFile:
                            raise exceptions.UserError(
                                _('Zip64 is not supported(' +
                                  sharedFile.filename + ')'))
                smb_conn.close()
            else:
                logger.info("""Failed to list files in imported \
                folder Imports oh the NAS in emplacement: {}""".format(
                    imported_letter_path))

            self.nber_letters = tmp

    ##########################################################################
    #                             VIEW CALLBACKS                             #
    ##########################################################################
    @api.multi
    def button_import(self):
        """
        Analyze the imports in order to create the letter's lines
        """
        if not self.manual_import:
            # when letters are in a folder on NAS redefine method
            for letters_import in self:
                letters_import.state = 'pending'
                if self.env.context.get('async_mode', True):
                    letters_import.with_delay()._run_analyze()
                else:
                    letters_import._run_analyze()
            return True
        else:
            # when letters selected by user, save them on NAS and call
            # super method
            for letters_import in self:
                if letters_import.data and self.env.context.get(
                        'async_mode', True):
                    for attachment in letters_import.data:
                        self._save_imported_letter(attachment)

            return super(ImportLettersHistory, self).button_import()

    @api.multi
    def button_save(self):
        """
        When saving the import_line as correspondences, move pdf file in the
        done folder on the NAS and remove image attachment (Web letter case)
        """
        try:
            for import_letters in self:
                if import_letters.config_id.name == \
                        self.env.ref('sbc_switzerland.web_letter').name:
                    for line in import_letters.import_line_ids:
                        # build part of filename, corresponding to this web
                        # import
                        # letters remove part after '-' caracter (including)
                        # corresponding to the page number
                        part_filename = (line.letter_image.name[:-4]).split(
                            '-')[0]

                        share_nas = self.env.ref(
                            'sbc_switzerland.share_on_nas').value

                        smb_conn = self._get_smb_connection()
                        if smb_conn and smb_conn.connect(
                                SmbConfig.smb_ip, SmbConfig.smb_port):
                            imported_letter_path = self.env.ref(
                                'sbc_switzerland.scan_letter_imported').value
                            listPaths = smb_conn.listPath(
                                share_nas,
                                imported_letter_path)

                            # loop in import folder and find pdf corresponding
                            # to this web letter and eventual attached image
                            image_ext = None
                            file_to_save = False
                            for sharedFile in listPaths:
                                ext = sharedFile.filename[-4:]
                                if part_filename == sharedFile.filename[:-4]:
                                    if ext == '.pdf':
                                        file_to_save = True
                                    else:
                                        image_ext = ext

                            # move web letter on 'Done' folder on the NAS
                            if file_to_save:
                                filename = part_filename + '.pdf'
                                file_obj = BytesIO()
                                smb_conn.retrieveFile(
                                    share_nas, imported_letter_path + filename,
                                    file_obj)
                                file_obj.seek(0)
                                self._copy_imported_to_done_letter(
                                    filename, file_obj, False)
                                # delete files corresponding to web letter in
                                # 'Import'
                                try:
                                    smb_conn.deleteFiles(
                                        share_nas,
                                        imported_letter_path + filename)
                                except Exception as inst:
                                    logger.info(
                                        'Failed to delete pdf web letter'
                                        .format(inst))

                                # image is attached to this letter so we
                                # remove it
                                if image_ext:
                                    try:
                                        smb_conn.deleteFiles(
                                            share_nas,
                                            imported_letter_path +
                                            part_filename + image_ext)
                                    except Exception as inst:
                                        logger.info(
                                            'Failed to delete attached '
                                            'image {}'.format(inst))

                if import_letters.manual_import:
                    self._manage_all_imported_files()

            super(ImportLettersHistory, self).button_save()

        except Exception as e:
            logger.info("Exception during letter import: {}", e)
            # bug during import, so we remove letter sent on translation
            # platform
            for letters in self:
                for corresp in letters.letters_ids:
                    if corresp.state == "Global Partner translation queue":
                        logger.info("LETTER DELETE FROM TRANSLATION PLATFORM")
                        tc = translate_connector.TranslateConnect()
                        tc.remove_translation_with_odoo_id(corresp.id)
            raise e

        return True

    #########################################################################
    #                             PRIVATE METHODS                           #
    #########################################################################
    @job(default_channel='root.sbc_compassion')
    @related_action(action='related_action_s2b_imports')
    def _run_analyze(self):
        """
        Analyze each attachment:
        - check for duplicate file names and skip them
        - decompress zip file if necessary
{'fraction_non_alphanumeric': 0.11058154787931788, 'fraction_numerical': 0.00209881941407958, 'mean_word_length': 1.2108264862252296, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 82, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '818720', 'n_tokens_mistral': 5115, 'n_tokens_neox': 4755, 'n_words': 1343}