<commit_before>"""Internal library."""

from io import BytesIO
import os
import random
import struct

from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.backends import default_backend

from django.conf import settings
from django.utils.encoding import force_bytes
from django.utils.translation import ugettext as _

from modoboa.lib.exceptions import InternalError
from modoboa.parameters import tools as param_tools


def init_storage_dir():
    """Create the directory whare documents will be stored."""
    storage_dir = param_tools.get_global_parameter("storage_dir")
    if os.path.exists(storage_dir):
        return
    try:
        os.mkdir(storage_dir)
    except (OSError, IOError) as inst:
        raise InternalError(
            _("Failed to create the directory that will contain "
              "PDF documents (%s)") % inst
        )


def get_creds_filename(account):
    """Return the full path of a document."""
    storage_dir = param_tools.get_global_parameter("storage_dir")
    return os.path.join(storage_dir, account.username + ".pdf")


def delete_credentials(account):
    """Try to delete a local file."""
    fname = get_creds_filename(account)
    if not os.path.exists(fname):
        return
    try:
        os.remove(fname)
    except OSError:
        pass


def _get_cipher(iv):
    """Return ready-to-user Cipher."""
    key = param_tools.get_global_parameter("secret_key", app="core")
    backend = default_backend()
    return Cipher(
        algorithms.AES(force_bytes(key)),
        modes.CBC(force_bytes(iv)),
        backend=backend
    )


def crypt_and_save_to_file(content, filename, length, chunksize=64*512):
    """Crypt content and save it to a file."""
    iv = "".join(chr(random.randint(0, 0xFF)) for i in range(16))
    cipher = _get_cipher(iv)
    encryptor = cipher.encryptor()
    with open(filename, "wb") as fp:
        fp.write(struct.pack("<Q", length))
        fp.write(iv)
        while True:
            chunk = content.read(chunksize)
            if not len(chunk):
                break
            elif len(chunk) % 16:
                chunk += " " * (16 - len(chunk) % 16)
            fp.write(encryptor.update(force_bytes(chunk)))
        fp.write(encryptor.finalize())


def decrypt_file(filename, chunksize=24*1024):
    """Decrypt the content of a file and return it."""
    buff = BytesIO()
    with open(filename, "rb") as fp:
        origsize = struct.unpack("<Q", fp.read(struct.calcsize("Q")))[0]
        iv = fp.read(16)
        cipher = _get_cipher(iv)
        decryptor = cipher.decryptor()
        while True:
            chunk = fp.read(chunksize)
            if not len(chunk):
                break<commit_msg>Fixed some python3 compat issues.<commit_after>"""Internal library."""

from __future__ import unicode_literals

from io import BytesIO
import os
import random
import struct

from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.backends import default_backend

from django.conf import settings
from django.utils.encoding import force_bytes
from django.utils.translation import ugettext as _

from modoboa.lib.exceptions import InternalError
from modoboa.parameters import tools as param_tools


def init_storage_dir():
    """Create the directory whare documents will be stored."""
    storage_dir = param_tools.get_global_parameter("storage_dir")
    if os.path.exists(storage_dir):
        return
    try:
        os.mkdir(storage_dir)
    except (OSError, IOError) as inst:
        raise InternalError(
            _("Failed to create the directory that will contain "
              "PDF documents (%s)") % inst
        )


def get_creds_filename(account):
    """Return the full path of a document."""
    storage_dir = param_tools.get_global_parameter("storage_dir")
    return os.path.join(storage_dir, account.username + ".pdf")


def delete_credentials(account):
    """Try to delete a local file."""
    fname = get_creds_filename(account)
    if not os.path.exists(fname):
        return
    try:
        os.remove(fname)
    except OSError:
        pass


def _get_cipher(iv):
    """Return ready-to-user Cipher."""
    key = param_tools.get_global_parameter("secret_key", app="core")
    backend = default_backend()
    return Cipher(
        algorithms.AES(force_bytes(key)),
        modes.CBC(iv),
        backend=backend
    )


def crypt_and_save_to_file(content, filename, length, chunksize=64*512):
    """Crypt content and save it to a file."""
    iv = os.urandom(16)
    cipher = _get_cipher(iv)
    encryptor = cipher.encryptor()
    with open(filename, "wb") as fp:
        fp.write(struct.pack("<Q", length))
        fp.write(iv)
        while True:
            chunk = content.read(chunksize)
            if not len(chunk):
                break
            elif len(chunk) % 16:
                chunk += b" " * (16 - len(chunk) % 16)
            fp.write(encryptor.update(force_bytes(chunk)))
        fp.write(encryptor.finalize())


def decrypt_file(filename, chunksize=24*1024):
    """Decrypt the content of a file and return it."""
    buff = BytesIO()
    with open(filename, "rb") as fp:
        origsize = struct.unpack("<Q", fp.read(struct.calcsize("Q")))[0]
        iv = fp.read(16)
        cipher = _get_cipher(iv)
        decryptor = cipher.decryptor()
        while True:
            chunk = fp.read(chunksize)
            if not len(chunk):
                break
{'fraction_non_alphanumeric': 0.10941197888221373, 'fraction_numerical': 0.008556344438376115, 'mean_word_length': 2.8285714285714287, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '944286', 'n_tokens_mistral': 1708, 'n_tokens_neox': 1633, 'n_words': 433}