<commit_before>#!/usr/bin/env python
""" Exchange cron job downloads exchange domain content on regular basis.

"""
import os
import time
import shutil
import multiprocessing

from exchangescan import settings
from exchangescan.export_exchange_content import ExchangeServerExport, read_users

from utils import load_webscanner_settings, run_django_setup

load_webscanner_settings()

run_django_setup()

from os2webscanner.models.domain_model import Domain
from os2webscanner.models.exchangescanner_model import ExchangeScanner


def start_exchange_export():
    """Starts the exchange server export"""
    exchange_scanner.is_exporting = True
    exchange_scanner.is_ready_to_scan = False
    exchange_scanner.save()
    valid_domains = exchange_scanner.domains.filter(
        validation_status=Domain.VALID
    )

    for domain in valid_domains:
        # Collect credentials
        credentials = (domain.authentication.username,
                       domain.authentication.get_password())

        # Create use queue and load in users.
        user_queue = multiprocessing.Queue()
        read_users(user_queue,
                   domain.exchangedomain.get_userlist_file_path())

        done_queue = multiprocessing.Queue()

        mail_ending = domain.url

        # TODO: Make an intelligent usage of last_export_date, where you consider how to handle user_list is updated.
        last_export_date = None

        export_dir = os.path.join(settings.EXCHANGE_EXPORT_DIR_PREFIX,
                                  mail_ending.replace('@', ''))
        # Make export dir export ready
        if os.path.isdir(export_dir):
            shutil.rmtree(export_dir)
        os.mkdir(export_dir)

        scanners = {}
        for i in range(0, settings.NUMBER_OF_EMAIL_THREADS):
            scanners[i] = ExchangeServerExport(credentials,
                                               user_queue,
                                               done_queue,
                                               export_dir + '/',
                                               mail_ending,
                                               start_date=
                                               last_export_date)<commit_msg>* Exchange scanner object is now updated after export_dir is handled.<commit_after>#!/usr/bin/env python
""" Exchange cron job downloads exchange domain content on regular basis.

"""
import os
import time
import shutil
import multiprocessing

from exchangescan import settings
from exchangescan.export_exchange_content import ExchangeServerExport, read_users

from utils import load_webscanner_settings, run_django_setup

load_webscanner_settings()

run_django_setup()

from os2webscanner.models.domain_model import Domain
from os2webscanner.models.exchangescanner_model import ExchangeScanner


def start_exchange_export():
    """Starts the exchange server export"""

    valid_domains = exchange_scanner.domains.filter(
        validation_status=Domain.VALID
    )

    for domain in valid_domains:
        # Collect credentials
        credentials = (domain.authentication.username,
                       domain.authentication.get_password())

        # Create use queue and load in users.
        user_queue = multiprocessing.Queue()
        read_users(user_queue,
                   domain.exchangedomain.get_userlist_file_path())

        done_queue = multiprocessing.Queue()

        mail_ending = domain.url

        # TODO: Make an intelligent usage of last_export_date, where you consider how to handle user_list is updated.
        last_export_date = None

        export_dir = os.path.join(settings.EXCHANGE_EXPORT_DIR_PREFIX,
                                  mail_ending.replace('@', ''))
        # Make export dir export ready
        if os.path.isdir(export_dir):
            shutil.rmtree(export_dir)
        os.mkdir(export_dir)

        exchange_scanner.is_exporting = True
        exchange_scanner.is_ready_to_scan = False
        exchange_scanner.save()
        scanners = {}

        for i in range(0, settings.NUMBER_OF_EMAIL_THREADS):
            scanners[i] = ExchangeServerExport(credentials,
                                               user_queue,
                                               done_queue,
                                               export_dir + '/',
                                               mail_ending,
                                               start_date=
                                               last_export_date)
{'fraction_non_alphanumeric': 0.06080781180648025, 'fraction_numerical': 0.0013315579227696406, 'mean_word_length': 2.0046666666666666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3208304', 'n_tokens_mistral': 1130, 'n_tokens_neox': 1083, 'n_words': 263}