<commit_before>from __future__ import print_function
import bz2
from ftplib import FTP
import gzip
import hashlib
import logging
import os
import re
from subprocess import Popen, STDOUT, PIPE
import sys
import urllib
import yaml


class Mirror:

    # Setup class vars and logger
    def __init__(self, mirror_path, mirror_url,
                 temp_indices=None, log_file=None, log_level=None):

        if not temp_indices:
            self.temp_indices = '/tmp/dists-indices'

        if not log_level:
            log_level='ERROR'

        self.mirror_path = mirror_path
        self.mirror_url = mirror_url
        self.temp_indices = temp_indices

        self.logger = logging.getLogger()
        if log_level.upper() == 'DEBUG':
            self.logger.setLevel(logging.DEBUG)

        elif log_level.upper() == 'INFO':
            self.logger.setLevel(logging.INFO)

        elif log_level.upper() == 'WARNING':
            self.logger.setLevel(logging.WARNING)

        elif log_level.upper() == 'ERROR':
            self.logger.setLevel(logging.ERROR)

        elif log_level.upper() == 'CRITICAL':
            self.logger.setLevel(logging.CRITICAL)

        else:
            print("Bad log level entered, defaulting to 'INFO'")
            self.logger.setLevel(logging.INFO)

        log_format = "%(asctime)s [%(levelname)-5.5s]  %(message)s"
        logFormatter = logging.Formatter(log_format)

        console = logging.StreamHandler()
        console.setFormatter(logFormatter)

        fileHandler = logging.FileHandler(filename=log_file)
        fileHandler.setFormatter(logFormatter)

        self.logger.addHandler(fileHandler)
        self.logger.addHandler(console)

    # Sync the whole mirror
    def sync(self):
        self.logger.info("=======================================")
        self.logger.info("= Starting Sync of Mirror             =")
        self.logger.info("=======================================")
        self.update_pool()
        self.get_dists_indices()
        self.get_zzz_dists()
        self.check_release_files()
        self.check_indices()
        self.update_mirrors()
        self.update_indices()
        self.update_project_dir()
        self.gen_lslR()

    # Update the pool directory of the mirror
    # NOTE: This does not delete old packages, so it is safe to run at any time
    def update_pool(self):
        rsync_command = "rsync --recursive --times --links --hard-links \
                --contimeout=10 --timeout=10 --no-motd --stats \
                --progress \
                -vz rsync://{mirror_url}/pool {mirror_path}/"
        rsync_command = rsync_command.format(
                mirror_url=self.mirror_url,
                mirror_path=self.mirror_path
            )

        self.logger.info("Downloading new packages")
        rsync_status = Popen(rsync_command, stdout=PIPE, stderr=PIPE,
                             shell=True)

        for line in rsync_status.stdout:
            self.logger.debug(line)

    # Update the entire mirror, excluding package, source, and release indices
    def update_mirrors(self):
        rsync_command = "rsync --recursive --times --links --hard-links \
                --exclude 'Packages*' --exclude 'Sources*' \
                --exclude 'Release*' --exclude 'ls-lR.gz' \
                --contimeout=10 --timeout=10 --no-motd --delete --stats \<commit_msg>Exclude pool on second update<commit_after>from __future__ import print_function
import bz2
from ftplib import FTP
import gzip
import hashlib
import logging
import os
import re
from subprocess import Popen, STDOUT, PIPE
import sys
import urllib
import yaml


class Mirror:

    # Setup class vars and logger
    def __init__(self, mirror_path, mirror_url,
                 temp_indices=None, log_file=None, log_level=None):

        if not temp_indices:
            self.temp_indices = '/tmp/dists-indices'

        if not log_level:
            log_level='ERROR'

        self.mirror_path = mirror_path
        self.mirror_url = mirror_url
        self.temp_indices = temp_indices

        self.logger = logging.getLogger()
        if log_level.upper() == 'DEBUG':
            self.logger.setLevel(logging.DEBUG)

        elif log_level.upper() == 'INFO':
            self.logger.setLevel(logging.INFO)

        elif log_level.upper() == 'WARNING':
            self.logger.setLevel(logging.WARNING)

        elif log_level.upper() == 'ERROR':
            self.logger.setLevel(logging.ERROR)

        elif log_level.upper() == 'CRITICAL':
            self.logger.setLevel(logging.CRITICAL)

        else:
            print("Bad log level entered, defaulting to 'INFO'")
            self.logger.setLevel(logging.INFO)

        log_format = "%(asctime)s [%(levelname)-5.5s]  %(message)s"
        logFormatter = logging.Formatter(log_format)

        console = logging.StreamHandler()
        console.setFormatter(logFormatter)

        fileHandler = logging.FileHandler(filename=log_file)
        fileHandler.setFormatter(logFormatter)

        self.logger.addHandler(fileHandler)
        self.logger.addHandler(console)

    # Sync the whole mirror
    def sync(self):
        self.logger.info("=======================================")
        self.logger.info("= Starting Sync of Mirror             =")
        self.logger.info("=======================================")
        self.update_pool()
        self.get_dists_indices()
        self.get_zzz_dists()
        self.check_release_files()
        self.check_indices()
        self.update_mirrors()
        self.update_indices()
        self.update_project_dir()
        self.gen_lslR()

    # Update the pool directory of the mirror
    # NOTE: This does not delete old packages, so it is safe to run at any time
    def update_pool(self):
        rsync_command = "rsync --recursive --times --links --hard-links \
                --contimeout=10 --timeout=10 --no-motd --stats \
                --progress \
                -vz rsync://{mirror_url}/pool {mirror_path}/"
        rsync_command = rsync_command.format(
                mirror_url=self.mirror_url,
                mirror_path=self.mirror_path
            )

        self.logger.info("Downloading new packages")
        rsync_status = Popen(rsync_command, stdout=PIPE, stderr=PIPE,
                             shell=True)

        for line in rsync_status.stdout:
            self.logger.debug(line)

    # Update the entire mirror, excluding package, source, and release indices
    def update_mirrors(self):
        rsync_command = "rsync --recursive --times --links --hard-links \
                --exclude 'Packages*' --exclude 'Sources*' \
                --exclude 'Release*' --exclude 'ls-lR.gz' --exclude 'pool' \
                --contimeout=10 --timeout=10 --no-motd --delete --stats \
{'fraction_non_alphanumeric': 0.12479764532744665, 'fraction_numerical': 0.003237674760853569, 'mean_word_length': 2.2102031176192725, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4597925', 'n_tokens_mistral': 2010, 'n_tokens_neox': 1915, 'n_words': 436}