<commit_before>from binascii import hexlify
import datetime
import logging
logger = logging.getLogger(__name__)
import os
import shutil
import sys
import time

from .helpers import get_keys_dir, get_cache_dir
from .locking import UpgradableLock
from .repository import Repository, MAGIC
from .key import KeyfileKey, KeyfileNotFoundError

ATTIC_MAGIC = b'ATTICSEG'


class AtticRepositoryUpgrader(Repository):
    def upgrade(self, dryrun=True, inplace=False):
        """convert an attic repository to a borg repository

        those are the files that need to be upgraded here, from most
        important to least important: segments, key files, and various
        caches, the latter being optional, as they will be rebuilt if
        missing.

        we nevertheless do the order in reverse, as we prefer to do
        the fast stuff first, to improve interactivity.
        """
        backup = None
        if not inplace:
            backup = '{}.upgrade-{:%Y-%m-%d-%H:%M:%S}'.format(self.path, datetime.datetime.now())
            logger.info('making a hardlink copy in %s', backup)
            if not dryrun:
                shutil.copytree(self.path, backup, copy_function=os.link)
        logger.info("opening attic repository with borg and converting")
        # we need to open the repo to load configuration, keyfiles and segments
        self.open(self.path, exclusive=False)
        segments = [filename for i, filename in self.io.segment_iterator()]
        try:
            keyfile = self.find_attic_keyfile()
        except KeyfileNotFoundError:
            logger.warning("no key file found for repository")
        else:
            self.convert_keyfiles(keyfile, dryrun)
        self.close()
        # partial open: just hold on to the lock
        self.lock = UpgradableLock(os.path.join(self.path, 'lock'),
                                   exclusive=True).acquire()
        try:
            self.convert_cache(dryrun)
            self.convert_segments(segments, dryrun=dryrun, inplace=inplace)
        finally:
            self.lock.release()
            self.lock = None
        return backup

    @staticmethod
    def convert_segments(segments, dryrun=True, inplace=False):
        """convert repository segments from attic to borg

        replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in
        `$ATTIC_REPO/data/**`.

        luckily the magic string length didn't change so we can just
        replace the 8 first bytes of all regular files in there."""
        logger.info("converting %d segments..." % len(segments))
        i = 0
        for filename in segments:
            i += 1
            print("\rconverting segment %d/%d, %.2f%% done (%s)"
                  % (i, len(segments), 100*float(i)/len(segments), filename),
                  end='', file=sys.stderr)
            if dryrun:
                time.sleep(0.001)
            else:
                AtticRepositoryUpgrader.header_replace(filename, ATTIC_MAGIC, MAGIC, inplace=inplace)
        print(file=sys.stderr)

    @staticmethod
    def header_replace(filename, old_magic, new_magic, inplace=True):
        with open(filename, 'r+b') as segment:
            segment.seek(0)
            # only write if necessary
            if segment.read(len(old_magic)) == old_magic:
                if inplace:
                    segment.seek(0)
                    segment.write(new_magic)
                else:
                    # remove the hardlink and rewrite the file. this
                    # works because our old file handle is still open
                    # so even though the file is removed, we can still
                    # read it until the file is closed.
                    os.unlink(filename)
                    with open(filename, 'wb') as new_segment:
                        new_segment.write(new_magic)
                        new_segment.write(segment.read())

    def find_attic_keyfile(self):
        """find the attic keyfiles

        the keyfiles are loaded by `KeyfileKey.find_key_file()`. that
        finds the keys with the right identifier for the repo.

        this is expected to look into $HOME/.attic/keys or
        $ATTIC_KEYS_DIR for key files matching the given Borg
        repository.

        it is expected to raise an exception (KeyfileNotFoundError) if
        no key is found. whether that exception is from Borg or Attic
        is unclear.

        this is split in a separate function in case we want to use
        the attic code here directly, instead of our local
        implementation."""
        return AtticKeyfileKey.find_key_file(self)
<commit_msg>try to work around Windows idiocy of open files<commit_after>from binascii import hexlify
import datetime
import logging
logger = logging.getLogger(__name__)
import os
import shutil
import sys
import time

from .helpers import get_keys_dir, get_cache_dir
from .locking import UpgradableLock
from .repository import Repository, MAGIC
from .key import KeyfileKey, KeyfileNotFoundError

ATTIC_MAGIC = b'ATTICSEG'


class AtticRepositoryUpgrader(Repository):
    def upgrade(self, dryrun=True, inplace=False):
        """convert an attic repository to a borg repository

        those are the files that need to be upgraded here, from most
        important to least important: segments, key files, and various
        caches, the latter being optional, as they will be rebuilt if
        missing.

        we nevertheless do the order in reverse, as we prefer to do
        the fast stuff first, to improve interactivity.
        """
        backup = None
        if not inplace:
            backup = '{}.upgrade-{:%Y-%m-%d-%H:%M:%S}'.format(self.path, datetime.datetime.now())
            logger.info('making a hardlink copy in %s', backup)
            if not dryrun:
                shutil.copytree(self.path, backup, copy_function=os.link)
        logger.info("opening attic repository with borg and converting")
        # we need to open the repo to load configuration, keyfiles and segments
        self.open(self.path, exclusive=False)
        segments = [filename for i, filename in self.io.segment_iterator()]
        try:
            keyfile = self.find_attic_keyfile()
        except KeyfileNotFoundError:
            logger.warning("no key file found for repository")
        else:
            self.convert_keyfiles(keyfile, dryrun)
        self.close()
        # partial open: just hold on to the lock
        self.lock = UpgradableLock(os.path.join(self.path, 'lock'),
                                   exclusive=True).acquire()
        try:
            self.convert_cache(dryrun)
            self.convert_segments(segments, dryrun=dryrun, inplace=inplace)
        finally:
            self.lock.release()
            self.lock = None
        return backup

    @staticmethod
    def convert_segments(segments, dryrun=True, inplace=False):
        """convert repository segments from attic to borg

        replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in
        `$ATTIC_REPO/data/**`.

        luckily the magic string length didn't change so we can just
        replace the 8 first bytes of all regular files in there."""
        logger.info("converting %d segments..." % len(segments))
        i = 0
        for filename in segments:
            i += 1
            print("\rconverting segment %d/%d, %.2f%% done (%s)"
                  % (i, len(segments), 100*float(i)/len(segments), filename),
                  end='', file=sys.stderr)
            if dryrun:
                time.sleep(0.001)
            else:
                AtticRepositoryUpgrader.header_replace(filename, ATTIC_MAGIC, MAGIC, inplace=inplace)
        print(file=sys.stderr)

    @staticmethod
    def header_replace(filename, old_magic, new_magic, inplace=True):
        with open(filename, 'r+b') as segment:
            segment.seek(0)
            # only write if necessary
            if segment.read(len(old_magic)) == old_magic:
                if inplace:
                    segment.seek(0)
                    segment.write(new_magic)
                else:
                    # remove the hardlink and rewrite the file. this
                    # works because our old file handle is still open
                    # so even though the file is removed, we can still
                    # read it until the file is closed.
                    os.rename(filename, filename + '.tmp')
                    with open(filename, 'wb') as new_segment:
                        new_segment.write(new_magic)
                        new_segment.write(segment.read())
                    # the little dance with the .tmp file is necessary
                    # because Windows won't allow overwriting an open
                    # file.
                    os.unlink(filename + '.tmp')

    def find_attic_keyfile(self):
        """find the attic keyfiles

        the keyfiles are loaded by `KeyfileKey.find_key_file()`. that
        finds the keys with the right identifier for the repo.

        this is expected to look into $HOME/.attic/keys or
        $ATTIC_KEYS_DIR for key files matching the given Borg
        repository.

        it is expected to raise an exception (KeyfileNotFoundError) if
        no key is found. whether that exception is from Borg or Attic
        is unclear.

        this is split in a separate function in case we want to use
        the attic code here directly, instead of our local
        implementation."""
        return AtticKeyfileKey.find_key_file(self)

{'fraction_non_alphanumeric': 0.07715757384631557, 'fraction_numerical': 0.0027331020708504152, 'mean_word_length': 2.065077319587629, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6368974', 'n_tokens_mistral': 2582, 'n_tokens_neox': 2456, 'n_words': 871}