<commit_before>from binascii import hexlify
import os
import time

from .helpers import get_keys_dir
from .locking import UpgradableLock
from .repository import Repository, MAGIC
from .key import KeyfileKey, KeyfileNotFoundError

ATTIC_MAGIC = b'ATTICSEG'

class AtticRepositoryConverter(Repository):
    def convert(self, dryrun=True):
        """convert an attic repository to a borg repository

        those are the files that need to be converted here, from most
        important to least important: segments, key files, and various
        caches, the latter being optional, as they will be rebuilt if
        missing."""
        print("reading segments from attic repository using borg")
        segments = [ filename for i, filename in self.io.segment_iterator() ]
        try:
            keyfile = self.find_attic_keyfile()
        except KeyfileNotFoundError:
            print("no key file found for repository")
        else:
            self.convert_keyfiles(keyfile, dryrun)
        self.close()
        # partial open: just hold on to the lock
        self.lock = UpgradableLock(os.path.join(self.path, 'lock'),
                                   exclusive=True).acquire()
        try:
            self.convert_segments(segments, dryrun)
        finally:
            self.lock.release()
            self.lock = None
        self.convert_cache(dryrun)

    @staticmethod
    def convert_segments(segments, dryrun):
        """convert repository segments from attic to borg

        replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in
        `$ATTIC_REPO/data/**`.

        luckily the magic string length didn't change so we can just
        replace the 8 first bytes of all regular files in there."""
        print("converting %d segments..." % len(segments))
        i = 0
        for filename in segments:
            print("\rconverting segment %d/%d in place, %.2f%% done (%s)"
                  % (i, len(segments), float(i)/len(segments), filename), end='')
            i += 1
            if dryrun:
                time.sleep(0.001)
            else:
                with open(filename, 'r+b') as segment:<commit_msg>cosmetic: show 100% when done, not n-1/n%<commit_after>from binascii import hexlify
import os
import time

from .helpers import get_keys_dir
from .locking import UpgradableLock
from .repository import Repository, MAGIC
from .key import KeyfileKey, KeyfileNotFoundError

ATTIC_MAGIC = b'ATTICSEG'

class AtticRepositoryConverter(Repository):
    def convert(self, dryrun=True):
        """convert an attic repository to a borg repository

        those are the files that need to be converted here, from most
        important to least important: segments, key files, and various
        caches, the latter being optional, as they will be rebuilt if
        missing."""
        print("reading segments from attic repository using borg")
        segments = [ filename for i, filename in self.io.segment_iterator() ]
        try:
            keyfile = self.find_attic_keyfile()
        except KeyfileNotFoundError:
            print("no key file found for repository")
        else:
            self.convert_keyfiles(keyfile, dryrun)
        self.close()
        # partial open: just hold on to the lock
        self.lock = UpgradableLock(os.path.join(self.path, 'lock'),
                                   exclusive=True).acquire()
        try:
            self.convert_segments(segments, dryrun)
        finally:
            self.lock.release()
            self.lock = None
        self.convert_cache(dryrun)

    @staticmethod
    def convert_segments(segments, dryrun):
        """convert repository segments from attic to borg

        replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in
        `$ATTIC_REPO/data/**`.

        luckily the magic string length didn't change so we can just
        replace the 8 first bytes of all regular files in there."""
        print("converting %d segments..." % len(segments))
        i = 0
        for filename in segments:
            i += 1
            print("\rconverting segment %d/%d in place, %.2f%% done (%s)"
                  % (i, len(segments), float(i)/len(segments), filename), end='')
            if dryrun:
                time.sleep(0.001)
            else:
                with open(filename, 'r+b') as segment:
{'fraction_non_alphanumeric': 0.08649398704902868, 'fraction_numerical': 0.004625346901017576, 'mean_word_length': 2.2494365138993238, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6565739', 'n_tokens_mistral': 1199, 'n_tokens_neox': 1158, 'n_words': 401}