<commit_before>#!/usr/bin/python3
"""
Clone all Git repositories for a GitHub user or organisation.
"""

import argparse
import concurrent.futures
import fnmatch
import os
import subprocess
import sys
import threading
from operator import itemgetter
from configparser import ConfigParser

import requests
import requests_cache


__author__ = '<NAME> <<EMAIL>>'
__licence__ = 'MIT'
__url__ = 'https://github.com/mgedmin/cloneall'
__version__ = '1.5.dev0'


# Only one of the following two can be set
DEFAULT_ORGANIZATION = 'ZopeFoundation'
DEFAULT_USER = None

DEFAULT_PATTERN = '*'


class Error(Exception):
    """An error that is not a bug in this script."""


def get_json_and_headers(url):
    """Perform HTTP GET for a URL, return deserialized JSON and headers.

    Returns a tuple (json_data, headers) where headers is something dict-like.
    """
    r = requests.get(url)
    if 400 <= r.status_code < 500:
        raise Error("Failed to fetch {}:\n{}".format(url, r.json()['message']))
    return r.json(), r.headers


def get_github_list(url, batch_size=100, progress_callback=None):
    """Perform (a series of) HTTP GETs for a URL, return deserialized JSON.

    Format of the JSON is documented at
    http://developer.github.com/v3/repos/#list-organization-repositories

    Supports batching (which GitHub indicates by the presence of a Link header,
    e.g. ::

        Link: <https://api.github.com/resource?page=2>; rel="next",
              <https://api.github.com/resource?page=5>; rel="last"

    """
    # API documented at http://developer.github.com/v3/#pagination
    res, headers = get_json_and_headers('{}?per_page={}'.format(
                                                url, batch_size))
    page = 1
    while 'rel="next"' in headers.get('Link', ''):
        page += 1
        if progress_callback:
            progress_callback(len(res))
        more, headers = get_json_and_headers('{}?page={}&per_page={}'.format(
                                                    url, page, batch_size))
        res += more
    return res


def synchronized(method):
    def wrapper(self, *args, **kw):
        with self.lock:
            return method(self, *args, **kw)
    return wrapper


class Progress(object):
    """A progress bar.

    There are two parts of progress output:

    - a scrolling list of items
    - a progress bar (or status message) at the bottom

    These are controlled by the following API methods:

    - status(msg) replaces the progress bar with a status message
    - clear() clears the progress bar/status message
    - set_total(n) defines how many items there will be in total
    - item(text) shows an item and updates the progress bar
    - update(extra_text) updates the last item (and highlights it in a
      different color)
    - finish(msg) clear the progress bar/status message and print a summary

    """

    progress_bar_format = '[{bar}] {cur}/{total}'
    bar_width = 20
    full_char = '#'
    empty_char = '.'

    t_cursor_up = '\033[%dA'
    t_cursor_down = '\033[%dB'
    t_insert_lines = '\033[%dL'
    t_reset = '\033[m'
    t_green = '\033[32m'
    t_red = '\033[31m'
    t_brown = '\033[33m'

    def __init__(self, stream=sys.stdout):
        self.stream = stream
        self.last_status = ''  # so we know how many characters to erase
        self.cur = self.total = 0
        self.items = []
        self.lock = threading.Lock()

    def status(self, message):
        """Replace the status message."""
        self.clear()
        if message:
            self.stream.write('\r')
            self.stream.write(message)
            self.stream.write('\r')
            self.stream.flush()
            self.last_status = message

    def clear(self):
        """Clear the status message."""
        if self.last_status:
            self.stream.write(
                '\r{}\r'.format(' ' * len(self.last_status.rstrip())))
            self.stream.flush()
            self.last_status = ''

    def finish(self, msg=''):
        """Clear the status message and print a summary.

        Differs from status(msg) in that it leaves the cursor on a new line
        and cannot be cleared.
        """
        self.clear()
        if msg:
            print(msg, file=self.stream)

    def progress(self):
        self.status(self.format_progress_bar(self.cur, self.total))

    def format_progress_bar(self, cur, total):
        return self.progress_bar_format.format(
            cur=cur, total=total, bar=self.bar(cur, total))

    def scale(self, range, cur, total):
        return range * cur // max(total, 1)

    def bar(self, cur, total):
        n = min(self.scale(self.bar_width, cur, total), self.bar_width)
        return (self.full_char * n).ljust(self.bar_width, self.empty_char)

    def set_limit(self, total):
        """Specify the expected total number of items.

        E.g. if you set_limit(10), this means you expect to call item() ten
        times.
        """
        self.total = total
        self.progress()

    @synchronized
    def item(self, msg=''):
        """Show an item and update the progress bar."""
        item = self.Item(self, msg, len(self.items))
        self.items.append(item)
        if msg:
            self.clear()
            self.draw_item(item)
        self.cur += 1
        self.progress()
        return item

    def draw_item(self, item, prefix='', suffix='\n', flush=True):
        self.stream.write(''.join([
            prefix,
            item.color,
            item.msg,
            item.reset,
            suffix,
        ]))
        if flush:
            self.stream.flush()

    @synchronized
    def update_item(self, item):
        n = sum(i.height for i in self.items[item.idx:])
        self.draw_item(item, self.t_cursor_up % n, '\n' * n)

    @synchronized
    def extra_info(self, item, lines):
        n = sum(i.height for i in self.items[item.idx + 1:])
        if n:
            self.stream.write(self.t_cursor_up % n)
        self.stream.write(self.t_insert_lines % len(lines))
        for indent, color, line, reset in lines:
            self.stream.write(''.join([indent, color, line, reset, '\n']))
        for i in self.items[item.idx + 1:]:
            self.draw_item(i, flush=False)
            for indent, color, line, reset in i.extra_info_lines:
                self.stream.write(''.join([indent, color, line, reset, '\n']))
        self.progress()

    class Item(object):
        def __init__(self, progress, msg, idx):
            self.progress = progress
            self.msg = msg
            self.idx = idx
            self.extra_info_lines = []
            self.color = self.progress.t_brown
            self.reset = self.progress.t_reset
            self.updated = False
            self.failed = False

        @property
        def height(self):
            return 1 + len(self.extra_info_lines)

        def update(self, msg, failed=False):
            """Update the last shown item and highlight it."""
            self.updated = True
            if failed:
                self.failed = True
                self.color = self.progress.t_red
            elif not self.failed:
                self.color = self.progress.t_green
            self.reset = self.progress.t_reset
            self.msg += msg
            self.progress.update_item(self)

        def finished(self):
            """Mark the item as finished."""
            if not self.updated and not self.failed:
                self.color = ''
                self.reset = ''
            self.progress.update_item(self)

        def extra_info(self, msg, color='', reset='', indent='    '):
            """Print some extra information."""
            lines = [(indent, color, line, reset) for line in msg.splitlines()]
            if not lines:
                return
            self.extra_info_lines += lines
            self.progress.extra_info(self, lines)

        def error_info(self, msg):
            """Print some extra information about an error."""
            self.extra_info(msg, color=self.progress.t_red,
                            reset=self.progress.t_reset)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self.clear()
        if exc_type is KeyboardInterrupt:
            print('Interrupted', file=self.stream)


class RepoWrangler(object):

    def __init__(self, dry_run=False, verbose=0, progress=None):
        self.n_repos = 0
        self.n_updated = 0
        self.n_new = 0
        self.n_dirty = 0
        self.dry_run = dry_run
        self.verbose = verbose or 0
        self.progress = progress if progress else Progress()
        self.lock = threading.Lock()

    def list_repos(self, *, user=None, organization=None, pattern=None):
        if organization and not user:
            owner = organization
            list_url = 'https://api.github.com/orgs/{}/repos'.format(owner)
        elif user and not organization:
            owner = user
            list_url = 'https://api.github.com/users/{}/repos'.format(owner)
        else:
            raise ValueError('specify either user or organization, not both')

        message = "Fetching list of {}'s repositories from GitHub...".format(
            owner)
        self.progress.status(message)

        def progress_callback(n):
            self.progress.status("{} ({})".format(message, n))

        repos = get_github_list(list_url, progress_callback=progress_callback)
        if pattern:
            repos = (r for r in repos if fnmatch.fnmatch(r['name'], pattern))
        return sorted(repos, key=itemgetter('name'))

    def process_task(self, repo):
        item = self.progress.item("+ {name}".format(**repo))
        task = RepoTask(repo, item, self, self.task_finished)
        return task

    @synchronized
    def task_finished(self, task):
        self.n_repos += 1
        self.n_new += task.new
        self.n_updated += task.updated
        self.n_dirty += task.dirty


class RepoTask(object):

    def __init__(self, repo, progress_item, options, finished_callback):
        self.repo = repo
        self.progress_item = progress_item
        self.options = options
        self.finished_callback = finished_callback
        self.updated = False
        self.new = False
        self.dirty = False

    def repo_dir(self, repo):
        return repo['name']

    def repo_url(self, repo):
        # use repo['git_url'] for anonymous checkouts, but they'e slower
        # (at least as long as you use SSH connection multiplexing)
        return repo['ssh_url']

    def decode(self, output):
        return output.decode('UTF-8', 'replace')

    def branch_name(self, head):
        if head.startswith('refs/'):
            head = head[len('refs/'):]
        if head.startswith('heads/'):
            head = head[len('heads/'):]
        return head

    def pretty_command(self, args):
        if self.options.verbose:
            return ' '.join(args)
        else:
            return ' '.join(args[:2])  # 'git diff' etc.

    def call(self, args, **kwargs):
        """Call a subprocess and return its exit code.

        The subprocess is expected to produce no output.  If any output is
        seen, it'll be displayed as an error.
        """
        p = subprocess.Popen(args, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT, **kwargs)
        output, _ = p.communicate()
        retcode = p.wait()
        if output:
            self.progress_item.error_info(self.decode(output))
            self.progress_item.error_info(
                '{command} exited with {rc}'.format(
                    command=self.pretty_command(args), rc=retcode))
        return retcode

    def check_call(self, args, **kwargs):
        """Call a subprocess.

        The subprocess is expected to produce no output.  If any output is
        seen, it'll be displayed as an error.

        The subprocess is expected to return exit code 0.  If it returns
        non-zero, that'll be displayed as an error.
        """
        p = subprocess.Popen(args, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT, **kwargs)
        output, _ = p.communicate()
        retcode = p.wait()
        if retcode != 0:
            self.progress_item.update(' (failed)', failed=True)
        if output or retcode != 0:
            self.progress_item.error_info(self.decode(output))
            self.progress_item.error_info(
                '{command} exited with {rc}'.format(
                    command=self.pretty_command(args), rc=retcode))

    def check_output(self, args, **kwargs):
        """Call a subprocess and return its standard output code.

        The subprocess is expected to produce no output on stderr.  If any
        output is seen, it'll be displayed as an error.

        The subprocess is expected to return exit code 0.  If it returns
        non-zero, that'll be displayed as an error.
        """
        p = subprocess.Popen(args, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, **kwargs)
        stdout, stderr = p.communicate()
        retcode = p.wait()
        if retcode != 0:
            self.progress_item.error_info(self.decode(stderr))
            self.progress_item.error_info(
                '{command} exited with {rc}'.format(
                    command=self.pretty_command(args), rc=retcode))
        return self.decode(stdout)

    def run(self):
        dir = self.repo_dir(self.repo)
        if os.path.exists(dir):
            self.update(self.repo, dir)
            self.verify(self.repo, dir)
        else:
            self.clone(self.repo, dir)
        self.progress_item.finished()
        if self.finished_callback:
            self.finished_callback(self)

    def clone(self, repo, dir):
        self.progress_item.update(' (new)')
        if not self.options.dry_run:
            url = self.repo_url(repo)
            self.check_call(['git', 'clone', '-q', url])
        self.new = True

    def update(self, repo, dir):
        if not self.options.dry_run:
            old_sha = self.get_current_commit(dir)
            self.check_call(['git', 'pull', '-q', '--ff-only'], cwd=dir)
            new_sha = self.get_current_commit(dir)
            if old_sha != new_sha:
                self.progress_item.update(' (updated)')
                self.updated = True

    def verify(self, repo, dir):
        if self.has_local_changes(dir):
            self.progress_item.update(' (local changes)')
            self.dirty = True
        if self.has_staged_changes(dir):
            self.progress_item.update(' (staged changes)')
            self.dirty = True<commit_msg>Handle exceptions while processing tasks<commit_after>#!/usr/bin/python3
"""
Clone all Git repositories for a GitHub user or organisation.
"""

import argparse
import concurrent.futures
import fnmatch
import os
import subprocess
import sys
import threading
from operator import itemgetter
from configparser import ConfigParser

import requests
import requests_cache


__author__ = '<NAME> <<EMAIL>>'
__licence__ = 'MIT'
__url__ = 'https://github.com/mgedmin/cloneall'
__version__ = '1.5.dev0'


# Only one of the following two can be set
DEFAULT_ORGANIZATION = 'ZopeFoundation'
DEFAULT_USER = None

DEFAULT_PATTERN = '*'


class Error(Exception):
    """An error that is not a bug in this script."""


def get_json_and_headers(url):
    """Perform HTTP GET for a URL, return deserialized JSON and headers.

    Returns a tuple (json_data, headers) where headers is something dict-like.
    """
    r = requests.get(url)
    if 400 <= r.status_code < 500:
        raise Error("Failed to fetch {}:\n{}".format(url, r.json()['message']))
    return r.json(), r.headers


def get_github_list(url, batch_size=100, progress_callback=None):
    """Perform (a series of) HTTP GETs for a URL, return deserialized JSON.

    Format of the JSON is documented at
    http://developer.github.com/v3/repos/#list-organization-repositories

    Supports batching (which GitHub indicates by the presence of a Link header,
    e.g. ::

        Link: <https://api.github.com/resource?page=2>; rel="next",
              <https://api.github.com/resource?page=5>; rel="last"

    """
    # API documented at http://developer.github.com/v3/#pagination
    res, headers = get_json_and_headers('{}?per_page={}'.format(
                                                url, batch_size))
    page = 1
    while 'rel="next"' in headers.get('Link', ''):
        page += 1
        if progress_callback:
            progress_callback(len(res))
        more, headers = get_json_and_headers('{}?page={}&per_page={}'.format(
                                                    url, page, batch_size))
        res += more
    return res


def synchronized(method):
    def wrapper(self, *args, **kw):
        with self.lock:
            return method(self, *args, **kw)
    return wrapper


class Progress(object):
    """A progress bar.

    There are two parts of progress output:

    - a scrolling list of items
    - a progress bar (or status message) at the bottom

    These are controlled by the following API methods:

    - status(msg) replaces the progress bar with a status message
    - clear() clears the progress bar/status message
    - set_total(n) defines how many items there will be in total
    - item(text) shows an item and updates the progress bar
    - update(extra_text) updates the last item (and highlights it in a
      different color)
    - finish(msg) clear the progress bar/status message and print a summary

    """

    progress_bar_format = '[{bar}] {cur}/{total}'
    bar_width = 20
    full_char = '#'
    empty_char = '.'

    t_cursor_up = '\033[%dA'
    t_cursor_down = '\033[%dB'
    t_insert_lines = '\033[%dL'
    t_reset = '\033[m'
    t_green = '\033[32m'
    t_red = '\033[31m'
    t_brown = '\033[33m'

    def __init__(self, stream=sys.stdout):
        self.stream = stream
        self.last_status = ''  # so we know how many characters to erase
        self.cur = self.total = 0
        self.items = []
        self.lock = threading.Lock()

    def status(self, message):
        """Replace the status message."""
        self.clear()
        if message:
            self.stream.write('\r')
            self.stream.write(message)
            self.stream.write('\r')
            self.stream.flush()
            self.last_status = message

    def clear(self):
        """Clear the status message."""
        if self.last_status:
            self.stream.write(
                '\r{}\r'.format(' ' * len(self.last_status.rstrip())))
            self.stream.flush()
            self.last_status = ''

    def finish(self, msg=''):
        """Clear the status message and print a summary.

        Differs from status(msg) in that it leaves the cursor on a new line
        and cannot be cleared.
        """
        self.clear()
        if msg:
            print(msg, file=self.stream)

    def progress(self):
        self.status(self.format_progress_bar(self.cur, self.total))

    def format_progress_bar(self, cur, total):
        return self.progress_bar_format.format(
            cur=cur, total=total, bar=self.bar(cur, total))

    def scale(self, range, cur, total):
        return range * cur // max(total, 1)

    def bar(self, cur, total):
        n = min(self.scale(self.bar_width, cur, total), self.bar_width)
        return (self.full_char * n).ljust(self.bar_width, self.empty_char)

    def set_limit(self, total):
        """Specify the expected total number of items.

        E.g. if you set_limit(10), this means you expect to call item() ten
        times.
        """
        self.total = total
        self.progress()

    @synchronized
    def item(self, msg=''):
        """Show an item and update the progress bar."""
        item = self.Item(self, msg, len(self.items))
        self.items.append(item)
        if msg:
            self.clear()
            self.draw_item(item)
        self.cur += 1
        self.progress()
        return item

    def draw_item(self, item, prefix='', suffix='\n', flush=True):
        self.stream.write(''.join([
            prefix,
            item.color,
            item.msg,
            item.reset,
            suffix,
        ]))
        if flush:
            self.stream.flush()

    @synchronized
    def update_item(self, item):
        n = sum(i.height for i in self.items[item.idx:])
        self.draw_item(item, self.t_cursor_up % n, '\n' * n)

    @synchronized
    def extra_info(self, item, lines):
        n = sum(i.height for i in self.items[item.idx + 1:])
        if n:
            self.stream.write(self.t_cursor_up % n)
        self.stream.write(self.t_insert_lines % len(lines))
        for indent, color, line, reset in lines:
            self.stream.write(''.join([indent, color, line, reset, '\n']))
        for i in self.items[item.idx + 1:]:
            self.draw_item(i, flush=False)
            for indent, color, line, reset in i.extra_info_lines:
                self.stream.write(''.join([indent, color, line, reset, '\n']))
        self.progress()

    class Item(object):
        def __init__(self, progress, msg, idx):
            self.progress = progress
            self.msg = msg
            self.idx = idx
            self.extra_info_lines = []
            self.color = self.progress.t_brown
            self.reset = self.progress.t_reset
            self.updated = False
            self.failed = False

        @property
        def height(self):
            return 1 + len(self.extra_info_lines)

        def update(self, msg, failed=False):
            """Update the last shown item and highlight it."""
            self.updated = True
            if failed:
                self.failed = True
                self.color = self.progress.t_red
            elif not self.failed:
                self.color = self.progress.t_green
            self.reset = self.progress.t_reset
            self.msg += msg
            self.progress.update_item(self)

        def finished(self):
            """Mark the item as finished."""
            if not self.updated and not self.failed:
                self.color = ''
                self.reset = ''
            self.progress.update_item(self)

        def extra_info(self, msg, color='', reset='', indent='    '):
            """Print some extra information."""
            lines = [(indent, color, line, reset) for line in msg.splitlines()]
            if not lines:
                return
            self.extra_info_lines += lines
            self.progress.extra_info(self, lines)

        def error_info(self, msg):
            """Print some extra information about an error."""
            self.extra_info(msg, color=self.progress.t_red,
                            reset=self.progress.t_reset)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self.clear()
        if exc_type is KeyboardInterrupt:
            print('Interrupted', file=self.stream)


class RepoWrangler(object):

    def __init__(self, dry_run=False, verbose=0, progress=None):
        self.n_repos = 0
        self.n_updated = 0
        self.n_new = 0
        self.n_dirty = 0
        self.dry_run = dry_run
        self.verbose = verbose or 0
        self.progress = progress if progress else Progress()
        self.lock = threading.Lock()

    def list_repos(self, *, user=None, organization=None, pattern=None):
        if organization and not user:
            owner = organization
            list_url = 'https://api.github.com/orgs/{}/repos'.format(owner)
        elif user and not organization:
            owner = user
            list_url = 'https://api.github.com/users/{}/repos'.format(owner)
        else:
            raise ValueError('specify either user or organization, not both')

        message = "Fetching list of {}'s repositories from GitHub...".format(
            owner)
        self.progress.status(message)

        def progress_callback(n):
            self.progress.status("{} ({})".format(message, n))

        repos = get_github_list(list_url, progress_callback=progress_callback)
        if pattern:
            repos = (r for r in repos if fnmatch.fnmatch(r['name'], pattern))
        return sorted(repos, key=itemgetter('name'))

    def process_task(self, repo):
        item = self.progress.item("+ {name}".format(**repo))
        task = RepoTask(repo, item, self, self.task_finished)
        return task

    @synchronized
    def task_finished(self, task):
        self.n_repos += 1
        self.n_new += task.new
        self.n_updated += task.updated
        self.n_dirty += task.dirty


class RepoTask(object):

    def __init__(self, repo, progress_item, options, finished_callback):
        self.repo = repo
        self.progress_item = progress_item
        self.options = options
        self.finished_callback = finished_callback
        self.updated = False
        self.new = False
        self.dirty = False

    def repo_dir(self, repo):
        return repo['name']

    def repo_url(self, repo):
        # use repo['git_url'] for anonymous checkouts, but they'e slower
        # (at least as long as you use SSH connection multiplexing)
        return repo['ssh_url']

    def decode(self, output):
        return output.decode('UTF-8', 'replace')

    def branch_name(self, head):
        if head.startswith('refs/'):
            head = head[len('refs/'):]
        if head.startswith('heads/'):
            head = head[len('heads/'):]
        return head

    def pretty_command(self, args):
        if self.options.verbose:
            return ' '.join(args)
        else:
            return ' '.join(args[:2])  # 'git diff' etc.

    def call(self, args, **kwargs):
        """Call a subprocess and return its exit code.

        The subprocess is expected to produce no output.  If any output is
        seen, it'll be displayed as an error.
        """
        p = subprocess.Popen(args, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT, **kwargs)
        output, _ = p.communicate()
        retcode = p.wait()
        if output:
            self.progress_item.error_info(self.decode(output))
            self.progress_item.error_info(
                '{command} exited with {rc}'.format(
                    command=self.pretty_command(args), rc=retcode))
        return retcode

    def check_call(self, args, **kwargs):
        """Call a subprocess.

        The subprocess is expected to produce no output.  If any output is
        seen, it'll be displayed as an error.

        The subprocess is expected to return exit code 0.  If it returns
        non-zero, that'll be displayed as an error.
        """
        p = subprocess.Popen(args, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT, **kwargs)
        output, _ = p.communicate()
        retcode = p.wait()
        if retcode != 0:
            self.progress_item.update(' (failed)', failed=True)
        if output or retcode != 0:
            self.progress_item.error_info(self.decode(output))
            self.progress_item.error_info(
                '{command} exited with {rc}'.format(
                    command=self.pretty_command(args), rc=retcode))

    def check_output(self, args, **kwargs):
        """Call a subprocess and return its standard output code.

        The subprocess is expected to produce no output on stderr.  If any
        output is seen, it'll be displayed as an error.

        The subprocess is expected to return exit code 0.  If it returns
        non-zero, that'll be displayed as an error.
        """
        p = subprocess.Popen(args, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, **kwargs)
        stdout, stderr = p.communicate()
        retcode = p.wait()
        if retcode != 0:
            self.progress_item.error_info(self.decode(stderr))
            self.progress_item.error_info(
                '{command} exited with {rc}'.format(
                    command=self.pretty_command(args), rc=retcode))
        return self.decode(stdout)

    def run(self):
        try:
            dir = self.repo_dir(self.repo)
            if os.path.exists(dir):
                self.update(self.repo, dir)
                self.verify(self.repo, dir)
            else:
                self.clone(self.repo, dir)
        except Exception as e:
            self.progress_item.error_info(
                '{}: {}'.format(e.__class__.__name__, e))
        finally:
            self.progress_item.finished()
            if self.finished_callback:
                self.finished_callback(self)

    def clone(self, repo, dir):
        self.progress_item.update(' (new)')
        if not self.options.dry_run:
            url = self.repo_url(repo)
            self.check_call(['git', 'clone', '-q', url])
        self.new = True

    def update(self, repo, dir):
        if not self.options.dry_run:
            old_sha = self.get_current_commit(dir)
            self.check_call(['git', 'pull', '-q', '--ff-only'], cwd=dir)
            new_sha = self.get_current_commit(dir)
            if old_sha != new_sha:
                self.progress_item.update(' (updated)')
                self.updated = True

    def verify(self, repo, dir):
        if self.has_local_changes(dir):
            self.progress_item.update(' (local changes)')
            self.dirty = True
        if self.has_staged_changes(dir):
            self.progress_item.update(' (staged changes)')
            self.dirty = True
{'fraction_non_alphanumeric': 0.1140579073617352, 'fraction_numerical': 0.004729889523294706, 'mean_word_length': 2.0496600041211623, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3650930', 'n_tokens_mistral': 8767, 'n_tokens_neox': 8386, 'n_words': 2342}