<commit_before>#!/usr/bin/env python
from __future__ import print_function

import argparse
import csv
import re
import shutil
import sys
import time
import traceback

from collections import defaultdict
from itertools import count
from functools import partial

import gouda
import gouda.util

from gouda.engines.options import engine_options
from gouda.gouda_error import GoudaError
from gouda.util import expand_wildcard, read_image
from gouda.strategies.roi.roi import roi
from gouda.strategies.resize import resize


def decode(paths, strategies, engine, visitors, read_greyscale):
    """Finds and decodes barcodes in images given in pathss
    """
    for p in paths:
        if p.is_dir():
            # Descend into directory
            decode(p.iterdir(), strategies, engine, visitors, read_greyscale)
        else:
            # Process file
            try:
                img = read_image(p, read_greyscale)
                if img is None:
                    # Most likely not an image
                    for visitor in visitors:
                        visitor.result(p, [None, []])
                else:
                    # Read barcodes
                    for strategy in strategies:
                        result = strategy(img, engine)
                        if result:
                            # Found a barcode
                            break
                    else:
                        # No barcode was found
                        result = [None, []]

                    for visitor in visitors:
                        visitor.result(p, result)
            except Exception:
                print('Error processing [{0}]'.format(p))
                traceback.print_exc()


class BasicReportVisitor(object):
    """Writes a line-per-file and a line-per-barcode to stdout
    """
    def result(self, path, result):
        print(path)
        strategy, barcodes = result
        print('Found [{0}] barcodes:'.format(len(barcodes)))
        for index, barcode in enumerate(barcodes):
            print('[{0}] [{1}] [{2}]'.format(index, barcode.type, barcode.data))


class TerseReportVisitor(object):
    """Writes a line-per-file to stdout
    """
    def result(self, path, result):
        strategy, barcodes = result
        values = [b.data for b in barcodes]
        print(path, ' '.join(['[{0}]'.format(v) for v in values]))


class CSVReportVisitor(object):
    """Writes a CSV report
    """
    def __init__(self, engine, greyscale, file=sys.stdout):
        self.w = csv.writer(file, lineterminator='\n')
        self.w.writerow([
            'OS', 'Engine', 'Directory', 'File', 'Image.conversion',
            'Elapsed', 'N.found', 'Types', 'Values', 'Strategy'
        ])
        self.engine = engine
        self.image_conversion = 'Greyscale' if greyscale else 'Unchanged'
        self.start_time = time.time()

    def result(self, path, result):
        strategy, barcodes = result
        types = '|'.join(b.type for b in barcodes)
        values = '|'.join(b.data for b in barcodes)
        self.w.writerow([sys.platform,
                         self.engine,
                         path.parent.name,
                         path.name,
                         self.image_conversion,
                         time.time()-self.start_time,
                         len(barcodes),
                         types,
                         values,
                         strategy])


class RenameVisitor(object):
    """Renames files based on their barcodes
    """
    def __init__(self, avoid_collisions):
        self.avoid_collisions = avoid_collisions
        # Mapping from path to iterator of integer suffixes, used to avoid
        # collisions - see self._destination
        self.suffix = defaultdict(partial(count, start=1))

    def _destination(self, path):
        """Returns path possibly with a suffix appended to the name to avoid
        collisions with existing files, iff self.avoid_collisions is True,
        otherwise path is returned unaltered.
        """
        destination = path
        if self.avoid_collisions:
            while destination.is_file():
                fname = u'{0}-{1}{2}'.format(
                    path.stem,
                    next(self.suffix[path.name]),
                    path.suffix
                )
                destination = path.with_name(fname)

        return destination

    def result(self, path, result):
        strategy, barcodes = result
        print(path)
        if not barcodes:
            print('  No barcodes')
        else:
            # TODO How best to sanitize filenames?
            values = [re.sub('[^a-zA-Z0-9_-]', '_', b.data) for b in barcodes]

            # The first time round the loop, the file will be renamed and
            # first_destination set to the new filename.
            # On subsequent iterations, first_destination will copied
            # to new destinations.
            first_destination = None
            for value in values:
                dest = path.with_name(u'{0}{1}'.format(value, path.suffix))
                dest = self._destination(dest)
                source = first_destination if first_destination else path
                rename = not bool(first_destination)
                if source == dest:
                    print('  Already correctly named')
                elif dest.is_file() and not self.avoid_collisions:
                    msg = '  Cannot rename to [{0}] because destination exists'
                    print(msg.format(dest))
                elif rename:
                    path.rename(dest)<commit_msg>Better logic for avoiding collisions<commit_after>#!/usr/bin/env python
from __future__ import print_function

import argparse
import csv
import re
import shutil
import sys
import time
import traceback

from collections import defaultdict
from itertools import count
from functools import partial

import gouda
import gouda.util

from gouda.engines.options import engine_options
from gouda.gouda_error import GoudaError
from gouda.util import expand_wildcard, read_image
from gouda.strategies.roi.roi import roi
from gouda.strategies.resize import resize


def decode(paths, strategies, engine, visitors, read_greyscale):
    """Finds and decodes barcodes in images given in pathss
    """
    for p in paths:
        if p.is_dir():
            # Descend into directory
            decode(p.iterdir(), strategies, engine, visitors, read_greyscale)
        else:
            # Process file
            try:
                img = read_image(p, read_greyscale)
                if img is None:
                    # Most likely not an image
                    for visitor in visitors:
                        visitor.result(p, [None, []])
                else:
                    # Read barcodes
                    for strategy in strategies:
                        result = strategy(img, engine)
                        if result:
                            # Found a barcode
                            break
                    else:
                        # No barcode was found
                        result = [None, []]

                    for visitor in visitors:
                        visitor.result(p, result)
            except Exception:
                print('Error processing [{0}]'.format(p))
                traceback.print_exc()


class BasicReportVisitor(object):
    """Writes a line-per-file and a line-per-barcode to stdout
    """
    def result(self, path, result):
        print(path)
        strategy, barcodes = result
        print('Found [{0}] barcodes:'.format(len(barcodes)))
        for index, barcode in enumerate(barcodes):
            print('[{0}] [{1}] [{2}]'.format(index, barcode.type, barcode.data))


class TerseReportVisitor(object):
    """Writes a line-per-file to stdout
    """
    def result(self, path, result):
        strategy, barcodes = result
        values = [b.data for b in barcodes]
        print(path, ' '.join(['[{0}]'.format(v) for v in values]))


class CSVReportVisitor(object):
    """Writes a CSV report
    """
    def __init__(self, engine, greyscale, file=sys.stdout):
        self.w = csv.writer(file, lineterminator='\n')
        self.w.writerow([
            'OS', 'Engine', 'Directory', 'File', 'Image.conversion',
            'Elapsed', 'N.found', 'Types', 'Values', 'Strategy'
        ])
        self.engine = engine
        self.image_conversion = 'Greyscale' if greyscale else 'Unchanged'
        self.start_time = time.time()

    def result(self, path, result):
        strategy, barcodes = result
        types = '|'.join(b.type for b in barcodes)
        values = '|'.join(b.data for b in barcodes)
        self.w.writerow([sys.platform,
                         self.engine,
                         path.parent.name,
                         path.name,
                         self.image_conversion,
                         time.time()-self.start_time,
                         len(barcodes),
                         types,
                         values,
                         strategy])


class RenameVisitor(object):
    """Renames files based on their barcodes
    """
    def __init__(self, avoid_collisions):
        self.avoid_collisions = avoid_collisions
        # Mapping from path to iterator of integer suffixes, used to avoid
        # collisions - see self._destination
        self.suffix = defaultdict(partial(count, start=1))

    def _destination(self, path):
        """Returns path possibly with a suffix appended to the name to avoid
        collisions with existing files, iff self.avoid_collisions is True,
        otherwise path is returned unaltered.
        """
        destination = path
        if self.avoid_collisions:
            while destination.is_file():
                fname = u'{0}-{1}{2}'.format(
                    path.stem,
                    next(self.suffix[path.name]),
                    path.suffix
                )
                destination = path.with_name(fname)

        return destination

    def result(self, path, result):
        strategy, barcodes = result
        print(path)
        if not barcodes:
            print('  No barcodes')
        else:
            # TODO How best to sanitize filenames?
            values = [re.sub('[^a-zA-Z0-9_-]', '_', b.data) for b in barcodes]

            # The first time round the loop, the file will be renamed and
            # first_destination set to the new filename.
            # On subsequent iterations, first_destination will copied
            # to new destinations.
            first_destination = None
            for value in values:
                dest = path.with_name(u'{0}{1}'.format(value, path.suffix))
                dest = self._destination(dest)
                source = first_destination if first_destination else path
                rename = not bool(first_destination)
                if source == dest:
                    print('  Already correctly named')
                elif dest.is_file():
                    msg = '  Cannot rename to [{0}] because destination exists'
                    print(msg.format(dest))
                elif rename:
                    path.rename(dest)
{'fraction_non_alphanumeric': 0.09060133630289532, 'fraction_numerical': 0.0026726057906458797, 'mean_word_length': 1.6888622754491018, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1985746', 'n_tokens_mistral': 3003, 'n_tokens_neox': 2848, 'n_words': 874}