<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse
import logging
import mimetypes
import os
import sys

sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath('.'))

import cv2
import numpy as np
from pyfann import libfann
import yaml

import common
import features as ft

def main():
    if sys.flags.debug:
        # Print debug messages if the -d flag is set for the Python interpreter.
        logging.basicConfig(level=logging.DEBUG, format='%(levelname)s %(message)s')
    else:
        # Otherwise just show log messages of type INFO.
        logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s')

    parser = argparse.ArgumentParser(description='Generate training data and train artificial neural networks.')

    # Create a sub parser for sub-commands.
    subparsers = parser.add_subparsers(help='Specify which task to start.')

    # Create an argument parser for sub-command 'traindata'.
    help_data = """Create a tab separated file with training data. Preprocessing
    steps and features to extract must be set in a YAML file. See orchids.yml
    for an example.
    """
    parser_data = subparsers.add_parser('data',
        help=help_data,
        description=help_data)
    parser_data.add_argument("--conf", metavar="FILE", required=True, help="Path to a YAML file with feature extraction parameters.")
    parser_data.add_argument("--output", "-o", metavar="FILE", required=True, help="Output file name for training data. Any existing file with same name will be overwritten.")
    parser_data.add_argument("images", metavar="PATH", help="Directory path to load image files from. Images must be in level one sub directories. Sub directory names will be used as class names.")

    # Create an argument parser for sub-command 'trainann'.
    help_ann = """Train an artificial neural network. Optional training
    parameters can be set in a separate YAML file. See orchids.yml
    for an example file.
    """
    parser_ann = subparsers.add_parser('ann',
        help=help_ann,
        description=help_ann)
    parser_ann.add_argument("--test-data", metavar="FILE", help="Path to tab separated file with test data.")
    parser_ann.add_argument("--conf", metavar="FILE", help="Path to a YAML file with ANN training parameters.")
    parser_ann.add_argument("--output", "-o", metavar="FILE", required=True, help="Output file name for the artificial neural network. Any existing file with same name will be overwritten.")
    parser_ann.add_argument("data", metavar="FILE", help="Path to tab separated file with training data.")

    # Create an argument parser for sub-command 'test-ann'.
    help_test_ann = """Test an artificial neural network. If `--output` is
    set, then `--conf` must also be set. See orchids.yml for an example YAML
    file with class names."""
    parser_test_ann = subparsers.add_parser('test-ann',
        help=help_test_ann,
        description=help_test_ann)
    parser_test_ann.add_argument("--ann", metavar="FILE", required=True, help="A trained artificial neural network.")
    parser_test_ann.add_argument("--output", "-o", metavar="FILE", help="Output file name for the test results. Specifying this option will output a table with the classification result for each sample.")
    parser_test_ann.add_argument("--conf", metavar="FILE", help="Path to a YAML file with class names.")
    parser_test_ann.add_argument("--error", metavar="N", type=float, default=0.01, help="The maximum error for classification. Default is 0.01")
    parser_test_ann.add_argument("data", metavar="FILE", help="Path to tab separated file containing test data.")

    # Create an argument parser for sub-command 'classify'.
    help_classify = """Classify an image. See orchids.yml for an example YAML
    file with class names."""
    parser_classify = subparsers.add_parser('classify',
        help=help_classify,
        description=help_classify)
    parser_classify.add_argument("--ann", metavar="FILE", required=True, help="Path to a trained artificial neural network file.")
    parser_classify.add_argument("--conf", metavar="FILE", required=True, help="Path to a YAML file with class names.")
    parser_classify.add_argument("--error", metavar="N", type=float, default=0.01, help="The maximum error for classification. Default is 0.01")
    parser_classify.add_argument("image", metavar="FILE", help="Path to image file to be classified.")

    # Parse arguments.
    args = parser.parse_args()

    if sys.argv[1] == 'data':
        train_data(args.images, args.conf, args.output)
    elif sys.argv[1] == 'ann':
        train_ann(args.data, args.output, args.test_data, args.conf)
    elif sys.argv[1] == 'test-ann':
        if args.output and not args.conf:
            sys.stderr.write("Option '--conf' must be set when '--output' is set.\n")
            sys.exit()
        test_ann(args.ann, args.data, args.output, args.conf, args.error)
    elif sys.argv[1] == 'classify':
        classify(args.image, args.ann, args.conf, args.error)

    sys.exit()

def train_data(images_path, conf_path, output_path):
    for path in (images_path, conf_path):
        if not os.path.exists(path):
            sys.stderr.write("Cannot open %s (no such file or directory)\n" % path)
            return 1

    yml = open_yaml(conf_path)
    if not yml:
        return 1

    if 'preprocess' in yml and 'segmentation' in yml.preprocess:
        path = getattr(yml.preprocess.segmentation, 'output_folder', None)
        if path and not os.path.exists(path):
            sys.stderr.write("Cannot open %s (no such file or directory)\n" % path)
            return 1

    out_file = open(output_path, 'w')

    # Get list of image files and set the classes.
    images = {}
    classes = []
    for item in os.listdir(images_path):
        path = os.path.join(images_path, item)
        if os.path.isdir(path):
            classes.append(item)
            images[item] = get_image_files(path)

    # Make codeword for each class.
    codewords = get_codewords(classes, -1, 1)

    # Construct the header row.
    header_primer = ["ID"]
    header_data = []
    header_out = []

    if 'color_histograms' in yml.features:
        for colorspace, bins in vars(yml.features.color_histograms).iteritems():
            for ch, n in enumerate([bins] * len(colorspace)):
                for i in range(n):
                    header_data.append("%s.%d" % (colorspace[ch], i+1))

    if 'shape_outline' in yml.features:
        n = 2 * getattr(yml.features.shape_outline, 'resolution', 15)
        for i in range(n):
            header_data.append("OL.%d" % (i+1,))

    if 'shape_360' in yml.features:
        n = 360 / getattr(yml.features.shape_360, 'step', 1)
        for i in range(n):
            header_data.append("MEAN.%d" % i)
            header_data.append("SD.%d" % i)

    for i in range(len(classes)):
        header_out.append("OUT.%d" % (i+1,))

    # Write the header row.
    out_file.write( "%s\n" % "\t".join(header_primer + header_data + header_out) )

    # Set the training data.
    training_data = common.TrainData(len(header_data), len(classes))
    fp = Fingerprint()
    for im_class, files in images.items():
        for im_path in files:
            if fp.open(im_path, yml) == None:
                logging.info("Failed to read %s. Skipping." % im_path)
                continue

            try:
                data = fp.make()
            except:
                logging.info("Fingerprint failed. Skipping.")
                continue

            assert len(data) == len(header_data), "Data length mismatch"

            training_data.append(data, codewords[im_class], label=im_path)

    training_data.finalize()

    # Round all values.
    training_data.round_input(4)

    # Write data rows.
    for label, input_data, output_data in training_data:
        row = []
        row.append( label )
        row.extend( input_data.astype(str) )
        row.extend( output_data.astype(str) )
        out_file.write( "%s\n" % "\t".join(row) )

    out_file.close()
    logging.info("Training data written to %s" % output_path)

def train_ann(train_data_path, output_path, test_data_path=None, conf_path=None):
    for path in (train_data_path, test_data_path, conf_path):
        if path and not os.path.exists(path):<commit_msg>Print a list of files that failed to process<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse
import logging
import mimetypes
import os
import sys

sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath('.'))

import cv2
import numpy as np
from pyfann import libfann
import yaml

import common
import features as ft

def main():
    if sys.flags.debug:
        # Print debug messages if the -d flag is set for the Python interpreter.
        logging.basicConfig(level=logging.DEBUG, format='%(levelname)s %(message)s')
    else:
        # Otherwise just show log messages of type INFO.
        logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s')

    parser = argparse.ArgumentParser(description='Generate training data and train artificial neural networks.')

    # Create a sub parser for sub-commands.
    subparsers = parser.add_subparsers(help='Specify which task to start.')

    # Create an argument parser for sub-command 'traindata'.
    help_data = """Create a tab separated file with training data. Preprocessing
    steps and features to extract must be set in a YAML file. See orchids.yml
    for an example.
    """
    parser_data = subparsers.add_parser('data',
        help=help_data,
        description=help_data)
    parser_data.add_argument("--conf", metavar="FILE", required=True, help="Path to a YAML file with feature extraction parameters.")
    parser_data.add_argument("--output", "-o", metavar="FILE", required=True, help="Output file name for training data. Any existing file with same name will be overwritten.")
    parser_data.add_argument("images", metavar="PATH", help="Directory path to load image files from. Images must be in level one sub directories. Sub directory names will be used as class names.")

    # Create an argument parser for sub-command 'trainann'.
    help_ann = """Train an artificial neural network. Optional training
    parameters can be set in a separate YAML file. See orchids.yml
    for an example file.
    """
    parser_ann = subparsers.add_parser('ann',
        help=help_ann,
        description=help_ann)
    parser_ann.add_argument("--test-data", metavar="FILE", help="Path to tab separated file with test data.")
    parser_ann.add_argument("--conf", metavar="FILE", help="Path to a YAML file with ANN training parameters.")
    parser_ann.add_argument("--output", "-o", metavar="FILE", required=True, help="Output file name for the artificial neural network. Any existing file with same name will be overwritten.")
    parser_ann.add_argument("data", metavar="FILE", help="Path to tab separated file with training data.")

    # Create an argument parser for sub-command 'test-ann'.
    help_test_ann = """Test an artificial neural network. If `--output` is
    set, then `--conf` must also be set. See orchids.yml for an example YAML
    file with class names."""
    parser_test_ann = subparsers.add_parser('test-ann',
        help=help_test_ann,
        description=help_test_ann)
    parser_test_ann.add_argument("--ann", metavar="FILE", required=True, help="A trained artificial neural network.")
    parser_test_ann.add_argument("--output", "-o", metavar="FILE", help="Output file name for the test results. Specifying this option will output a table with the classification result for each sample.")
    parser_test_ann.add_argument("--conf", metavar="FILE", help="Path to a YAML file with class names.")
    parser_test_ann.add_argument("--error", metavar="N", type=float, default=0.01, help="The maximum error for classification. Default is 0.01")
    parser_test_ann.add_argument("data", metavar="FILE", help="Path to tab separated file containing test data.")

    # Create an argument parser for sub-command 'classify'.
    help_classify = """Classify an image. See orchids.yml for an example YAML
    file with class names."""
    parser_classify = subparsers.add_parser('classify',
        help=help_classify,
        description=help_classify)
    parser_classify.add_argument("--ann", metavar="FILE", required=True, help="Path to a trained artificial neural network file.")
    parser_classify.add_argument("--conf", metavar="FILE", required=True, help="Path to a YAML file with class names.")
    parser_classify.add_argument("--error", metavar="N", type=float, default=0.01, help="The maximum error for classification. Default is 0.01")
    parser_classify.add_argument("image", metavar="FILE", help="Path to image file to be classified.")

    # Parse arguments.
    args = parser.parse_args()

    if sys.argv[1] == 'data':
        train_data(args.images, args.conf, args.output)
    elif sys.argv[1] == 'ann':
        train_ann(args.data, args.output, args.test_data, args.conf)
    elif sys.argv[1] == 'test-ann':
        if args.output and not args.conf:
            sys.stderr.write("Option '--conf' must be set when '--output' is set.\n")
            sys.exit()
        test_ann(args.ann, args.data, args.output, args.conf, args.error)
    elif sys.argv[1] == 'classify':
        classify(args.image, args.ann, args.conf, args.error)

    sys.exit()

def train_data(images_path, conf_path, output_path):
    for path in (images_path, conf_path):
        if not os.path.exists(path):
            sys.stderr.write("Cannot open %s (no such file or directory)\n" % path)
            return 1

    yml = open_yaml(conf_path)
    if not yml:
        return 1

    if 'preprocess' in yml and 'segmentation' in yml.preprocess:
        path = getattr(yml.preprocess.segmentation, 'output_folder', None)
        if path and not os.path.exists(path):
            sys.stderr.write("Cannot open %s (no such file or directory)\n" % path)
            return 1

    out_file = open(output_path, 'w')

    # Get list of image files and set the classes.
    images = {}
    classes = []
    for item in os.listdir(images_path):
        path = os.path.join(images_path, item)
        if os.path.isdir(path):
            classes.append(item)
            images[item] = get_image_files(path)

    # Make codeword for each class.
    codewords = get_codewords(classes, -1, 1)

    # Construct the header row.
    header_primer = ["ID"]
    header_data = []
    header_out = []

    if 'color_histograms' in yml.features:
        for colorspace, bins in vars(yml.features.color_histograms).iteritems():
            for ch, n in enumerate([bins] * len(colorspace)):
                for i in range(n):
                    header_data.append("%s.%d" % (colorspace[ch], i+1))

    if 'shape_outline' in yml.features:
        n = 2 * getattr(yml.features.shape_outline, 'resolution', 15)
        for i in range(n):
            header_data.append("OL.%d" % (i+1,))

    if 'shape_360' in yml.features:
        n = 360 / getattr(yml.features.shape_360, 'step', 1)
        for i in range(n):
            header_data.append("MEAN.%d" % i)
            header_data.append("SD.%d" % i)

    for i in range(len(classes)):
        header_out.append("OUT.%d" % (i+1,))

    # Write the header row.
    out_file.write( "%s\n" % "\t".join(header_primer + header_data + header_out) )

    # Set the training data.
    training_data = common.TrainData(len(header_data), len(classes))
    fp = Fingerprint()
    failed = []
    for im_class, files in images.items():
        for im_path in files:
            if fp.open(im_path, yml) == None:
                logging.warning("Failed to read %s. Skipping." % im_path)
                failed.append(im_path)
                continue

            try:
                data = fp.make()
            except:
                logging.warning("Fingerprint failed. Skipping.")
                failed.append(im_path)
                continue

            assert len(data) == len(header_data), "Data length mismatch"

            training_data.append(data, codewords[im_class], label=im_path)

    training_data.finalize()

    # Round all values.
    training_data.round_input(4)

    # Write data rows.
    for label, input_data, output_data in training_data:
        row = []
        row.append( label )
        row.extend( input_data.astype(str) )
        row.extend( output_data.astype(str) )
        out_file.write( "%s\n" % "\t".join(row) )

    out_file.close()
    logging.info("Training data written to %s" % output_path)

    if len(failed) > 0:
        logging.warning("Some files could not be processed:")
        for path in failed:
            logging.warning("- %s" % path)

def train_ann(train_data_path, output_path, test_data_path=None, conf_path=None):
    for path in (train_data_path, test_data_path, conf_path):
        if path and not os.path.exists(path):
{'fraction_non_alphanumeric': 0.11941878934054945, 'fraction_numerical': 0.005000294134949115, 'mean_word_length': 3.043767840152236, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3345979', 'n_tokens_mistral': 5263, 'n_tokens_neox': 5068, 'n_words': 1526}