<commit_before>#!/usr/bin/env python

import celery

import nyc3dcars

import scores

import logging
import argparse

from detect import *
from geo_rescore import *
from nms import *

def flatten(task):
    if task is None:
        return None 
    else:
        parent = flatten(task.parent)
        if parent is None:
            return [task.task_id]
        else:
            return [task.task_id] + parent

def test(model, remote, methods, dataset_id):
    session = nyc3dcars.SESSION()
    try:
        test_set = session.query(nyc3dcars.Photo) \
            .filter(nyc3dcars.Photo.test == True) \
            .filter(nyc3dcars.Photo.dataset_id == dataset_id) \

        session.query(nyc3dcars.Model) \
            .filter_by(filename=model) \
            .one()

        for photo in test_set:
            logging.info('PID: %d'%photo.id)

            celery_list = [detect.s(photo.id, model)]

            for method in methods:
                celery_list += [geo_rescore.s(model, method)]

            for method in methods:
                celery_list += [nms.s(model, method)]


            celery_task = celery.chain(celery_list)

            if remote:
                task_handle = celery_task.apply_async()
            else:
                celery_task.apply()

    except:
        session.rollback()
        raise

    finally:
        session.close()

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)

    parser = argparse.ArgumentParser()
    parser.add_argument('--model', required=True)
    parser.add_argument('--methods', nargs='+', default=scores.Methods.keys())
    parser.add_argument('--dataset-id', required=True, type=int)
    parser.add_argument('--remote', action='store_true')
    args = parser.parse_args()

    test(**vars(args))
<commit_msg>Cleaned up test.<commit_after>#!/usr/bin/env python

import celery

import nyc3dcars

import scores

import logging
import argparse

from detect import detect
from geo_rescore import geo_rescore
from nms import nms


def flatten(task):
    if task is None:
        return None
    else:
        parent = flatten(task.parent)
        if parent is None:
            return [task.task_id]
        else:
            return [task.task_id] + parent


def test(model, remote, methods, dataset_id):
    session = nyc3dcars.SESSION()
    try:
        test_set = session.query(nyc3dcars.Photo) \
            .filter(nyc3dcars.Photo.test == True) \
            .filter(nyc3dcars.Photo.dataset_id == dataset_id) \

        session.query(nyc3dcars.Model) \
            .filter_by(filename=model) \
            .one()

        for photo in test_set:
            logging.info('PID: %d' % photo.id)

            celery_list = [detect.s(photo.id, model)]

            for method in methods:
                celery_list += [geo_rescore.s(model, method)]

            for method in methods:
                celery_list += [nms.s(model, method)]

            celery_task = celery.chain(celery_list)

            if remote:
                celery_task.apply_async()
            else:
                celery_task.apply()

    except:
        session.rollback()
        raise

    finally:
        session.close()

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)

    parser = argparse.ArgumentParser()
    parser.add_argument('--model', required=True)
    parser.add_argument('--methods', nargs='+', default=scores.METHODS.keys())
    parser.add_argument('--dataset-id', required=True, type=int)
    parser.add_argument('--remote', action='store_true')
    args = parser.parse_args()

    test(**vars(args))

{'fraction_non_alphanumeric': 0.11329639889196676, 'fraction_numerical': 0.0033240997229916896, 'mean_word_length': 2.17311072056239, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1927001', 'n_tokens_mistral': 1146, 'n_tokens_neox': 1097, 'n_words': 233}