<commit_before>import argparse
import copy
import numpy as np

import chainer
from chainer.datasets import TransformDataset
from chainer import training
from chainer.training import extensions
from chainer.training import triggers

from chainercv.datasets import VOCDetectionDataset
from chainercv.datasets import voc_detection_label_names
from chainercv.extensions import DetectionVOCEvaluator
from chainercv.links.model.ssd import ConcatenatedDataset
from chainercv.links.model.ssd import MultiboxTrainChain
from chainercv.links.model.ssd import random_transform
from chainercv.links.model.ssd import SelectiveWeightDecay
from chainercv.links import SSD300
from chainercv import transforms


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--batchsize', type=int, default=32)
    parser.add_argument('--gpu', type=int, default=-1)
    parser.add_argument('--out', default='result')
    args = parser.parse_args()

    model = SSD300(
        n_fg_class=len(voc_detection_label_names),
        pretrained_model='imagenet')
    model.use_preset('evaluate')
    train_chain = MultiboxTrainChain(model)
    if args.gpu >= 0:
        chainer.cuda.get_device(args.gpu).use()
        model.to_gpu()

    coder = copy.copy(model.coder)
    coder.to_cpu()
    size = model.insize
    mean = model.mean

    def transform(in_data):
        img, bbox, label = in_data
        img, bbox, label = random_transform(img, bbox, label, size, mean)
        img -= np.array(mean)[:, np.newaxis, np.newaxis]
        mb_loc, mb_label = coder.encode(
            transforms.resize_bbox(bbox, (size, size), (1, 1)), label)
        return img, mb_loc, mb_label

    train = TransformDataset(
        ConcatenatedDataset(
            VOCDetectionDataset(year='2007', split='trainval'),
            VOCDetectionDataset(year='2012', split='trainval')
        ),
        transform)
    train_iter = chainer.iterators.MultiprocessIterator(
        train, args.batchsize, n_processes=2)

    test = VOCDetectionDataset(year='2007', split='test')
    test_iter = chainer.iterators.SerialIterator(
        test, args.batchsize, repeat=False, shuffle=False)

    optimizer = chainer.optimizers.MomentumSGD()
    optimizer.setup(train_chain)
    optimizer.add_hook(SelectiveWeightDecay(0.0005, b={'lr': 2, 'decay': 0}))
<commit_msg>return difficult<commit_after>import argparse
import copy
import numpy as np

import chainer
from chainer.datasets import TransformDataset
from chainer import training
from chainer.training import extensions
from chainer.training import triggers

from chainercv.datasets import VOCDetectionDataset
from chainercv.datasets import voc_detection_label_names
from chainercv.extensions import DetectionVOCEvaluator
from chainercv.links.model.ssd import ConcatenatedDataset
from chainercv.links.model.ssd import MultiboxTrainChain
from chainercv.links.model.ssd import random_transform
from chainercv.links.model.ssd import SelectiveWeightDecay
from chainercv.links import SSD300
from chainercv import transforms


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--batchsize', type=int, default=32)
    parser.add_argument('--gpu', type=int, default=-1)
    parser.add_argument('--out', default='result')
    args = parser.parse_args()

    model = SSD300(
        n_fg_class=len(voc_detection_label_names),
        pretrained_model='imagenet')
    model.use_preset('evaluate')
    train_chain = MultiboxTrainChain(model)
    if args.gpu >= 0:
        chainer.cuda.get_device(args.gpu).use()
        model.to_gpu()

    coder = copy.copy(model.coder)
    coder.to_cpu()
    size = model.insize
    mean = model.mean

    def transform(in_data):
        img, bbox, label = in_data
        img, bbox, label = random_transform(img, bbox, label, size, mean)
        img -= np.array(mean)[:, np.newaxis, np.newaxis]
        mb_loc, mb_label = coder.encode(
            transforms.resize_bbox(bbox, (size, size), (1, 1)), label)
        return img, mb_loc, mb_label

    train = TransformDataset(
        ConcatenatedDataset(
            VOCDetectionDataset(year='2007', split='trainval'),
            VOCDetectionDataset(year='2012', split='trainval')
        ),
        transform)
    train_iter = chainer.iterators.MultiprocessIterator(
        train, args.batchsize, n_processes=2)

    test = VOCDetectionDataset(
        year='2007', split='test',
        use_difficult=True, return_difficult=True)
    test_iter = chainer.iterators.SerialIterator(
        test, args.batchsize, repeat=False, shuffle=False)

    optimizer = chainer.optimizers.MomentumSGD()
    optimizer.setup(train_chain)
    optimizer.add_hook(SelectiveWeightDecay(0.0005, b={'lr': 2, 'decay': 0}))

{'fraction_non_alphanumeric': 0.101716465352829, 'fraction_numerical': 0.0135621953803772, 'mean_word_length': 3.850976361767729, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3909653', 'n_tokens_mistral': 1542, 'n_tokens_neox': 1475, 'n_words': 316}