<commit_before>#  Numenta Platform for Intelligent Computing (NuPIC)
#  Copyright (C) 2019, Numenta, Inc.  Unless you have an agreement
#  with Numenta, Inc., for a separate license for this software code, the
#  following terms and conditions apply:
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero Public License version 3 as
#  published by the Free Software Foundation.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#  See the GNU Affero Public License for more details.
#
#  You should have received a copy of the GNU Affero Public License
#  along with this program.  If not, see http://www.gnu.org/licenses.
#
#  http://numenta.org/licenses/
#
# Original Code here:
# https://github.com/pytorch/examples/blob/master/mnist/main.py

import argparse
import json
import os

import torch
from torchvision import datasets

from nupic.research.support.parse_config import parse_config
from projects.whydense.cifar.cifar_experiment import TinyCIFAR


def train_models(configs, project_dir):
    """Run all the training experiments specified in configs."""
    # download dataset
    data_dir = os.path.join(project_dir, "data")
    datasets.CIFAR10(data_dir, download=True, train=True)

    # Run all experiments in serial
    if len(configs) == 0:
        print("No experiments to run!")

    for exp in configs:
        config = configs[exp]
        if "name" not in config:
            config["name"] = exp

        # Make sure local directories are relative to the project location
        path = config.get("path", "results")
        if not os.path.isabs(path):
            config["path"] = os.path.join(project_dir, path)

        data_dir = config.get("data_dir", "data")
        if not os.path.isabs(data_dir):
            config["data_dir"] = os.path.join(project_dir, data_dir)

        model = TinyCIFAR()
        model.model_setup(config)
        for epoch in range(config["iterations"]):
            ret = model.train_epoch(epoch)
            print("epoch=", epoch, ":", ret)
            # if ret['stop'] == 1:
            #   print("Stopping early!")
            #   break

        if config.get("checkpoint_at_end", False):
            model.model_save(path)


def parse_options():
    """parses the command line options for different settings."""
    optparser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )
    optparser.add_argument(
        "-c",
        "--config",
        dest="config",
        type=str,
        default="",
        help="your experiment config file",
    )
    optparser.add_argument(<commit_msg>Data dir now in nta<commit_after>#  Numenta Platform for Intelligent Computing (NuPIC)
#  Copyright (C) 2019, Numenta, Inc.  Unless you have an agreement
#  with Numenta, Inc., for a separate license for this software code, the
#  following terms and conditions apply:
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero Public License version 3 as
#  published by the Free Software Foundation.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#  See the GNU Affero Public License for more details.
#
#  You should have received a copy of the GNU Affero Public License
#  along with this program.  If not, see http://www.gnu.org/licenses.
#
#  http://numenta.org/licenses/
#
# Original Code here:
# https://github.com/pytorch/examples/blob/master/mnist/main.py

import argparse
import json
import os

import torch
from torchvision import datasets

from nupic.research.support.parse_config import parse_config
from projects.whydense.cifar.cifar_experiment import TinyCIFAR


def train_models(configs, project_dir):
    """Run all the training experiments specified in configs."""
    # download dataset
    data_dir = os.path.join(os.environ["HOME"], "nta", "data")
    datasets.CIFAR10(data_dir, download=True, train=True)

    # Run all experiments in serial
    if len(configs) == 0:
        print("No experiments to run!")

    for exp in configs:
        config = configs[exp]
        if "name" not in config:
            config["name"] = exp

        # Make sure local directories are relative to the project location
        path = config.get("path", "results")
        if not os.path.isabs(path):
            config["path"] = os.path.join(project_dir, path)

        if "data_dir" not in config:
            config["data_dir"] = data_dir
        if not os.path.isabs(config["data_dir"]):
            config["data_dir"] = os.path.abspath(config["data_dir"])

        model = TinyCIFAR()
        model.model_setup(config)
        for epoch in range(config["iterations"]):
            ret = model.train_epoch(epoch)
            print("epoch=", epoch, ":", ret)
            # if ret['stop'] == 1:
            #   print("Stopping early!")
            #   break

        if config.get("checkpoint_at_end", False):
            model.model_save(path)


def parse_options():
    """parses the command line options for different settings."""
    optparser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )
    optparser.add_argument(
        "-c",
        "--config",
        dest="config",
        type=str,
        default="",
        help="your experiment config file",
    )
    optparser.add_argument(
{'fraction_non_alphanumeric': 0.09881352930759696, 'fraction_numerical': 0.0031875332034708697, 'mean_word_length': 2.919500346981263, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3814553', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1607, 'n_words': 537}