<commit_before>import os
import tqdm
from pathlib import Path
from thinc.api import use_ops
from timeit import default_timer as timer
import shutil
import srsly
from wasabi import msg
import contextlib
import random

from ..util import create_default_optimizer
from ..util import use_gpu as set_gpu
from ..gold import GoldCorpus
from ..lookups import Lookups
from .. import util
from .. import about


def train(
    # fmt: off
    lang: ("Model language", "positional", None, str),
    output_path: ("Output directory to store model in", "positional", None, Path),
    train_path: ("Location of JSON-formatted training data", "positional", None, Path),
    dev_path: ("Location of JSON-formatted development data", "positional", None, Path),
    raw_text: ("Path to jsonl file with unlabelled text documents.", "option", "rt", Path) = None,
    base_model: ("Name of model to update (optional)", "option", "b", str) = None,
    pipeline: ("Comma-separated names of pipeline components", "option", "p", str) = "tagger,parser,ner",
    vectors: ("Model to load vectors from", "option", "v", str) = None,
    replace_components: ("Replace components from base model", "flag", "R", bool) = False,
    n_iter: ("Number of iterations", "option", "n", int) = 30,
    n_early_stopping: ("Maximum number of training epochs without dev accuracy improvement", "option", "ne", int) = None,
    n_examples: ("Number of examples", "option", "ns", int) = 0,
    use_gpu: ("Use GPU", "option", "g", int) = -1,
    version: ("Model version", "option", "V", str) = "0.0.0",
    meta_path: ("Optional path to meta.json to use as base.", "option", "m", Path) = None,
    init_tok2vec: ("Path to pretrained weights for the token-to-vector parts of the models. See 'spacy pretrain'. Experimental.", "option", "t2v", Path) = None,
    parser_multitasks: ("Side objectives for parser CNN, e.g. 'dep' or 'dep,tag'", "option", "pt", str) = "",
    entity_multitasks: ("Side objectives for NER CNN, e.g. 'dep' or 'dep,tag'", "option", "et", str) = "",
    noise_level: ("Amount of corruption for data augmentation", "option", "nl", float) = 0.0,
    orth_variant_level: ("Amount of orthography variation for data augmentation", "option", "ovl", float) = 0.0,
    eval_beam_widths: ("Beam widths to evaluate, e.g. 4,8", "option", "bw", str) = "",
    gold_preproc: ("Use gold preprocessing", "flag", "G", bool) = False,
    learn_tokens: ("Make parser learn gold-standard tokenization", "flag", "T", bool) = False,
    textcat_multilabel: ("Textcat classes aren't mutually exclusive (multilabel)", "flag", "TML", bool) = False,
    textcat_arch: ("Textcat model architecture", "option", "ta", str) = "bow",
    textcat_positive_label: ("Textcat positive label for binary classes with two labels", "option", "tpl", str) = None,
    tag_map_path: ("Location of JSON-formatted tag map", "option", "tm", Path) = None,
    omit_extra_lookups: ("Don't include extra lookups in model", "flag", "OEL", bool) = False,
    verbose: ("Display more information for debug", "flag", "VV", bool) = False,
    debug: ("Run data diagnostics before training", "flag", "D", bool) = False,
    # fmt: on
):
    """
    Train or update a spaCy model. Requires data to be formatted in spaCy's
    JSON format. To convert data from other formats, use the `spacy convert`
    command.
    """
    util.fix_random_seed()
    util.set_env_log(verbose)

    # Make sure all files and paths exists if they are needed
    train_path = util.ensure_path(train_path)
    dev_path = util.ensure_path(dev_path)
    meta_path = util.ensure_path(meta_path)
    output_path = util.ensure_path(output_path)
    if raw_text is not None:
        raw_text = list(srsly.read_jsonl(raw_text))
    if not train_path or not train_path.exists():
        msg.fail("Training data not found", train_path, exits=1)
    if not dev_path or not dev_path.exists():
        msg.fail("Development data not found", dev_path, exits=1)
    if meta_path is not None and not meta_path.exists():
        msg.fail("Can't find model meta.json", meta_path, exits=1)
    meta = srsly.read_json(meta_path) if meta_path else {}
    if output_path.exists() and [p for p in output_path.iterdir() if p.is_dir()]:
        msg.warn(
            "Output directory is not empty",
            "This can lead to unintended side effects when saving the model. "
            "Please use an empty directory or a different path instead. If "
            "the specified output path doesn't exist, the directory will be "
            "created for you.",
        )
    if not output_path.exists():
        output_path.mkdir()
        msg.good(f"Created output directory: {output_path}")

    tag_map = {}
    if tag_map_path is not None:
        tag_map = srsly.read_json(tag_map_path)
    # Take dropout and batch size as generators of values -- dropout
    # starts high and decays sharply, to force the optimizer to explore.
    # Batch size starts at 1 and grows, so that we make updates quickly
    # at the beginning of training.
    dropout_rates = util.decaying(
        util.env_opt("dropout_from", 0.2),
        util.env_opt("dropout_to", 0.2),
        util.env_opt("dropout_decay", 0.0),
    )
    batch_sizes = util.compounding(
        util.env_opt("batch_from", 100.0),
        util.env_opt("batch_to", 1000.0),
        util.env_opt("batch_compound", 1.001),
    )

    if not eval_beam_widths:
        eval_beam_widths = [1]
    else:
        eval_beam_widths = [int(bw) for bw in eval_beam_widths.split(",")]
        if 1 not in eval_beam_widths:
            eval_beam_widths.append(1)
        eval_beam_widths.sort()
    has_beam_widths = eval_beam_widths != [1]

    default_dir = Path(__file__).parent.parent / "ml" / "models" / "defaults"

    # Set up the base model and pipeline. If a base model is specified, load
    # the model and make sure the pipeline matches the pipeline setting. If
    # training starts from a blank model, intitalize the language class.
    pipeline = [p.strip() for p in pipeline.split(",")]
    msg.text(f"Training pipeline: {pipeline}")
    disabled_pipes = None
    pipes_added = False
    if use_gpu >= 0:
        activated_gpu = None
        try:
            activated_gpu = set_gpu(use_gpu)
        except Exception as e:
            msg.warn(f"Exception: {e}")
        if activated_gpu is not None:
            msg.text(f"Using GPU: {use_gpu}")
        else:
            msg.warn(f"Unable to activate GPU: {use_gpu}")
            msg.text("Using CPU only")
            use_gpu = -1
    if base_model:
        msg.text(f"Starting with base model '{base_model}'")<commit_msg>Update default cfg dir in train CLI<commit_after>import os
import tqdm
from pathlib import Path
from thinc.api import use_ops
from timeit import default_timer as timer
import shutil
import srsly
from wasabi import msg
import contextlib
import random

from ..util import create_default_optimizer
from ..util import use_gpu as set_gpu
from ..gold import GoldCorpus
from ..lookups import Lookups
from .. import util
from .. import about


def train(
    # fmt: off
    lang: ("Model language", "positional", None, str),
    output_path: ("Output directory to store model in", "positional", None, Path),
    train_path: ("Location of JSON-formatted training data", "positional", None, Path),
    dev_path: ("Location of JSON-formatted development data", "positional", None, Path),
    raw_text: ("Path to jsonl file with unlabelled text documents.", "option", "rt", Path) = None,
    base_model: ("Name of model to update (optional)", "option", "b", str) = None,
    pipeline: ("Comma-separated names of pipeline components", "option", "p", str) = "tagger,parser,ner",
    vectors: ("Model to load vectors from", "option", "v", str) = None,
    replace_components: ("Replace components from base model", "flag", "R", bool) = False,
    n_iter: ("Number of iterations", "option", "n", int) = 30,
    n_early_stopping: ("Maximum number of training epochs without dev accuracy improvement", "option", "ne", int) = None,
    n_examples: ("Number of examples", "option", "ns", int) = 0,
    use_gpu: ("Use GPU", "option", "g", int) = -1,
    version: ("Model version", "option", "V", str) = "0.0.0",
    meta_path: ("Optional path to meta.json to use as base.", "option", "m", Path) = None,
    init_tok2vec: ("Path to pretrained weights for the token-to-vector parts of the models. See 'spacy pretrain'. Experimental.", "option", "t2v", Path) = None,
    parser_multitasks: ("Side objectives for parser CNN, e.g. 'dep' or 'dep,tag'", "option", "pt", str) = "",
    entity_multitasks: ("Side objectives for NER CNN, e.g. 'dep' or 'dep,tag'", "option", "et", str) = "",
    noise_level: ("Amount of corruption for data augmentation", "option", "nl", float) = 0.0,
    orth_variant_level: ("Amount of orthography variation for data augmentation", "option", "ovl", float) = 0.0,
    eval_beam_widths: ("Beam widths to evaluate, e.g. 4,8", "option", "bw", str) = "",
    gold_preproc: ("Use gold preprocessing", "flag", "G", bool) = False,
    learn_tokens: ("Make parser learn gold-standard tokenization", "flag", "T", bool) = False,
    textcat_multilabel: ("Textcat classes aren't mutually exclusive (multilabel)", "flag", "TML", bool) = False,
    textcat_arch: ("Textcat model architecture", "option", "ta", str) = "bow",
    textcat_positive_label: ("Textcat positive label for binary classes with two labels", "option", "tpl", str) = None,
    tag_map_path: ("Location of JSON-formatted tag map", "option", "tm", Path) = None,
    omit_extra_lookups: ("Don't include extra lookups in model", "flag", "OEL", bool) = False,
    verbose: ("Display more information for debug", "flag", "VV", bool) = False,
    debug: ("Run data diagnostics before training", "flag", "D", bool) = False,
    # fmt: on
):
    """
    Train or update a spaCy model. Requires data to be formatted in spaCy's
    JSON format. To convert data from other formats, use the `spacy convert`
    command.
    """
    util.fix_random_seed()
    util.set_env_log(verbose)

    # Make sure all files and paths exists if they are needed
    train_path = util.ensure_path(train_path)
    dev_path = util.ensure_path(dev_path)
    meta_path = util.ensure_path(meta_path)
    output_path = util.ensure_path(output_path)
    if raw_text is not None:
        raw_text = list(srsly.read_jsonl(raw_text))
    if not train_path or not train_path.exists():
        msg.fail("Training data not found", train_path, exits=1)
    if not dev_path or not dev_path.exists():
        msg.fail("Development data not found", dev_path, exits=1)
    if meta_path is not None and not meta_path.exists():
        msg.fail("Can't find model meta.json", meta_path, exits=1)
    meta = srsly.read_json(meta_path) if meta_path else {}
    if output_path.exists() and [p for p in output_path.iterdir() if p.is_dir()]:
        msg.warn(
            "Output directory is not empty",
            "This can lead to unintended side effects when saving the model. "
            "Please use an empty directory or a different path instead. If "
            "the specified output path doesn't exist, the directory will be "
            "created for you.",
        )
    if not output_path.exists():
        output_path.mkdir()
        msg.good(f"Created output directory: {output_path}")

    tag_map = {}
    if tag_map_path is not None:
        tag_map = srsly.read_json(tag_map_path)
    # Take dropout and batch size as generators of values -- dropout
    # starts high and decays sharply, to force the optimizer to explore.
    # Batch size starts at 1 and grows, so that we make updates quickly
    # at the beginning of training.
    dropout_rates = util.decaying(
        util.env_opt("dropout_from", 0.2),
        util.env_opt("dropout_to", 0.2),
        util.env_opt("dropout_decay", 0.0),
    )
    batch_sizes = util.compounding(
        util.env_opt("batch_from", 100.0),
        util.env_opt("batch_to", 1000.0),
        util.env_opt("batch_compound", 1.001),
    )

    if not eval_beam_widths:
        eval_beam_widths = [1]
    else:
        eval_beam_widths = [int(bw) for bw in eval_beam_widths.split(",")]
        if 1 not in eval_beam_widths:
            eval_beam_widths.append(1)
        eval_beam_widths.sort()
    has_beam_widths = eval_beam_widths != [1]

    default_dir = Path(__file__).parent.parent / "pipeline" / "defaults"

    # Set up the base model and pipeline. If a base model is specified, load
    # the model and make sure the pipeline matches the pipeline setting. If
    # training starts from a blank model, intitalize the language class.
    pipeline = [p.strip() for p in pipeline.split(",")]
    msg.text(f"Training pipeline: {pipeline}")
    disabled_pipes = None
    pipes_added = False
    if use_gpu >= 0:
        activated_gpu = None
        try:
            activated_gpu = set_gpu(use_gpu)
        except Exception as e:
            msg.warn(f"Exception: {e}")
        if activated_gpu is not None:
            msg.text(f"Using GPU: {use_gpu}")
        else:
            msg.warn(f"Unable to activate GPU: {use_gpu}")
            msg.text("Using CPU only")
            use_gpu = -1
    if base_model:
        msg.text(f"Starting with base model '{base_model}'")
{'fraction_non_alphanumeric': 0.12395322519803847, 'fraction_numerical': 0.006639004149377593, 'mean_word_length': 3.298313878080415, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5773694', 'n_tokens_mistral': 4216, 'n_tokens_neox': 4073, 'n_words': 1426}