<commit_before>import copy
import os
import warnings

import dill
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

from metal.contrib.modules.lstm_module import EmbeddingsEncoder, LSTMModule
from metal.end_model import IdentityModule
from metal.mmtl.auxiliary_tasks import SPACY_TAGS, auxiliary_task_functions
from metal.mmtl.glue.glue_datasets import get_glue_dataset
from metal.mmtl.glue.glue_metrics import acc_f1, matthews_corr, mse, pearson_spearman
from metal.mmtl.glue.glue_slices import create_slice_labels
from metal.mmtl.modules import (
    BertExtractCls,
    BertRaw,
    BertTokenClassificationHead,
    BinaryHead,
    MulticlassHead,
    RegressionHead,
    SoftAttentionModule,
)
from metal.mmtl.payload import Payload
from metal.mmtl.scorer import Scorer
from metal.mmtl.slicing import create_slice_task
from metal.mmtl.task import ClassificationTask, RegressionTask, TokenClassificationTask
from metal.utils import recursive_merge_dicts, set_seed

ALL_TASKS = [
    "COLA",
    "SST2",
    "MNLI",
    "SNLI",
    "RTE",
    "WNLI",
    "QQP",
    "MRPC",
    "STSB",
    "QNLI",
]

# List of tasks requiring Spacy tokenization
SPACY_TASKS = ["SPACY_NER", "SPACY_POS"]

task_defaults = {
    # General
    "split_prop": None,
    "splits": ["train", "valid", "test"],
    "max_len": 200,
    "max_datapoints": -1,
    "seed": None,
    "preprocessed": False,  # If True, load the cached datasets with spacy tokens saved
    "dl_kwargs": {
        "batch_size": 16,
        "shuffle": True,  # Used only when split_prop is None; otherwise, use Sampler
    },
    "task_dl_kwargs": None,  # Overwrites dl kwargs e.g. {"STSB": {"batch_size": 2}}
    # NOTE: This dropout only applies to the output of the pooler; it will not change
    # the dropout rate of BERT (defaults to 0.1) or add dropout to other modules.
    # The main BERT module ends with a dropout layer already, so token-based tasks
    # that do not use BertExtractCls middle module do not need additional dropout first
    "dropout": 0.1,
    # BERT
    "encoder_type": "bert",
    "bert_model": "bert-base-uncased",  # Required for all encoders for BertTokenizer
    "bert_kwargs": {
        "freeze_bert": False,
        "pooler": True,  # If True, include the [768, 768] linear on top of [CLS] token
    },
    # LSTM
    "lstm_config": {
        "emb_size": 300,
        "hidden_size": 512,
        "vocab_size": 30522,  # bert-base-uncased-vocab.txt
        "bidirectional": True,
        "lstm_num_layers": 1,
    },
    "attention_config": {
        "attention_module": None,  # None, soft currently accepted
        "nonlinearity": "tanh",  # tanh, sigmoid currently accepted
    },
    # Auxiliary Tasks
    "auxiliary_task_dict": {  # A map of each aux. task to the payloads it applies to
        "THIRD": ALL_TASKS,
        "BLEU": ["MNLI", "RTE", "WNLI", "QQP", "MRPC", "STSB", "QNLI"],  # sent pairs
        "SPACY_NER": ALL_TASKS,
        "SPACY_POS": ALL_TASKS,
    },
    "auxiliary_loss_multiplier": 1.0,
    # Slicing
    "slice_dict": {  # A map of the slices that apply to each task
        "COLA": ["ends_with_question"]
    },
}


def create_tasks_and_payloads(task_names, **kwargs):
    assert len(task_names) > 0

    config = recursive_merge_dicts(task_defaults, kwargs)

    if config["seed"] is None:
        config["seed"] = np.random.randint(1e6)
        print(f"Using random seed: {config['seed']}")
    set_seed(config["seed"])
<commit_msg>Make default slice dict empty<commit_after>import copy
import os
import warnings

import dill
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

from metal.contrib.modules.lstm_module import EmbeddingsEncoder, LSTMModule
from metal.end_model import IdentityModule
from metal.mmtl.auxiliary_tasks import SPACY_TAGS, auxiliary_task_functions
from metal.mmtl.glue.glue_datasets import get_glue_dataset
from metal.mmtl.glue.glue_metrics import acc_f1, matthews_corr, mse, pearson_spearman
from metal.mmtl.glue.glue_slices import create_slice_labels
from metal.mmtl.modules import (
    BertExtractCls,
    BertRaw,
    BertTokenClassificationHead,
    BinaryHead,
    MulticlassHead,
    RegressionHead,
    SoftAttentionModule,
)
from metal.mmtl.payload import Payload
from metal.mmtl.scorer import Scorer
from metal.mmtl.slicing import create_slice_task
from metal.mmtl.task import ClassificationTask, RegressionTask, TokenClassificationTask
from metal.utils import recursive_merge_dicts, set_seed

ALL_TASKS = [
    "COLA",
    "SST2",
    "MNLI",
    "SNLI",
    "RTE",
    "WNLI",
    "QQP",
    "MRPC",
    "STSB",
    "QNLI",
]

# List of tasks requiring Spacy tokenization
SPACY_TASKS = ["SPACY_NER", "SPACY_POS"]

task_defaults = {
    # General
    "split_prop": None,
    "splits": ["train", "valid", "test"],
    "max_len": 200,
    "max_datapoints": -1,
    "seed": None,
    "preprocessed": False,  # If True, load the cached datasets with spacy tokens saved
    "dl_kwargs": {
        "batch_size": 16,
        "shuffle": True,  # Used only when split_prop is None; otherwise, use Sampler
    },
    "task_dl_kwargs": None,  # Overwrites dl kwargs e.g. {"STSB": {"batch_size": 2}}
    # NOTE: This dropout only applies to the output of the pooler; it will not change
    # the dropout rate of BERT (defaults to 0.1) or add dropout to other modules.
    # The main BERT module ends with a dropout layer already, so token-based tasks
    # that do not use BertExtractCls middle module do not need additional dropout first
    "dropout": 0.1,
    # BERT
    "encoder_type": "bert",
    "bert_model": "bert-base-uncased",  # Required for all encoders for BertTokenizer
    "bert_kwargs": {
        "freeze_bert": False,
        "pooler": True,  # If True, include the [768, 768] linear on top of [CLS] token
    },
    # LSTM
    "lstm_config": {
        "emb_size": 300,
        "hidden_size": 512,
        "vocab_size": 30522,  # bert-base-uncased-vocab.txt
        "bidirectional": True,
        "lstm_num_layers": 1,
    },
    "attention_config": {
        "attention_module": None,  # None, soft currently accepted
        "nonlinearity": "tanh",  # tanh, sigmoid currently accepted
    },
    # Auxiliary Tasks
    "auxiliary_task_dict": {  # A map of each aux. task to the payloads it applies to
        "THIRD": ALL_TASKS,
        "BLEU": ["MNLI", "RTE", "WNLI", "QQP", "MRPC", "STSB", "QNLI"],  # sent pairs
        "SPACY_NER": ALL_TASKS,
        "SPACY_POS": ALL_TASKS,
    },
    "auxiliary_loss_multiplier": 1.0,
    # Slicing
    "slice_dict": {},  # A map of the slices that apply to each task
}


def create_tasks_and_payloads(task_names, **kwargs):
    assert len(task_names) > 0

    config = recursive_merge_dicts(task_defaults, kwargs)

    if config["seed"] is None:
        config["seed"] = np.random.randint(1e6)
        print(f"Using random seed: {config['seed']}")
    set_seed(config["seed"])

{'fraction_non_alphanumeric': 0.1094334196146103, 'fraction_numerical': 0.010353753235547885, 'mean_word_length': 3.3852459016393444, 'pattern_counts': {'":': 69, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3686717', 'n_tokens_mistral': 2465, 'n_tokens_neox': 2362, 'n_words': 664}