<jupyter_start><jupyter_text><jupyter_code>#@title
!git clone https://github.com/NVIDIA/apex.git
%cd apex
!pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" .
%cd ..

import torch
import apex
from fastai.text import *
import datetime
run_start_time = datetime.datetime.today().strftime('%Y-%m-%d_%H-%M-%S')

LOG_PATH=Path('logs/')  
MODEL_PATH=Path('models/') 

if not LOG_PATH.exists():
  LOG_PATH.mkdir()
import logging

args = {
    "run_text": "my_test",
    "max_seq_length": 512,
    "do_lower_case": True,
    "train_batch_size": 16,
    "learning_rate": 6e-5,
    "num_train_epochs": 12.0,
    "warmup_proportion": 0.002,
    "local_rank": -1,
    "gradient_accumulation_steps": 1,
    "fp16": True,
    "loss_scale": 128
}

logfile = str(LOG_PATH/'log-{}-{}.txt'.format(run_start_time, args["run_text"]))

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
    datefmt='%m/%d/%Y %H:%M:%S',
    handlers=[
        logging.FileHandler(logfile),
        logging.StreamHandler(sys.stdout)
    ])

logger = logging.getLogger()

device = torch.device('cuda')

if torch.cuda.device_count() > 1:
    multi_gpu = True
else:
    multi_gpu = False
!git clone https://github.com/wshuyi/demo-multi-label-classification-bert.git
!pip install fast-bert
from fast_bert.data import *
from fast_bert.learner import *
from fast_bert.metrics import *
from pytorch_pretrained_bert.tokenization import BertTokenizer
DATA_PATH = Path('demo-multi-label-classification-bert/sample/data/')     
LABEL_PATH = Path('demo-multi-label-classification-bert/sample/labels/')  

BERT_PRETRAINED_MODEL = "bert-base-uncased"

args["do_lower_case"] = True
args["train_batch_size"] = 16
args["learning_rate"] = 6e-5
args["max_seq_length"] = 512
args["fp16"] = True
tokenizer = BertTokenizer.from_pretrained(BERT_PRETRAINED_MODEL, 
                                          do_lower_case=args['do_lower_case'])
label_cols = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
databunch = BertDataBunch(DATA_PATH, LABEL_PATH, tokenizer, train_file='train.csv', val_file='valid.csv',
                          test_data='test.csv', label_file="labels.csv",
                          text_col="comment_text", label_col=label_cols,
                          bs=args['train_batch_size'], maxlen=args['max_seq_length'], 
                          multi_gpu=multi_gpu, multi_label=True)
metrics = [{'name': 'accuracy', 'function': accuracy_multilabel}]
logger.info(args)
learner = BertLearner.from_pretrained_model(databunch, BERT_PRETRAINED_MODEL, metrics, device, logger, 
                                            is_fp16=args['fp16'], loss_scale=args['loss_scale'], 
                                            multi_gpu=multi_gpu,  multi_label=True)
learner.fit(4, lr=args['learning_rate'], schedule_type="warmup_linear")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14848793747876315, 'fraction_numerical': 0.012232415902140673, 'mean_word_length': 3.578538102643857, 'pattern_counts': {'":': 11, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '466829', 'license': 'Apache-2.0', 'n_tokens_mistral': 1066, 'n_tokens_neox': 1006, 'n_words': 159, 'path': 'demo_fastbert_sample.ipynb', 'repo_name': 'HaiLongEmb/tpu', 'size': '2943'}