<jupyter_start><jupyter_text>Utility Methods<jupyter_code># converts string representation to class
def str_to_class(classname):
    return getattr(sys.modules[__name__], classname)<jupyter_output><empty_output><jupyter_text>Class Inspection & Querying **Notes**:[1] There are "three standard classes required to use each model: **configuration, models and tokenizer**."[2] All three standard classes can be initialized [via] `from_pretrained()`.  This method will download (as needed), cache, and load the pre-trained instace from the library or via the filesystem. **Models**: All derive from `nn.Module` (e.g., `BertModel`)**Configuration**: Stores configuration required to **build a model** (e.g., `BertConfig`). "*If you are using a pretrained model* without any modification, *creating the model will automatically take care of instantiating the configuration* (which is part of the model)."**Tokenizer**: Stores the vocab for each model and provides methods to encode/decode strings and provide the various embeddings required to be fed into a model.**`from_pretrained()`**: To instantiate any of the above classes using a friendly name included in the library (`bert-base-uncased`) or from a path.**`save_pretrained()`**: To save any of the classes locally so it can be re-loaded using `from_pretrained()`<jupyter_code>transformer_classes = inspect.getmembers(sys.modules[__name__], 
                                         lambda member: inspect.isclass(member)
                                                         and member.__module__.startswith('transformers.'))

transformer_classes[:5]
df = pd.DataFrame(transformer_classes, columns=['class_name', 'class_location'])
df.head()
df['module'] = df.class_location.apply(lambda v: v.__module__); df.head()
df.drop(labels=['class_location'], axis=1, inplace=True)
df.head()
module_parts_df = df.module.str.split(".", n = -1, expand = True) 
for i in range(len(module_parts_df.columns)):
    df[f'module_part_{i}'] = module_parts_df[i]
    
df.head()
module_part_1_df = df.module_part_1.str.split("_", n = 1, expand = True) 
module_part_1_df.head()
df[['functional_area', 'arch']] = module_part_1_df
df.head()<jupyter_output><empty_output><jupyter_text>Look custom, task-based implementations of models (indicated by `For`)<jupyter_code>model_type_df = df[(df.functional_area == 'modeling')].class_name.str.split('For', n=1, expand=True)
model_type_df.head()
model_type_df[1] = np.where(model_type_df[1].notnull(), 'For' + model_type_df[1].astype(str), model_type_df[1])
df['model_task'] = model_type_df[1]<jupyter_output><empty_output><jupyter_text>Look custom, task-based implementations of models (indicated by `With`)<jupyter_code>model_type_df = df[(df.functional_area == 'modeling')].class_name.str.split('With', n=1, expand=True)
model_type_df.head()
model_type_df[1] = np.where(model_type_df[1].notnull(), 
                            'With' + model_type_df[1].astype(str), 
                            df[(df.functional_area == 'modeling')].model_task)

df['model_task'] = model_type_df[1]
df.head()
print(list(df.model_task.unique()))
print(list(df.functional_area.unique()))
print(list(df.module_part_2.unique()))
print(list(df.module_part_3.unique()))
# look at what we're going to remove (use to verify we're just getting rid of stuff we want too)
# df[~df['hf_class_type'].isin(['modeling', 'configuration', 'tokenization'])]
df = df[df['functional_area'].isin(['modeling', 'configuration', 'tokenization'])]<jupyter_output><empty_output><jupyter_text>Get included architectures<jupyter_code>def get_architectures(): 
    return df[(df.arch.notna()) & (df.arch != None)].arch.unique().tolist()
print(get_architectures())
TRANSFORMER_ARCHITECTURES = Enum('TRANSFORMER_ARCHITECTURES', get_architectures())
print(L(TRANSFORMER_ARCHITECTURES))<jupyter_output><empty_output><jupyter_text>Get an architecture's config<jupyter_code>def get_config(arch): 
    return df[(df.functional_area == 'configuration') & (df.arch == arch)].class_name.values[0]
print(get_config('bert'))<jupyter_output><empty_output><jupyter_text>Get an architecture's tokenizersThere may be multiple so this returns a list<jupyter_code>def get_tokenizers(arch): 
    return df[(df.functional_area == 'tokenization') & (df.arch == arch)].class_name.values
print(get_tokenizers('electra'))<jupyter_output><empty_output><jupyter_text>Get included custom model tasksGet the type of tasks for which there is a custom model for (*optional: by architecture*). There are a number of customized models built for specific tasks like token classification, question/answering, LM, etc....<jupyter_code>def get_tasks(arch=None): 
    query = ['model_task.notna()']
    if (arch): query.append(f'arch == "{arch}"')
        
    return df.query(' & '.join(query)).model_task.unique().tolist()
print(get_tasks())
print(get_tasks('bart'))
TRANSFORMER_TASKS_ALL = Enum('TRANSFORMER_TASKS_ALL', get_tasks())
TRANSFORMER_TASKS_AUTO = Enum('TRANSFORMER_TASKS_AUTO', get_tasks('auto'))
print('--- all tasks ---')
print(L(TRANSFORMER_TASKS_ALL))
print('\n--- auto only ---')
print(L(TRANSFORMER_TASKS_AUTO))<jupyter_output><empty_output><jupyter_text>Get included modelsThe transformer models available for use (*optional: by architecture | task*)<jupyter_code>def get_models(arch=None, task=None): 
    query = ['functional_area == "modeling"']
    if (arch): query.append(f'arch == "{arch}"')
    if (task): query.append(f'model_task == "{task}"')
        
    return df.query(' & '.join(query)).class_name.tolist()
print(L(get_models()))
print(get_models(arch='bert'))
print(get_models(task='ForTokenClassification'))
print(get_models(arch='bert', task='ForTokenClassification'))
TRANSFORMER_MODELS = Enum('TRANSFORMER_MODELS', get_models())
print(L(TRANSFORMER_MODELS))<jupyter_output><empty_output><jupyter_text>Get tokenizers, config, and model for a given model name / enum<jupyter_code>def get_classes_for_model(model_name_or_enum):
    model_name = model_name_or_enum if isinstance(model_name_or_enum, str) else model_name_or_enum.name
    
    meta = df[df.class_name == model_name]
    tokenizers = get_tokenizers(meta.arch.values[0])
    config = get_config(meta.arch.values[0])
    
    return ([str_to_class(tok) for tok in tokenizers], str_to_class(config), str_to_class(model_name))
tokenizers, config, model = get_classes_for_model('RobertaForSequenceClassification')

print(tokenizers[0])
print(config)
print(model)
tokenizers, config, model = get_classes_for_model(TRANSFORMER_MODELS.DistilBertModel)

print(tokenizers[0])
print(config)
print(model)
def get_model_architecture(model_name_or_enum):
    model_name = model_name_or_enum if isinstance(model_name_or_enum, str) else model_name_or_enum.name
    return df[df.class_name == model_name].arch.values[0]
get_model_architecture('RobertaForSequenceClassification')<jupyter_output><empty_output><jupyter_text>Loading Pre-Trained (configs, tokenizer, model)<jupyter_code>tokenizer = AutoTokenizer.from_pretrained("bert-base-cased-finetuned-mrpc")
model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased-finetuned-mrpc")
def get_auto_hf_objects(pretrained_model_name_or_path,
                        task=TRANSFORMER_TASKS_AUTO.ForSequenceClassification, 
                        config=None):
    
    tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path)
    config = AutoConfig.from_pretrained(pretrained_model_name_or_path) if (config is None) else config
    
    model = str_to_class(f'AutoModel{task.name}').from_pretrained(pretrained_model_name_or_path, 
                                                                  config=config)
    arch = get_model_architecture(type(model).__name__)
    
    return (arch, tokenizer, config, model)
arch, tokenizer, config, model = get_auto_hf_objects("bert-base-cased-finetuned-mrpc", 
                                                     task=TRANSFORMER_TASKS_AUTO.WithLMHead)

print(arch)
print(type(tokenizer))
print(type(config))
print(type(model))
arch, tokenizer, config, model = get_auto_hf_objects("fmikaelian/flaubert-base-uncased-squad", 
                                                              task=TRANSFORMER_TASKS_AUTO.ForQuestionAnswering)

print(arch)
print(type(tokenizer))
print(type(config))
print(type(model))
def get_transformer_objects(pretrained_model_name_or_path, 
                            tokenizer_cls=BertTokenizer,
                            model_cls=TRANSFORMER_MODELS.BertModel,
                            config_cls=BertConfig):
    
    tokenizer = tokenizer_cls.from_pretrained(pretrained_model_name_or_path)
    
    if (config_cls is None):
        model = str_to_class(model_cls.name).from_pretrained(pretrained_model_name_or_path)
        config = None
    else:
        config = config_cls.from_pretrained(pretrained_model_name_or_path)
        model = str_to_class(model_cls.name).from_pretrained(pretrained_model_name_or_path, config=config)
    
    arch = get_model_architecture(type(model).__name__)
        
    return (arch, tokenizer, config, model)
arch, tokenizer, config, model = get_transformer_objects("bert-base-cased-finetuned-mrpc",
                                                         tokenizer_cls=BertTokenizer, 
                                                         config_cls=None,
                                                         model_cls=TRANSFORMER_MODELS.BertForNextSentencePrediction)
print(arch)
print(type(tokenizer))
print(type(config))
print(type(model))<jupyter_output><empty_output><jupyter_text>Tokenizers Terms:**Input IDs**: \"The input ids are often the only required parameters to be passed to the model as input. They are *token indices, numerical representations of tokens* building the sequences that will be used as input by the model."`tokenizer.tokenize(sequence)` => Splits the sequnce into tokens based on vocab`tokenizer.encode(sequence)` => Converts tokens to their numerical IDs (add `add_special_tokens=False` to exclude special tokens)`tokenizer.encode_plus(sequence)` => Returnes a dictionary of "input_ids", "token_type_ids", and "attention_mask"**Attention Mask**: \"This argument indicates to the model which tokens should be attended to, and which should not ... a binary tensor indicating the position of the padded indices so that the model does not attend to them. For the BertTokenizer, 1 indicate a value that should be attended to while 0 indicate a padded value." (optional)`tokenizer.encode(sequence, max_length=20, pad_to_max_length=True)`**Token Type IDs**: \"Some models’ purpose is to do sequence classification or question answering. These require two different sequences to be encoded in the same input IDs. They are usually separated by special tokens, such as the classifier and separator tokens.... The Token Type IDs are a binary mask identifying the different sequences (segments) in the model."`tokenizer.encode(sequence_a, sequence_b)`"The first sequence, the “context” used for the question, has all its tokens represented by 0, whereas the question has all its tokens represented by 1. Some models, like `XLNetModel` use an additional token represented by a 2."**Position IDs**: \"The position IDs are used by the model to identify which token is at which position. Contrary to RNNs that have the position of each token embedded within them, transformers are unaware of the position of each token.... If no position IDs are passed to the model, they are automatically created as absolute positional embeddings." (optional)"Absolute positional embeddings are selected in the range `[0, config.max_position_embeddings - 1]`. Some models use other types of positional embeddings, such as sinusoidal position embeddings or relative position embeddings."<jupyter_code>tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
print(tokenizer.tokenize("Hi! You like the Bert Tokenizer?"))
print(tokenizer.encode("Hi! You like the Bert Tokenizer?"))
print(tokenizer.encode("Hi! You like the Bert Tokenizer?", add_special_tokens=False))
print(tokenizer.encode_plus("Hi! You like the Bert Tokenizer?"))
print(tokenizer.encode_plus("Hi!", "You like the Bert Tokenizer?"))
print(tokenizer.encode("Hi! You like the Bert Tokenizer?", add_special_tokens=False))
# ALBERT
tokenizer = AlbertTokenizer.from_pretrained("albert-base-v2")
tok_results =  tokenizer.encode_plus("Hi!", "You like the Bert Tokenizer?")
print(tok_results)
print(tokenizer.decode(tok_results['input_ids']))
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)
# BART
tokenizer = BartTokenizer.from_pretrained("bart-large-cnn")
tok_results =  tokenizer.encode_plus("Hi!", "You like the Bert Tokenizer?")
print(tok_results)
print(tokenizer.decode(tok_results['input_ids']))
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)
# BERT
tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
tok_results =  tokenizer.encode_plus("Hi!", "You like the Bert Tokenizer?")
print(tok_results)
print(tokenizer.decode(tok_results['input_ids']))
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)
print(tokenizer.prepare_for_model([101, 8790, 106, 102, 1192, 1176, 1103, 15035, 1706, 6378, 17260, 136, 102],None))
# CTRL
tokenizer = CTRLTokenizer.from_pretrained("ctrl")
tok_results =  tokenizer.encode_plus("Hi!", "You like the Bert Tokenizer?")
print(tok_results)
print(tokenizer.decode(tok_results['input_ids']))
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)
# CAMBERT
tokenizer = CamembertTokenizer.from_pretrained("camembert-base")
tok_results =  tokenizer.encode_plus("Hi!", "You like the Bert Tokenizer?")
print(tok_results)
print(tokenizer.decode(tok_results['input_ids']))
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)
# ELECTRA
tokenizer = ElectraTokenizer.from_pretrained('google/electra-small-discriminator')
tok_results =  tokenizer.encode_plus("Hi!", "You like the Bert Tokenizer?")
print(tok_results)
print(tokenizer.decode(tok_results['input_ids']))
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)
# GPT-2
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
tok_results =  tokenizer.encode_plus("Hi!", "You like the Bert Tokenizer?")
print(tok_results)
print(tokenizer.decode(tok_results['input_ids']))
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)
# GPT
tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt')
tok_results =  tokenizer.encode_plus("Hi!", "You like the Bert Tokenizer?")
print(tok_results)
print(tokenizer.decode(tok_results['input_ids']))
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)
# RobertaTokenizer
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
tok_results =  tokenizer.encode_plus("Hi!", "You like the Bert Tokenizer?")
print(tok_results)
print(tokenizer.decode(tok_results['input_ids']))
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)
# T5
tokenizer = T5Tokenizer.from_pretrained('t5-small')
tok_results =  tokenizer.encode_plus("Hi!", "You like the Bert Tokenizer?")
print(tok_results)
print(tokenizer.decode(tok_results['input_ids']))
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)
# TransfoXLTokenizer
tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103')
tok_results =  tokenizer.encode_plus("Hi!", "You like the Bert Tokenizer?", add_space_before_punct_symbol=True)
print(tok_results)
print(tokenizer.decode(tok_results['input_ids']))
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)
# XLMRobertaTokenizer

# XLM
tokenizer = XLMTokenizer.from_pretrained("xlm-mlm-en-2048")
tok_results =  tokenizer.encode_plus("Hi!", None)
print(tok_results)
print(tokenizer.decode(tok_results['input_ids']))
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)
# XLNet
tokenizer = XLNetTokenizer.from_pretrained("xlnet-base-cased")
tok_results =  tokenizer.encode_plus("Hi! what do you thing of this thing we are doing")

print(tok_results)
print(tokenizer.pad_token_id, tokenizer.pad_token_type_id)

tokenizer.batch_encode_plus(['Hi! what do you thing of this thing we are doing'], 
                            max_length=10, stride=5,
                            pad_to_max_length=True,
                               return_overflowing_tokens=True,
                               return_special_tokens_masks=True,
                            return_input_lengths=True)
encoded_ids = tokenizer.encode("Hi!", "You like the Bert Tokenizer?")
print(encoded_ids)
toks = tokenizer.convert_ids_to_tokens(encoded_ids)
print(toks)
sep_idxs = [idx for idx, tok in enumerate(toks) if tok == tokenizer.sep_token]
print(len(sep_idxs), sep_idxs)
toks_modified = toks if len(sep_idxs) == 1 else [toks[:sep_idxs[0]+1], toks[sep_idxs[0]+1:]]
print(toks_modified)
tokenizer.get_special_tokens_mask(*toks_modified)

tokenizer.encode('.', add_special_tokens=False)
tokenizer.get_vocab()['.']
tok_a =tokenizer.tokenize("Hi!")
tok_b =tokenizer.tokenize("You like the Bert Tokenizer?")
tok_a, tok_b
a = tokenizer.convert_tokens_to_ids(tokenizer.tokenize("Hi!"))
b = tokenizer.convert_tokens_to_ids(tokenizer.tokenize("You like the Bert Tokenizer?"))

print(tokenizer.build_inputs_with_special_tokens(a,b))
print(tokenizer.create_token_type_ids_from_sequences(a,b))

# ddd = tokenizer.build_inputs_with_special_tokens(a,b)
# [0 if idx == tokenizer.pad_token_id else 1 for idx in ddd]

tokenizer.pad_token_id, tokenizer.pad_token_type_id
d = tokenizer.prepare_for_model(a,b, max_length=25, pad_to_max_length=True, return_tensors='pt')
e = tokenizer.prepare_for_model(a,b, max_length=25, pad_to_max_length=True, return_tensors='pt')
f = tokenizer.prepare_for_model(a,b, max_length=25, pad_to_max_length=True, return_tensors='pt')

x = [d['input_ids'], e['input_ids'], f['input_ids']]
d['input_ids'].shape, torch.cat(x).shape<jupyter_output><empty_output><jupyter_text>Models "See the models docstrings for the detail of the inputs" ... `outputs = model(tokens_tensor, token_type_ids=segments_tensors)`"Transformers models always output tuples. See the models docstrings for the detail of all the outputs. In our case, the first element is the hidden state of the last layer of the Bert model" ... `encoded_layers = outputs[0]``GPT-2`, `GPT`, `XLNet`, `Transfo-XL`, `CTRL` (and some others) "make use of a `past` or `mems` attribute which can be used to prevent re-computing the key/value pairs when using sequential decoding. It is useful when generating sequences as a big part of the attention mechanism benefits from previous computations.""If you want to fine-tune a model on a specific task, you can leverage one of the `run_$TASK.py` script in the examples directory.**AutoModel**:"These examples leverage auto-models, which are classes that will instantiate a model according to a given checkpoint, automatically selecting the correct model architecture. Please check the `AutoModel` documentation for more information"- AutoConfig- AutoTokenizer- AutoModel- AutoModelForPreTraining- AutoModelWithLMHead- AutoModelForQuestionAnswering- AutoModelForSequenceClassification- AutoModelForTokenClassification**Inference**:Option 1: Use `Pipelines`Option 2: Use the model directly with the tokenizer  Question-Answer<jupyter_code>tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForQuestionAnswering.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')
question, text = "Who was <NAME>?", "<NAME> was a nice puppet"
input_ids = tokenizer.encode(question, text)
token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))]
start_scores, end_scores = model(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids]))
input_ids, start_scores.shape
all_tokens = tokenizer.convert_ids_to_tokens(input_ids)
answer = ' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])
answer
tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048')
model = XLMForQuestionAnsweringSimple.from_pretrained('xlm-mlm-en-2048')
question, text = "Who was <NAME>?", "<NAME> was a nice puppet"
input_ids = tokenizer.encode(question, text, add_special_tokens=True)
outputs = model(torch.tensor([input_ids]))
len(outputs)
all_tokens = tokenizer.convert_ids_to_tokens(input_ids)
answer = ' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])
answer, all_tokens
input_ids = torch.tensor(tokenizer.encode("Who was <NAME>?", "<NAME> was a nice puppet", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
start_positions = torch.tensor([1])
end_positions = torch.tensor([3])
outputs = model(input_ids)
# loss = outputs[0]
torch.argmax(outputs[1]), len(input_ids[0]), input_ids<jupyter_output><empty_output><jupyter_text>Navigating nn hierarchy<jupyter_code># layer_groups = hft_splitter(temp_arch, tmp_model)
# print(len(layer_groups))

# for g in layer_groups:
#     print(len(g))

# layer_groups[3][3].shape

# tmp_model

# for g in layer_groups:
#     print(len(g))

# x = list(hft_model.named_children())[0]

# len(list(x[1].named_children()))
# for m in x[1].named_children():
#     print(m[0])

# for m in tmp_model.named_children():
#     print(m[0])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11655309483248154, 'fraction_numerical': 0.007098239636570131, 'mean_word_length': 5.691893603546548, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '34950', 'license': 'Apache-2.0', 'n_tokens_mistral': 6841, 'n_tokens_neox': 6655, 'n_words': 1499, 'path': 'nbs/_transformers-playground.ipynb', 'repo_name': 'techthiyanes/blurr', 'size': '21156'}