<jupyter_start><jupyter_text>Creation of the environment<jupyter_code>%tensorflow_version 2.x
!pip3 install --upgrade pip
!pip install -qU t5


import functools
import os
import time
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)

import tensorflow.compat.v1 as tf
import tensorflow_datasets as tfds

import t5

#Set the base dir(Google cloud bucket)
BASE_DIR = "gs://bucket_block_completion" 

if not BASE_DIR or BASE_DIR == "gs://":
  raise ValueError("You must enter a BASE_DIR.")
ON_CLOUD = True

!pip install tensorflow-gcs-config

if ON_CLOUD:
  import tensorflow_gcs_config
  from google.colab import auth
  # Set credentials for GCS reading/writing from Colab and TPU.
  TPU_TOPOLOGY = "2x2"
  try:
    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection
    TPU_ADDRESS = tpu.get_master()
    print('Running on TPU:', TPU_ADDRESS)
  except ValueError:
    raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')
  auth.authenticate_user()
  tf.config.experimental_connect_to_host(TPU_ADDRESS)
  tensorflow_gcs_config.configure_gcs_from_colab_auth()

tf.disable_v2_behavior()

# Improve logging.
from contextlib import contextmanager
import logging as py_logging

if ON_CLOUD:
  tf.get_logger().propagate = False
  py_logging.root.setLevel('INFO')

@contextmanager
def tf_verbosity_level(level):
  og_level = tf.logging.get_verbosity()
  tf.logging.set_verbosity(level)
  yield
  tf.logging.set_verbosity(og_level)<jupyter_output>Requirement already satisfied: pip in /usr/local/lib/python3.7/dist-packages (21.1.3)
Collecting pip
  Downloading pip-21.3.1-py3-none-any.whl (1.7 MB)
[K     |████████████████████████████████| 1.7 MB 5.1 MB/s 
[?25hInstalling collected packages: pip
  Attempting uninstall: pip
    Found existing installation: pip 21.1.3
    Uninstalling pip-21.1.3:
      Successfully uninstalled pip-21.1.3
Successfully installed pip-21.3.1
     |████████████████████████████████| 153 kB 5.2 MB/s            
     |████████████████████████████████| 1.2 MB 60.7 MB/s            
     |████████████████████████████████| 3.1 MB 86.8 MB/s            
     |████████████████████████████████| 4.0 MB 66.0 MB/s            
     |████████████████████████████████| 4.9 MB 69.8 MB/s            
     |████████████████████████████████| 366 kB 92.1 MB/s            
     |████████████████████████████████| 90 kB 11.6 MB/s            
     |████████████████████████████████| 286 kB 104.8 MB/s            
     |█████████████[...]<jupyter_text>Loading of tsv filesWith this script you can load each tsv file for finetuning.Please be sure that the path to all tsv files are correct<jupyter_code>nq_tsv_path = {
    "train":      'gs://bucket_block_completion/dataset_08_07/train.tsv',
    # we don't use them
    "validation": 'gs://bucket_block_completion/dataset_08_07/test.tsv',
}

num_nq_examples = dict(train=1197310, validation=15783)<jupyter_output><empty_output><jupyter_text>Preprocess of the datasetIn this step we preprocess the dataset.  You have to change the path to vocab files (*vocab_model_path* and *vocab_path*)We're going to preprocess all the tsv file so that T5 can use them for evaluation.<jupyter_code>from t5.data import postprocessors as t5_postprocessors
from t5.seqio import Feature,SentencePieceVocabulary


# # Set the path of sentencepiece model and vocab files
# # Must be the same used for the pre-trained phase
vocab_model_path = 'gs://bucket_block_completion/code.model'
vocab_path = 'gs://bucket_block_completion/code.vocab'


TaskRegistry = t5.data.TaskRegistry
TfdsTask = t5.data.TfdsTask


def get_default_vocabulary():
  return SentencePieceVocabulary(vocab_model_path, 100)

DEFAULT_OUTPUT_FEATURES = {
    "inputs": Feature(
        vocabulary=get_default_vocabulary(), add_eos=True, required=False),

    "targets": Feature(
        vocabulary=get_default_vocabulary(), add_eos=True)
}
prefix=''

def nq_dataset_task(split, shuffle_files=True):
  # We only have one file for each split.
  del shuffle_files

  # Load lines from the text file as examples.

  ds = tf.data.TextLineDataset(nq_tsv_path[split])
  ds = ds.map(
      functools.partial(tf.io.decode_csv, record_defaults=["string","string"],
                        field_delim="\t", use_quote_delim=False),
      num_parallel_calls=tf.data.experimental.AUTOTUNE)
  
  ds = ds.map(lambda *ex: dict(zip(["input", "output"], ex)))
  return ds

print("A few raw train examples...")
for ex in tfds.as_numpy(nq_dataset_task("train").take(5)):
    print(ex)


def preprocessing(ds):
  
  def to_inputs_and_targets(ex):
        inputs = tf.strings.join([prefix + ex['input']], separator=' ')
        class_label = tf.strings.join([ex['output']], separator=' ')
        return {'inputs': inputs, 'targets': class_label}
    
  return ds.map(to_inputs_and_targets, 
                num_parallel_calls=tf.data.experimental.AUTOTUNE)
    
      
t5.data.TaskRegistry.remove('finetuning')
t5.data.TaskRegistry.add(
    "finetuning",
    dataset_fn=nq_dataset_task,
    splits=["validation"],
    text_preprocessor=preprocessing,
    output_features = DEFAULT_OUTPUT_FEATURES,
    metric_fns=[t5.evaluation.metrics.accuracy]
)
nq_task = t5.data.TaskRegistry.get("finetuning")
ds = nq_task.get_dataset(split="validation", sequence_length={"inputs": 512, "targets": 512})
print("A few preprocessed training examples...")
for ex in tfds.as_numpy(ds.take(5)):
  print(ex)<jupyter_output>A few preprocessed training examples...
{'inputs_pretokenized': b'public void handleWindowUpdate(int streamId, int deltaWindowSize) throws IOException { if (streamId == 0) { if (deltaWindowSize == 0) <extra_id_0> synchronized (flowControlLock) { boolean exhausted = sendWindowSize <= FLOW_CONTROL_MIN_WINDOW; sendWindowSize += deltaWindowSize; if (exhausted) { notifyFlowControlAllowed(); } if (sendWindowSize > Integer.MAX_VALUE) { sendGoAway(ERROR_FLOW_CONTROL_ERROR); } } } else { if (deltaWindowSize == 0) { UndertowLogger.REQUEST_IO_LOGGER.debug("Invalid flow-control window increment of 0 received with WINDOW_UPDATE frame for stream " + streamId); sendRstStream(streamId, ERROR_PROTOCOL_ERROR); return; } StreamHolder holder = currentStreams.get(streamId); Http2StreamSinkChannel stream = holder != null ? holder.sinkChannel : null; if (stream == null) { if(isIdle(streamId)) { sendGoAway(ERROR_PROTOCOL_ERROR); } } else { stream.updateFlowControlWindow(deltaWindowSize); } } }', 'inputs': arr[...]<jupyter_text>CHOOSE THE DATASET YOU WANT TO PROCESS<jupyter_code>dataset="0807"

checkpoints=dict()
checkpoints["0302"]=220400
checkpoints["0403"]=240800
checkpoints["0605"]=424400
checkpoints["0807"]=648800

checkpoint=checkpoints[dataset]

from t5 import models

MODEL_SIZE = "small" 

# Set the folder where the checkpoints and all the others information will be writed
MODEL_DIR = 'gs://bucket_block_completion/finetuning/{}/model/'.format(dataset)

# Specify the pre-trained dir which must contain the pre-trained models, the operative_config.gin file and the checkpoint file as well
PRETRAINED_DIR='gs://bucket_block_completion/pretrained_model'


model_parallelism, train_batch_size, keep_checkpoint_max = {
    "small": (1, 256, 5000),
    "base": (2, 128, 8),
    "large": (8, 64, 4),
    "3B": (8, 16, 1),
    "11B": (8, 16, 1)}[MODEL_SIZE]

tf.io.gfile.makedirs(MODEL_DIR)

model = models.mtf_model.MtfModel(
    model_dir=MODEL_DIR,
    tpu=TPU_ADDRESS,
    tpu_topology=TPU_TOPOLOGY,
    model_parallelism=model_parallelism,
    batch_size=train_batch_size,
    sequence_length={"inputs": 512, "targets": 512},
    save_checkpoints_steps=5000,
    keep_checkpoint_max=keep_checkpoint_max if ON_CLOUD else None,
    iterations_per_loop=100,
)


input_folder=MODEL_DIR
output_folder="gs://bucket_block_completion/score/{}".format(dataset)

import os
input_file='gs://bucket_block_completion/input_testset/finetuning_inputs'
# you can keep the same output file; by default when predicting it added the number of steps so the file is not overwritten
output_file=os.path.join(output_folder, "scores.txt")
print(output_file)

prediction_file='gs://bucket_block_completion/input_testset/{}/predictions.txt-{}'.format(dataset, checkpoint)
print(prediction_file)
from t5.seqio import SentencePieceVocabulary

# # Set the path of sentencepiece model and vocab files
# # Must be the same used for the pre-trained phase
vocab_model_path = 'gs://bucket_block_completion/code.model'
vocab_path = 'gs://bucket_block_completion/code.vocab'

def get_default_vocabulary():
  return SentencePieceVocabulary(vocab_model_path, 100)

vocabulary_predict=get_default_vocabulary()

model.batch_size = 512

# we use the prediction file as target file (since we want to see how much the prediction is likely)
model.score(inputs=input_file, 
            targets=prediction_file,
            scores_file=output_file,
            checkpoint_steps=checkpoint, 
            vocabulary=vocabulary_predict)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13588039867109636, 'fraction_numerical': 0.025027685492801773, 'mean_word_length': 4.748567791215786, 'pattern_counts': {'":': 14, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '196265', 'license': 'MIT', 'n_tokens_mistral': 3401, 'n_tokens_neox': 3065, 'n_words': 690, 'path': 'Finetuning/score/evaluate_score.ipynb', 'repo_name': 'DLCloning/DLClone', 'size': '9030'}