<jupyter_start><jupyter_text>Baseline Model Training* Custom Tokenizer* GRU Encoder / Decoder  Imports<jupyter_code>import os
import io
import sys


import pandas as pd 
import numpy as np 

sys.path.append('D:\PROJECT\Level-4-Project')
os.environ["TF_FORCE_GPU_ALLOW_GROWTH"]="true"
from code2text.models.baseline.model import seq2seqTrain, MaskedLoss
from code2text.helper.model import BatchLogs
from code2text.helper.preprocess import tf_lower_and_split_punct
import ijson
import tensorflow as tf
import tensorflow_text as text
#tf.debugging.set_log_device_placement(True)
tf.config.experimental.set_memory_growth(tf.config.experimental.list_physical_devices('GPU')[0], True)
#tf.config.experimental.set_virtual_device_configuration(tf.config.experimental.list_physical_devices('GPU')[0],[tf.config.experimental.VirtualDeviceConfiguration(memory_limit=3072)])
tf.config.run_functions_eagerly(False)<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>path = "D:\PROJECT\data\CodeSearchNet"
langs = ["python"]
format = ["train.jsonl", "test.jsonl", "valid.jsonl"]
def read_data(path):
    with open(path, encoding="UTF-8") as json_file:
        cursor = 0
        code_set = []
        string_set = []
        lang_set = []
        for _, line in enumerate(json_file):
            code = None
            string = None
            line_as_file = io.StringIO(line)
            json_parser = ijson.parse(line_as_file)

            for prefix, _, value in json_parser:
                if prefix == "code":  
                    code = value
                if prefix == "docstring":
                    string = value
                if prefix == "language":
                    lang = value
            if (code is not None and string is not None):
                code_set.append(code)
                string_set.append(string)
                lang_set.append(lang)

            cursor += len(line)
    return pd.DataFrame(data={'code': code_set, 'docstring': string_set, 'language': lang_set})
train = []
test = []
valid = []

for lang in langs:
    tmp = os.path.join(path, lang)
    for file in format:
        print("Processing ", lang, " ", file, "... ")
        data = read_data(os.path.join(tmp, file))
        if file == "train.jsonl":
            train.append(data)
        if file == "test.jsonl":
            test.append(data)
        if file == "valid.jsonl":
            valid.append(data)
train = pd.concat(train)
test = pd.concat(test)
valid = pd.concat(valid)
data = pd.concat([train, test, valid])
train.reset_index(inplace=True)
train.to_json("D:\PROJECT\data\CodeSearchNet\py\\train.jsonl", orient="records", lines=True)
test.reset_index(inplace=True)
test.to_json("D:\PROJECT\data\CodeSearchNet\py\\test.jsonl", orient="records", lines=True)
valid.reset_index(inplace=True)
valid.to_json("D:\PROJECT\data\CodeSearchNet\Azure\\valid.jsonl", orient="records", lines=True)
data.reset_index(inplace=True)
data.to_json("D:\PROJECT\data\CodeSearchNet\Combine_clean\data.json")<jupyter_output><empty_output><jupyter_text>Dataset Initialization<jupyter_code>data = pd.read_json("D:\PROJECT\data\CodeSearchNet\Combine_clean\data.json")
train = pd.read_json("D:\PROJECT\data\CodeSearchNet\Combine_clean\\train.json")
valid = pd.read_json("D:\PROJECT\data\CodeSearchNet\Combine_clean\\valid.json")
test = pd.read_json("D:\PROJECT\data\CodeSearchNet\Combine_clean\\test.json")
batch_size = 256
buffer = 2048
dataset = (
    tf.data.Dataset.from_tensor_slices(
        (
            tf.cast(data["code"].values, tf.string),
            tf.cast(data["docstring"].values, tf.string)
        )
    )
).shuffle(buffer).batch(batch_size)
train_set = (
    tf.data.Dataset.from_tensor_slices(
        (
            tf.cast(train["code"].values, tf.string),
            tf.cast(train["docstring"].values, tf.string)
        )
    )
).shuffle(buffer).batch(batch_size, drop_remainder=True).cache().prefetch(tf.data.AUTOTUNE)
test_set = (
    tf.data.Dataset.from_tensor_slices(
        (
        tf.cast(test["code"].values, tf.string),
        tf.cast(test["docstring"].values, tf.string)
        )
    )
).shuffle(buffer).batch(batch_size, drop_remainder=True).cache().prefetch(tf.data.AUTOTUNE)
valid_set = (
    tf.data.Dataset.from_tensor_slices(
        (
        tf.cast(valid["code"].values, tf.string),
        tf.cast(valid["docstring"].values, tf.string)
        )  
    )
).shuffle(buffer).batch(batch_size, drop_remainder=True).cache().prefetch(tf.data.AUTOTUNE)<jupyter_output><empty_output><jupyter_text>Config & Build  Adapt Process<jupyter_code>tokens = 30000
input_processor = input_text_processor = tf.keras.layers.TextVectorization(
    standardize=tf_lower_and_split_punct,
    max_tokens=tokens)

output_processor = tf.keras.layers.TextVectorization(
    standardize=tf_lower_and_split_punct,
    max_tokens=tokens)
input_processor.adapt(train["code"])
output_processor.adapt(train["docstring"])
with open("D:\PROJECT\Level-4-Project\data\invocab.txt", "w") as infile:
    infile.write("\n".join(map(str, input_processor.get_vocabulary())))

with open("D:\PROJECT\Level-4-Project\data\outvocab.txt", "w") as outfile:
    outfile.write("\n".join(map(str, output_processor.get_vocabulary())))<jupyter_output><empty_output><jupyter_text>Build<jupyter_code>input_processor = input_text_processor = tf.keras.layers.TextVectorization(
    standardize=tf_lower_and_split_punct,
    vocabulary="D:\PROJECT\Level-4-Project\data\outvocab.txt")

output_processor = tf.keras.layers.TextVectorization(
    standardize=tf_lower_and_split_punct,
    vocabulary="D:\PROJECT\Level-4-Project\data\outvocab.txt")
train_model = seq2seqTrain(112, 64, input_text_processor=input_processor,
    output_text_processor=output_processor)
train_model.compile(
    optimizer=tf.optimizers.Adam(learning_rate=0.0004),
    loss=MaskedLoss(),
    metrics=['acc', text.metrics.rouge_l]
)
batch_loss = BatchLogs('batch_loss')
chkpt = tf.keras.callbacks.ModelCheckpoint(
    "D:\PROJECT\Level-4-Project\notebooks\training\chkpt\baseline", monitor='loss', save_best_only=True, save_freq=1000
)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>tf.test.is_gpu_available(
    cuda_only=False,
    min_cuda_compute_capability=None
)
tf.keras.backend.clear_session()
history = train_model.fit(train_set, epochs=3, validation_data=valid_set, callbacks=[batch_loss, chkpt])<jupyter_output>Epoch 1/3
  999/14190 [=>............................] - ETA: 16:18:16 - batch_loss: 5.7883WARNING:tensorflow:Can save best model only with loss available, skipping.
 1999/14190 [===>..........................] - ETA: 14:53:02 - batch_loss: 5.4793WARNING:tensorflow:Can save best model only with loss available, skipping.
 2050/14190 [===>..........................] - ETA: 14:46:15 - batch_loss: 5.4566<jupyter_text>Validation & Testing<jupyter_code>history<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14627775359164127, 'fraction_numerical': 0.015527499637207953, 'mean_word_length': 3.898365316275764, 'pattern_counts': {'":': 6, '<': 28, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '337467', 'license': 'CC0-1.0', 'n_tokens_mistral': 2422, 'n_tokens_neox': 2326, 'n_words': 322, 'path': 'notebooks/training/baseline.ipynb', 'repo_name': 'Parry-Parry/Level-4-Project', 'size': '6891'}