<jupyter_start><jupyter_text>Vocab generator<jupyter_code>import os
from typing import List
from typing import Tuple
import logging
from collections import defaultdict
from collections import Counter
import json
import torch
import numpy as np

import sys
sys.path.append(os.path.join(os.path.dirname("__file__"), '..', '..', 'dataset'))
def isnotebook():
    try:
        shell = get_ipython().__class__.__name__
        if shell == 'ZMQInteractiveShell':
            return True   # Jupyter notebook or qtconsole
        elif shell == 'TerminalInteractiveShell':
            return False  # Terminal running IPython
        else:
            return False  # Other type (?)
    except NameError:
        return False      # Probably standard Python interpreter
if isnotebook():
    device = torch.device("cpu")
else:
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

FORMAT = "%(asctime)-15s %(message)s"
logging.basicConfig(format=FORMAT, level=logging.INFO,
                    datefmt="%Y-%m-%d %H:%M")
logger = logging.getLogger(__name__)

from ReaSCAN_dataset import *
condition = "p1"

path_to_data = f"../../../data-files-updated/ReaSCAN-compositional-{condition}/data-compositional-splits.txt"
logger.info(f"Reading dataset from file: {path_to_data}...")
data_json = json.load(open(path_to_data, "r"))
train_dataset = ReaSCANDataset(
    data_json=data_json, 
    save_directory=f"../../../data-files-updated/ReaSCAN-compositional-{condition}/", 
    k=0, 
    split="train", 
    generate_vocabulary=True
)
train_dataset.save_vocabularies(
    input_vocabulary_file="input_vocabulary.txt", 
    target_vocabulary_file="target_vocabulary.txt"
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12412177985948478, 'fraction_numerical': 0.00234192037470726, 'mean_word_length': 3.8140845070422533, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '27083', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 558, 'n_tokens_neox': 542, 'n_words': 114, 'path': 'code/models/seq2seq/ReaSCAN-vocab-generator.ipynb', 'repo_name': 'anirbanl/Reason-SCAN', 'size': '1708'}