<jupyter_start><jupyter_text>This notebook tests the generation of the CLEVR with masks dataset.  Image generation (example)<jupyter_code>%cd image_generation
!./blender/blender --background \
    --python render_images.py -- \
    --num_images 10 --use_gpu 1 --min_objects 2 --max_objects 6
%cd ..<jupyter_output><empty_output><jupyter_text>Single CLEVR_scenes.json generation<jupyter_code>%cd image_generation
!python collect_scenes.py --date "$(date)" \
    --input_dir /dfs/user/tailin/.results/CLEVR_relation/clevr-relation-easy-mpi-0-100000/scenes \
--output /dfs/user/tailin/.results/CLEVR_relation/clevr-relation-easy-mpi-0-100000/partial/CLEVR_scenes.json
%cd ..<jupyter_output><empty_output><jupyter_text>Question template generation<jupyter_code>!pip install pyjson5
%cd relation_generator
!python generate_relations.py
%cd ..<jupyter_output><empty_output><jupyter_text>Question generation<jupyter_code>%cd question_generation/
!python generate_questions.py \
    --input_scene_file /dfs/user/tailin/.results/CLEVR_relation/mpi-0-10000/CLEVR_scenes.json \
    --output_questions_file ./questions.json \
    --template_dir babyarc_easy --max-num-objects 6
%cd ..<jupyter_output><empty_output><jupyter_text>Question analysis<jupyter_code>from collections import defaultdict
import json
from typing import List

from relation_generator.generate_relations import RELATIONS

def get_unique_task_string(program: List[str]):
    """
    Parses the program for a given question and returns a unique string that identifies the 
    babyARC task that it embodies.

    This function is somewhat hacky in that it doesn't deal with the AST directly, but it
    works for the generated babyARC template programs.
    """
    inputs = []
    object_str = []
    for node in program:
        # Generate a new object str every time we see a new "scene" (which implies
        # a new object)
        if node["type"] == "scene":
            if len(object_str) != 0:
                inputs.append(",".join(object_str))
                object_str = []
            continue

        # If we're not at a scene, then we're in the middle of an object
        if node["type"].startswith("filter_"):
            # This node filters some property of the input. Let's consider it.
            object_str.append(node["type"][7:] + "=" + node["value_inputs"][0])
    inputs.append(",".join(object_str))
    relations = sorted([node["type"] for node in program if node["type"] in RELATIONS])
    
    return "+".join(relations) + "-" + ";".join(inputs)


# Load the question data
file = "question_generation/questions.json"
with open(file) as f:
    data = json.load(f)
question_list = data["questions"]

observed_question_types = dict()

# Count the number of times each question type occurs
for question in question_list:
    template_filename = question["template_filename"]
    question_family_index = question["question_family_index"]
    program = question["program"]
    image = question["image"]

    task_str = get_unique_task_string(program)

    if task_str not in observed_question_types:
        observed_question_types[task_str] = {"count": 0, "questions": [], "images": []}
    
    observed_question_types[task_str]["count"] += 1
    observed_question_types[task_str]["questions"].append(question)
    observed_question_types[task_str]["images"].append(image)
for task_str, data in observed_question_types.items():
    print("{} - {}".format(task_str, data["count"]))
import pandas as pd

df = pd.DataFrame.from_dict(observed_question_types, orient='index')
df.sort_values(by=["count"], ascending=False).images[0]
observed_question_types["same_size-size=large,color=purple,material=metal"]<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code>%load_ext autoreload
%autoreload 2
from dataset import ClevrRelationDataset
dataset = ClevrRelationDataset(image_dir="/dfs/user/tailin/.results/CLEVR_relation/mpi-0-10000/images",
                               question_dir="/dfs/user/tailin/.results/CLEVR_relation/mpi-0-10000",
                               output_type="full-color")
dataset.save("/dfs/user/tailin/.results/CLEVR_relation/relations-dataset-2021-08-18-608-tasks.pt")
dataset = ClevrRelationDataset(image_dir="/dfs/user/tailin/.results/CLEVR_relation/mpi-0-10000/images",
                              question_dir="./question_generation/")
# Visualize the dataset

import matplotlib.pyplot as plt

el = dataset[-3]

# Find a compound task
# for el in dataset:
#     if "+" in el["task_str"]:
#         break
# else:
#     assert False
    
print(el["task_str"])
# print(el["questions"][0]["question"])a

plt.figure(figsize=(15,50)) # specifying the overall grid size

for i in range(min(len(el["inputs"]), 5)):
    plt.subplot(len(el["inputs"]),2, 2 * i + 1)
    plt.imshow(el["inputs"][i]["image"].permute(1, 2, 0))
    plt.subplot(len(el["inputs"]),2, 2 * i + 2)
    plt.imshow(el["outputs"][i].permute(1, 2, 0))

plt.show()
from PIL import Image
plt.imshow(Image.open("/dfs/user/tailin/.results/CLEVR_relation/test1/0/images/CLEVR_new_000000.png"))<jupyter_output><empty_output><jupyter_text>CLEVR relation "easy" dataset  Partial dataset  Question generation<jupyter_code>%cd question_generation/
!python generate_questions.py \
    --input_scene_file /dfs/user/tailin/.results/CLEVR_relation/clevr-relation-easy-mpi-0-100000/partial/CLEVR_scenes.json \
    --output_questions_file /dfs/user/tailin/.results/CLEVR_relation/clevr-relation-easy-mpi-100000/partial/questions.json \
    --template_dir babyarc_easy --max-num-objects 6
%cd ..
dataset = ClevrRelationDataset(image_dir="/dfs/user/tailin/.results/CLEVR_relation/clevr-relation-easy-mpi-0-100000/images",
                               question_dir="/dfs/user/tailin/.results/CLEVR_relation/clevr-relation-easy-mpi-0-100000/partial",
                               output_type="full-color")<jupyter_output><empty_output><jupyter_text>Full dataset  Single CLEVR_scenes.json generation<jupyter_code>%cd image_generation
!python collect_scenes.py --date "$(date)" \
    --input_dir /dfs/user/tailin/.results/CLEVR_relation/clevr-relation-easy-mpi-0-100000/scenes \
--output /dfs/user/tailin/.results/CLEVR_relation/clevr-relation-easy-mpi-0-100000/full/CLEVR_scenes.json
%cd ..
!mkdir /dfs/user/tailin/.results/CLEVR_relation/clevr-relation-easy-mpi-0-100000/full<jupyter_output><empty_output><jupyter_text>Generate questions<jupyter_code>%cd question_generation/
!python generate_questions.py \
    --input_scene_file /dfs/user/tailin/.results/CLEVR_relation/mpi-0-10000/CLEVR_scenes.json \
    --output_questions_file /dfs/user/tailin/.results/CLEVR_relation/mpi-0-10000/easy-questions/questions.json \
    --template_dir babyarc_easy --max-num-objects 6
%cd ..<jupyter_output><empty_output><jupyter_text>Test dataset<jupyter_code>dataset = ClevrRelationDataset(image_dir="/dfs/user/tailin/.results/CLEVR_relation/mpi-0-10000/images",
                               question_dir="/dfs/user/tailin/.results/CLEVR_relation/mpi-0-10000/easy-questions",
                               output_type="mask-only", is_easy_dataset=True)
dataset.save("/dfs/user/tailin/.results/CLEVR_relation/relations-dataset-easy-2021-09-16-461-tasks.pt")
len(dataset)<jupyter_output><empty_output><jupyter_text>Helper functions<jupyter_code>from dataset import create_full_dataset, create_easy_dataset

train_set, val_set, test_set = create_full_dataset()
assert len(train_set) + len(val_set) + len(test_set) == 608
train_set, val_set, test_set = create_easy_dataset()
assert len(train_set) + len(val_set) + len(test_set) == 461<jupyter_output>Loaded 461 tasks.
{'fraction_non_alphanumeric': 0.13059456621603885, 'fraction_numerical': 0.024150150938443367, 'mean_word_length': 4.180149558123725, 'pattern_counts': {'":': 4, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '140459', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2699, 'n_tokens_neox': 2477, 'n_words': 429, 'path': 'generation.ipynb', 'repo_name': 'tailintalent/clevr-dataset-gen', 'size': '7619'}