<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import pickle\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import argparse\n",
    "#parser = argparse.ArgumentParser(description='PyTorch Relations-from-Stream sort-of-CLVR Example')\n",
    "#args = parser.parse_args()\n",
    "\n",
    "from attrdict import AttrDict\n",
    "args = AttrDict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "args.batch_size = 32\n",
    "args.cuda = torch.cuda.is_available()\n",
    "args.lr   = 0.0001\n",
    "args.seed = 5\n",
    "args.process_coords=False\n",
    "args.debug = True\n",
    "args.rnn_hidden_size = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(args.seed)\n",
    "if args.cuda:\n",
    "    torch.cuda.manual_seed(args.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load in a batch of images to test dimensions, etc...\n",
    "bs = args.batch_size\n",
    "\n",
    "input_img = torch.FloatTensor(bs, 3, 75, 75)\n",
    "input_qst = torch.FloatTensor(bs, 11)\n",
    "label = torch.LongTensor(bs)\n",
    "\n",
    "if args.cuda:\n",
    "    input_img = input_img.cuda()\n",
    "    input_qst = input_qst.cuda()\n",
    "    label = label.cuda()\n",
    "    \n",
    "input_img = Variable(input_img)\n",
    "input_qst = Variable(input_qst)\n",
    "label = Variable(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dirs = './data'\n",
    "\n",
    "filename = os.path.join(data_dirs, 'sort-of-clevr.pickle')\n",
    "with open(filename, 'rb') as f:\n",
    "  train_datasets, test_datasets = pickle.load(f)\n",
    "\n",
    "def cvt_data_axis(data):\n",
    "    img = [e[0] for e in data]\n",
    "    qst = [e[1] for e in data]\n",
    "    ans = [e[2] for e in data]\n",
    "    return (img,qst,ans)\n",
    "\n",
    "rel_train, norel_train = [], []\n",
    "for img, relations, norelations in train_datasets:\n",
    "    img = np.swapaxes(img,0,2)\n",
    "    for qst,ans in zip(relations[0], relations[1]):\n",
    "        rel_train.append((img,qst,ans))\n",
    "    for qst,ans in zip(norelations[0], norelations[1]):\n",
    "        norel_train.append((img,qst,ans))\n",
    "\n",
    "rel   = cvt_data_axis(rel_train)\n",
    "norel = cvt_data_axis(norel_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tensor_data(data, i):\n",
    "    img = torch.from_numpy(np.asarray(data[0][bs*i:bs*(i+1)]))\n",
    "    qst = torch.from_numpy(np.asarray(data[1][bs*i:bs*(i+1)]))\n",
    "    ans = torch.from_numpy(np.asarray(data[2][bs*i:bs*(i+1)]))\n",
    "\n",
    "    input_img.data.resize_(img.size()).copy_(img)\n",
    "    input_qst.data.resize_(qst.size()).copy_(qst)\n",
    "    label.data.resize_(ans.size()).copy_(ans)\n",
    "\n",
    "tensor_data(rel, 0)    # Loads batch 0 into input_img, input_qst amd label\n",
    "#tensor_data(norel, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_image(img):\n",
    "    im = np.swapaxes( img.cpu().data.numpy(), 0,2)  # Undo the np->pytorch swap\n",
    "    plt.figure(figsize=(6,6))\n",
    "    plt.imshow(im[:,:,::-1], interpolation='nearest')  # BGR-> RGB\n",
    "    #plt.axis('off')\n",
    "\n",
    "def show_question(q):\n",
    "    colors = ['red ', 'green ', 'blue ', 'orange ', 'gray ', 'yellow ']\n",
    "    question = list(q.cpu().data.numpy())\n",
    "    query = colors[question[0:6].index(1)]\n",
    "\n",
    "    if question[6] == 1:  # NonRel Questions\n",
    "        if question[8] == 1:\n",
    "            query += 'shape?'\n",
    "        if question[9] == 1:\n",
    "            query += 'left?'\n",
    "        if question[10] == 1:\n",
    "            query += 'up?'\n",
    "    if question[7] == 1:  # Rel questions\n",
    "        if question[8] == 1:\n",
    "            query += 'closest shape?'\n",
    "        if question[9] == 1:\n",
    "            query += 'furthest shape?'\n",
    "        if question[10] == 1:\n",
    "            query += 'count?'\n",
    "    return query\n",
    "\n",
    "def show_answer(a):\n",
    "    answer_sheet = ['yes', 'no', 'rectangle', 'circle', '1', '2', '3', '4', '5', '6']\n",
    "    answer = a.cpu().data.numpy()[0]\n",
    "    return answer_sheet[answer]\n",
    "\n",
    "def show_example(i):\n",
    "    print( show_question( input_qst[i] ), show_answer( label[i] ) )\n",
    "    show_image(input_img[i])\n",
    "\n",
    "show_example(18)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [<commit_msg>Question interpretation (and images) make more sense now<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import pickle\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import argparse\n",
    "#parser = argparse.ArgumentParser(description='PyTorch Relations-from-Stream sort-of-CLVR Example')\n",
    "#args = parser.parse_args()\n",
    "\n",
    "from attrdict import AttrDict\n",
    "args = AttrDict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "args.batch_size = 32\n",
    "args.cuda = torch.cuda.is_available()\n",
    "args.lr   = 0.0001\n",
    "args.seed = 5\n",
    "args.process_coords=False\n",
    "args.debug = True\n",
    "args.rnn_hidden_size = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(args.seed)\n",
    "if args.cuda:\n",
    "    torch.cuda.manual_seed(args.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load in a batch of images to test dimensions, etc...\n",
    "bs = args.batch_size\n",
    "\n",
    "input_img = torch.FloatTensor(bs, 3, 75, 75)\n",
    "input_qst = torch.FloatTensor(bs, 11)\n",
    "label = torch.LongTensor(bs)\n",
    "\n",
    "if args.cuda:\n",
    "    input_img = input_img.cuda()\n",
    "    input_qst = input_qst.cuda()\n",
    "    label = label.cuda()\n",
    "    \n",
    "input_img = Variable(input_img)\n",
    "input_qst = Variable(input_qst)\n",
    "label = Variable(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dirs = './data'\n",
    "\n",
    "filename = os.path.join(data_dirs, 'sort-of-clevr.pickle')\n",
    "with open(filename, 'rb') as f:\n",
    "  train_datasets, test_datasets = pickle.load(f)\n",
    "\n",
    "def cvt_data_axis(data):\n",
    "    img = [e[0] for e in data]\n",
    "    qst = [e[1] for e in data]\n",
    "    ans = [e[2] for e in data]\n",
    "    return (img,qst,ans)\n",
    "\n",
    "rel_train, norel_train = [], []\n",
    "for img, relations, norelations in train_datasets:\n",
    "    img = np.swapaxes(img,0,2)\n",
    "    for qst,ans in zip(relations[0], relations[1]):\n",
    "        rel_train.append((img,qst,ans))\n",
    "    for qst,ans in zip(norelations[0], norelations[1]):\n",
    "        norel_train.append((img,qst,ans))\n",
    "\n",
    "rel   = cvt_data_axis(rel_train)\n",
    "norel = cvt_data_axis(norel_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tensor_data(data, i):\n",
    "    img = torch.from_numpy(np.asarray(data[0][bs*i:bs*(i+1)]))\n",
    "    qst = torch.from_numpy(np.asarray(data[1][bs*i:bs*(i+1)]))\n",
    "    ans = torch.from_numpy(np.asarray(data[2][bs*i:bs*(i+1)]))\n",
    "\n",
    "    input_img.data.resize_(img.size()).copy_(img)\n",
    "    input_qst.data.resize_(qst.size()).copy_(qst)\n",
    "    label.data.resize_(ans.size()).copy_(ans)\n",
    "\n",
    "tensor_data(rel, 0)    # Loads batch 0 into input_img, input_qst amd label\n",
    "#tensor_data(norel, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_image(img):\n",
    "    im = np.swapaxes( img.cpu().data.numpy(), 0,2)  # Undo the np->pytorch swap\n",
    "    plt.figure(figsize=(6,6))\n",
    "    plt.imshow(im[:,:,::-1], interpolation='nearest')  # BGR-> RGB\n",
    "    #plt.axis('off')\n",
    "\n",
    "def show_question(q):\n",
    "    colors = ['red ', 'green ', 'blue ', 'orange ', 'gray ', 'yellow ']\n",
    "    question = list(q.cpu().data.numpy())\n",
    "    query = colors[question[0:6].index(1)]\n",
    "\n",
    "    if question[6] == 1:  # NonRel Questions\n",
    "        if question[8] == 1:\n",
    "            query += 'shape?'\n",
    "        if question[9] == 1:\n",
    "            query += 'left?'\n",
    "        if question[10] == 1:\n",
    "            query += 'up?'\n",
    "    if question[7] == 1:  # Rel questions\n",
    "        if question[8] == 1:\n",
    "            query += 'closest shape?'\n",
    "        if question[9] == 1:\n",
    "            query += 'furthest shape?'\n",
    "        if question[10] == 1:\n",
    "            query += 'shape count?'\n",
    "    return query\n",
    "\n",
    "def show_answer(a):\n",
    "    answer_sheet = ['yes', 'no', 'rectangle', 'circle', '1', '2', '3', '4', '5', '6']\n",
    "    answer = a.cpu().data.numpy()[0]\n",
    "    return answer_sheet[answer]\n",
    "\n",
    "def show_example(i):\n",
    "    print( show_question( input_qst[i] ), show_answer( label[i] ) )\n",
    "    show_image(input_img[i])\n",
    "\n",
    "show_example(14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
{'fraction_non_alphanumeric': 0.2302046455401088, 'fraction_numerical': 0.012088766082376306, 'mean_word_length': 2.417527294187076, 'pattern_counts': {'":': 122, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6570837', 'n_tokens_mistral': 4733, 'n_tokens_neox': 4595, 'n_words': 772}