<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import json\n",
    "import uuid\n",
    "\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper for displaying images.\n",
    "\n",
    "# source: http://nbviewer.ipython.org/gist/deeplook/5162445\n",
    "from io import BytesIO\n",
    "\n",
    "from IPython import display\n",
    "from PIL import Image\n",
    "\n",
    "\n",
    "def display_pil_image(im):\n",
    "   \"\"\"Displayhook function for PIL Images, rendered as PNG.\"\"\"\n",
    "\n",
    "   b = BytesIO()\n",
    "   im.save(b, format='png')\n",
    "   data = b.getvalue()\n",
    "\n",
    "   ip_img = display.Image(data=data, format='png', embed=True)\n",
    "   return ip_img._repr_png_()\n",
    "\n",
    "\n",
    "# register display func with PNG formatter:\n",
    "png_formatter = get_ipython().display_formatter.formatters['image/png']\n",
    "dpi = png_formatter.for_type(Image.Image, display_pil_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [<commit_msg>WIP: Add generic graph manipulation classes.<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import json\n",
    "import uuid\n",
    "\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper for displaying images.\n",
    "\n",
    "# source: http://nbviewer.ipython.org/gist/deeplook/5162445\n",
    "from io import BytesIO\n",
    "\n",
    "from IPython import display\n",
    "from PIL import Image\n",
    "\n",
    "\n",
    "def display_pil_image(im):\n",
    "   \"\"\"Displayhook function for PIL Images, rendered as PNG.\"\"\"\n",
    "\n",
    "   b = BytesIO()\n",
    "   im.save(b, format='png')\n",
    "   data = b.getvalue()\n",
    "\n",
    "   ip_img = display.Image(data=data, format='png', embed=True)\n",
    "   return ip_img._repr_png_()\n",
    "\n",
    "\n",
    "# register display func with PNG formatter:\n",
    "png_formatter = get_ipython().display_formatter.formatters['image/png']\n",
    "dpi = png_formatter.for_type(Image.Image, display_pil_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Edge:\n",
    "    def __init__(self, graph, label, target):\n",
    "        pass\n",
    "\n",
    "\n",
    "class Node:\n",
    "    def __init__(self, graph, guid=None):\n",
    "        if guid is None:\n",
    "            guid = uuid.uuid4().hex\n",
    "        self.guid = guid\n",
    "        self._graph = graph\n",
    "    \n",
    "    @property\n",
    "    def edges(self):\n",
    "        return self.graph.get_edges(self)\n",
    "\n",
    "\n",
    "class JSONGraph:\n",
    "    def __init__(self):\n",
    "        self._properties = {'guidMap': {}}\n",
    "    \n",
    "    @property\n",
    "    def _guid_map(self):\n",
    "        return self._properties['guidMap']\n",
    "    \n",
    "    @classmethod\n",
    "    def load(cls, fp):\n",
    "        graph = cls()\n",
    "        graph._properties = json.load(f)\n",
    "    \n",
    "    def save(self, fp):\n",
    "        json.dump(core, f, indent=2, sort_keys=True)\n",
    "\n",
    "    def _to_json(self, target):\n",
    "        # TODO: switch on type\n",
    "\n",
    "    def get_edges(self, node):\n",
    "        pass\n",
    "        \n",
    "    def add_edge(self, source, target, label):\n",
    "        if source.guid not in self._guid_map:\n",
    "            self._guid_map[source.guid] = {}    \n",
    "        self._guid_map[source.guid][label.guid] = self._to_json(target)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
{'fraction_non_alphanumeric': 0.21121000721674285, 'fraction_numerical': 0.005292278085157566, 'mean_word_length': 2.1404833836858006, 'pattern_counts': {'":': 37, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4528909', 'n_tokens_mistral': 1607, 'n_tokens_neox': 1571, 'n_words': 290}