<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Characterwise Double-Stacked LSTM as Author\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy\n",
    "import theano\n",
    "from theano import tensor\n",
    "\n",
    "from blocks.bricks import Tanh\n",
    "from blocks.bricks.recurrent import GatedRecurrent\n",
    "from blocks.bricks.sequence_generators import (SequenceGenerator, Readout, SoftmaxEmitter, LookupFeedback)\n",
    "from blocks.graph import ComputationGraph\n",
    "from fuel.streams import DataStream\n",
    "from fuel.schemes import ConstantScheme\n",
    "from blocks.algorithms import GradientDescent, Scale\n",
    "from blocks.initialization import Orthogonal, IsotropicGaussian, Constant\n",
    "from blocks.model import Model\n",
    "from blocks.monitoring import aggregation\n",
    "from blocks.extensions import FinishAfter, Printing\n",
    "from blocks.extensions.saveload import Checkpoint\n",
    "from blocks.extensions.monitoring import TrainingDataMonitoring\n",
    "from blocks.main_loop import MainLoop\n",
    "from blocks.select import Selector\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Dictionaries\n",
    "import string\n",
    "\n",
    "all_chars = [ a for a in string.printable]+['<UNK>']\n",
    "code2char = dict(enumerate(all_chars))\n",
    "char2code = {v: k for k, v in code2char.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#from fuel.datasets import Dataset\n",
    "from fuel.datasets.TextFile\n",
    "\n",
    "data_path = \n",
    "\n",
    "dataset = TextFile(data_path, dictionary=char2code)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rng = numpy.random.RandomState(1)\n",
    "batch_size = 50\n",
    "seq_len = 100\n",
    "\n",
    "num_states=256\n",
    "dim = 10\n",
    "feedback_dim = 8\n",
    "\n",
    "transition = GatedRecurrent(name=\"transition\", dim=dim, activation=Tanh())\n",
    "generator =  SequenceGenerator(\n",
    "                Readout(readout_dim=num_states, source_names=[\"states\"],\n",
    "                        emitter=SoftmaxEmitter(name=\"emitter\"),\n",
    "                        feedback_brick=LookupFeedback(\n",<commit_msg>Dataset reading<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Characterwise Double-Stacked LSTM as Author\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy\n",
    "import theano\n",
    "from theano import tensor\n",
    "\n",
    "from blocks.bricks import Tanh\n",
    "from blocks.bricks.recurrent import GatedRecurrent\n",
    "from blocks.bricks.sequence_generators import (SequenceGenerator, Readout, SoftmaxEmitter, LookupFeedback)\n",
    "from blocks.graph import ComputationGraph\n",
    "from fuel.streams import DataStream\n",
    "from fuel.schemes import ConstantScheme\n",
    "from blocks.algorithms import GradientDescent, Scale\n",
    "from blocks.initialization import Orthogonal, IsotropicGaussian, Constant\n",
    "from blocks.model import Model\n",
    "from blocks.monitoring import aggregation\n",
    "from blocks.extensions import FinishAfter, Printing\n",
    "from blocks.extensions.saveload import Checkpoint\n",
    "from blocks.extensions.monitoring import TrainingDataMonitoring\n",
    "from blocks.main_loop import MainLoop\n",
    "from blocks.select import Selector\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Dictionaries\n",
    "import string\n",
    "\n",
    "all_chars = [ a for a in string.printable]+['<UNK>']\n",
    "code2char = dict(enumerate(all_chars))\n",
    "char2code = {v: k for k, v in code2char.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#from fuel.datasets import Dataset\n",
    "from fuel.datasets import TextFile\n",
    "\n",
    "data_files = ['../data/Shakespeare.poetry.txt']\n",
    "\n",
    "dataset = TextFile(data_files, bos_token=None, eos_token=None, level=\"character\", dictionary=char2code)\n",
    "data_stream = dataset.get_example_stream()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#rng = numpy.random.RandomState(1)\n",
    "batch_size = 50\n",
    "seq_len = 100\n",
    "\n",
    "num_states=len(char2code)\n",
    "dim = 10\n",
    "feedback_dim = 8\n",
    "\n",
    "transition = GatedRecurrent(name=\"transition\", dim=dim, activation=Tanh())\n",
    "generator =  SequenceGenerator(\n",
    "                Readout(readout_dim=num_states, source_names=[\"states\"],\n",
    "                        emitter=SoftmaxEmitter(name=\"emitter\"),\n",
    "                        feedback_brick=LookupFeedback(\n",
{'fraction_non_alphanumeric': 0.16970702045328911, 'fraction_numerical': 0.007002026902524415, 'mean_word_length': 2.956268221574344, 'pattern_counts': {'":': 56, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5738977', 'n_tokens_mistral': 1909, 'n_tokens_neox': 1849, 'n_words': 365}