<commit_before>{
 "metadata": {
  "name": "",
  "signature": "sha256:fea953554885d80a64755da7882cdd1b1c3bb0a71255a28042713d3d91603fe5"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Does not work on Windows, probably\n",
      "import multiprocessing\n",
      "import os\n",
      "import subprocess\n",
      "import tempfile  # cannot use\n",
      "\n",
      "import pexpect  # requires github pexpect"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "my_app = '../../broken_app/broken_app.py'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Here we define the wrapper"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def run_batch(pipe, temp_dir, app, op):\n",
      "    os.chdir(temp_dir)\n",
      "    #This is enough to run simple interactive processes\n",
      "    proc = subprocess.Popen(['../' + app], bufsize=1, universal_newlines=True,\n",
      "                            stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n",
      "    out, err = proc.communicate('%d\\n' % op)\n",
      "    pipe.send(out.split('\\n')[-1].rstrip())\n",
      "\n",
      "def run_interactive(pipe, temp_dir, app, op):\n",
      "    os.chdir(temp_dir)\n",
      "    c = pexpect.spawnu('../' + app)\n",
      "    c.expect(u'\\?')\n",
      "    #Explain the problem with popen\n",
      "    c.sendline(u'%d' % op)\n",
      "    pipe.send(c.readlines()[-1].rstrip())\n",
      "    os.chdir('..')  # not necessary\n",
      "\n",
      "\n",
      "def run_app(app, op, in_file, out_file, interactive=False):\n",
      "    parent, child = multiprocessing.Pipe()\n",
      "    temp_dir = tempfile.mkdtemp(dir='.')  # Dir should be on the local FS\n",
      "    os.symlink('../%s' % in_file, '%s/in.params' % temp_dir)  # Link, not copy - could be gigantic file\n",
      "    if not interactive:\n",
      "        p = multiprocessing.Process(target=run_batch, args=(child, temp_dir, app, op))\n",
      "    else:\n",
      "        p = multiprocessing.Process(target=run_interactive, args=(child, temp_dir, app, op))\n",
      "    p.start()\n",
      "    line = parent.recv()\n",
      "    p.join()\n",
      "    os.rmdir(temp_dir)\n",
      "    #No command-line arguments\n",
      "    #instead of call\n",
      "    #Pipe\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Using the wrapper"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def create_dummy_input(fname, n1, n2):\n",
      "    w = open(fname, 'w')\n",
      "    w.write('%f\\n%f\\n' % (n1, n2))\n",
      "    w.close()\n",
      "\n",
      "create_dummy_input('my.params', 1.5, 2)\n",
      "op = 1  # subtraction\n",
      "run_app(my_app, op, 'my.params', 'out.result', interactive=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "The result of the multiplication is: 3.000000\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}<commit_msg>shutil<commit_after>{
 "metadata": {
  "name": "",
  "signature": "sha256:0fb4773789ccbda22cdfe0a1bba42e4285980843eeca9e088a8284723ba8e918"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Does not work on Windows, probably\n",
      "import multiprocessing\n",
      "import os\n",
      "import subprocess\n",
      "import tempfile  # cannot use\n",
      "\n",
      "import pexpect  # requires github pexpect"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "my_app = '../../broken_app/broken_app.py'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Here we define the wrapper"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def run_batch(pipe, temp_dir, app, op):\n",
      "    os.chdir(temp_dir)\n",
      "    #This is enough to run simple interactive processes\n",
      "    proc = subprocess.Popen(['../' + app], bufsize=1, universal_newlines=True,\n",
      "                            stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n",
      "    out, err = proc.communicate('%d\\n' % op)\n",
      "    pipe.send(out.split('\\n')[-1].rstrip())\n",
      "\n",
      "def run_interactive(pipe, temp_dir, app, op):\n",
      "    os.chdir(temp_dir)\n",
      "    c = pexpect.spawnu('../' + app)\n",
      "    c.expect(u'\\?')\n",
      "    #Explain the problem with popen\n",
      "    c.sendline(u'%d' % op)\n",
      "    pipe.send(c.readlines()[-1].rstrip())\n",
      "    os.chdir('..')  # not necessary\n",
      "\n",
      "\n",
      "def run_app(app, op, in_file, out_file, interactive=False):\n",
      "    parent, child = multiprocessing.Pipe()\n",
      "    temp_dir = tempfile.mkdtemp(dir='.')  # Dir should be on the local FS\n",
      "    os.symlink('../%s' % in_file, '%s/in.params' % temp_dir)  # Link, not copy - could be gigantic file\n",
      "    if not interactive:\n",
      "        p = multiprocessing.Process(target=run_batch, args=(child, temp_dir, app, op))\n",
      "    else:\n",
      "        p = multiprocessing.Process(target=run_interactive, args=(child, temp_dir, app, op))\n",
      "    p.start()\n",
      "    line = parent.recv()\n",
      "    p.join()\n",
      "    os.rmdir(temp_dir)\n",
      "    #No command-line arguments\n",
      "    #instead of call\n",
      "    #Pipe\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Using the wrapper"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def create_dummy_input(fname, n1, n2):\n",
      "    w = open(fname, 'w')\n",
      "    w.write('%f\\n%f\\n' % (n1, n2))\n",
      "    w.close()\n",
      "\n",
      "create_dummy_input('my.params', 1.5, 2)\n",
      "op = 1  # subtraction\n",
      "run_app(my_app, op, 'my.params', 'out.result', interactive=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "OSError",
       "evalue": "[Errno 39] Directory not empty: './tmp7t9tvH'",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mOSError\u001b[0m                                   Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-5-a08b97bffb51>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mcreate_dummy_input\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'my.params'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1.5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[0mop\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1\u001b[0m  \u001b[1;31m# subtraction\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[0mrun_app\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmy_app\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'my.params'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'out.result'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minteractive\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[1;32m<ipython-input-3-c99d5605d9b8>\u001b[0m in \u001b[0;36mrun_app\u001b[1;34m(app, op, in_file, out_file, interactive)\u001b[0m\n\u001b[0;32m     28\u001b[0m     \u001b[0mline\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     29\u001b[0m     \u001b[0mp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 30\u001b[1;33m     \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrmdir\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtemp_dir\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     31\u001b[0m     \u001b[1;31m#No command-line arguments\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     32\u001b[0m     \u001b[1;31m#instead of call\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mOSError\u001b[0m: [Errno 39] Directory not empty: './tmp7t9tvH'"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "os.listdir('../../broken_app')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}
{'fraction_non_alphanumeric': 0.21349848824734224, 'fraction_numerical': 0.0888520433043987, 'mean_word_length': 2.2854854213393145, 'pattern_counts': {'":': 119, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6711448', 'n_tokens_mistral': 4835, 'n_tokens_neox': 4332, 'n_words': 580}