<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Jenkins Configuration Notebook\n",
    "==============================\n",
    "\n",
    "This notebook should be used to update the Jenkins configuration for the Menpo project en-masse. It works by completing three template projects with the state that changes between projects - namely, the name of the project (and hence the github URL) and the Python versions that need to be built.\n",
    "\n",
    "Dependencies\n",
    "------------\n",
    "\n",
    "- java\n",
    "- an internet connection (no SSH tunnel etc required!)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set your jenkins credentials here - and do not commit them!!\n",
    "USERNAME = 'username'\n",
    "PASSWORD = 'password'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "from subprocess import check_output, Popen, PIPE, STDOUT, CalledProcessError\n",
    "from utils import PROJECTS, download_file\n",
    "\n",
    "# downlad the CLI file for use\n",
    "download_file('https://jenkins.menpo.org/jnlpJars/jenkins-cli.jar', 'jenkins-cli.jar')\n",
    "\n",
    "T_OFF = '<disabled>true</disabled>'\n",
    "T_ON = '<disabled>false</disabled>'\n",
    "T_VERSIONS = '<string>@PYTHON_VERSIONS@</string>\\n'\n",
    "T_NAME = '@PROJECT_NAME@'\n",
    "\n",
    "VERSION_TO_STR = {k: '<string>{}</string>\\n        '.format(v) for k, v in \n",
    "                  {2: '2.7', 34: '3.4', 35: '3.5'}.items()}\n",
    "\n",
    "JENKINS_CMD = ['java', '-jar', 'jenkins-cli.jar', \n",
    "               '-noKeyAuth', '-noCertificateCheck', \n",
    "               '-s', 'https://jenkins.menpo.org']\n",
    "AUTH = [\n",
    "    '--username', USERNAME,\n",
    "    '--password', <PASSWORD>",
    "]\n",
    "\n",
    "cmd_for_args = lambda args: JENKINS_CMD + list(args) + AUTH\n",
    "\n",
    "    \n",
    "def jcall(*args, **kwargs):\n",
    "    verbose = kwargs.get('verbose', False)\n",
    "    cmd = cmd_for_args(args)\n",
    "    if verbose:\n",
    "        print(' '.join(cmd))\n",
    "    try:\n",
    "        output = check_output(cmd).decode()\n",
    "        return output.split('Skipping HTTPS certificate checks altogether. Note that this is not secure at all.\\n')[-1]\n",
    "    except CalledProcessError as e:\n",
    "        print(e.returncode)\n",
    "        print(e.output)\n",
    "\n",
    "\n",
    "def jinput(stdin, cmd):\n",
    "    p = Popen(cmd_for_args(cmd), stdout=PIPE, stdin=PIPE, stderr=STDOUT)\n",
    "    try:\n",
    "        return p.communicate(input=stdin.encode())[0].decode()\n",
    "    except CalledProcessError as e:\n",
    "        print(e.returncode)\n",
    "        print(e.output)\n",
    "\n",
    "def fill_template(t, name, versions):\n",
    "    return t.replace(T_NAME, name).replace(T_VERSIONS, versions).replace(T_OFF, T_ON)\n",
    "\n",
    "\n",
    "get_job = partial(jcall, 'get-job')\n",
    "create_job = lambda name, job: jinput(job, ['create-job', name])\n",
    "update_job = lambda name, job: jinput(job, ['update-job', name])\n",
    "delete_job = lambda name: jcall('delete-job', name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Check we are successfully logged in:\n",
    "print(jcall('who-am-i'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print('loading templates...')\n",
    "\n",
    "def load_text(name):\n",
    "    with open(name, 'rt') as f:\n",
    "        content = f.read()\n",
    "    return content\n",
    "\n",
    "T, T_PR = (load_text('./jenkins/templates/TEMPLATE.xml'), \n",
    "           load_text('./jenkins/templates/TEMPLATE-PR.xml'))\n",
    "\n",
    "SUFFIX_TO_TEMPLATE = { '': T, '-pr': T_PR }\n",
    "print('done.')\n",
    "print('acquiring current jobs...')\n",
    "JOBS = set(x for x in jcall('list-jobs').split('\\n') if not x == '')\n",
    "print('done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "non_pr_jobs = {j for j in JOBS if not j.endswith('-pr')}\n",
    "known_jobs = {p.name for p in PROJECTS}\n",
    "extra_jobs = non_pr_jobs - known_jobs - { 'TEMPLATE' }\n",
    "if len(extra_jobs) != 0:\n",
    "    raise ValueError(\"Warning - a Jenkins jobs is present that isn't managed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for p in PROJECTS:\n",
    "    versions = ' '.join([VERSION_TO_STR[v] for v in p.versions])\n",
    "    for suffix, template in SUFFIX_TO_TEMPLATE.items():\n",
    "        job = fill_template(template, p.name, versions)\n",
    "        job_name = p.name + suffix\n",
    "        if job_name in JOBS:\n",
    "            print('{} already exists, updating.'.format(job_name))\n",
    "            update_job(job_name, job)\n",
    "        else:\n",
    "            print('creating job: {}'.format(job_name))\n",
    "            create_job(job_name, job)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}<commit_msg>Add print to un-managed projects<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Jenkins Configuration Notebook\n",
    "==============================\n",
    "\n",
    "This notebook should be used to update the Jenkins configuration for the Menpo project en-masse. It works by completing three template projects with the state that changes between projects - namely, the name of the project (and hence the github URL) and the Python versions that need to be built.\n",
    "\n",
    "Dependencies\n",
    "------------\n",
    "\n",
    "- java\n",
    "- an internet connection (no SSH tunnel etc required!)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set your jenkins credentials here - and do not commit them!!\n",
    "USERNAME = 'USER'\n",
    "PASSWORD = '<PASSWORD>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "from subprocess import check_output, Popen, PIPE, STDOUT, CalledProcessError\n",
    "from utils import PROJECTS, download_file\n",
    "\n",
    "# downlad the CLI file for use\n",
    "download_file('https://jenkins.menpo.org/jnlpJars/jenkins-cli.jar', 'jenkins-cli.jar')\n",
    "\n",
    "T_OFF = '<disabled>true</disabled>'\n",
    "T_ON = '<disabled>false</disabled>'\n",
    "T_VERSIONS = '<string>@PYTHON_VERSIONS@</string>\\n'\n",
    "T_NAME = '@PROJECT_NAME@'\n",
    "\n",
    "VERSION_TO_STR = {k: '<string>{}</string>\\n        '.format(v) for k, v in \n",
    "                  {2: '2.7', 34: '3.4', 35: '3.5'}.items()}\n",
    "\n",
    "JENKINS_CMD = ['java', '-jar', 'jenkins-cli.jar', \n",
    "               '-noKeyAuth', '-noCertificateCheck', \n",
    "               '-s', 'https://jenkins.menpo.org']\n",
    "AUTH = [\n",
    "    '--username', USERNAME,\n",
    "    '--password', <PASSWORD>",
    "]\n",
    "\n",
    "cmd_for_args = lambda args: JENKINS_CMD + list(args) + AUTH\n",
    "\n",
    "    \n",
    "def jcall(*args, **kwargs):\n",
    "    verbose = kwargs.get('verbose', False)\n",
    "    cmd = cmd_for_args(args)\n",
    "    if verbose:\n",
    "        print(' '.join(cmd))\n",
    "    try:\n",
    "        output = check_output(cmd).decode()\n",
    "        return output.split('Skipping HTTPS certificate checks altogether. Note that this is not secure at all.\\n')[-1]\n",
    "    except CalledProcessError as e:\n",
    "        print(e.returncode)\n",
    "        print(e.output)\n",
    "\n",
    "\n",
    "def jinput(stdin, cmd):\n",
    "    p = Popen(cmd_for_args(cmd), stdout=PIPE, stdin=PIPE, stderr=STDOUT)\n",
    "    try:\n",
    "        return p.communicate(input=stdin.encode())[0].decode()\n",
    "    except CalledProcessError as e:\n",
    "        print(e.returncode)\n",
    "        print(e.output)\n",
    "\n",
    "def fill_template(t, name, versions):\n",
    "    return t.replace(T_NAME, name).replace(T_VERSIONS, versions).replace(T_OFF, T_ON)\n",
    "\n",
    "\n",
    "get_job = partial(jcall, 'get-job')\n",
    "create_job = lambda name, job: jinput(job, ['create-job', name])\n",
    "update_job = lambda name, job: jinput(job, ['update-job', name])\n",
    "delete_job = lambda name: jcall('delete-job', name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Check we are successfully logged in:\n",
    "print(jcall('who-am-i'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print('loading templates...')\n",
    "\n",
    "def load_text(name):\n",
    "    with open(name, 'rt') as f:\n",
    "        content = f.read()\n",
    "    return content\n",
    "\n",
    "T, T_PR = (load_text('./jenkins/templates/TEMPLATE.xml'), \n",
    "           load_text('./jenkins/templates/TEMPLATE-PR.xml'))\n",
    "\n",
    "SUFFIX_TO_TEMPLATE = { '': T, '-pr': T_PR }\n",
    "print('done.')\n",
    "print('acquiring current jobs...')\n",
    "JOBS = set(x for x in jcall('list-jobs').split('\\n') if not x == '')\n",
    "print('done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "non_pr_jobs = {j for j in JOBS if not j.endswith('-pr')}\n",
    "known_jobs = {p.name for p in PROJECTS}\n",
    "extra_jobs = non_pr_jobs - known_jobs - { 'TEMPLATE' }\n",
    "if len(extra_jobs) != 0:\n",
    "    print(extra_jobs)\n",
    "    raise ValueError(\"Warning - a Jenkins jobs is present that isn't managed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for p in PROJECTS:\n",
    "    versions = ' '.join([VERSION_TO_STR[v] for v in p.versions])\n",
    "    for suffix, template in SUFFIX_TO_TEMPLATE.items():\n",
    "        job = fill_template(template, p.name, versions)\n",
    "        job_name = p.name + suffix\n",
    "        if job_name in JOBS:\n",
    "            print('{} already exists, updating.'.format(job_name))\n",
    "            update_job(job_name, job)\n",
    "        else:\n",
    "            print('creating job: {}'.format(job_name))\n",
    "            create_job(job_name, job)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
{'fraction_non_alphanumeric': 0.2126068376068376, 'fraction_numerical': 0.004026955950032873, 'mean_word_length': 2.4571022727272727, 'pattern_counts': {'":': 125, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '491274', 'n_tokens_mistral': 4556, 'n_tokens_neox': 4453, 'n_words': 936}