<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%pylab notebook\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%cd /Users/brodzik/projects/CHARIS/charistools_test_data/calibration_stats/\n",
    "%ls\n",
    "filename = 'AM_Vakhsh_calibration-1494195-1.out'\n",
    "#filename = 'IN_Hunza_calibration-1493941-2.out'\n",
    "#filename = 'GA_Karnali_calibration-1494189-0.out'\n",
    "#filename = 'GA_Narayani_calibration-1494190-0.out'\n",
    "#filename = 'GA_SaptaKosi_calibration-1494191-0.out'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_table(filename, sep='\\s+')\n",
    "df\n",
    "subdf = df[['DRAINAGEID','YYYY','min_snow_ddf','max_snow_ddf','min_ice_ddf','max_ice_ddf','Monthly_rmse_km3','Annual_voldiff_pcent']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "subdf[\"model\"] = (\n",
    "    subdf[\"min_snow_ddf\"].map(str) + \"_\" + \n",
    "    subdf[\"max_snow_ddf\"].map(str) + \"_\" +\n",
    "    subdf[\"min_ice_ddf\"].map(str) + \"_\" +\n",
    "    subdf[\"max_ice_ddf\"].map(str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "subdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#subdf.describe().loc[['min','max'],['min_snow_ddf','max_snow_ddf','min_ice_ddf','max_ice_ddf']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# for name, group in subdf.groupby(['model']):\n",
    "#    print(name)\n",
    "#    print(group)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "mean_vol_diff = subdf.groupby(['model']).mean()['Annual_voldiff_pcent']\n",
    "mean_rmse = subdf.groupby(['model']).mean()['Monthly_rmse_km3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "new = mean_rmse.to_frame()\n",
    "new['Annual_voldiff_pcent'] = mean_vol_diff\n",
    "#new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "min_vol_diff = np.min(mean_vol_diff)\n",
    "max_vol_diff = np.max(mean_vol_diff)\n",
    "print(\"min vol_diff=%f\" % min_vol_diff)\n",
    "print(\"max_vol_diff=%f\" % max_vol_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "min_rmse = np.min(mean_rmse)\n",
    "max_rmse = np.max(mean_rmse)\n",
    "print(\"min rmse=%f\" % min_rmse)\n",
    "print(\"max rmse=%f\" % max_rmse)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "<h2>Now, normalize the two variables so they range from 0.0 to 1.0</h2>\n",
    "This should map min_vol_diff 0. and max_vol_diff to 1.0\n",
    "\n",
    "and             min_rmse to 0. and max_rmse to 1.0:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "new['z_Vol_Diff'] = (\n",
    "    (new['Annual_voldiff_pcent'] - min_vol_diff) / \n",
    "    (max_vol_diff - min_vol_diff))\n",
    "new['z_RMSE'] = (\n",
    "    (new['Monthly_rmse_km3'] - min_rmse) / \n",
    "    (max_rmse - min_rmse))\n",
    "new['z'] = new['z_Vol_Diff'] + new['z_RMSE']\n",
    "new"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now calculate the combined statistic (z_vol_diff + z_rmse) and find the minimum:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# new['z'].plot(kind='barh',figsize=(12,20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sorted = new.sort(['z'], ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"DDF ranges included in this file:\")\n",
    "subdf.describe().loc[['min','max'],['min_snow_ddf','max_snow_ddf','min_ice_ddf','max_ice_ddf']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(sorted.iloc[0])\n",
    "print(\"Best model is %s\" % sorted.index[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"<commit_msg>Added function to move header to top of file<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%pylab notebook\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%cd /Users/brodzik/projects/CHARIS/charistools_test_data/calibration_stats/\n",
    "%ls\n",
    "#filename = 'AM_Vakhsh_calibration-1493953-0.out'\n",
    "filename = 'IN_Hunza_calibration-1509472.out'\n",
    "#filename = 'GA_Karnali_calibration-1493201.out'\n",
    "#filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def move_header_to_top(filename):\n",
    "    # Read the whole file\n",
    "    with open(filename) as f:\n",
    "        lines = f.readlines()\n",
    "\n",
    "    # Find the header and its position\n",
    "    re_header = re.compile(r'DRAINAGE')\n",
    "    header_pos = 0\n",
    "    for i, line in enumerate(lines):\n",
    "        if re_header.match(line):\n",
    "            print(\"Found header at position=%d\" % i)\n",
    "            header_pos = i\n",
    "            header = line\n",
    "            break\n",
    "            \n",
    "    # Delete the header from its original position\n",
    "    # And insert it at the beginnin\n",
    "    if header_pos > 0:\n",
    "        del lines[header_pos]\n",
    "        lines.insert(0, header)\n",
    "        \n",
    "    # Write out the new file\n",
    "    f = open(filename, 'w')\n",
    "    for line in lines:\n",
    "        f.write(line)\n",
    "    f.close\n",
    "    \n",
    "    print(\"Moved header to top of file.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "move_header_to_top(filename)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_table(filename, sep='\\s+')\n",
    "df\n",
    "subdf = df[['DRAINAGEID','YYYY','min_snow_ddf','max_snow_ddf','min_ice_ddf','max_ice_ddf','Monthly_rmse_km3','Annual_voldiff_pcent']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "subdf[\"model\"] = (\n",
    "    subdf[\"min_snow_ddf\"].map(str) + \"_\" + \n",
    "    subdf[\"max_snow_ddf\"].map(str) + \"_\" +\n",
    "    subdf[\"min_ice_ddf\"].map(str) + \"_\" +\n",
    "    subdf[\"max_ice_ddf\"].map(str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#subdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "subdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#subdf.describe().loc[['min','max'],['min_snow_ddf','max_snow_ddf','min_ice_ddf','max_ice_ddf']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for name, group in subdf.groupby(['model']):\n",
    "#    print(name)\n",
    "#    print(group)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mean_vol_diff = subdf.groupby(['model']).mean()['Annual_voldiff_pcent']\n",
    "mean_rmse = subdf.groupby(['model']).mean()['Monthly_rmse_km3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "new = mean_rmse.to_frame()\n",
    "new['Annual_voldiff_pcent'] = mean_vol_diff\n",
    "#new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "min_vol_diff = np.min(mean_vol_diff)\n",
    "max_vol_diff = np.max(mean_vol_diff)\n",
    "print(\"min vol_diff=%f\" % min_vol_diff)\n",
    "print(\"max_vol_diff=%f\" % max_vol_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "min_rmse = np.min(mean_rmse)\n",
    "max_rmse = np.max(mean_rmse)\n",
    "print(\"min rmse=%f\" % min_rmse)\n",
    "print(\"max rmse=%f\" % max_rmse)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "<h2>Now, normalize the two variables so they range from 0.0 to 1.0</h2>\n",
    "This should map min_vol_diff 0. and max_vol_diff to 1.0\n",
    "\n",
    "and             min_rmse to 0. and max_rmse to 1.0:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "new['z_Vol_Diff'] = (\n",
    "    (new['Annual_voldiff_pcent'] - min_vol_diff) / \n",
    "    (max_vol_diff - min_vol_diff))\n",
    "new['z_RMSE'] = (\n",
    "    (new['Monthly_rmse_km3'] - min_rmse) / \n",
    "    (max_rmse - min_rmse))\n",
    "new['z'] = new['z_Vol_Diff'] + new['z_RMSE']\n",
    "new"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now calculate the combined statistic (z_vol_diff + z_rmse) and find the minimum:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# new['z'].plot(kind='barh',figsize=(12,20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sorted = new.sort(['z'], ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(\"DDF ranges included in this file:\")\n",
    "subdf.describe().loc[['min','max'],['min_snow_ddf','max_snow_ddf','min_ice_ddf','max_ice_ddf']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(sorted.iloc[0])\n",
    "print(\"Best model is %s\" % sorted.index[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
{'fraction_non_alphanumeric': 0.22182481751824817, 'fraction_numerical': 0.008978102189781022, 'mean_word_length': 2.6322905620360553, 'pattern_counts': {'":': 300, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5301080', 'n_tokens_mistral': 5635, 'n_tokens_neox': 5532, 'n_words': 862}