<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}<commit_msg>Ädded all the appropriate code. No consideration of empty mass bins<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import division, absolute_import\n",
    "\n",
    "import astropy.stats\n",
    "import glob\n",
    "import math\n",
    "import matplotlib.pyplot as plt \n",
    "from matplotlib import ticker\n",
    "from matplotlib.ticker import FormatStrFormatter\n",
    "import numpy as np \n",
    "import os\n",
    "import pandas as pd\n",
    "from scipy import integrate,optimize,spatial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "__author__     =['<NAME>']\n",
    "__copyright__  =[\"Copyright 2016 <NAME>, Index function\"]\n",
    "__email__      =['<EMAIL>']\n",
    "__maintainer__ =['<NAME>']\n",
    "\n",
    "def Index(directory, datatype):\n",
    "    \"\"\"\n",
    "    Indexes the files in a directory `directory' with a\n",
    "    specific data type.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    directory: str\n",
    "            Absolute path to the folder that is indexed.\n",
    "\n",
    "    datatype: str\n",
    "            Data type of the files to be indexed in the folder.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    file_array: array_like \n",
    "            np.array of indexed files in the folder 'directory' \n",
    "            with specific datatype.\n",
    "\n",
    "    Examples\n",
    "    --------\n",
    "    >>> Index('~/data', '.txt')\n",
    "    >>> array(['A.txt', 'Z'.txt', ...])\n",
    "    \"\"\"\n",
    "    assert(os.path.exists(directory))\n",
    "    files = np.array(glob.glob('{0}/*{1}'.format(directory, datatype)))\n",
    "\n",
    "    return files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def myceil(x, base=10):\n",
    "    \"\"\"\n",
    "    Returns the upper-bound integer of 'x' in base 'base'.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    x: float\n",
    "        number to be approximated to closest number to 'base'\n",
    "\n",
    "    base: float\n",
    "        base used to calculate the closest 'largest' number\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    n_high: float\n",
    "        Closest float number to 'x', i.e. upper-bound float.\n",
    "\n",
    "    Example\n",
    "    -------\n",
    "    >>>> myceil(12,10)\n",
    "      20\n",
    "    >>>>\n",
    "    >>>> myceil(12.05, 0.1)\n",
    "     12.10000 \n",
    "    \"\"\"\n",
    "    n_high = float(base*math.ceil(float(x)/base))\n",
    "\n",
    "    return n_high\n",
    "\n",
    "###############################################################################    \n",
    "\n",
    "def myfloor(x, base=10):\n",
    "    \"\"\"\n",
    "    Returns the lower-bound integer of 'x' in base 'base'\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    x: float\n",
    "        number to be approximated to closest number of 'base'\n",
    "\n",
    "    base: float\n",
    "        base used to calculate the closest 'smallest' number\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    n_low: float\n",
    "        Closest float number to 'x', i.e. lower-bound float.\n",
    "\n",
    "    Example\n",
    "    -------\n",
    "    >>>> myfloor(12, 5)\n",
    "    >>>> 10\n",
    "    \"\"\"\n",
    "    n_low = float(base*math.floor(float(x)/base))\n",
    "\n",
    "    return n_low\n",
    "\n",
    "###############################################################################\n",
    "\n",
    "def Bins_array_create(arr, base=10):\n",
    "    \"\"\"\n",
    "    Generates array between [arr.min(), arr.max()] in steps of `base`.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    arr: array_like, Shape (N,...), One-dimensional\n",
    "        Array of numerical elements\n",
    "\n",
    "    base: float, optional (default=10)\n",
    "        Interval between bins\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    bins_arr: array_like\n",
    "        Array of bin edges for given arr\n",
    "\n",
    "    \"\"\"\n",
    "    base = float(base)\n",
    "    arr  = np.array(arr)\n",
    "    assert(arr.ndim==1)\n",
    "    arr_min  = myfloor(arr.min(), base=base)\n",
    "    arr_max  = myceil( arr.max(), base=base)\n",
    "    bins_arr = np.arange(arr_min, arr_max+0.5*base, base)\n",
    "\n",
    "    return bins_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sph_to_cart(ra,dec,cz):\n",
    "    \"\"\"\n",
    "    Converts spherical coordinates to Cartesian coordinates.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    ra: array-like\n",
    "        right-ascension of galaxies in degrees\n",
    "    dec: array-like\n",
    "        declination of galaxies in degrees\n",
    "    cz: array-like\n",
    "        velocity of galaxies in km/s\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    coords: array-like, shape = N by 3\n",
    "        x, y, and z coordinates\n",
    "    \"\"\"\n",
    "    cz_dist = cz/70. #converts velocity into distance\n",
    "    x_arr   = cz_dist*np.cos(np.radians(ra))*np.cos(np.radians(dec))\n",
    "    y_arr   = cz_dist*np.sin(np.radians(ra))*np.cos(np.radians(dec))\n",
    "    z_arr   = cz_dist*np.sin(np.radians(dec))\n",
    "    coords  = np.column_stack((x_arr,y_arr,z_arr))\n",
    "\n",
    "    return coords\n",
    "\n",
    "############################################################################\n",
    "\n",
    "def calc_dens(n_val,r_val):\n",
    "    \"\"\"\n",
    "    Returns densities of spheres with radius being the distance to the \n",
    "        nth nearest neighbor.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    n_val = integer\n",
    "        The 'N' from Nth nearest neighbor\n",
    "    r_val = array-like\n",
    "        An array with the distances to the Nth nearest neighbor for\n",
    "        each galaxy\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    dens: array-like\n",
    "        An array with the densities of the spheres created with radii\n",
    "        to the Nth nearest neighbor.\n",
    "    \"\"\"\n",
    "    dens = np.array([(3.*(n_val+1)/(4.*np.pi*r_val[hh]**3)) \\\n",
    "                     for hh in range(len(r_val))])\n",
    "\n",
    "    return dens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_calcs(mass,bins,dlogM):\n",
    "    \"\"\"\n",
    "    Returns values for plotting the stellar mass function and \n",
    "        mass ratios\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    mass: array-like\n",
    "        A 1D array with mass values, assumed to be in order\n",
    "    bins: array=like\n",
    "        A 1D array with the values which will be used as the bin edges\n",
    "        by the histogram function\n",
    "    dlogM: float-like\n",
    "        The log difference between bin edges\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    bin_centers: array-like\n",
    "        An array with the medians mass values of the mass bins\n",
    "    mass-freq: array-like\n",
    "        Contains the number density values of each mass bin\n",
    "    ratio_dict: dictionary-like\n",
    "        A dictionary with three keys, corresponding to the divisors\n",
    "        2,4, and 10 (as the percentile cuts are based on these \n",
    "        divisions). Each key has the density-cut, mass ratios for\n",
    "        that specific cut (50/50 for 2; 25/75 for 4; 10/90 for 10).\n",
    "    \"\"\"\n",
    "\n",
    "    mass_counts, edges = np.histogram(mass,bins)\n",
    "    bin_centers        = 0.5*(edges[:-1]+edges[1:])\n",
    "\n",
    "    mass_freq  = mass_counts/float(len(mass))/dlogM\n",
    "\n",
    "    ratio_dict = {}\n",
    "    frac_val   = [2,4,10]\n",
    "\n",
    "    yerr = []\n",
    "\n",
    "    for ii in frac_val:\n",
    "        ratio_dict[ii] = {}\n",
    "\n",
    "        # Calculations for the lower density cut\n",
    "        frac_data      = int(len(mass)/ii)\n",
    "        frac_mass      = mass[0:frac_data]\n",
    "        counts, edges  = np.histogram(frac_mass,bins)\n",
    "\n",
    "        # Calculations for the higher density cut\n",
    "        frac_mass_2       = mass[-frac_data:]\n",
    "        counts_2, edges_2 = np.histogram(frac_mass_2,bins)\n",
    "\n",
    "        # Ratio determination\n",
    "        ratio_counts   = (1.*counts_2)/(1.*counts)\n",
    "        ratio_dict[ii] = ratio_counts\n",
    "\n",
    "        yerr.append((counts_2*1.)/(counts*1.)*\\\n",
    "            np.sqrt(1./counts + 1./counts_2))\n",
    "\n",
    "    mass_freq_list     = [[] for xx in xrange(2)]\n",
    "    mass_freq_list[0]  = mass_freq\n",
    "    mass_freq_list[1]  = np.sqrt(mass_counts)/float(len(mass))/dlogM\n",
    "    mass_freq          = np.array(mass_freq_list)\n",
    "\n",
    "    ratio_dict_list    = [[] for xx in range(2)]\n",
    "    ratio_dict_list[0] = ratio_dict\n",
    "    ratio_dict_list[1] = yerr\n",
    "    ratio_dict         = ratio_dict_list\n",
    "\n",
    "    return bin_centers, mass_freq, ratio_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bin_func(mass_dist,bins,kk,bootstrap=False):\n",
    "    \"\"\"\n",
    "    Returns median distance to Nth nearest neighbor\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    mass_dist: array-like\n",
    "        An array with mass values in at index 0 (when transformed) and distance \n",
    "        to the Nth nearest neighbor in the others\n",
    "        Example: 6239 by 7\n",
    "            Has mass values and distances to 6 Nth nearest neighbors  \n",
    "    bins: array=like\n",
    "        A 1D array with the values which will be used as the bin edges     \n",
    "    kk: integer-like\n",
    "        The index of mass_dist (transformed) where the appropriate distance \n",
    "        array may be found\n",
    "\n",
    "    Optional\n",
    "    --------\n",
    "    bootstrap == True\n",
    "        Calculates the bootstrap errors associated with each median distance \n",
    "        value. Creates an array housing arrays containing the actual distance \n",
    "        values associated with every galaxy in a specific bin. Bootstrap error\n",
    "        is then performed using astropy, and upper and lower one sigma values \n",
    "        are found for each median value.  These are added to a list with the \n",
    "        median distances, and then converted to an array and returned in place \n",
    "        of just 'medians.'\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    medians: array-like\n",
    "        An array with the median distance to the Nth nearest neighbor from \n",
    "        all the galaxies in each of the bins\n",
    "    \"\"\"\n",
    "    frac_vals    = np.array([2,4,10])\n",
    "    \n",
    "    edges        = bins\n",
    "\n",
    "    # print 'length bins:'\n",
    "    # print len(bins)\n",
    "\n",
    "    digitized    = np.digitize(mass_dist.T[0],edges)\n",
    "    digitized   -= int(1)\n",
    "\n",
    "    bin_nums          = np.unique(digitized)\n",
    "    bin_nums_list     = list(bin_nums)\n",
    "\n",
    "    if (len(bins)) in bin_nums_list:\n",
    "        bin_nums_list.remove(len(bins))\n",
    "    # print 'removed'\n",
    "    # print bin_nums_list\n",
    "\n",
    "    bin_nums = np.array(bin_nums_list)\n",
    "\n",
    "    for ii in bin_nums:\n",
    "        if len(mass_dist.T[kk][digitized==ii]) == 0:\n",
    "            temp_list = list(mass_dist.T[kk]\\\n",
    "                                             [digitized==ii])\n",
    "            temp_list.append(np.zeros(len(bin_nums)))\n",
    "            mass_dist.T[kk][digitized==ii] = np.array(temp_list)\n",
    "\n",
    "    # print bin_nums\n",
    "    # print len(bin_nums)\n",
    "    \n",
    "    medians  = np.array([np.median(mass_dist.T[kk][digitized==ii]) \\\n",
    "                for ii in bin_nums])\n",
    "\n",
    "    # print len(medians)\n",
    "\n",
    "    if bootstrap == True:\n",
    "        dist_in_bin    = np.array([(mass_dist.T[kk][digitized==ii]) \\\n",
    "                for ii in bin_nums])\n",
    "        for vv in range(len(dist_in_bin)):\n",
    "            if len(dist_in_bin[vv]) == 0:\n",
    "                dist_in_bin_list = list(dist_in_bin[vv])\n",
    "                dist_in_bin[vv] = np.zeros(len(dist_in_bin[0]))\n",
    "        low_err_test   = np.array([np.percentile(astropy.stats.bootstrap\\\n",
    "                        (dist_in_bin[vv],bootnum=1000,bootfunc=np.median),16) \\\n",
    "                        for vv in range(len(dist_in_bin))])\n",
    "        high_err_test  = np.array([np.percentile(astropy.stats.bootstrap\\\n",
    "                        (dist_in_bin[vv],bootnum=1000,bootfunc=np.median),84) \\\n",
    "                        for vv in range(len(dist_in_bin))])\n",
    "\n",
    "        med_list    = [[] for yy in range(len(frac_vals))]\n",
    "        med_list[0] = medians\n",
    "        med_list[1] = low_err_test\n",
    "        med_list[2] = high_err_test\n",
    "        medians     = np.array(med_list)\n",
    "\n",
    "    return medians    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def hist_calcs(mass,bins,dlogM):\n",
    "    \"\"\"\n",
    "    Returns dictionaries with the counts for the upper\n",
    "        and lower density portions; calculates the \n",
    "        three different percentile cuts for each mass\n",
    "        array given\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    mass: array-like\n",
    "        A 1D array with log stellar mass values, assumed\n",
    "        to be an order which corresponds to the ascending \n",
    "        densities; (necessary, as the index cuts are based \n",
    "        on this)\n",
    "    bins: array-like\n",
    "        A 1D array with the values which will be used as the bin edges   \n",
    "    dlogM: float-like\n",
    "        The log difference between bin edges\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    hist_dict_low: dictionary-like\n",
    "        A dictionary with three keys (the frac vals), with arrays\n",
    "        as values. The values for the lower density cut\n",
    "    hist_dict_high: dictionary like\n",
    "        A dictionary with three keys (the frac vals), with arrays\n",
    "        as values. The values for the higher density cut\n",
    "    \"\"\"\n",
    "    hist_dict_low  = {}\n",
    "    hist_dict_high = {}\n",
    "    frac_val  = np.array([2,4,10])\n",
    "    frac_dict = {2:0,4:1,10:2}\n",
    "    \n",
    "    low_err   = [[] for xx in xrange(len(frac_val))]\n",
    "    high_err  = [[] for xx in xrange(len(frac_val))]\n",
    "    \n",
    "    for ii in frac_val:\n",
    "        hist_dict_low[ii]  = {}\n",
    "        hist_dict_high[ii] = {}\n",
    "    \n",
    "        frac_data     = int(len(mass)/ii)\n",
    "        \n",
    "        frac_mass     = mass[0:frac_data]\n",
    "        counts, edges = np.histogram(frac_mass,bins)\n",
    "        low_counts    = (counts/float(len(frac_mass))/dlogM)\n",
    "        \n",
    "        low_err[frac_dict[ii]]    = np.sqrt(counts)/len(frac_mass)/dlogM\n",
    "        \n",
    "        hist_dict_low[ii]  = low_counts\n",
    "\n",
    "        frac_mass_2        = mass[-frac_data:]\n",
    "        counts_2, edges_2  = np.histogram(frac_mass_2,bins)\n",
    "        high_counts        = (counts_2/float(len(frac_mass_2))/dlogM)\n",
    "\n",
    "        high_err[frac_dict[ii]] = np.sqrt(counts_2)/len(frac_mass_2)/\\\n",
    "                                            dlogM\n",
    "        \n",
    "        hist_dict_high[ii] = high_counts\n",
    "        \n",
    "    hist_dict_low['low_err']      = low_err\n",
    "    hist_dict_high['high_err']    = high_err\n",
    "    \n",
    "    return hist_dict_low, hist_dict_high"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def mean_bin_mass(mass_dist,bins):\n",
    "    \"\"\"\n",
    "    Returns median distance to Nth nearest neighbor\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    mass_dist: array-like\n",
    "        An array with mass values in at index 0 (when transformed) \n",
    "    bins: array=like\n",
    "        A 1D array with the values which will be used as the bin edges     \n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "\n",
    "    \"\"\"\n",
    "    edges        = bins\n",
    "\n",
    "    digitized    = np.digitize(mass_dist.T[0],edges)\n",
    "    digitized   -= int(1)\n",
    "\n",
    "    bin_nums          = np.unique(digitized)\n",
    "    bin_nums_list     = list(bin_nums)\n",
    "\n",
    "    for jj in range(len(bins)-1):\n",
    "        if jj not in bin_nums:\n",
    "            bin_nums_list.append(jj)\n",
    "\n",
    "    if (len(bins)-1) in bin_nums_list:\n",
    "        bin_nums_list.remove(len(bins)-1)\n",
    "\n",
    "    if (len(bins)) in bin_nums_list:\n",
    "        bin_nums_list.remove(len(bins))\n",
    "\n",
    "    bin_nums = np.array(bin_nums_list)\n",
    "\n",
    "    mean_mass = np.array([np.mean(mass_dist.T[0][digitized==ii]) \\\n",
    "                for ii in bin_nums])\n",
    "\n",
    "    return mean_mass "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def schechter_real_func(mean_of_mass_bin,bins,phi_star,alpha,Mstar):\n",
    "    M_over_mstar = mean_of_mass_bin/Mstar\n",
    "    temp_val     = phi_star/Mstar * M_over_mstar**(alpha+1) * \\\n",
    "                        np.exp(- M_over_mstar)\n",
    "    int_val = [[] for ii in range(len(temp_val))]\n",
    "    for ii in range(len(temp_val)):\n",
    "        int_val[ii]      = integrate.quad(temp_val[ii],bins[ii],bins[ii+1])\n",
    "\n",
    "    return int_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_params(bin_int,mean_mass,count_err):\n",
    "    \"\"\"\n",
    "    Parameters\n",
    "    ----------\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    opt_v: array-like\n",
    "        Array with three values: phi_star, alpha, and M_star\n",
    "    res_arr: array-like\n",
    "        Array with two values: alpha and log_M_star\n",
    "\n",
    "\n",
    "    \"\"\"\n",
    "    xdata = 10**mean_mass\n",
    "    p0    = (1,-1.05,10**10.64)\n",
    "    opt_v,est_cov = optimize.curve_fit(schechter_real_func,xdata,\\\n",
    "                            bin_int,p0=p0,sigma=count_err)\n",
    "    alpha   = opt_v[1]\n",
    "    log_m_star    = np.log10(opt_v[2])\n",
    "    res_arr = np.array([alpha,log_m_star])\n",
    "\n",
    "    return opt_v, res_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dirpath  = r\"C:\\Users\\Hannah\\Desktop\\Vanderbilt_REU\\Stellar_mass_env_density\"\n",
    "dirpath += r\"\\Catalogs\\Resolve_plk_5001_so_mvir_scatter_ECO_Mocks_\"\n",
    "dirpath += r\"scatter_mocks\\Resolve_plk_5001_so_mvir_scatter0p2_ECO_Mocks\"\n",
    "\n",
    "usecols  = (0,1,8,13)\n",
    "dlogM    = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ECO_cats = (Index(dirpath,'.dat'))\n",
    "\n",
    "names    = ['ra','dec','cz','logMstar']\n",
    "PD       = [(pd.read_csv(ECO_cats[ii],sep=\"\\s+\", usecols= usecols,header=None,\\\n",
    "                   skiprows=2,names=names)) for ii in range(len(ECO_cats))]\n",
    "PD_comp  = [(PD[ii][PD[ii].logMstar >= 9.1]) for ii in range(len(ECO_cats))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "min_max_mass_arr = []\n",
    "\n",
    "for ii in range(len(PD_comp)):\n",
    "    min_max_mass_arr.append(max(PD_comp[ii].logMstar))\n",
    "    min_max_mass_arr.append(min(PD_comp[ii].logMstar))\n",
    "\n",
    "min_max_mass_arr = np.array(min_max_mass_arr)\n",
    "\n",
    "bins = Bins_array_create(min_max_mass_arr,dlogM)\n",
    "bins+= 0.1\n",
    "bins_list = list(bins)\n",
    "for ii in bins:\n",
    "    if ii > 11.7:\n",
    "        bins_list.remove(ii)\n",
    "\n",
    "bins = np.array(bins_list)\n",
    "\n",
    "num_of_bins = int(len(bins) - 1) \n",
    "\n",
    "ra_arr   = np.array([(np.array(PD_comp[ii])).T[0] \\\n",
    "    for ii in range(len(PD_comp))])\n",
    "dec_arr  = np.array([(np.array(PD_comp[ii])).T[1] \\\n",
    "    for ii in range(len(PD_comp))])\n",
    "cz_arr   = np.array([(np.array(PD_comp[ii])).T[2] \\\n",
    "    for ii in range(len(PD_comp))])\n",
    "mass_arr = np.array([(np.array(PD_comp[ii])).T[3] \\\n",
    "    for ii in range(len(PD_comp))])\n",
    "\n",
    "coords_test = np.array([sph_to_cart(ra_arr[vv],dec_arr[vv],cz_arr[vv]) \\\n",
    "                for vv in range(len(ECO_cats))])\n",
    "\n",
    "neigh_vals  = np.array([1,2,3,5,10,20])\n",
    "\n",
    "nn_arr      = [[] for xx in xrange(len(coords_test))]\n",
    "nn_arr_nn   = [[] for yy in xrange(len(neigh_vals))]\n",
    "\n",
    "for vv in range(len(coords_test)):\n",
    "    nn_arr[vv] = spatial.cKDTree(coords_test[vv])\n",
    "    nn_arr[vv] = np.array(nn_arr[vv].query(coords_test[vv],21)[0])\n",
    "\n",
    "nn_specs       = [(np.array(nn_arr).T[ii].T[neigh_vals].T) for ii in \\\n",
    "                    range(len(coords_test))]\n",
    "nn_mass_dist   = np.array([(np.column_stack((mass_arr[qq],nn_specs[qq]))) \\\n",
    "                    for qq in range(len(coords_test))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nn_dist    = {}\n",
    "nn_dens    = {}\n",
    "mass_dat   = {}\n",
    "ratio_info = {}\n",
    "\n",
    "mass_freq  = [[] for xx in xrange(len(coords_test))]\n",
    "\n",
    "for ii in range(len(coords_test)):\n",
    "    nn_dist[ii]    = {}\n",
    "    nn_dens[ii]    = {}\n",
    "    mass_dat[ii]   = {}\n",
    "    ratio_info[ii] = {}\n",
    "\n",
    "    nn_dist[ii]['mass'] = nn_mass_dist[ii].T[0]\n",
    "\n",
    "    for jj in range(len(neigh_vals)):\n",
    "        nn_dist[ii][(neigh_vals[jj])]  = np.array(nn_mass_dist[ii].T\\\n",
    "                                            [range(1,len(neigh_vals)+1)[jj]])\n",
    "        nn_dens[ii][(neigh_vals[jj])]  = np.column_stack((nn_mass_dist[ii].T\\\n",
    "                                            [0],calc_dens(neigh_vals[jj],\\\n",
    "                                            nn_mass_dist[ii].T[range(1,len\\\n",
    "                                                (neigh_vals)+1)[jj]])))\n",
    "\n",
    "        idx = np.array([nn_dens[ii][neigh_vals[jj]].T[1].argsort()])\n",
    "        mass_dat[ii][(neigh_vals[jj])] = (nn_dens[ii][neigh_vals[jj]]\\\n",
    "                                            [idx].T[0])\n",
    "\n",
    "        bin_centers, mass_freq[ii], ratio_info[ii][neigh_vals[jj]] = \\\n",
    "                            plot_calcs(mass_dat[ii][neigh_vals[jj]],bins,dlogM)\n",
    "\n",
    "all_mock_meds = [[] for xx in range(len(nn_mass_dist))]\n",
    "all_mock_mass_means = [[] for xx in range(len(nn_mass_dist))]\n",
    "\n",
    "for vv in range(len(nn_mass_dist)):\n",
    "    all_mock_meds[vv] = np.array([bin_func(nn_mass_dist[vv],bins,(jj+1)) \\\n",
    "                                for jj in range(len(nn_mass_dist[vv].T)-1)])\n",
    "    all_mock_mass_means[vv] = (mean_bin_mass(nn_mass_dist[vv],bins))  \n",
    "    \n",
    "med_plot_arr = [([[] for yy in xrange(len(nn_mass_dist))]) \\\n",
    "                                            for xx in xrange(len(neigh_vals))]\n",
    "\n",
    "for ii in range(len(neigh_vals)):\n",
    "    for jj in range(len(nn_mass_dist)):\n",
    "        med_plot_arr[ii][jj] = all_mock_meds[jj][ii]    \n",
    "\n",
    "# for ii in range(len(neigh_vals)):\n",
    "#     for jj in range(len(nn_mass_dist)):\n",
    "#         print len(all_mock_meds[jj][ii])\n",
    "\n",
    "# mass_freq_plot  = (np.array(mass_freq))\n",
    "# max_lim = [[] for xx in range(len(mass_freq_plot.T))]\n",
    "# min_lim = [[] for xx in range(len(mass_freq_plot.T))]\n",
    "# for jj in range(len(mass_freq_plot.T)):\n",
    "#     max_lim[jj] = max(mass_freq_plot.T[jj])\n",
    "#     min_lim[jj] = min(mass_freq_plot.T[jj])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[ 0.86946387,  0.72649573,  0.63636364,  0.56876457,  0.47707848,\n",
       "          0.48018648,  0.44289044,  0.35042735,  0.24941725,  0.11810412,\n",
       "          0.06216006,  0.01631702,  0.001554  ],\n",
       "        [ 0.02599181,  0.02375895,  0.02223635,  0.02102214,  0.01925332,\n",
       "          0.01931593,  0.01855064,  0.01650098,  0.01392111,  0.00957951,\n",
       "          0.00694971,  0.00356066,  0.00109885]]),\n",
       " array([[ 0.88682746,  0.72820037,  0.6270872 ,  0.52504638,  0.5       ,\n",
       "          0.48701299,  0.42764378,  0.35992579,  0.25974026,  0.1270872 ,\n",
       "          0.05473098,  0.01113173,  0.00463822],\n",
       "        [ 0.02868205,  0.02599059,  0.02411874,  0.02206935,  0.02153652,\n",
       "          0.02125499,  0.01991736,  0.01827246,  0.01552245,  0.01085779,\n",
       "          0.00712537,  0.00321345,  0.00207427]]),\n",
       " array([[  8.70452227e-01,   7.30193812e-01,   6.22237334e-01,\n",
       "           5.37232234e-01,   5.06630398e-01,   4.82828970e-01,\n",
       "           4.41176471e-01,   3.75722543e-01,   2.22713363e-01,\n",
       "           1.28357701e-01,   6.29037742e-02,   1.87011221e-02,\n",
       "           8.50051003e-04],\n",
       "        [  2.72016321e-02,   2.49138913e-02,   2.29985537e-02,\n",
       "           2.13699509e-02,   2.07523897e-02,   2.02590535e-02,\n",
       "           1.93654977e-02,   1.78712989e-02,   1.37592775e-02,\n",
       "           1.04456016e-02,   7.31241522e-03,   3.98709262e-03,\n",
       "           8.50051003e-04]]),\n",
       " array([[ 0.88676506,  0.69683039,  0.66053714,  0.53532543,  0.48451488,\n",
       "          0.43551899,  0.42765546,  0.37079603,  0.26373095,  0.13972901,\n",
       "          0.07137672,  0.02117106,  0.0048391 ],\n",
       "        [ 0.02316016,  0.02053056,  0.01998876,  0.01799477,  0.01711949,\n",
       "          0.01623083,  0.01608364,  0.01497631,  0.01263042,  0.00919349,\n",
       "          0.00657076,  0.00357856,  0.00171088]]),\n",
       " array([[ 0.83522683,  0.70829742,  0.60800752,  0.53318185,  0.50105775,\n",
       "          0.4646243 ,  0.4591397 ,  0.36981901,  0.26208572,  0.15121837,\n",
       "          0.07208337,  0.02585599,  0.00783515],\n",
       "        [ 0.01808884,  0.01665775,  0.01543345,  0.01445261,  0.01401046,\n",
       "          0.01349148,  0.01341161,  0.01203658,  0.01013282,  0.00769681,\n",
       "          0.00531406,  0.00318265,  0.00175199]]),\n",
       " array([[ 0.82980525,  0.71065683,  0.58969396,  0.54614733,  0.50139107,\n",
       "          0.46207814,  0.45723963,  0.38889561,  0.26430386,  0.15967098,\n",
       "          0.06411032,  0.02056369,  0.00483851],\n",
       "        [ 0.02240264,  0.02073199,  0.01888532,  0.01817465,  0.01741403,\n",
       "          0.0167174 ,  0.01662964,  0.01533655,  0.01264337,  0.00982707,\n",
       "          0.00622694,  0.00352664,  0.00171067]]),\n",
       " array([[ 0.86603742,  0.7054148 ,  0.6284153 ,  0.54479218,  0.51581388,\n",
       "          0.47690015,  0.42722305,  0.38251366,  0.24341778,  0.13661202,\n",
       "          0.05547276,  0.01490313,  0.00248385],\n",
       "        [ 0.02677755,  0.02416711,  0.02281003,  0.02123821,  0.02066565,\n",
       "          0.01987084,  0.01880745,  0.01779615,  0.01419641,  0.01063523,\n",
       "          0.00677708,  0.0035127 ,  0.00143405]]),\n",
       " array([[  8.54069674e-01,   7.57746027e-01,   5.76336491e-01,\n",
       "           5.38609729e-01,   4.65564296e-01,   4.87237117e-01,\n",
       "           4.46299567e-01,   3.82083802e-01,   2.53652272e-01,\n",
       "           1.54920533e-01,   6.74265532e-02,   1.52512442e-02,\n",
       "           8.02697062e-04],\n",
       "        [  2.61831858e-02,   2.46625325e-02,   2.15086868e-02,\n",
       "           2.07927980e-02,   1.93315052e-02,   1.97763445e-02,\n",
       "           1.89273176e-02,   1.75127823e-02,   1.42690551e-02,\n",
       "           1.11514240e-02,   7.35684009e-03,   3.49887538e-03,\n",
       "           8.02697062e-04]])]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mass_freq\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eco_path  = r\"C:\\Users\\Hannah\\Desktop\\Vanderbilt_REU\\Stellar_mass_env_density\"\n",
    "eco_path += r\"\\Catalogs\\ECO_true\"\n",
    "eco_cols  = np.array([0,1,2,4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ECO_true = (Index(eco_path,'.txt'))\n",
    "names    = ['ra','dec','cz','logMstar']\n",
    "PD_eco   = pd.read_csv(ECO_true[0],sep=\"\\s+\", usecols=(eco_cols),header=None,\\\n",
    "                skiprows=1,names=names)\n",
    "eco_comp = PD_eco[PD_eco.logMstar >= 9.1]\n",
    "\n",
    "ra_eco   = (np.array(eco_comp)).T[0]\n",
    "dec_eco  = (np.array(eco_comp)).T[1] \n",
    "cz_eco   = (np.array(eco_comp)).T[2] \n",
    "mass_eco = (np.array(eco_comp)).T[3]\n",
    "\n",
    "coords_eco        = sph_to_cart(ra_eco,dec_eco,cz_eco)\n",
    "eco_neighbor_tree = spatial.cKDTree(coords_eco)\n",
    "eco_tree_dist     = np.array(eco_neighbor_tree.query(coords_eco,\\\n",
    "                    (neigh_vals[-1]+1))[0])\n",
    "\n",
    "eco_mass_dist = np.column_stack((mass_eco,eco_tree_dist.T[neigh_vals].T))\n",
    "##range 1,7 because of the six nearest neighbors (and fact that 0 is mass)\n",
    "##the jj is there to specify which index in the [1,6] array\n",
    "eco_dens = ([calc_dens(neigh_vals[jj],\\\n",
    "            (eco_mass_dist.T[range(1,7)[jj]])) for jj in range\\\n",
    "            (len(neigh_vals))])\n",
    "\n",
    "eco_mass_dens = [(np.column_stack((mass_eco,eco_dens[ii]))) for ii in \\\n",
    "                range(len(neigh_vals))]\n",
    "eco_idx  = [(eco_mass_dens[jj].T[1].argsort()) for jj in \\\n",
    "            range(len(neigh_vals))]\n",
    "eco_mass_dat  = [(eco_mass_dens[jj][eco_idx[jj]].T[0]) for jj in \\\n",
    "                range(len(neigh_vals))]\n",
    "\n",
    "eco_ratio_info    = [[] for xx in xrange(len(eco_mass_dat))]\n",
    "\n",
    "\n",
    "for qq in range(len(eco_mass_dat)):\n",
    "    bin_centers, eco_freq, eco_ratio_info[qq] = plot_calcs(eco_mass_dat[qq],\\\n",
    "                                    bins,dlogM)\n",
    "\n",
    "eco_medians   = [[] for xx in xrange(len(eco_mass_dat))]    \n",
    "\n",
    "for jj in (range(len(eco_mass_dat))):\n",
    "    eco_medians[jj] = np.array(bin_func(eco_mass_dist,bins,(jj+1),\\\n",
    "        bootstrap=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hist_low_info  = {}\n",
    "hist_high_info = {}\n",
    "\n",
    "for ii in xrange(len(coords_test)):\n",
    "    hist_low_info[ii]  = {}\n",
    "    hist_high_info[ii] = {}\n",
    "    \n",
    "    for jj in range(len(neigh_vals)):\n",
    "        hist_low_info[ii][neigh_vals[jj]],hist_high_info[ii][neigh_vals[jj]] \\\n",
    "        = hist_calcs(mass_dat[ii][neigh_vals[jj]],bins,dlogM)\n",
    "        \n",
    "frac_vals     = [2,4,10]\n",
    "hist_low_arr  = [[[] for yy in xrange(len(nn_mass_dist))] for xx in \\\n",
    "    xrange(len(neigh_vals))]\n",
    "hist_high_arr = [[[] for yy in xrange(len(nn_mass_dist))] for xx in \\\n",
    "    xrange(len(neigh_vals))]\n",
    "\n",
    "for ii in range(len(neigh_vals)):\n",
    "    for jj in range(len(nn_mass_dist)):\n",
    "        hist_low_arr[ii][jj]  = (hist_low_info[jj][neigh_vals[ii]])\n",
    "        hist_high_arr[ii][jj] = (hist_high_info[jj][neigh_vals[ii]])\n",
    "        \n",
    "        \n",
    "        plot_low_hist  = [[[[] for yy in xrange(len(nn_mass_dist))] \\\n",
    "                         for zz in xrange(len(frac_vals))] for xx in \\\n",
    "                         xrange(len(hist_low_arr))]\n",
    "        \n",
    "        plot_high_hist = [[[[] for yy in xrange(len(nn_mass_dist))] \\\n",
    "                 for zz in xrange(len(frac_vals))] for xx in \\\n",
    "                 xrange(len(hist_high_arr))]\n",
    "\n",
    "for jj in range(len(nn_mass_dist)):\n",
    "    for hh in range(len(frac_vals)):\n",
    "        for ii in range(len(neigh_vals)):\n",
    "            plot_low_hist[ii][hh][jj]  = hist_low_arr[ii][jj][frac_vals[hh]]        \n",
    "            plot_high_hist[ii][hh][jj] = hist_high_arr[ii][jj][frac_vals[hh]] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "eco_low  = {}\n",
    "eco_high = {}\n",
    "for jj in range(len(neigh_vals)):\n",
    "    eco_low[neigh_vals[jj]]  = {}\n",
    "    eco_high[neigh_vals[jj]] = {}\n",
    "    eco_low[neigh_vals[jj]], eco_high[neigh_vals[jj]] = hist_calcs\\\n",
    "    (eco_mass_dat[jj],bins,dlogM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: {2: array([ 0.86711409,  0.78389262,  0.65503356,  0.6442953 ,  0.5409396 ,\n",
       "          0.52751678,  0.39731544,  0.27919463,  0.19060403,  0.08456376,\n",
       "          0.02550336,  0.00402685,  0.        ]),\n",
       "  4: array([ 0.87540279,  0.84049409,  0.66863588,  0.66058002,  0.56122449,\n",
       "          0.55585392,  0.38668099,  0.23093448,  0.15843179,  0.04296455,\n",
       "          0.01611171,  0.00268528,  0.        ]),\n",
       "  10: array([ 0.95302013,  0.87919463,  0.72483221,  0.68456376,  0.55704698,\n",
       "          0.59060403,  0.29530201,  0.18120805,  0.10738255,  0.02013423,\n",
       "          0.00671141,  0.        ,  0.        ]),\n",
       "  'low_err': [array([ 0.03411615,  0.03243771,  0.02965198,  0.02940792,  0.02694612,\n",
       "           0.0266097 ,  0.02309349,  0.01935866,  0.01599513,  0.01065403,\n",
       "           0.00585087,  0.0023249 ,  0.        ]),\n",
       "   array([ 0.04848408,  0.04750753,  0.04237308,  0.04211704,  0.03882071,\n",
       "           0.03863452,  0.03222342,  0.02490231,  0.02062606,  0.01074114,\n",
       "           0.00657758,  0.00268528,  0.        ]),\n",
       "   array([ 0.07997567,  0.07681559,  0.06974701,  0.06778191,  0.06114385,\n",
       "           0.0629586 ,  0.04451845,  0.03487351,  0.02684564,  0.0116245 ,\n",
       "           0.00671141,  0.        ,  0.        ])]},\n",
       " 2: {2: array([ 0.84697987,  0.79060403,  0.65771812,  0.63892617,  0.54228188,\n",
       "          0.54899329,  0.41073826,  0.26308725,  0.19463087,  0.07516779,\n",
       "          0.02818792,  0.00268456,  0.        ]),\n",
       "  4: array([ 0.93984962,  0.82706767,  0.6820623 ,  0.67132116,  0.52900107,\n",
       "          0.5424275 ,  0.37325456,  0.23361976,  0.14769066,  0.04027927,\n",
       "          0.01342642,  0.        ,  0.        ]),\n",
       "  10: array([ 0.91275168,  0.95302013,  0.77852349,  0.67785235,  0.4966443 ,\n",
       "          0.55704698,  0.30872483,  0.18120805,  0.11409396,  0.02013423,\n",
       "          0.        ,  0.        ,  0.        ]),\n",
       "  'low_err': [array([ 0.03371774,  0.03257627,  0.02971268,  0.02928513,  0.02697953,\n",
       "           0.02714597,  0.02348034,  0.01879195,  0.01616321,  0.01004472,\n",
       "           0.00615111,  0.00189827,  0.        ]),\n",
       "   array([ 0.05023708,  0.04712655,  0.04279639,  0.04245808,  0.03768977,\n",
       "           0.03816507,  0.03165904,  0.02504667,  0.0199146 ,  0.01040006,\n",
       "           0.00600448,  0.        ,  0.        ]),\n",
       "   array([ 0.07826781,  0.07997567,  0.07228409,  0.06744883,  0.05773373,\n",
       "           0.06114385,  0.04551899,  0.03487351,  0.02767185,  0.0116245 ,\n",
       "           0.        ,  0.        ,  0.        ])]},\n",
       " 3: {2: array([ 0.85234899,  0.78791946,  0.67651007,  0.61342282,  0.54362416,\n",
       "          0.55033557,  0.40536913,  0.28053691,  0.17449664,  0.08053691,\n",
       "          0.03221477,  0.00268456,  0.        ]),\n",
       "  4: array([ 0.88077336,  0.79752954,  0.73308271,  0.67669173,  0.55048335,\n",
       "          0.54779807,  0.349087  ,  0.2443609 ,  0.15574651,  0.04296455,\n",
       "          0.02148228,  0.        ,  0.        ]),\n",
       "  10: array([ 0.95973154,  0.91275168,  0.75838926,  0.63087248,  0.53691275,\n",
       "          0.55033557,  0.31543624,  0.18120805,  0.12751678,  0.01342282,\n",
       "          0.01342282,  0.        ,  0.        ]),\n",
       "  'low_err': [array([ 0.03382444,  0.03252092,  0.03013415,  0.02869471,  0.0270129 ,\n",
       "           0.02717914,  0.02332637,  0.01940514,  0.01530437,  0.01039727,\n",
       "           0.00657581,  0.00189827,  0.        ]),\n",
       "   array([ 0.04863257,  0.04627736,  0.04436818,  0.04262757,  0.03844742,\n",
       "           0.03835354,  0.03061696,  0.02561598,  0.02045052,  0.01074114,\n",
       "           0.00759513,  0.        ,  0.        ]),\n",
       "   array([ 0.08025678,  0.07826781,  0.07134326,  0.06506953,  0.06002867,\n",
       "           0.0607744 ,  0.0460111 ,  0.03487351,  0.02925436,  0.00949137,\n",
       "           0.00949137,  0.        ,  0.        ])]},\n",
       " 5: {2: array([ 0.86174497,  0.77718121,  0.64697987,  0.62013423,  0.55167785,\n",
       "          0.55973154,  0.39597315,  0.29530201,  0.17583893,  0.08322148,\n",
       "          0.03087248,  0.00134228,  0.        ]),\n",
       "  4: array([ 0.8915145 ,  0.79752954,  0.71965628,  0.64178303,  0.5424275 ,\n",
       "          0.56390977,  0.37325456,  0.26852846,  0.1396348 ,  0.04027927,\n",
       "          0.02148228,  0.        ,  0.        ]),\n",
       "  10: array([ 0.93959732,  0.87919463,  0.72483221,  0.60402685,  0.57718121,\n",
       "          0.59060403,  0.3557047 ,  0.20805369,  0.10067114,  0.01342282,\n",
       "          0.00671141,  0.        ,  0.        ]),\n",
       "  'low_err': [array([ 0.03401036,  0.03229855,  0.02946913,  0.02885126,  0.02721226,\n",
       "           0.02741017,  0.02305445,  0.01990926,  0.01536312,  0.01056914,\n",
       "           0.00643736,  0.00134228,  0.        ]),\n",
       "   array([ 0.04892821,  0.04627736,  0.04396   ,  0.04151349,  0.03816507,\n",
       "           0.03891347,  0.03165904,  0.02685285,  0.01936386,  0.01040006,\n",
       "           0.00759513,  0.        ,  0.        ]),\n",
       "   array([ 0.07941047,  0.07681559,  0.06974701,  0.06367002,  0.06223905,\n",
       "           0.0629586 ,  0.0488598 ,  0.03736755,  0.02599318,  0.00949137,\n",
       "           0.00671141,  0.        ,  0.        ])]},\n",
       " 10: {2: array([ 0.85771812,  0.76912752,  0.65100671,  0.61073826,  0.53288591,\n",
       "          0.56241611,  0.40134228,  0.31006711,  0.18926174,  0.07651007,\n",
       "          0.03624161,  0.00268456,  0.        ]),\n",
       "  4: array([ 0.92105263,  0.77873255,  0.72234157,  0.6471536 ,  0.52094522,\n",
       "          0.55048335,  0.35982814,  0.28464017,  0.14769066,  0.04296455,\n",
       "          0.02416756,  0.        ,  0.        ]),\n",
       "  10: array([ 0.97986577,  0.84563758,  0.73825503,  0.61744966,  0.55033557,\n",
       "          0.54362416,  0.30872483,  0.26845638,  0.10067114,  0.03355705,\n",
       "          0.01342282,  0.        ,  0.        ]),\n",
       "  'low_err': [array([ 0.0339308 ,  0.03213076,  0.02956069,  0.02863185,  0.02674478,\n",
       "           0.02747582,  0.02321022,  0.02040092,  0.01593871,  0.01013401,\n",
       "           0.0069747 ,  0.00189827,  0.        ]),\n",
       "   array([ 0.04973217,  0.04572875,  0.04404194,  0.04168683,  0.03740169,\n",
       "           0.03844742,  0.03108442,  0.0276467 ,  0.0199146 ,  0.01074114,\n",
       "           0.00805585,  0.        ,  0.        ]),\n",
       "   array([ 0.08109427,  0.07533538,  0.07038986,  0.06437358,  0.0607744 ,\n",
       "           0.06040268,  0.04551899,  0.04244668,  0.02599318,  0.01500717,\n",
       "           0.00949137,  0.        ,  0.        ])]},\n",
       " 20: {2: array([ 0.84563758,  0.75838926,  0.63221477,  0.60536913,  0.54228188,\n",
       "          0.57583893,  0.40402685,  0.30872483,  0.20805369,  0.07651007,\n",
       "          0.03892617,  0.00402685,  0.        ]),\n",
       "  4: array([ 0.9264232 ,  0.76799141,  0.67132116,  0.60687433,  0.53168636,\n",
       "          0.53437164,  0.37862513,  0.30343716,  0.19065521,  0.06176155,\n",
       "          0.02416756,  0.00268528,  0.        ]),\n",
       "  10: array([ 1.        ,  0.87248322,  0.71812081,  0.59731544,  0.52348993,\n",
       "          0.53691275,  0.34228188,  0.24161074,  0.12751678,  0.03355705,\n",
       "          0.00671141,  0.        ,  0.        ]),\n",
       "  'low_err': [array([ 0.03369101,  0.03190568,  0.02913092,  0.02850572,  0.02697953,\n",
       "           0.02780177,  0.02328772,  0.02035671,  0.01671127,  0.01013401,\n",
       "           0.00722841,  0.0023249 ,  0.        ]),\n",
       "   array([ 0.04987695,  0.04541228,  0.04245808,  0.04036868,  0.0377853 ,\n",
       "           0.0378806 ,  0.03188599,  0.02854497,  0.02262661,  0.01287817,\n",
       "           0.00805585,  0.00268528,  0.        ]),\n",
       "   array([ 0.08192319,  0.07652184,  0.06942336,  0.06331531,  0.05927356,\n",
       "           0.06002867,  0.04792905,  0.04026846,  0.02925436,  0.01500717,\n",
       "           0.00671141,  0.        ,  0.        ])]}}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eco_low\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
{'fraction_non_alphanumeric': 0.2021758436944938, 'fraction_numerical': 0.1336367673179396, 'mean_word_length': 1.7690274191565227, 'pattern_counts': {'":': 172, '<': 7, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3947914', 'n_tokens_mistral': 21816, 'n_tokens_neox': 18481, 'n_words': 2990}