<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Meredith/Applications/Ureka/variants/common/lib/python2.7/site-packages/IPython/kernel/__init__.py:13: ShimWarning: The `IPython.kernel` package has been deprecated. You should import from ipykernel or jupyter_client instead.\n",
      "  \"You should import from ipykernel or jupyter_client instead.\", ShimWarning)\n",
      "/Users/Meredith/Applications/Ureka/variants/common/lib/python2.7/site-packages/Crypto/Util/number.py:57: PowmInsecureWarning: Not using mpz_powm_sec.  You should rebuild using libgmp >= 5 to avoid timing attack vulnerability.\n",
      "  _warn(\"Not using mpz_powm_sec.  You should rebuild using libgmp >= 5 to avoid timing attack vulnerability.\", PowmInsecureWarning)\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from astropy.io import fits\n",
    "import paramiko"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is specifically for analyzing 1-meter photometry data, which I have stored in a specific place."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "connecting to astronomy first\n",
      "success! connected\n"
     ]
    }
   ],
   "source": [
    "fullFilePath = '/mnt/mrawls/1m_obs/'\n",
    "hostname = 'seismo'\n",
    "username = 'mrawls'\t\t# password is magically saved in SSH file-land\n",
    "client = paramiko.SSHClient()\n",
    "client.set_missing_host_key_policy(paramiko.AutoAddPolicy())\n",
    "client.load_system_host_keys()\n",
    "try:\t\t# maybe we're on campus\n",
    "\tclient.connect(hostname, 22, username)\n",
    "\tprint('success! connected')\n",
    "except:\t\t# maybe we're not on campus\n",
    "\tprint('connecting to astronomy first')\n",
    "\tclient.connect('astronomy.nmsu.edu', 22, username)\n",
    "\tproxy_command = 'ssh %<EMAIL> nc %s %s' % (username, hostname+'.nmsu.edu', 22)\n",
    "\tproxy = paramiko.ProxyCommand(proxy_command)\n",
    "\tclient = paramiko.SSHClient()\n",
    "\tclient.set_missing_host_key_policy(paramiko.AutoAddPolicy())\n",
    "\tclient.connect(hostname, 22, username, sock=proxy)\n",
    "\tprint('success! connected')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sftp = client.open_sftp()\n",
    "listOfAllFiles = sftp.listdir(fullFilePath)\t# get all files and directories\n",
    "listOfDirs = []\n",
    "for item in listOfAllFiles:\t\t\t\t\t# get directories that are dates (eg '140101')\n",
    "\tif item[0:2] == ('14' or '15'):\n",
    "\t\tlistOfDirs.append(fullFilePath + str(item) + '/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/mnt/mrawls/1m_obs/140626/', '/mnt/mrawls/1m_obs/140628/', '/mnt/mrawls/1m_obs/140629/', '/mnt/mrawls/1m_obs/140630/', '/mnt/mrawls/1m_obs/140829/', '/mnt/mrawls/1m_obs/140831/', '/mnt/mrawls/1m_obs/140901/', '/mnt/mrawls/1m_obs/140902/', '/mnt/mrawls/1m_obs/140903/', '/mnt/mrawls/1m_obs/140910/', '/mnt/mrawls/1m_obs/140414/', '/mnt/mrawls/1m_obs/140418/', '/mnt/mrawls/1m_obs/140421/', '/mnt/mrawls/1m_obs/140422/', '/mnt/mrawls/1m_obs/140424/', '/mnt/mrawls/1m_obs/140425/', '/mnt/mrawls/1m_obs/140429/', '/mnt/mrawls/1m_obs/140430/', '/mnt/mrawls/1m_obs/140504/', '/mnt/mrawls/1m_obs/140507/', '/mnt/mrawls/1m_obs/140508/', '/mnt/mrawls/1m_obs/140509/', '/mnt/mrawls/1m_obs/140515/', '/mnt/mrawls/1m_obs/140518/', '/mnt/mrawls/1m_obs/140520/', '/mnt/mrawls/1m_obs/140522/']\n",
      "successfully accessed /mnt/mrawls/1m_obs/140626/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140628/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140629/ directory"
     ]
    }
   ],
   "source": [
    "listOfFitsFiles = []\n",
    "#test = sftp.file('/mnt/mrawls/1m_obs/140522/140522.108.fits')\n",
    "#print(test)\n",
    "#print(listOfDirs)\n",
    "for dir in listOfDirs:\n",
    "    try:\n",
    "        listOfFilesInDir = sftp.listdir(dir)\n",
    "        print('successfully accessed {0} directory'.format(dir))\n",
    "    except:\n",
    "        print('skipping {0} directory'.format(dir))\n",
    "        #continue\n",
    "    for file in listOfFilesInDir:\n",
    "        if file[-4:] == ('fits' or 'FITS'):\n",
    "            fullfilepath = dir+file\n",
    "            try:\n",
    "                sftp.file(fullfilepath)\n",
    "                listOfFitsFiles.append( sftp.file(fullfilepath))\n",
    "            except:\n",
    "                #print('Can\\'t access file {0} with SFTP'.format(fullfilepath))\n",
    "                continue\n",
    "#print(fits.open(listOfFitsFiles[0], ignore_missing_end=True)[0].header) #example\n",
    "for object in listOfFitsFiles:\n",
    "    hdu = fits.open(object, ignore_missing_end = True)\n",
    "    print(hdu[0].header['date-obs'], hdu[0].header['RA'], hdu[0].header['filtname'][-9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
<commit_msg>nested loops properly, still only accessing 1st dir<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Meredith/Applications/Ureka/variants/common/lib/python2.7/site-packages/IPython/kernel/__init__.py:13: ShimWarning: The `IPython.kernel` package has been deprecated. You should import from ipykernel or jupyter_client instead.\n",
      "  \"You should import from ipykernel or jupyter_client instead.\", ShimWarning)\n",
      "/Users/Meredith/Applications/Ureka/variants/common/lib/python2.7/site-packages/Crypto/Util/number.py:57: PowmInsecureWarning: Not using mpz_powm_sec.  You should rebuild using libgmp >= 5 to avoid timing attack vulnerability.\n",
      "  _warn(\"Not using mpz_powm_sec.  You should rebuild using libgmp >= 5 to avoid timing attack vulnerability.\", PowmInsecureWarning)\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from astropy.io import fits\n",
    "import paramiko"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is specifically for analyzing 1-meter photometry data, which I have stored in a specific place."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "connecting to astronomy first\n",
      "success! connected\n"
     ]
    }
   ],
   "source": [
    "fullFilePath = '/mnt/mrawls/1m_obs/'\n",
    "hostname = 'seismo'\n",
    "username = 'mrawls'\t\t# password is magically saved in SSH file-land\n",
    "client = paramiko.SSHClient()\n",
    "client.set_missing_host_key_policy(paramiko.AutoAddPolicy())\n",
    "client.load_system_host_keys()\n",
    "try:\t\t# maybe we're on campus\n",
    "\tclient.connect(hostname, 22, username)\n",
    "\tprint('success! connected')\n",
    "except:\t\t# maybe we're not on campus\n",
    "\tprint('connecting to astronomy first')\n",
    "\tclient.connect('astronomy.nmsu.edu', 22, username)\n",
    "\tproxy_command = 'ssh %s@astr<EMAIL> nc %s %s' % (username, hostname+'.nmsu.edu', 22)\n",
    "\tproxy = paramiko.ProxyCommand(proxy_command)\n",
    "\tclient = paramiko.SSHClient()\n",
    "\tclient.set_missing_host_key_policy(paramiko.AutoAddPolicy())\n",
    "\tclient.connect(hostname, 22, username, sock=proxy)\n",
    "\tprint('success! connected')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sftp = client.open_sftp()\n",
    "listOfAllFiles = sftp.listdir(fullFilePath)\t# get all files and directories\n",
    "listOfDirs = []\n",
    "for item in listOfAllFiles:\t\t\t\t\t# get directories that are dates (eg '140101')\n",
    "\tif item[0:2] == ('14' or '15'):\n",
    "\t\tlistOfDirs.append(fullFilePath + str(item) + '/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "successfully accessed /mnt/mrawls/1m_obs/140628/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140629/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140630/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140829/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140831/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140901/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140902/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140903/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140910/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140414/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140418/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140421/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140422/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140424/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140425/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140429/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140430/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140504/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140507/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140508/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140509/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140515/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140518/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140520/ directory\n",
      "skipping /mnt/mrawls/1m_obs/140522/ directory\n",
      "2014-06-28 19:58:18.6 B\n",
      "2014-06-28 19:58:18.6 V\n",
      "2014-06-28 19:58:18.6 R\n",
      "2014-06-28 19:58:18.6 I\n",
      "2014-06-28 19:58:18.6 B\n",
      "2014-06-28 19:58:18.6 V\n",
      "2014-06-28 19:58:18.5 R\n",
      "2014-06-28 19:58:18.6 I\n",
      "2014-06-28 19:58:18.6 B\n",
      "2014-06-28 19:58:18.6 V\n",
      "2014-06-28 19:58:18.6 R\n",
      "2014-06-28 19:58:18.6 I\n",
      "2014-06-28 19:58:18.7 B\n",
      "2014-06-28 19:58:18.6 V\n",
      "2014-06-28 19:58:18.6 R\n",
      "2014-06-28 19:58:18.6 I\n",
      "2014-06-28 19:58:18.6 B\n",
      "2014-06-28 19:58:18.6 V\n",
      "2014-06-28 19:58:18.6 R\n",
      "2014-06-28 19:58:18.6 I\n",
      "2014-06-28 19:58:18.6 B\n",
      "2014-06-28 19:58:18.6 V\n",
      "2014-06-28 19:58:18.6 R\n",
      "2014-06-28 19:58:18.6 I\n",
      "2014-06-28 19:58:18.6 B\n",
      "2014-06-28 19:58:18.6 V\n",
      "2014-06-28 19:58:18.7 R\n",
      "2014-06-28 19:58:18.6 I\n",
      "2014-06-28 19:58:18.6 B\n",
      "2014-06-28 19:58:18.6 V\n",
      "2014-06-28 19:58:18.6 R\n",
      "2014-06-28 19:58:18.6 I\n",
      "2014-06-28 19:50:38.5 V\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.7 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:49:42.8 I\n",
      "2014-06-28 19:54:50.5 B\n",
      "2014-06-28 19:54:50.4 V\n",
      "2014-06-28 19:54:50.4 R\n",
      "2014-06-28 19:54:50.4 I\n",
      "2014-06-28 19:54:50.4 B\n",
      "2014-06-28 19:54:50.3 V\n",
      "2014-06-28 19:54:50.4 R\n",
      "2014-06-28 19:54:50.4 I\n",
      "2014-06-28 19:54:50.4 B\n",
      "2014-06-28 19:54:50.4 V\n",
      "2014-06-28 19:54:50.4 R\n",
      "2014-06-28 19:54:50.4 I\n",
      "2014-06-28 19:54:50.4 B\n",
      "2014-06-28 19:54:50.4 V\n",
      "2014-06-28 19:54:50.4 R\n",
      "2014-06-28 19:54:50.4 I\n",
      "2014-06-28 19:54:50.4 B\n",
      "2014-06-28 19:54:50.4 V\n",
      "2014-06-28 19:54:50.4 R\n",
      "2014-06-28 19:54:50.3 I\n",
      "2014-06-28 19:54:50.4 B\n",
      "2014-06-28 19:54:50.4 V\n",
      "2014-06-28 19:54:50.4 R\n",
      "2014-06-28 19:54:50.4 I\n",
      "2014-06-28 19:54:50.4 B\n",
      "2014-06-28 19:54:50.4 V\n",
      "2014-06-28 19:54:50.4 R\n",
      "2014-06-28 19:54:50.4 I\n",
      "2014-06-28 19:54:50.4 B\n",
      "2014-06-28 19:54:50.4 V\n",
      "2014-06-28 19:54:50.4 R\n",
      "2014-06-28 19:54:50.4 I\n",
      "2014-06-28 20:18:26.8 V\n",
      "2014-06-28 20:18:26.6 I\n",
      "2014-06-28 20:18:26.6 I\n",
      "2014-06-28 20:18:26.6 I\n",
      "2014-06-28 20:18:26.6 I\n",
      "2014-06-28 20:18:26.6 I\n",
      "2014-06-28 20:18:26.6 I\n",
      "2014-06-28 20:18:26.6 I\n",
      "2014-06-28 19:58:18.6 B\n",
      "2014-06-28 19:58:18.6 V\n",
      "2014-06-28 19:58:18.6 R\n",
      "2014-06-28 19:58:18.5 I\n"
     ]
    }
   ],
   "source": [
    "listOfFitsFiles = []\n",
    "#test = sftp.file('/mnt/mrawls/1m_obs/140522/140522.108.fits')\n",
    "#print(test)\n",
    "#print(listOfDirs)\n",
    "for dir in listOfDirs[1:]: # THIS IS BREAKING... it can only access the first dir in this list for some reason.\n",
    "    try:\n",
    "        listOfFilesInDir = sftp.listdir(dir)\n",
    "        print('successfully accessed {0} directory'.format(dir))\n",
    "        for file in listOfFilesInDir:\n",
    "            if file[-4:] == ('fits' or 'FITS'):\n",
    "                fullfilepath = dir+file\n",
    "                try:\n",
    "                    sftp.file(fullfilepath)\n",
    "                    listOfFitsFiles.append(sftp.file(fullfilepath))\n",
    "                except:\n",
    "                    #print('Can\\'t access file {0} with SFTP'.format(fullfilepath))\n",
    "                    continue\n",
    "    except:\n",
    "        print('skipping {0} directory'.format(dir))\n",
    "        #continue\n",
    "\n",
    "#print(fits.open(listOfFitsFiles[0], ignore_missing_end=True)[0].header) #example\n",
    "for object in listOfFitsFiles:\n",
    "    hdu = fits.open(object, ignore_missing_end = True)\n",
    "    print(hdu[0].header['date-obs'], hdu[0].header['RA'], hdu[0].header['filtname'][-9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}

{'fraction_non_alphanumeric': 0.2018291929304165, 'fraction_numerical': 0.12309974045235447, 'mean_word_length': 2.510412147505423, 'pattern_counts': {'":': 122, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4581305', 'n_tokens_mistral': 8017, 'n_tokens_neox': 6793, 'n_words': 1115}