<commit_before>import logging
from pymongo import MongoClient
import numpy as np
import urllib2
from StringIO import StringIO
from gzip import GzipFile
import json
from ast import literal_eval
import os
from astropy.io import fits
from astropy import wcs, coordinates as coord, units as u
import astroquery
from astroquery.irsa import Irsa
import time
import argparse

# Custom modules for the RGZ catalog

import catalogFunctions as fn #contains custom functions
import contourNode as c #contains Node class
from updateConsensus import updateConsensus #replaces the current consensus collection with a specified csv

rgz_path = '/Users/willettk/Astronomy/Research/GalaxyZoo/rgz-analysis'
data_path = '/Volumes/REISEPASS/'

@profile
def RGZcatalog():

    #begin logging even if not run from command line
    logging.basicConfig(filename='RGZcatalog.log', level=logging.DEBUG, format='%(asctime)s %(message)s')
    logging.captureWarnings(True)

    #check if consensus collection needs to be updated
    parser = argparse.ArgumentParser()
    parser.add_argument('--consensus', help='replace the current consensus collection with a specified csv')
    args = parser.parse_args()<commit_msg>Paths option for both zrak and tabernacle<commit_after>import logging
import urllib2
import json
import os
import time
import argparse
import numpy as np
from StringIO import StringIO
from gzip import GzipFile
from ast import literal_eval

from pymongo import MongoClient
from astropy.io import fits
from astropy import wcs, coordinates as coord, units as u
import astroquery
from astroquery.irsa import Irsa

# Custom modules for the RGZ catalog

import catalogFunctions as fn #contains custom functions
import contourNode as c #contains Node class
from updateConsensus import updateConsensus #replaces the current consensus collection with a specified csv

# Set up the local data paths. Currently works from UMN servers on tabernacle, plus
# <NAME>'s laptop.

def determine_paths(paths):

    found_path = False
    for path in paths:
        if os.path.exists(path):
            found_path = True
            return path

    if found_path == False:
        print "Unable to find the hardcoded local path:"
        print paths
        return None

rgz_path = determine_paths(('/Users/willettk/Astronomy/Research/GalaxyZoo/rgz-analysis',
                           '/data/tabernacle/larry/RGZdata/rgz-analysis'))
data_path = determine_paths(('/Volumes/REISEPASS/','/data/extragal/willett'))

@profile
def RGZcatalog():

    #begin logging even if not run from command line
    logging.basicConfig(filename='RGZcatalog.log', level=logging.DEBUG, format='%(asctime)s %(message)s')
    logging.captureWarnings(True)

    #check if consensus collection needs to be updated
    parser = argparse.ArgumentParser()
    parser.add_argument('--consensus', help='replace the current consensus collection with a specified csv')
    args = parser.parse_args()
{'fraction_non_alphanumeric': 0.06732403944236655, 'fraction_numerical': 0.0006800408024481469, 'mean_word_length': 4.398165137614679, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '490501', 'n_tokens_mistral': 827, 'n_tokens_neox': 798, 'n_words': 307}