<commit_before>from datetime import timedelta
from datetime import datetime
import os
import ocfiledates
from dateutil.relativedelta import relativedelta
import urllib2
import bz2


class Processor():
	"""
	Contains all processing functions general to all Oceancolor products. 
	"""

	def __init__(self, start_date, end_date, res, time_composite, filenames):
		self.start_date     = start_date
		self.end_date       = end_date
		self.res            = res
		self.time_composite = time_composite
		self.fns            = filenames



	def extract(self, targetfile):
                """
                Method to download and extract a file 
                """
                f_download = 'http://oceandata.sci.gsfc.nasa.gov/cgi/getfile/{}.bz2'.format(os.path.basename(targetfile))
                print f_download
		f_compress = '{}.bz2'.format(targetfile)
                f_uncompress = targetfile
		print 'f_uncompress', f_uncompress


		try:
                	thefile = urllib2.urlopen(f_download)
                	f = open(f_compress, 'wb')
                	f.write(thefile.read())
              	  	f.close()
                	uncom = bz2.BZ2File(f_compress, 'r').read()
               	 	output = open(f_uncompress, 'w')
                	output.write(uncom)
                	output.close()
                	return f_uncompress

	        except:
                        return 1


        def createfilenames(self):
                """
                Creates a list of filenames based on inputs 
                """
                minyear = self.start_date.year
                maxyear = self.end_date.year

                leap_years = [x for x in range(minyear, maxyear+1)
                        if (x % 400 == 0) or (x % 4 == 0 and not x % 100 == 0)]

                filenames = []
                if self.time_composite == 'Annual':

                        minyear = self.start_date.year
                        maxyear = self.end_date.year

                        for year in range(minyear, maxyear + 1):
                                if year in leap_years:
                                        filenames.append(self.fns['fn_annual'].format(year, 366, self.res))
                                else:
                                        filenames.append(self.fns['fn_annual'].format(year, 365, self.res))
<commit_msg>fixed uncompressing files so works in Windows now<commit_after>from datetime import timedelta
from datetime import datetime
import os
import ocfiledates
from dateutil.relativedelta import relativedelta
import urllib2
import bz2


class Processor():
	"""
	Contains all processing functions general to all Oceancolor products. 
	"""

	def __init__(self, start_date, end_date, res, time_composite, filenames):
		self.start_date     = start_date
		self.end_date       = end_date
		self.res            = res
		self.time_composite = time_composite
		self.fns            = filenames



	def extract(self, targetfile):
                """
                Method to download and extract a file 
                """
                f_download = 'http://oceandata.sci.gsfc.nasa.gov/cgi/getfile/{}.bz2'.format(os.path.basename(targetfile))
		f_compress = '{}.bz2'.format(targetfile)
                f_uncompress = targetfile


		try:
                	thefile = urllib2.urlopen(f_download)
                	f = open(f_compress, 'wb')
                	f.write(thefile.read())
              	  	f.close()
                	uncom = bz2.BZ2File(f_compress, 'rb').read()
               	 	output = open(f_uncompress, 'wb')
                	output.write(uncom)
                	output.close()
                	return f_uncompress

	        except:
                        return 1


        def createfilenames(self):
                """
                Creates a list of filenames based on inputs 
                """
                minyear = self.start_date.year
                maxyear = self.end_date.year

                leap_years = [x for x in range(minyear, maxyear+1)
                        if (x % 400 == 0) or (x % 4 == 0 and not x % 100 == 0)]

                filenames = []
                if self.time_composite == 'Annual':

                        minyear = self.start_date.year
                        maxyear = self.end_date.year

                        for year in range(minyear, maxyear + 1):
                                if year in leap_years:
                                        filenames.append(self.fns['fn_annual'].format(year, 366, self.res))
                                else:
                                        filenames.append(self.fns['fn_annual'].format(year, 365, self.res))

{'fraction_non_alphanumeric': 0.08803271631510977, 'fraction_numerical': 0.011192423590185106, 'mean_word_length': 1.4393700787401575, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6801229', 'n_tokens_mistral': 1407, 'n_tokens_neox': 1265, 'n_words': 300}