<commit_before>import json
import random
import datetime
import calendar
import time
import zipfile
import requests
import urlparse
import os

"""
Local file system data provider
A home for functions so code is not duplicated
"""

class Filesystem(object):
	
	def __init__(self, tmp_dir):
		self.tmp_dir = tmp_dir
	
	def read_document_to_content(self, document):
		mode = "r"
		
		try:
			o = urlparse.urlparse(document)
			if(o.scheme != ""):
				document = self.download_document(document)
				self.document = document
		except AttributeError:
			pass
		
		if(self.is_zip(document)):
			document = self.unzip_document(document)
			self.document = document
		
		f = open(document, mode)
		self.content = f.read()
		f.close()

	def download_document(self, document, filename = None, validate_url = True, scheme = None, netloc = None):
		"""
		Attempt to download the document, with some simple
		URL validation built in
		"""

		o = urlparse.urlparse(document)
		
		if(validate_url == True):
			if(scheme is not None and netloc is not None):
				# User defined scheme and netloc to validate
				pass
			else:<commit_msg>Add content writing function from ArticleToFluidinfo to the filesystem provider.<commit_after>import json
import random
import datetime
import calendar
import time
import zipfile
import requests
import urlparse
import os

"""
Local file system data provider
A home for functions so code is not duplicated
"""

class Filesystem(object):
	
	def __init__(self, tmp_dir):
		self.tmp_dir = tmp_dir
	
	def read_document_to_content(self, document):
		mode = "r"
		
		try:
			o = urlparse.urlparse(document)
			if(o.scheme != ""):
				document = self.download_document(document)
				self.document = document
		except AttributeError:
			pass
		
		if(self.is_zip(document)):
			document = self.unzip_document(document)
			self.document = document
		
		f = open(document, mode)
		self.content = f.read()
		f.close()

	def write_content_to_document(self, filename):
		mode = "w"
		
		f = self.open_file_from_tmp_dir(filename, mode)
		f.write(self.content)
		f.close()
		
		# Reset the object document
		tmp_dir = self.get_tmp_dir()
		if(tmp_dir):
			self.document = tmp_dir + os.sep + filename
		else:
			self.document = filename

	def download_document(self, document, filename = None, validate_url = True, scheme = None, netloc = None):
		"""
		Attempt to download the document, with some simple
		URL validation built in
		"""

		o = urlparse.urlparse(document)
		
		if(validate_url == True):
			if(scheme is not None and netloc is not None):
				# User defined scheme and netloc to validate
				pass
			else:
{'fraction_non_alphanumeric': 0.0916030534351145, 'fraction_numerical': 0.0, 'mean_word_length': 3.8091743119266055, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2812500', 'n_tokens_mistral': 927, 'n_tokens_neox': 786, 'n_words': 254}