<commit_before>from __future__ import print_function
from os import makedirs

__author__ = "<NAME>"
__date__ = "$Dec 10, 2016 2:23:14 PM$"

import os
import sys
import shutil
import tempfile
from urllib.parse import urlparse, urlunparse
from os import listdir
from os.path import isfile, join, isdir

try:
    from hdfs import InsecureClient
except:
    pass

class PosixFileSystem():
    
    def makedirs(self, path):
        if not os.path.exists(path):
            os.makedirs(path) 
        return path
    
    def remove(self, path):
        if os.path.isdir(path):
            shutil.rmtree(path)
        elif os.path.isfile(path):
            os.remove(path)
            
    def addfolder(self, path):
        return self.makedirs(path)
    
    def rename(self, oldpath, newpath):
        os.rename(oldpath, newpath)
    
    def get_files(self, path):
        return [f for f in listdir(path) if isfile(join(path, f))]
    
    def get_folders(self, path):
        return [f for f in listdir(path) if isdir(join(path, f))]
    
    def read(self, path):
        with open(path) as reader:
            return reader.read()
        
    def write(self, path, content):
        with open(path, 'w') as writer:
            return writer.write(content)
            
class HadoopFileSystem():
    def __init__(self, addr, user):
        self.client = InsecureClient(addr, user=user)
    
    def normaize_path(self, path):
        u = urlparse(path)
        return u.path
        
    def makedirs(self, path):<commit_msg>normalize filesystem path<commit_after>from __future__ import print_function
from os import makedirs

__author__ = "<NAME>"
__date__ = "$Dec 10, 2016 2:23:14 PM$"

import os
import sys
import shutil
import tempfile
from urllib.parse import urlparse, urlunparse
from os import listdir
from os.path import isfile, join, isdir

try:
    from hdfs import InsecureClient
except:
    pass

class PosixFileSystem():
    
    def __init__(self):
        self.localdir = path.join(path.abspath(path.dirname(__file__)), 'storage')
    
    def normaize_path(self, path):
        return join(self.localdir, path)
        
    def makedirs(self, path):
        path = self.normaize_path(path)
        if not os.path.exists(path):
            os.makedirs(path) 
        return path
    
    def remove(self, path):
        path = self.normaize_path(path)
        if os.path.isdir(path):
            shutil.rmtree(path)
        elif os.path.isfile(path):
            os.remove(path)
            
    def addfolder(self, path):
        path = self.normaize_path(path)
        return self.makedirs(path)
    
    def rename(self, oldpath, newpath):
        path = self.normaize_path(path)
        os.rename(oldpath, newpath)
    
    def get_files(self, path):
        path = self.normaize_path(path)
        return [f for f in listdir(path) if isfile(join(path, f))]
    
    def get_folders(self, path):
        path = self.normaize_path(path)
        return [f for f in listdir(path) if isdir(join(path, f))]
    
    def read(self, path):
        path = self.normaize_path(path)
        with open(path) as reader:
            return reader.read()
        
    def write(self, path, content):
        path = self.normaize_path(path)
        with open(path, 'w') as writer:
            return writer.write(content)
            
class HadoopFileSystem():
    def __init__(self, addr, user):
        self.client = InsecureClient(addr, user=user)
    
    def normaize_path(self, path):
        u = urlparse(path)
        return u.path
        
    def makedirs(self, path):
{'fraction_non_alphanumeric': 0.099581589958159, 'fraction_numerical': 0.006136680613668061, 'mean_word_length': 2.3389199255121045, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1906888', 'n_tokens_mistral': 1179, 'n_tokens_neox': 1153, 'n_words': 308}