<commit_before>import datetime
from dateutil.parser import parse
import errno
import hashlib
import os
import requests


def basePath(path):
    """
    Returns the path of files in the sync folder
    without the leading system folder separation
    """
    if path.startswith("/"):
        path = path.replace("/", "", 1)
    if path.startswith("\\"):
        path = path.replace("\\", "", 1)
    return path


def createLocalDirs(path):
    """Tries to create local directories"""
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise


def getFileHash(filepath):
    """
    Returns the MD5 hash of a local file
    """
    fileToHash = open(filepath)
    md5 = hashlib.md5()
    while True:
        currentLine = fileToHash.readline()
        if not currentLine:
            break
        md5.update(currentLine)
    return md5.hexdigest()


def get_server_time():
    """
    Returns the time of the SmartFile servers
    """
    #TODO: cache the time
    response = requests.get('https://www.smartfile.com')
    time = parse(response.headers['Date']).replace(tzinfo=None, second=0)
    return time


def calculate_time_offset():
    """
    Calculates a time offset, duh!
    This should be tested more. If the offset calculation is incorrect,
    unchanged files will be put into sync up/down queues, and since a sync up
    on a file that has not been changed will cause corruption.
    """
    server_time = get_server_time()
    now = datetime.datetime.now().replace(microsecond=0, second=0)
    offset = now - server_time
    return offset<commit_msg>Cached the time offset in common<commit_after>import datetime
from dateutil.parser import parse
import errno
import hashlib
import os
import requests

_cache = {}


def cache(key):
    def _decorating_wrapper(func):
        def _caching_wrapper(*args, **kwargs):
            if _cache.has_key(key):
                return _cache[key]
            value = func(*args, **kwargs)
            _cache[key] = value
            return value
        return _caching_wrapper
    return _decorating_wrapper


def basePath(path):
    """
    Returns the path of files in the sync folder
    without the leading system folder separation
    """
    if path.startswith("/"):
        path = path.replace("/", "", 1)
    if path.startswith("\\"):
        path = path.replace("\\", "", 1)
    return path


def createLocalDirs(path):
    """Tries to create local directories"""
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise


def getFileHash(filepath):
    """
    Returns the MD5 hash of a local file
    """
    fileToHash = open(filepath)
    md5 = hashlib.md5()
    while True:
        currentLine = fileToHash.readline()
        if not currentLine:
            break
        md5.update(currentLine)
    return md5.hexdigest()


def get_server_time():
    """
    Returns the time of the SmartFile servers
    """
    response = requests.get('https://www.smartfile.com')
    time = parse(response.headers['Date']).replace(tzinfo=None, second=0)
    return time


@cache('server_time_offset')
def calculate_time_offset():
    """
    This should be tested more. If the offset calculation is incorrect,
    unchanged files will be put into sync up/down queues, and since a sync up
    on a file that has not been changed will cause corruption.
    """
    server_time = get_server_time()
    now = datetime.datetime.now().replace(microsecond=0, second=0)
    offset = now - server_time
    return offset
{'fraction_non_alphanumeric': 0.10198379435596536, 'fraction_numerical': 0.005588153115395362, 'mean_word_length': 2.641912512716175, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7442006', 'n_tokens_mistral': 1062, 'n_tokens_neox': 1037, 'n_words': 335}