<commit_before>import os
import errno
import json
import re
import hashlib
import time
import webbrowser

from functools import wraps

try:
    from urllib.parse import urlparse
    assert urlparse
except ImportError:
    from urlparse import urlparse

try:
    from .. import editor
    from . import shared as G
    from .exc_fmt import str_e
    from . import msg
    from .lib import DMP
    assert G and DMP
except ImportError:
    import editor
    import msg
    from exc_fmt import str_e
    import shared as G
    from lib import DMP


class JOIN_ACTION(object):
    PROMPT = 1
    UPLOAD = 2
    DOWNLOAD = 3


class FlooPatch(object):
    def __init__(self, current, buf):
        self.buf = buf
        self.current = current
        self.previous = buf['buf']
        if buf['encoding'] == 'base64':
            self.md5_before = hashlib.md5(self.previous).hexdigest()
            self.md5_after = hashlib.md5(self.current).hexdigest()
        else:
            try:
                self.md5_before = hashlib.md5(self.previous.encode('utf-8')).hexdigest()
            except Exception as e:
                # Horrible fallback if for some reason encoding doesn't agree with actual object
                self.md5_before = hashlib.md5(self.previous).hexdigest()
                msg.log('Error calculating md5_before for ', str(self), ': ', str_e(e))
            try:
                self.md5_after = hashlib.md5(self.current.encode('utf-8')).hexdigest()
            except Exception as e:
                # Horrible fallback if for some reason encoding doesn't agree with actual object
                self.md5_after = hashlib.md5(self.current).hexdigest()
                msg.log('Error calculating md5_after for ', str(self), ': ', str_e(e))

    def __str__(self):
        return '%s - %s' % (self.buf['id'], self.buf['path'])

    def patches(self):
        return DMP.patch_make(self.previous, self.current)

    def to_json(self):
        patches = self.patches()
        if len(patches) == 0:
            return None
        patch_str = ''
        for patch in patches:
            patch_str += str(patch)

        return {
            'id': self.buf['id'],
            'md5_after': self.md5_after,
            'md5_before': self.md5_before,
            'path': self.buf['path'],
            'patch': patch_str,
            'name': 'patch'
        }


def reload_settings():
    floorc_settings = load_floorc_json()
    for name, val in floorc_settings.items():
        setattr(G, name, val)
    validate_auth(G.AUTH)
    if G.SHARE_DIR:
        G.BASE_DIR = G.SHARE_DIR
    G.BASE_DIR = os.path.realpath(os.path.expanduser(G.BASE_DIR))
    G.COLAB_DIR = os.path.join(G.BASE_DIR, 'share')
    G.COLAB_DIR = os.path.realpath(G.COLAB_DIR)
    if G.DEBUG:
        msg.LOG_LEVEL = msg.LOG_LEVELS['DEBUG']
    else:
        msg.LOG_LEVEL = msg.LOG_LEVELS['MSG']
    mkdir(G.COLAB_DIR)
    return floorc_settings


def load_floorc_json():
    # Expose a few settings for curious users to tweak
    s = {
        'expert_mode': False,
        'debug': False,
    }
    try:
        with open(G.FLOORC_JSON_PATH, 'r') as fd:
            floorc_json = fd.read()
    except IOError as e:
        if e.errno == errno.ENOENT:
            return s
        raise

    try:
        default_settings = json.loads(floorc_json)
    except ValueError:
        return s

    for k, v in default_settings.items():
        s[k.upper()] = v
    return s


def save_floorc_json(s):
    floorc_json = {}
    for k, v in s.items():
        floorc_json[k.lower()] = v
    msg.log('Writing ', floorc_json)
    with open(G.FLOORC_JSON_PATH, 'w') as fd:
        fd.write(json.dumps(floorc_json, indent=4, sort_keys=True, separators=(',', ': ')))


def validate_auth(auth):
    if type(auth) != dict:
        msg.error('floorc.json validation error: Auth section is not an object!')
        return False
    for k, v in auth.items():
        if type(v) != dict:
            msg.error('floorc.json validation error: host "', k, '" has invalid auth credentials. Did you put a setting in the auth section?')
            return False
        for key in ['username', 'api_key', 'secret']:
            if not v.get(key):
                msg.error('floorc.json validation error: host "', k, '" missing "', key, '"')
                return False
    return True


def can_auth(host=None):
    if host is None:
        host = len(G.AUTH) and list(G.AUTH.keys())[0] or G.DEFAULT_HOST
    auth = G.AUTH.get(host)
    if type(auth) == dict:
        return bool((auth.get('username') or auth.get('api_key')) and auth.get('secret'))
    return False

<commit_msg>Even nicer fallback when parsing floorc.json. Figure out auth correctly even if user has put dumb stuff in auth info.<commit_after>import os
import errno
import json
import re
import hashlib
import time
import webbrowser

from functools import wraps

try:
    from urllib.parse import urlparse
    assert urlparse
except ImportError:
    from urlparse import urlparse

try:
    from .. import editor
    from . import shared as G
    from .exc_fmt import str_e
    from . import msg
    from .lib import DMP
    assert G and DMP
except ImportError:
    import editor
    import msg
    from exc_fmt import str_e
    import shared as G
    from lib import DMP


class JOIN_ACTION(object):
    PROMPT = 1
    UPLOAD = 2
    DOWNLOAD = 3


class FlooPatch(object):
    def __init__(self, current, buf):
        self.buf = buf
        self.current = current
        self.previous = buf['buf']
        if buf['encoding'] == 'base64':
            self.md5_before = hashlib.md5(self.previous).hexdigest()
            self.md5_after = hashlib.md5(self.current).hexdigest()
        else:
            try:
                self.md5_before = hashlib.md5(self.previous.encode('utf-8')).hexdigest()
            except Exception as e:
                # Horrible fallback if for some reason encoding doesn't agree with actual object
                self.md5_before = hashlib.md5(self.previous).hexdigest()
                msg.log('Error calculating md5_before for ', str(self), ': ', str_e(e))
            try:
                self.md5_after = hashlib.md5(self.current.encode('utf-8')).hexdigest()
            except Exception as e:
                # Horrible fallback if for some reason encoding doesn't agree with actual object
                self.md5_after = hashlib.md5(self.current).hexdigest()
                msg.log('Error calculating md5_after for ', str(self), ': ', str_e(e))

    def __str__(self):
        return '%s - %s' % (self.buf['id'], self.buf['path'])

    def patches(self):
        return DMP.patch_make(self.previous, self.current)

    def to_json(self):
        patches = self.patches()
        if len(patches) == 0:
            return None
        patch_str = ''
        for patch in patches:
            patch_str += str(patch)

        return {
            'id': self.buf['id'],
            'md5_after': self.md5_after,
            'md5_before': self.md5_before,
            'path': self.buf['path'],
            'patch': patch_str,
            'name': 'patch'
        }


def reload_settings():
    floorc_settings = load_floorc_json()
    for name, val in floorc_settings.items():
        setattr(G, name, val)
    validate_auth(G.AUTH)
    if G.SHARE_DIR:
        G.BASE_DIR = G.SHARE_DIR
    G.BASE_DIR = os.path.realpath(os.path.expanduser(G.BASE_DIR))
    G.COLAB_DIR = os.path.join(G.BASE_DIR, 'share')
    G.COLAB_DIR = os.path.realpath(G.COLAB_DIR)
    if G.DEBUG:
        msg.LOG_LEVEL = msg.LOG_LEVELS['DEBUG']
    else:
        msg.LOG_LEVEL = msg.LOG_LEVELS['MSG']
    mkdir(G.COLAB_DIR)
    return floorc_settings


def load_floorc_json():
    # Expose a few settings for curious users to tweak
    s = {
        'expert_mode': False,
        'debug': False,
    }
    try:
        with open(G.FLOORC_JSON_PATH, 'r') as fd:
            floorc_json = fd.read()
    except IOError as e:
        if e.errno == errno.ENOENT:
            return s
        raise

    try:
        default_settings = json.loads(floorc_json)
    except ValueError:
        return s

    for k, v in default_settings.items():
        s[k.upper()] = v
    return s


def save_floorc_json(s):
    floorc_json = {}
    for k, v in s.items():
        floorc_json[k.lower()] = v
    msg.log('Writing ', floorc_json)
    with open(G.FLOORC_JSON_PATH, 'w') as fd:
        fd.write(json.dumps(floorc_json, indent=4, sort_keys=True, separators=(',', ': ')))


def validate_auth(auth):
    if type(auth) != dict:
        msg.error('floorc.json validation error: Auth section is not an object!')
        return False
    to_delete = []
    for k, v in auth.items():
        if type(v) != dict:
            msg.error('floorc.json validation error: host "', k, '" has invalid auth credentials. Did you put a setting in the auth section?')
            to_delete.append(k)
            break
        for key in ['username', 'api_key', 'secret']:
            if not v.get(key):
                msg.error('floorc.json validation error: host "', k, '" missing "', key, '"')
                to_delete.append(k)
                break
    for k in to_delete:
        del auth[k]
    return len(to_delete) == 0


def can_auth(host=None):
    if host is None:
        host = len(G.AUTH) and list(G.AUTH.keys())[0] or G.DEFAULT_HOST
    auth = G.AUTH.get(host)
    if type(auth) == dict:
        return bool((auth.get('username') or auth.get('api_key')) and auth.get('secret'))
    return False


{'fraction_non_alphanumeric': 0.11472134595162986, 'fraction_numerical': 0.005993690851735016, 'mean_word_length': 2.2002018842530284, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '877870', 'n_tokens_mistral': 3081, 'n_tokens_neox': 2991, 'n_words': 807}