<commit_before>import os
import json
import tarfile

import shutil

from catalyst.data.bundles.core import download_without_progress
from catalyst.utils.deprecate import deprecated
from catalyst.utils.paths import data_root, ensure_directory


def get_marketplace_folder(environ=None):
    """
    The root path of the marketplace folder.

    Parameters
    ----------
    environ:

    Returns
    -------
    str

    """
    if not environ:
        environ = os.environ

    root = data_root(environ)
    marketplace_folder = os.path.join(root, 'marketplace')
    ensure_directory(marketplace_folder)

    return marketplace_folder


def get_data_source_folder(data_source_name, environ=None):
    """
    The root path of an data_source folder.

    Parameters
    ----------
    data_source_name: str
    environ:

    Returns
    -------
    str

    """
    if not environ:
        environ = os.environ

    root = data_root(environ)
    data_source_folder = os.path.join(root, 'marketplace', data_source_name)
    ensure_directory(data_source_folder)

    return data_source_folder


@deprecated
def get_bundle_folder(data_source_name, data_frequency, environ=None):
    data_source_folder = get_data_source_folder(data_source_name, environ)

    bundle_folder = os.path.join(data_source_folder, data_frequency)

    ensure_directory(bundle_folder)

    return bundle_folder


def get_temp_bundles_folder(environ=None):
    """
    The temp folder for bundle downloads by algo name.

    Parameters
    ----------
    ds_name: str
    environ:

    Returns
    -------
    str

    """
    root = data_root(environ)
    folder = os.path.join(root, 'marketplace', 'temp_bundles')
    ensure_directory(folder)

    return folder


def extract_bundle(tar_filename):
    """
    Extract a bcolz bundle.

    Parameters
    ----------
    ds_name

    Returns
    -------
    str

    """
    target_path = tar_filename.replace('.tar.gz', '')
    with tarfile.open(tar_filename, 'r') as tar:
        tar.extractall(target_path)

    return target_path


def get_user_pubaddr(environ=None):
    """
    The de-serialized contend of the user's addresses.json file.

    Parameters
    ----------
    environ:

    Returns
    -------
    Object

    """
    marketplace_folder = get_marketplace_folder(environ)
    filename = os.path.join(marketplace_folder, 'addresses.json')

    if os.path.isfile(filename):
        with open(filename) as data_file:
            data = json.load(data_file)
            try:
                d = data[0]['pubAddr']
            except Exception as e:
                return [data, ]
            return data
    else:
        data = dict(pubAddr='', desc='')
        with open(filename, 'w') as f:
            json.dump(data, f, sort_keys=False, indent=2,
                      separators=(',', ':'))
            return data


def save_user_pubaddr(data, environ=None):
    """
    Saves the user's public addresses and their related metadata in
    the corresponding addresses.json file.

    Parameters
    ----------
    data: dict

    Returns
    -------
    True

    """
    marketplace_folder = get_marketplace_folder(environ)
    filename = os.path.join(marketplace_folder, 'addresses.json')

    with open(filename, 'w') as f:
        json.dump(data, f, sort_keys=False, indent=2,
                  separators=(',', ':'))

    return True
<commit_msg>BUG: address json initialized with array of 1 dict, instead of dict<commit_after>import os
import json
import tarfile

import shutil

from catalyst.data.bundles.core import download_without_progress
from catalyst.utils.deprecate import deprecated
from catalyst.utils.paths import data_root, ensure_directory


def get_marketplace_folder(environ=None):
    """
    The root path of the marketplace folder.

    Parameters
    ----------
    environ:

    Returns
    -------
    str

    """
    if not environ:
        environ = os.environ

    root = data_root(environ)
    marketplace_folder = os.path.join(root, 'marketplace')
    ensure_directory(marketplace_folder)

    return marketplace_folder


def get_data_source_folder(data_source_name, environ=None):
    """
    The root path of an data_source folder.

    Parameters
    ----------
    data_source_name: str
    environ:

    Returns
    -------
    str

    """
    if not environ:
        environ = os.environ

    root = data_root(environ)
    data_source_folder = os.path.join(root, 'marketplace', data_source_name)
    ensure_directory(data_source_folder)

    return data_source_folder


@deprecated
def get_bundle_folder(data_source_name, data_frequency, environ=None):
    data_source_folder = get_data_source_folder(data_source_name, environ)

    bundle_folder = os.path.join(data_source_folder, data_frequency)

    ensure_directory(bundle_folder)

    return bundle_folder


def get_temp_bundles_folder(environ=None):
    """
    The temp folder for bundle downloads by algo name.

    Parameters
    ----------
    ds_name: str
    environ:

    Returns
    -------
    str

    """
    root = data_root(environ)
    folder = os.path.join(root, 'marketplace', 'temp_bundles')
    ensure_directory(folder)

    return folder


def extract_bundle(tar_filename):
    """
    Extract a bcolz bundle.

    Parameters
    ----------
    ds_name

    Returns
    -------
    str

    """
    target_path = tar_filename.replace('.tar.gz', '')
    with tarfile.open(tar_filename, 'r') as tar:
        tar.extractall(target_path)

    return target_path


def get_user_pubaddr(environ=None):
    """
    The de-serialized contend of the user's addresses.json file.

    Parameters
    ----------
    environ:

    Returns
    -------
    Object

    """
    marketplace_folder = get_marketplace_folder(environ)
    filename = os.path.join(marketplace_folder, 'addresses.json')

    if os.path.isfile(filename):
        with open(filename) as data_file:
            data = json.load(data_file)
            try:
                d = data[0]['pubAddr']
            except Exception as e:
                return [data, ]
            return data
    else:
        data = []
        data.append(dict(pubAddr='', desc=''))
        with open(filename, 'w') as f:
            json.dump(data, f, sort_keys=False, indent=2,
                      separators=(',', ':'))
            return data


def save_user_pubaddr(data, environ=None):
    """
    Saves the user's public addresses and their related metadata in
    the corresponding addresses.json file.

    Parameters
    ----------
    data: dict

    Returns
    -------
    True

    """
    marketplace_folder = get_marketplace_folder(environ)
    filename = os.path.join(marketplace_folder, 'addresses.json')

    with open(filename, 'w') as f:
        json.dump(data, f, sort_keys=False, indent=2,
                  separators=(',', ':'))

    return True

{'fraction_non_alphanumeric': 0.11458940011648223, 'fraction_numerical': 0.0010192195690157252, 'mean_word_length': 2.363858961802155, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4998393', 'n_tokens_mistral': 2076, 'n_tokens_neox': 2014, 'n_words': 471}