<commit_before># -*- coding: utf-8 -*-
"""functions for datapackage.json"""

import os
import re
import json
import csv
import logging
from datetime import datetime
from .model.package import Datapackage
from collections import OrderedDict


def get_datapackage(path, use_existing=True, update=False):
    """get the datapackage.json from a dataset path, create one if it's not exists

    Parameters
    ----------
    path : `str`
        the dataset path

    Keyword Args
    ------------
    use_existing : bool
        whether or not to use the existing datapackage
    update : bool
        if update is true, will update the resources and schema in existing datapackage.json. else just return existing
        datapackage.json
    """
    datapackage_path = os.path.join(path, 'datapackage.json')

    if os.path.exists(datapackage_path):
        with open(datapackage_path, encoding='utf8') as f:
            datapackage_old = json.load(f, object_pairs_hook=OrderedDict)

        if use_existing:
            if not update:
                return datapackage_old
            try:
                datapackage_old.pop('resources')  # don't use the old resources
                datapackage_old.pop('ddfSchema')  # and ddf schema
            except KeyError:
                logging.warning('no resources or ddfSchema in datapackage.json')
            datapackage_new = create_datapackage(path, **datapackage_old)
        else:
            datapackage_new = create_datapackage(path)
    else:
        if use_existing:
            print("WARNING: no existing datapackage.json")
        datapackage_new = create_datapackage(path)

    return datapackage_new


def get_ddf_files(path, root=None):
    info = next(os.walk(path))

    # don't include hidden and lang/etl dir.
    sub_dirs = [
        x for x in info[1] if (not x.startswith('.') and x not in ['lang', 'etl', 'langsplit'])
    ]
    files = list()
    for x in info[2]:
        if x.startswith('ddf--') and x != 'ddf--index.csv' and x.endswith('.csv'):
            files.append(x)
        else:
            logging.warning('skipping file {}'.format(x))

    for f in files:
        if root:
            yield os.path.join(root, f)
        else:
            yield f

    for sd in sub_dirs:
        for p in get_ddf_files(os.path.join(path, sd), root=sd):
            yield p


def create_datapackage(path, gen_schema=True, **kwargs):
    """create datapackage.json base on the files in `path`.

    If you want to set some attributes manually, you can pass them as
    keyword arguments to this function

    Note
    ----
    A DDFcsv datapackage MUST contain the fields `name` and `resources`.

    if name is not provided, then the base name of `path` will be used.

    Parameters
    ----------
    path : `str`
        the dataset path to create datapackage.json
    """

    datapackage = OrderedDict()

    # setting default name / lang
    try:
        name = kwargs.pop('name')
    except KeyError:
        # print('name not specified, using the path name')
        name = os.path.basename(os.path.normpath(os.path.abspath(path)))
    try:
        lang = kwargs.pop('language')
    except KeyError:
        lang = {'id': 'en'}

    datapackage['name'] = name
    datapackage['language'] = lang
    datapackage['last_updated'] = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")

    # add all optional settings
    for k in sorted(kwargs.keys()):
        datapackage[k] = kwargs[k]

    # generate resources
    resources = []
    names_sofar = dict()

    for f in get_ddf_files(path):
        path_res = f
        name_res = os.path.splitext(os.path.basename(f))[0]
<commit_msg>update the last_update time after loading other optional info<commit_after># -*- coding: utf-8 -*-
"""functions for datapackage.json"""

import csv
import json
import logging
import os
import re
from collections import OrderedDict
from datetime import datetime

from .model.package import Datapackage


def get_datapackage(path, use_existing=True, update=False):
    """get the datapackage.json from a dataset path, create one if it's not exists

    Parameters
    ----------
    path : `str`
        the dataset path

    Keyword Args
    ------------
    use_existing : bool
        whether or not to use the existing datapackage
    update : bool
        if update is true, will update the resources and schema in existing datapackage.json. else just return existing
        datapackage.json
    """
    datapackage_path = os.path.join(path, 'datapackage.json')

    if os.path.exists(datapackage_path):
        with open(datapackage_path, encoding='utf8') as f:
            datapackage_old = json.load(f, object_pairs_hook=OrderedDict)

        if use_existing:
            if not update:
                return datapackage_old
            try:
                datapackage_old.pop('resources')  # don't use the old resources
                datapackage_old.pop('ddfSchema')  # and ddf schema
            except KeyError:
                logging.warning('no resources or ddfSchema in datapackage.json')
            datapackage_new = create_datapackage(path, **datapackage_old)
        else:
            datapackage_new = create_datapackage(path)
    else:
        if use_existing:
            print("WARNING: no existing datapackage.json")
        datapackage_new = create_datapackage(path)

    return datapackage_new


def get_ddf_files(path, root=None):
    info = next(os.walk(path))

    # don't include hidden and lang/etl dir.
    sub_dirs = [
        x for x in info[1] if (not x.startswith('.') and x not in ['lang', 'etl', 'langsplit'])
    ]
    files = list()
    for x in info[2]:
        if x.startswith('ddf--') and x != 'ddf--index.csv' and x.endswith('.csv'):
            files.append(x)
        else:
            logging.warning('skipping file {}'.format(x))

    for f in files:
        if root:
            yield os.path.join(root, f)
        else:
            yield f

    for sd in sub_dirs:
        for p in get_ddf_files(os.path.join(path, sd), root=sd):
            yield p


def create_datapackage(path, gen_schema=True, **kwargs):
    """create datapackage.json base on the files in `path`.

    If you want to set some attributes manually, you can pass them as
    keyword arguments to this function

    Note
    ----
    A DDFcsv datapackage MUST contain the fields `name` and `resources`.

    if name is not provided, then the base name of `path` will be used.

    Parameters
    ----------
    path : `str`
        the dataset path to create datapackage.json
    """

    datapackage = OrderedDict()

    # setting default name / lang
    try:
        name = kwargs.pop('name')
    except KeyError:
        # print('name not specified, using the path name')
        name = os.path.basename(os.path.normpath(os.path.abspath(path)))
    try:
        lang = kwargs.pop('language')
    except KeyError:
        lang = {'id': 'en'}

    datapackage['name'] = name
    datapackage['language'] = lang

    # add all optional settings
    for k in sorted(kwargs.keys()):
        datapackage[k] = kwargs[k]

    # update the last updated time
    datapackage['last_updated'] = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")

    # generate resources
    resources = []
    names_sofar = dict()

    for f in get_ddf_files(path):
        path_res = f
        name_res = os.path.splitext(os.path.basename(f))[0]

{'fraction_non_alphanumeric': 0.10777552400270453, 'fraction_numerical': 0.001352265043948614, 'mean_word_length': 2.3541950113378687, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '84313', 'n_tokens_mistral': 2325, 'n_tokens_neox': 2199, 'n_words': 673}