<commit_before>from path import path
import hashlib
import json
import numpy as np
import os
import pandas as pd
from datetime import datetime


def md5(data_path):
    # we need to compute the md5 sum one chunk at a time, because some
    # files are too large to fit in memory
    md5 = hashlib.md5()
    with open(data_path, 'r') as fh:
        while True:
            chunk = fh.read(128)
            if not chunk:
                break
            md5.update(chunk)
    data_hash = md5.hexdigest()
    return data_hash


class DataPackage(dict):

    def __init__(self, name, licenses):
        self['name'] = name
        self['datapackage_version'] = '1.0-beta.5'
        self['licenses'] = []
        for l in licenses:
            if not isinstance(l, dict):
                if l == 'odc-by':
                    url = 'http://opendefinition.org/licenses/odc-by'
                else:
                    raise ValueError("unrecognized license: %s" % l)
                l = dict(id=l, url=url)
            self['licenses'].append(l)

        self['title'] = None
        self['description'] = None
        self['homepage'] = None
        self['version'] = '0.0.1'
        self['sources'] = []
        self['keywords'] = None
        self['last_modified'] = datetime.now().isoformat(" ")
        self['image'] = None
        self['contributors'] = []
        self['resources'] = []

        self._path = None
        self._resource_map = {}

    @property
    def abspath(self):
        return self._path.joinpath(self['name']).abspath()

    @classmethod
    def load(cls, pth):
        dpjson_pth = path(os.path.join(pth, "datapackage.json"))
        if not dpjson_pth.exists():
            raise IOError("No metadata file datapackage.json")
        with open(dpjson_pth, "r") as fh:
            dpjson = json.load(fh)

        name = dpjson['name']
        licenses = dpjson['licenses']
        resources = dpjson['resources']
        del dpjson['name']
        del dpjson['licenses']
        del dpjson['resources']

        dp = cls(name=name, licenses=licenses)
        dp._path = pth.splitpath()[0]
        dp.update(dpjson)

        if dp.abspath != pth.abspath():
            raise ValueError("malformed datapackage")
<commit_msg>Make sure to convert pth argument to a path object<commit_after>from path import path
import hashlib
import json
import numpy as np
import os
import pandas as pd
from datetime import datetime


def md5(data_path):
    # we need to compute the md5 sum one chunk at a time, because some
    # files are too large to fit in memory
    md5 = hashlib.md5()
    with open(data_path, 'r') as fh:
        while True:
            chunk = fh.read(128)
            if not chunk:
                break
            md5.update(chunk)
    data_hash = md5.hexdigest()
    return data_hash


class DataPackage(dict):

    def __init__(self, name, licenses):
        self['name'] = name
        self['datapackage_version'] = '1.0-beta.5'
        self['licenses'] = []
        for l in licenses:
            if not isinstance(l, dict):
                if l == 'odc-by':
                    url = 'http://opendefinition.org/licenses/odc-by'
                else:
                    raise ValueError("unrecognized license: %s" % l)
                l = dict(id=l, url=url)
            self['licenses'].append(l)

        self['title'] = None
        self['description'] = None
        self['homepage'] = None
        self['version'] = '0.0.1'
        self['sources'] = []
        self['keywords'] = None
        self['last_modified'] = datetime.now().isoformat(" ")
        self['image'] = None
        self['contributors'] = []
        self['resources'] = []

        self._path = None
        self._resource_map = {}

    @property
    def abspath(self):
        return self._path.joinpath(self['name']).abspath()

    @classmethod
    def load(cls, pth):
        pth = path(pth)

        dpjson_pth = pth.joinpath("datapackage.json")
        if not dpjson_pth.exists():
            raise IOError("No metadata file datapackage.json")
        with open(dpjson_pth, "r") as fh:
            dpjson = json.load(fh)

        name = dpjson['name']
        licenses = dpjson['licenses']
        resources = dpjson['resources']
        del dpjson['name']
        del dpjson['licenses']
        del dpjson['resources']

        dp = cls(name=name, licenses=licenses)
        dp._path = pth.splitpath()[0]
        dp.update(dpjson)

        if dp.abspath != pth.abspath():
            raise ValueError("malformed datapackage")

{'fraction_non_alphanumeric': 0.12387238723872387, 'fraction_numerical': 0.0070407040704070405, 'mean_word_length': 1.9986807387862797, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '294256', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1344, 'n_words': 352}