<commit_before># -*- coding:utf-8 -*-
# Author: hankcs
# Date: 2019-08-26 14:58
import json

from hanlp.utils.io_util import save_json, save_pickle, load_pickle, load_json, filename_is_json


class Serializable(object):
    """
    A super class for save/load operations.
    """

    def save(self, path, fmt=None):
        if not fmt:
            if filename_is_json(path):
                self.save_json(path)
            else:
                self.save_pickle(path)
        elif fmt in ['json', 'jsonl']:
            self.save_json(path)
        else:
            self.save_pickle(path)

    def load(self, path, fmt=None):
        if not fmt:
            if filename_is_json(path):
                self.load_json(path)
            else:
                self.load_json(path)
        elif fmt in ['json', 'jsonl']:
            self.load_json(path)
        else:
            self.load(path)
<commit_msg>Fix Serializable fix https://github.com/hankcs/HanLP/issues/1431<commit_after># -*- coding:utf-8 -*-
# Author: hankcs
# Date: 2019-08-26 14:58
import json

from hanlp.utils.io_util import save_json, save_pickle, load_pickle, load_json, filename_is_json


class Serializable(object):
    """
    A super class for save/load operations.
    """

    def save(self, path, fmt=None):
        if not fmt:
            if filename_is_json(path):
                self.save_json(path)
            else:
                self.save_pickle(path)
        elif fmt in ['json', 'jsonl']:
            self.save_json(path)
        else:
            self.save_pickle(path)

    def load(self, path, fmt=None):
        if not fmt:
            if filename_is_json(path):
                self.load_json(path)
            else:
                self.load_pickle(path)
        elif fmt in ['json', 'jsonl']:
            self.load_json(path)
        else:
            self.load_pickle(path)

{'fraction_non_alphanumeric': 0.11122994652406418, 'fraction_numerical': 0.016042780748663103, 'mean_word_length': 1.84779299847793, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '630740', 'n_tokens_mistral': 622, 'n_tokens_neox': 586, 'n_words': 133}