def refine_dict(full_mapping, clean_dict_filename, threshold, ignore_gap):
    " Clean dictionary based on frequency and gap of frequency.\n    For example,\n    {'s1': ['t1': 999, 't2': 199, 't3':1],\n     's2': ['m1': 2000, 'm2': 100]}\n     =>\n    {'s1': ['t1': 999, 't2': 199],\n     's2': ['m1': 2000]}\n\n    Args:\n        full_mapping:\n        clean_dict_filename:\n        threshold:\n        ignore_gap:\n\n    Returns:\n\n    "
    print('Refine dict to {}, threshold: {}, ignore_gap: {} ...'.format(clean_dict_filename, threshold, ignore_gap))
    full_mapping = sorted(full_mapping.items(), key=(lambda x: sum(x[1].values())), reverse=True)
    with open(clean_dict_filename, 'w') as fw:
        for (idx, src2trg) in enumerate(full_mapping):
            src = src2trg[0]
            trg = sorted(src2trg[1].items(), key=(lambda x: x[1]), reverse=True)
            total_count = sum((c[1] for c in trg))
            clean_trg = dict()
            p = trg[0][1]
            for (w, c) in trg:
                if ((c / total_count) < threshold):
                    break
                if (((p / c) > ignore_gap) and ((c / total_count) < (THRESHOLD * 5))):
                    break
                p = c
                clean_trg.update({w: round((c / total_count), 3)})
            fw.write('{}\n'.format(json.dumps({src: clean_trg}, ensure_ascii=False)))
Clean dictionary based on frequency and gap of frequency.
For example,
{'s1': ['t1': 999, 't2': 199, 't3':1],
 's2': ['m1': 2000, 'm2': 100]}
 =>
{'s1': ['t1': 999, 't2': 199],
 's2': ['m1': 2000]}

Args:
    full_mapping:
    clean_dict_filename:
    threshold:
    ignore_gap:

Returns:
def refine_dict(full_mapping, clean_dict_filename, threshold, ignore_gap):
    " Clean dictionary based on frequency and gap of frequency.\n    For example,\n    {'s1': ['t1': 999, 't2': 199, 't3':1],\n     's2': ['m1': 2000, 'm2': 100]}\n     =>\n    {'s1': ['t1': 999, 't2': 199],\n     's2': ['m1': 2000]}\n\n    Args:\n        full_mapping:\n        clean_dict_filename:\n        threshold:\n        ignore_gap:\n\n    Returns:\n\n    "
    print('Refine dict to {}, threshold: {}, ignore_gap: {} ...'.format(clean_dict_filename, threshold, ignore_gap))
    full_mapping = sorted(full_mapping.items(), key=(lambda x: sum(x[1].values())), reverse=True)
    with open(clean_dict_filename, 'w') as fw:
        for (idx, src2trg) in enumerate(full_mapping):
            src = src2trg[0]
            trg = sorted(src2trg[1].items(), key=(lambda x: x[1]), reverse=True)
            total_count = sum((c[1] for c in trg))
            clean_trg = dict()
            p = trg[0][1]
            for (w, c) in trg:
                if ((c / total_count) < threshold):
                    break
                if (((p / c) > ignore_gap) and ((c / total_count) < (THRESHOLD * 5))):
                    break
                p = c
                clean_trg.update({w: round((c / total_count), 3)})
            fw.write('{}\n'.format(json.dumps({src: clean_trg}, ensure_ascii=False)))