<commit_before>#!/usr/bin/env python3

# rekode
# (C) <NAME>
# 2021-04-29 2021-04-29 0.1
# not ready yet

# recode file f from encoding1 to encoding2
# params:
# -n --name file
# -f --from enc1  [cp1251]
# -t --to enc2    [utf8]
# -a --all
# -x --ext extension_to_add
# -u --unlist extension_to_exclude
# -h --help

import sys, getopt

def main(argv):

    try:
        opts, args = getopt.getopt (argv, 'hf:t:n:x:u:a',
            ["from=" "to=", "name=", "ext=", "unlist=", "help"])
    except:
        print('erros, check params')
        sys.exit(1)

    for opt, agr in opts:

        if opt in ('-h', '--help'):
            print("""
            Good program to recode it all
            """)
            sys.exit(0)

        elif opt in ('-f', '--from'):
            print(f"convert from {arg}")

        elif opt in ('-t', '--to'):
            print(f"convert to {arg}")

        elif opt in ('-n', '--name'):
            print(f"use file {opt}")

        elif opt in ('-a', '--all'):
            print("convert all files")

        elif opt in ('x', '--ext'):
            print(f"convert extension {arg}")

        else:
            print("whatever else...")

    print("here we process something...")

if __name__ == "__main__":
    main(sys.argv[1:])<commit_msg>good params processing for rekode.py<commit_after>#!/usr/bin/env python3

# rekode
# (C) <NAME>
# 2021-04-29 2021-04-29 0.1
# not ready yet

# recode file f from encoding1 to encoding2
# params:
# -n --name file
# -f --from enc1  [cp1251]
# -t --to enc2    [utf8]
# -a --all
# -x --ext extension_to_add
# -u --unlist extension_to_exclude
# -h --help

import sys, getopt

def main(argv):

    try:
        opts, args = getopt.getopt (argv, 'hf:t:n:x:u:ad',
            ["from=" "to=", "name=", "ext=", "unlist=", "help", "default"])
    except:
        print('errors, check params')
        sys.exit(1)

    fromenc  = 'cp1251'
    toenc = 'utf8'
    exts   = {'txt', 'bbs', 'ion'}
    flist = set()

    for opt, agr in opts:

        if opt in ('-h', '--help'):
            print("""
            Good program to recode it all.
            -h --help    help
            -f --from    from encoding ...
            -t --to      to encoding ...
            -a --all     all files
            -x --ext     add extension ... to list
            -u --unlist  unlist extension ...
            -n --name    add file ...
            -r --remove  remove file ... from list
            -d --default run in default mode
            """)
            sys.exit(0)

        elif opt in ('-f', '--from'):
            print(f"convert from {arg}")

        elif opt in ('-t', '--to'):
            print(f"convert to {arg}")

        elif opt in ('-n', '--name'):
            print(f"use file {opt}")

        elif opt in ('-a', '--all'):
            print("convert all files")

        elif opt in ('x', '--ext'):
            print(f"do convert extension {arg}")

        elif opt in ('u', '--unlist'):
            print(f"do not convert extension {arg}")

        else:
            print("whatever else...")

    print(f"here we process something as {fromenc=}, {toenc=}, {exts=}, {flist=}")

if __name__ == "__main__":
    main(sys.argv[1:])
{'fraction_non_alphanumeric': 0.18016890835157962, 'fraction_numerical': 0.020957147325617768, 'mean_word_length': 1.6473509933774835, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5064210', 'n_tokens_mistral': 1132, 'n_tokens_neox': 1083, 'n_words': 333}