<jupyter_start><jupyter_text>Parse Clustal Omega alignment output<jupyter_code>def read_clustal_omega(file_name: str) -> dict:
    with open(file_name, 'r') as reader:
        # skip the first 3 lines
        for _ in range(3):
            next(reader)

        res = {}
        for line in reader:
            line = line.strip()
            if not line.startswith(' ') and not line.startswith('\t') and line and not '*' in line:
                tokens = re.split(r'[ ]+|\t', line)

                if tokens[0] not in res:
                    res[tokens[0]] = list(tokens[1])
                else:
                    res[tokens[0]] = res[tokens[0]] + list(tokens[1])

        return res
seq = read_clustal_omega('data/VARFREQ_2021/clustalo-I20210309-103503-0178-93081341-p1m.clustal_num')
seq_df = pd.DataFrame.from_dict(seq, orient='index')
seq_df.head()<jupyter_output><empty_output><jupyter_text>Count nt occurrences per position<jupyter_code>counts = pd.DataFrame(0, index=['A','C','G','T','-','R','S','K','Y','M','N'], columns=seq_df.columns)

for colname, col in seq_df.iteritems():
    temp = seq_df[colname].value_counts()
       
    for tidx in temp.index:
        counts.at[tidx, colname] = temp[tidx]

counts.transpose().head(10)
counts.transpose().to_excel("nt_frequency.xlsx", sheet_name='nt frequency')<jupyter_output><empty_output><jupyter_text>Count missmatches<jupyter_code>mm = counts.transpose().sum(axis=1) - counts.transpose().max(axis=1)
mm.sum()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15512649800266312, 'fraction_numerical': 0.02929427430093209, 'mean_word_length': 3.356521739130435, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '616746', 'license': 'MIT', 'n_tokens_mistral': 567, 'n_tokens_neox': 536, 'n_words': 93, 'path': 'missmatch_freq_2021.ipynb', 'repo_name': 'mazzalab/playgrounds', 'size': '1502'}