<commit_before>__author__ = 'thor'

import pandas as pd
import os
import subprocess
from bs4 import BeautifulSoup
import copy
import itertools
import numpy as np
from hashlib import md5

import ut as ms
from ut.daf.manip import recursive_update
from ut.util.ulist import sort_as
from ut.daf.resources.disp_templates import inline_html_table_template


def map_strings_to_their_mp5s(df, cols_to_map):
    if isinstance(cols_to_map, basestring):
        cols_to_map = [cols_to_map]

    def md5_of(s):
        m = md5()
        m.update(s)
        return m.hexdigest()

    df = df.copy()
    for c in cols_to_map:
        df[c] = df[c].apply(md5_of)

    return df


def map_vals_to_ints_inplace(df, cols_to_map):
    """
    map_vals_to_ints_inplace(df, cols_to_map) will map columns of df inplace
    If cols_to_map is a string, a single column will be mapped to consecutive integers from 0 onwards.
    Elif cols_to_map is a dict, the function will attempt to map the columns dict.keys() using the (list-like)
        maps specified in dict.values()
    Else assumed that cols_to_map is list-like specifying columns to be mapped

    Note: If no mapping is specified (i.e. cols_to_map is not a dict), the function will return the mapping dict,
    which is of the form mapping_dict[col][val] := mapped val for col.<commit_msg>first working version of stroke ws<commit_after>__author__ = 'thor'

import pandas as pd
import os
import subprocess
from bs4 import BeautifulSoup
import copy
import itertools
import numpy as np
from hashlib import md5

import ut as ms
from ut.daf.manip import recursive_update
from ut.util.ulist import sort_as
from ut.daf.resources.disp_templates import inline_html_table_template

from collections import OrderedDict


def sr_to_ordered_dict(sr):
    return OrderedDict(sr.iteritems())


def map_strings_to_their_mp5s(df, cols_to_map):
    if isinstance(cols_to_map, basestring):
        cols_to_map = [cols_to_map]

    def md5_of(s):
        m = md5()
        m.update(s)
        return m.hexdigest()

    df = df.copy()
    for c in cols_to_map:
        df[c] = df[c].apply(md5_of)

    return df


def map_vals_to_ints_inplace(df, cols_to_map):
    """
    map_vals_to_ints_inplace(df, cols_to_map) will map columns of df inplace
    If cols_to_map is a string, a single column will be mapped to consecutive integers from 0 onwards.
    Elif cols_to_map is a dict, the function will attempt to map the columns dict.keys() using the (list-like)
        maps specified in dict.values()
    Else assumed that cols_to_map is list-like specifying columns to be mapped

    Note: If no mapping is specified (i.e. cols_to_map is not a dict), the function will return the mapping dict,
    which is of the form mapping_dict[col][val] := mapped val for col.
{'fraction_non_alphanumeric': 0.06467840459935321, 'fraction_numerical': 0.005030542579949695, 'mean_word_length': 3.6711409395973154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '38143', 'n_tokens_mistral': 945, 'n_tokens_neox': 903, 'n_words': 324}