<commit_before>import pandas as pd
import collections

def word_freq_table(docs):
    df = [[]] * len(docs)  # List to hold the tallies for each document

    # For each document, create a tally of the words that appear in it
    df = [collections.Counter(doc) for doc in docs]

    # Create a dataframe from the counter objects (and replace NAs with 0s)
    df = pd.DataFrame(df)
    df.fillna(0, inplace=True)

    return df
<commit_msg>NLP - Add docstring to word_freq_table()<commit_after>import pandas as pd
import collections

# ==============================================================================
#                                                                WORD_FREQ_TABLE
# ==============================================================================
def word_freq_table(docs):
    """
    Takes a list of documents that have been tokenised into a list of words, and
    returns a pandas dataframe that contains the word counts within each
    document.

    :param docs: (list of list of strings)
        A list of lists. each inner list represents the content of the document,
        and it should be composed of tokenized words.
    :return:
        A dataframe table with word counts for each document. The columns have
        the words, and the rows are the different documents
    :example:
        >>> reviews = [['i', 'liked', 'it'],
        >>>            ['i', 'hated', 'it'],
        >>>            ['so', 'so', 'boring'],
        >>>            ['so', 'good']]
        >>> print(word_freq_table(reviews))

           boring  good  hated  i  it  liked  so
        0       0     0      0  1   1      1   0
        1       0     0      1  1   1      0   0
        2       1     0      0  0   0      0   2
        3       0     1      0  0   0      0   1
    """
    # ==========================================================================
    df = [[]] * len(docs)  # List to hold the tallies for each document

    # For each document, create a tally of the words that appear in it
    df = [collections.Counter(doc) for doc in docs]

    # Create a dataframe from the counter objects (and replace NAs with 0s)
    df = pd.DataFrame(df)
    df.fillna(0, inplace=True)

    return df

{'fraction_non_alphanumeric': 0.17535971223021582, 'fraction_numerical': 0.01618705035971223, 'mean_word_length': 2.1338028169014085, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2344415', 'n_tokens_mistral': 610, 'n_tokens_neox': 577, 'n_words': 245}