<commit_before>#!/bin/env python3

'''
NOTE: I am not very confident that this particular class is working properly.
      Although it seems to do reasonable things (at least in the first iterations),
      I haven't tested it against the Percy Liang implementation (link below) 
      to see what the differences in output are.  Also, if you don't mind its
      research-only license, the Liang C++ implementation may be preferable
      since it is probably faster.

A little module for creating hierarchical word clusters.
This is based on the following papers.

* <NAME>; <NAME>; <NAME>; <NAME>; <NAME>.
  <NAME>; <NAME>. 1992.  Class-Based n-gram Models of Natural
  Language.  Computational Linguistics, Volume 18, Number 4.
  http://acl.ldc.upenn.edu/J/J92/J92-4003.pdf

* <NAME>. 2005.  Semi-supervised learning for natural language.  MIT.
  http://cs.stanford.edu/~pliang/papers/meng-thesis.pdf


Some additional references:

* See http://www.cs.columbia.edu/~cs4705/lectures/brown.pdf for a high-level
  overview of Brown clustering.

* Here is another implementation of Brown clustering:
  https://github.com/percyliang/brown-cluster


Author: <NAME> (<EMAIL>, <EMAIL>)

'''

import random
import argparse
import glob
import re
import itertools
import logging
from collections import defaultdict
from math import log, isnan, isinf

random.seed(1234567890)

logging.basicConfig(level=logging.INFO, format='%(asctime)s\t%(message)s')


def read_corpus(path):
    corpus = ""
    with open(path) as f:
        corpus = re.split(r'\s+', f.read())
    return corpus


def test_reviews():
    corpus = []
    for path in glob.glob('review_polarity/txt_sentoken/*/cv*'):
        with open(path) as f:<commit_msg>tweaked documentation<commit_after>#!/bin/env python3

'''
An implementation for creating hierarchical word clusters based on syntactic
context ("Brown clusters").  This is based on the following papers.

* <NAME>; <NAME>; <NAME>; <NAME>; <NAME>.
  <NAME>; <NAME>. 1992.  Class-Based n-gram Models of Natural
  Language.  Computational Linguistics, Volume 18, Number 4.
  http://acl.ldc.upenn.edu/J/J92/J92-4003.pdf

* <NAME>. 2005.  Semi-supervised learning for natural language.  MIT.
  http://cs.stanford.edu/~pliang/papers/meng-thesis.pdf


Some additional references:

* See http://www.cs.columbia.edu/~cs4705/lectures/brown.pdf for a high-level
  overview of Brown clustering.

* Here is another implementation of Brown clustering:
  https://github.com/percyliang/brown-cluster


NOTE: I am not very confident that this particular class is working properly.
      Although it seems to do reasonable things, I haven't tested it against
      implementation from  Percy Liang (link below) to see what the differences
      in output are.  Also, if you don't mind the research-only license of the
      Liang C++ implementation, that may be preferable since it is probably
      faster.

Author: <NAME> (<EMAIL>, <EMAIL>)

'''

import random
import argparse
import glob
import re
import itertools
import logging
from collections import defaultdict
from math import log, isnan, isinf

random.seed(1234567890)

logging.basicConfig(level=logging.INFO, format='%(asctime)s\t%(message)s')


def read_corpus(path):
    corpus = ""
    with open(path) as f:
        corpus = re.split(r'\s+', f.read())
    return corpus


def test_reviews():
    corpus = []
    for path in glob.glob('review_polarity/txt_sentoken/*/cv*'):
        with open(path) as f:
{'fraction_non_alphanumeric': 0.10793650793650794, 'fraction_numerical': 0.019624819624819625, 'mean_word_length': 3.747945205479452, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '308170', 'n_tokens_mistral': 1149, 'n_tokens_neox': 1045, 'n_words': 369}