<commit_before># imports

import sys
from functools import reduce

import read_data
import feature_extractor


# functions


# main script

sentences = read_data.read_sentences_from_file(sys.argv[1])
for sentence in sentences:
    print('--------------------------')
    print('Sentence: ' + ''.join(reduce((lambda x, y: x + ' ' + y), sentence)) + '\n')
    stopwords = feature_extractor.get_stop_words_in_sentence(sentence)
    print('stopwords: ' +
          ''.join(reduce((lambda x, y: x + ', ' + y), stopwords))
          if stopwords != []
          else '')
    pos_tagged_words = feature_extractor.get_pos_tagged_words(sentence)
    print('pos-tagged: ' +
          ''.join(
              reduce(
                  (lambda x, y: x + ', ' + y),
                  list(map((lambda x: '(' + x[0] + ', ' + x[1] + ')'), pos_tagged_words))))
          if pos_tagged_words != [] else '')
    for word in sentence:
        print(word + ' freq score: ' + str(feature_extractor.get_word_freq(word, sentence, sentences)))
    punct_freq = feature_extractor.get_punctuation_freq(sentence)
    print(punct_freq)
    print('--------------------------')<commit_msg>Prototype code for clustering with K-Means<commit_after># imports

import sys
import nltk.cluster
from functools import reduce
import numpy

import read_data
import feature_extractor


# functions
def distance(vector1, vector2):
    dist = 0
    for i in range(1, len(vector1)):
        dist += abs(vector1[i] - vector2[i])

    return dist

# main script

sentences = read_data.read_sentences_from_file(sys.argv[1])
# for sentence in sentences:
#     print('--------------------------')
#     print('Sentence: ' + ''.join(reduce((lambda x, y: x + ' ' + y), sentence)) + '\n')
#     stopwords = feature_extractor.get_stop_words_in_sentence(sentence)
#     print('stopwords: ' +
#           ''.join(reduce((lambda x, y: x + ', ' + y), stopwords))
#           if stopwords != []
#           else '')
#     pos_tagged_words = feature_extractor.get_pos_tagged_words(sentence)
#     print('pos-tagged: ' +
#           ''.join(
#               reduce(
#                   (lambda x, y: x + ', ' + y),
#                   list(map((lambda x: '(' + x[0] + ', ' + x[1] + ')'), pos_tagged_words))))
#           if pos_tagged_words != [] else '')
#     for word in sentence:
#         print(word + ' freq score: ' + str(feature_extractor.get_word_freq(word, sentence, sentences)))
#     punct_freq = feature_extractor.get_punctuation_freq(sentence)
#     print(punct_freq)
#     print('--------------------------')

feature_vectors = []
for sentence in sentences:
    punct_freq = feature_extractor.get_punctuation_freq(sentence)
    vector = [
        # number of stopwords
        len(feature_extractor.get_stop_words_in_sentence(sentence)),
        # number of non-stop words
        len(feature_extractor.get_non_stopwords_in_sentence(sentence)),
        # sum of score of each word
        reduce(lambda x, y: x + y, map(lambda w: feature_extractor.get_word_freq(w, sentence, sentences), sentence)),
        # freq
    ]
    feature_vectors.append(vector)

feature_vectors = [numpy.array(f) for f in feature_vectors]
clusterer = nltk.cluster.KMeansClusterer(num_means=3, distance=(lambda x, y: distance(x, y)))
clusters = clusterer.cluster(vectors=feature_vectors, assign_clusters=True)

numpy.set_printoptions(suppress=True)
print('Clustered: ')
print(feature_vectors)

print('Clusters: ')
print(clusters)

print('Means: ')
print(clusterer.means())
{'fraction_non_alphanumeric': 0.16309625821193943, 'fraction_numerical': 0.003998857469294488, 'mean_word_length': 2.8738938053097347, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6013934', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1107, 'n_words': 230}