<commit_before>#!/usr/bin/env python2.7

import re, sys

ord_offset = ord('a')

empty_word = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]

corpus = { 'words' : [], 'pairs' : {} }

def process_word(s):
  length = len(s)
  if length < 4:
    return None
  s = s.lower()
  word = { "length" : length, "original" : s, "data" : list(empty_word) }
  for c in s:
    word['data'][ord(c) - ord_offset] += 1

  return word


def is_duplicate(corpus, word):
  for w in corpus['words']:
    if w['original'] == word['original']:
      return True
  return False

def create_compare_string(words):
  compare = ""
  for i in range(26):
    char = chr(i + ord_offset)
    for word in words:
      compare += char*word["data"][i]
  return compare

def iterate_words(corpus, words, position, length, end):
  if length >= 1:<commit_msg>Better syntax for creating the empty_words list<commit_after>#!/usr/bin/env python2.7

import re, sys

ord_offset = ord('a')

empty_word = [0] * 26

corpus = { 'words' : [], 'pairs' : {} }

def process_word(s):
  length = len(s)
  if length < 4:
    return None
  s = s.lower()
  word = { "length" : length, "original" : s, "data" : list(empty_word) }
  for c in s:
    word['data'][ord(c) - ord_offset] += 1

  return word


def is_duplicate(corpus, word):
  for w in corpus['words']:
    if w['original'] == word['original']:
      return True
  return False

def create_compare_string(words):
  compare = ""
  for i in range(26):
    char = chr(i + ord_offset)
    for word in words:
      compare += char*word["data"][i]
  return compare

def iterate_words(corpus, words, position, length, end):
  if length >= 1:
{'fraction_non_alphanumeric': 0.1651705565529623, 'fraction_numerical': 0.02573309395571514, 'mean_word_length': 3.2762148337595907, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '162352', 'n_tokens_mistral': 675, 'n_tokens_neox': 630, 'n_words': 191}