<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Created on Feb 20, 2013

@author: maribelacosta
@author: <NAME>
'''

import hashlib
from codecs import encode

def calculateHash(text):
    encodedText = encode(text)
    return hashlib.md5(encodedText).hexdigest()


def splitIntoParagraphs(text):
    paragraphs = text.split("\n\n")

    return paragraphs


def splitIntoSentences(text):
    p = text

    p = p.replace('.', '.@@@@')
    p = p.replace('\n', '\n@@@@')
    p = p.replace(';', ';@@@@')
    p = p.replace('?', '?@@@@')
    p = p.replace('!', '!@@@@')
    #p = p.replace('.{', '.||{')
    #p = p.replace('!{', '!||{')
    #p = p.replace('?{', '?||{')<commit_msg>/ Fix #7 (thanks @wikiwho).<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Created on Feb 20, 2013

@author: maribelacosta
@author: <NAME>
'''

import hashlib
from codecs import encode

def calculateHash(text):
    encodedText = encode(text)
    return hashlib.md5(encodedText).hexdigest()


def splitIntoParagraphs(text):
    paragraphs = text.split("\n\n")

    return paragraphs


def splitIntoSentences(text):
    p = text

    p = p.replace('. ', '.@@@@')
    p = p.replace('\n', '\n@@@@')
    p = p.replace('; ', ';@@@@')
    p = p.replace('? ', '?@@@@')
    p = p.replace('! ', '!@@@@')
    #p = p.replace('.{', '.||{')
    #p = p.replace('!{', '!||{')
    #p = p.replace('?{', '?||{')
{'fraction_non_alphanumeric': 0.2633093525179856, 'fraction_numerical': 0.01223021582733813, 'mean_word_length': 3.1771771771771773, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7276435', 'n_tokens_mistral': 556, 'n_tokens_neox': 528, 'n_words': 106}