<commit_before>/*
 ** (C) 2015 <NAME> <<EMAIL>>
 ** This program is part of open-tamil library
 ** You may use this code under MIT License
 */
package com.opentamil.tamil;

import junit.framework.TestCase;
import java.util.Arrays;
import java.util.List;
import java.util.HashMap;


/**
 *
 * @author <NAME>
 */
public class utf8Test extends TestCase {
    
    public utf8Test(String testName) {
        super(testName);
    }
    
    @Override
    protected void setUp() throws Exception {
        super.setUp();
    }
    
    @Override
    protected void tearDown() throws Exception {
        super.tearDown();
    }
    
    public void testGetLength() {
        String inputWord = "எழில்";
        
        HashMap<String,Object> actual = utf8.get_length(inputWord);
        TestCase.assertEquals((int)actual.get("length"), 3);
        
        List<String> expected = Arrays.asList(new String [] {"மெ","ன்","பொ","ரு","ள்"} );
        TestCase.assertEquals(utf8.get_length("மென்பொருள்").get("letters"),expected);
    }
    
    public void testGetLetters_TA() {
        String inputWord = "எழில்";
        List<String> expected = Arrays.asList(new String [] {"எ","ழி","ல்"} );
        StringBuffer actual = new StringBuffer();
        for(String t : utf8.get_letters(inputWord)) {
            actual.append( t );
        }
        
        TestCase.assertEquals(inputWord,actual.toString());
        TestCase.assertEquals(utf8.get_letters("மென்பொருள்").size(),5);<commit_msg>Add new test for "கட்டளை" word length in Java, following the blog post @ https://ezhillang.wordpress.com/2015/06/07/open-tamil-java-usage/<commit_after>/*
 ** (C) 2015 <NAME> <<EMAIL>>
 ** This program is part of open-tamil library
 ** You may use this code under MIT License
 */
package com.opentamil.tamil;

import junit.framework.TestCase;
import java.util.Arrays;
import java.util.List;
import java.util.HashMap;


/**
 *
 * @author <NAME>
 */
public class utf8Test extends TestCase {
    
    public utf8Test(String testName) {
        super(testName);
    }
    
    @Override
    protected void setUp() throws Exception {
        super.setUp();
    }
    
    @Override
    protected void tearDown() throws Exception {
        super.tearDown();
    }
    
    public void testGetLength() {
        String inputWord = "எழில்",inputWord2 = "கட்டளை";
        
        HashMap<String,Object> actual = utf8.get_length(inputWord);
        TestCase.assertEquals((int)actual.get("length"), 3);
        
        actual = utf8.get_length(inputWord2);
        TestCase.assertEquals((int)actual.get("length"), 4);
        
        List<String> expected = Arrays.asList(new String [] {"மெ","ன்","பொ","ரு","ள்"} );
        TestCase.assertEquals(utf8.get_length("மென்பொருள்").get("letters"),expected);
    }
    
    public void testGetLetters_TA() {
        String inputWord = "எழில்";
        List<String> expected = Arrays.asList(new String [] {"எ","ழி","ல்"} );
        StringBuffer actual = new StringBuffer();
        for(String t : utf8.get_letters(inputWord)) {
            actual.append( t );
        }
        
        TestCase.assertEquals(inputWord,actual.toString());
        TestCase.assertEquals(utf8.get_letters("மென்பொருள்").size(),5);
{'fraction_non_alphanumeric': 0.1541318477251625, 'fraction_numerical': 0.011142061281337047, 'mean_word_length': 2.7934272300469485, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5039871', 'n_tokens_mistral': 1074, 'n_tokens_neox': 1083, 'n_words': 205}