<commit_before># -*- coding: utf-8 -*-
# 
# (C) 2013 <NAME>
# Licensed under GPL Version 3
# 

# 
# TSCII library provides the TSCII v1.7 symbol table and mapping to Unicode
# A converter, for example, could be written based on this information.
# 
# Ref: <NAME>, "Text conversion from TSCII 1.7 to Unicode," (2007).

# load ASCII 7-bit code page first
TSCII = map( lambda x:  x < 128 and u"%c"%x or u"?" , range(0,256))

# append TSCII tamil page on higher side

TSCII_DIRECT_LOOKUP = []

# Vowels, Consonants and Tamil numerals have bijective from TSCII into Unicode 
# Sec. 1 - Vowels
TSCII[0xAB:0xB8] = [u"அ",u"ஆ",u"இ",u"ஈ",u"உ",u"ஊ",u"எ",u"ஏ",u"ஐ",u"ஒ",u"ஓ",u"ஔ",u"ஃ"]
TSCII_DIRECT_LOOKUP = TSCII_DIRECT_LOOKUP + range(0xAB,0xB8)

# Sec. 2 - Consonants
TSCII[0xB8:0xCA] = [u"க",u"ங",u"ச",u"ஞ",u"ட",u"ண",u"த",u"ந",
                    u"ப",u"ம",u"ய",u"ர",u"ல",u"வ",u"ழ",u"ள",
                    u"ற",u"ன",]
TSCII_DIRECT_LOOKUP = TSCII_DIRECT_LOOKUP + range(0xB8,0xCA)<commit_msg>TSCII versions 1.0 and 1.7 are very different.<commit_after># -*- coding: utf-8 -*-
# 
# (C) 2013 <NAME>
# Licensed under GPL Version 3
# 

# 
# TSCII library provides the TSCII v1.7 symbol table and mapping to Unicode
# A converter, for example, could be written based on this information.
# 
# Ref: <NAME>, "Text conversion from TSCII 1.7 to Unicode," (2007).

VERSION = "1.7"

# load ASCII 7-bit code page first
TSCII = map( lambda x:  x < 128 and u"%c"%x or u"?" , range(0,256))

# append TSCII tamil page on higher side

TSCII_DIRECT_LOOKUP = []

# Vowels, Consonants and Tamil numerals have bijective from TSCII into Unicode 
# Sec. 1 - Vowels
TSCII[0xAB:0xB8] = [u"அ",u"ஆ",u"இ",u"ஈ",u"உ",u"ஊ",u"எ",u"ஏ",u"ஐ",u"ஒ",u"ஓ",u"ஔ",u"ஃ"]
TSCII_DIRECT_LOOKUP = TSCII_DIRECT_LOOKUP + range(0xAB,0xB8)

# Sec. 2 - Consonants
TSCII[0xB8:0xCA] = [u"க",u"ங",u"ச",u"ஞ",u"ட",u"ண",u"த",u"ந",
                    u"ப",u"ம",u"ய",u"ர",u"ல",u"வ",u"ழ",u"ள",
                    u"ற",u"ன",]
TSCII_DIRECT_LOOKUP = TSCII_DIRECT_LOOKUP + range(0xB8,0xCA)
{'fraction_non_alphanumeric': 0.18582755203171458, 'fraction_numerical': 0.03865213082259663, 'mean_word_length': 4.0475, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7609494', 'n_tokens_mistral': 1019, 'n_tokens_neox': 953, 'n_words': 204}