<commit_before>from nose.tools import *
from whitepy.lexerconstants import *
import unittest
import whitepy.lexer as lexer


class TestLexer(unittest.TestCase):

    def _get_lexer(self, line):
        return lexer.Lexer(line=line)

    def _sample_ws(self, ws_type):
        ws_samples = {
            'valid': "   \t\n",
            'invalid_int': "   \t"
        }
        return self._get_lexer(ws_samples[ws_type])

    def test_get_int(self):
        lexer = self._sample_ws('valid')
        r = lexer._get_int()
        assert r.get_type() == 'INT' and r.get_value() == 'POSITIVE 001'

    @raises(lexer.IntError)
    def test_invalid_int(self):
        lexer = self._sample_ws('invalid_int')
        lexer._get_int()

    def test_get_token(self):
        lexer = self._sample_ws('valid')
        lexer.pos = 1
        r = lexer._get_token(IMP_CONST)
        assert r.get_type() == 'STACK_MANIPULATION'

    def test_get_all_tokens(self):
        lexer = self._sample_ws('valid')
        lexer.get_all_tokens()
        t = lexer.tokens
        assert t[0].get_type() is 'STACK_MANIPULATION' and \
            t[1].get_type() is 'PUSH' and t[2].get_type() is 'INT'
<commit_msg>Add test to check non ws characters<commit_after>from nose.tools import *
from whitepy.lexerconstants import *
import unittest
import whitepy.lexer as lexer


class TestLexer(unittest.TestCase):

    def _get_lexer(self, line):
        return lexer.Lexer(line=line)

    def _sample_ws(self, ws_type):
        ws_samples = {
            'valid': "   \t\n",
            'invalid_int': "   \t",
            'non_ws': "A   \tB\nC"
        }
        return self._get_lexer(ws_samples[ws_type])

    def test_get_int(self):
        lexer = self._sample_ws('valid')
        r = lexer._get_int()
        assert r.get_type() == 'INT' and r.get_value() == 'POSITIVE 001'

    @raises(lexer.IntError)
    def test_invalid_int(self):
        lexer = self._sample_ws('invalid_int')
        lexer._get_int()

    def test_get_token(self):
        lexer = self._sample_ws('valid')
        lexer.pos = 1
        r = lexer._get_token(IMP_CONST)
        assert r.get_type() == 'STACK_MANIPULATION'

    def test_get_all_tokens(self):
        lexer = self._sample_ws('valid')
        lexer.get_all_tokens()
        t = lexer.tokens
        assert t[0].get_type() is 'STACK_MANIPULATION' and \
            t[1].get_type() is 'PUSH' and t[2].get_type() is 'INT'

    def test_get_all_tokens_with_non_ws(self):
        lexer = self._sample_ws('non_ws')
        lexer.get_all_tokens()
        t = lexer.tokens
        assert t[0].get_type() is 'STACK_MANIPULATION' and \
            t[1].get_type() is 'PUSH' and t[2].get_type() is 'INT'

{'fraction_non_alphanumeric': 0.12518518518518518, 'fraction_numerical': 0.006296296296296296, 'mean_word_length': 2.4806701030927836, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '192094', 'n_tokens_mistral': 1015, 'n_tokens_neox': 963, 'n_words': 184}