<commit_before>package com.python.pydev.analysis.tabnanny;

import junit.framework.TestCase;

import org.eclipse.jface.text.Document;

public class TabNannyIteratorTest extends TestCase {

    public static void main(String[] args) {
        try {
            TabNannyIteratorTest analyzer2 = new TabNannyIteratorTest();
            analyzer2.setUp();
//            analyzer2.testIterator13();
            analyzer2.tearDown();
            System.out.println("finished");
            
            junit.textui.TestRunner.run(TabNannyIteratorTest.class);
            System.out.println("finished all");
        } catch (Throwable e) {
            e.printStackTrace();
        }
        System.exit(0);
    }

    public void testIterator() throws Exception {
        Document doc = new Document("" +
        		"aaa\\\n" +
        		"bbbb\n" +
        		"ccc\n" +
        		""
                );
        
        TabNannyDocIterator iterator = new TabNannyDocIterator(doc);
        assertTrue(!iterator.hasNext()); //no indentations here...
    }
    

    public void testIterator2() throws Exception {
        String str = "" +
        "d\n" +
        "    pass\r" +
        "";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertTrue(!it.hasNext()); //no indentations here...
    }
    
    public void testIterator3() throws Exception {
        String str = "" +
        "d\n" +
        "    '''\r" +
        "    '''\r" +
        "\t" +
        "";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertEquals("\t",it.next().o1);
        assertTrue(!it.hasNext()); //no indentations here...
    }
    
    public void testIterator4() throws Exception {
        String str = "";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertTrue(!it.hasNext()); 
    }
    
    public void testIterator5() throws Exception {
        String str = "    #comment";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertTrue(!it.hasNext()); 
    }
    
    public void testIterator6() throws Exception {
        String str = 
                "    #comment   what's happening\\\n" + //escape is in comment... (so, it's not considered the same line)
        		"    pass";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertEquals("    ",it.next().o1);
        assertTrue(!it.hasNext()); 
    }
   
    
    public void testIterator7() throws Exception {
        String str = 
            "    g g g \t g\\\n" + //escape considered 
            "    pass";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertTrue(!it.hasNext()); 
    }
    
    public void testIterator8() throws Exception {
        String str = 
            "{g }\n" +
            "    pass";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertTrue(!it.hasNext()); 
    }
    
    public void testIterator9() throws Exception {<commit_msg>- Improved code formatter to deal with operators (+, -, *, etc) - Improved code formatter to handle '=' differently inside function calls / keyword args<commit_after>package com.python.pydev.analysis.tabnanny;

import junit.framework.TestCase;

import org.eclipse.jface.text.Document;

public class TabNannyIteratorTest extends TestCase {

    public static void main(String[] args) {
        try {
            TabNannyIteratorTest analyzer2 = new TabNannyIteratorTest();
            analyzer2.setUp();
//            analyzer2.testIterator13();
            analyzer2.tearDown();
            System.out.println("finished");
            
            junit.textui.TestRunner.run(TabNannyIteratorTest.class);
            System.out.println("finished all");
        } catch (Throwable e) {
            e.printStackTrace();
        }
        System.exit(0);
    }

    public void testIterator() throws Exception {
        Document doc = new Document("" +
        		"aaa\\\n" +
        		"bbbb\n" +
        		"ccc\n" +
        		""
                );
        
        TabNannyDocIterator iterator = new TabNannyDocIterator(doc);
        assertTrue(!iterator.hasNext()); //no indentations here...
    }
    

    public void testIterator2() throws Exception {
        String str = "" +
        "d\n" +
        "    pass\r" +
        "";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertTrue(!it.hasNext()); //no indentations here...
    }
    
    public void testIterator3() throws Exception {
        String str = "" +
        "d\n" +
        "    '''\r" +
        "    '''\r" +
        "\t" +
        "";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertEquals("\t",it.next().o1);
        assertTrue(!it.hasNext()); //no indentations here...
    }
    
    public void testIterator4() throws Exception {
        String str = "";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertTrue(!it.hasNext()); 
    }
    
    public void testIterator5() throws Exception {
        String str = "    #comment";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertTrue(!it.hasNext()); 
    }
    
    public void testIterator6() throws Exception {
        String str = 
                "    #comment   what's happening\\\n" + //escape is in comment... (so, it's not considered the same line)
        		"    pass";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertEquals("    ",it.next().o1);
        assertTrue(!it.hasNext()); 
    }
    
    public void testIterator6a() throws Exception {
        String str = 
            "    #comment   what's happening\r\n" + 
            "    #comment   what's happening2\r\n" + 
            "    pass\r\n";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertEquals("    ",it.next().o1);
        assertEquals("    ",it.next().o1);
        assertTrue(!it.hasNext()); 
    }
   
    
    public void testIterator7() throws Exception {
        String str = 
            "    g g g \t g\\\n" + //escape considered 
            "    pass";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertTrue(!it.hasNext()); 
    }
    
    public void testIterator8() throws Exception {
        String str = 
            "{g }\n" +
            "    pass";
        Document d = new Document(str);
        TabNannyDocIterator it = new TabNannyDocIterator(d);
        assertEquals("    ",it.next().o1);
        assertTrue(!it.hasNext()); 
    }
    
    public void testIterator9() throws Exception {
{'fraction_non_alphanumeric': 0.1273826738530301, 'fraction_numerical': 0.006704351255422637, 'mean_word_length': 1.8377471092875792, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '835234', 'n_tokens_mistral': 2169, 'n_tokens_neox': 2135, 'n_words': 496}