<commit_before>#!/usr/bin/env python

import unittest

from csvkit.cli import match_column_identifier, parse_column_identifiers

class TestCli(unittest.TestCase):
    def setUp(self):
        self.headers = ['id', 'name', 'i_work_here', '1', 'more-header-values', 'stuff', 'blueberry']

    def test_match_column_identifier_string(self):
        self.assertEqual(2, match_column_identifier(self.headers, 'i_work_here'))
        self.assertEqual(2, match_column_identifier(self.headers, 'i_work_here', zero_based=True))

    def test_match_column_identifier_numeric(self):
        self.assertEqual(2, match_column_identifier(self.headers, 3))
        self.assertEqual(3, match_column_identifier(self.headers, 3, zero_based=True))

    def test_match_column_which_could_be_integer_name_is_treated_as_positional_id(self):
        self.assertEqual(0, match_column_identifier(self.headers, '1'))
        self.assertEqual(1, match_column_identifier(self.headers, '1', zero_based=True))

    def test_parse_column_identifiers(self):
        self.assertEqual([2, 0, 1], parse_column_identifiers(' i_work_here, 1,name  ', self.headers))
        self.assertEqual([2, 1, 1], parse_column_identifiers(' i_work_here, 1,name  ', self.headers, zero_based=True))

    def test_range_notation(self):
        self.assertEqual([0,1,2], parse_column_identifiers('1:3', self.headers))
        self.assertEqual([1,2,3], parse_column_identifiers('1:3', self.headers, zero_based=True))
        self.assertEqual([1,2,3], parse_column_identifiers('2-4', self.headers))        
        self.assertEqual([2,3,4], parse_column_identifiers('2-4', self.headers, zero_based=True))
        self.assertEqual([0,1,2,3], parse_column_identifiers('1,2:4', self.headers))        
        self.assertEqual([1,2,3,4], parse_column_identifiers('1,2:4', self.headers, zero_based=True))
        self.assertEqual([4,2,5], parse_column_identifiers('more-header-values,3,stuff', self.headers))
        self.assertEqual([4,3,5], parse_column_identifiers('more-header-values,3,stuff', self.headers,zero_based=True))

    def test_range_notation_open_ended(self):
        self.assertEqual([0,1,2], parse_column_identifiers(':3', self.headers))
        target = range(3,len(self.headers) - 1) # protect against devs adding to self.headers
        target.insert(0,0)
        self.assertEqual(target, parse_column_identifiers('1,4:', self.headers))        
        <commit_msg>fix off-by-one in tests too; add test for full range<commit_after>#!/usr/bin/env python

import unittest

from csvkit.cli import match_column_identifier, parse_column_identifiers

class TestCli(unittest.TestCase):
    def setUp(self):
        self.headers = ['id', 'name', 'i_work_here', '1', 'more-header-values', 'stuff', 'blueberry']

    def test_match_column_identifier_string(self):
        self.assertEqual(2, match_column_identifier(self.headers, 'i_work_here'))
        self.assertEqual(2, match_column_identifier(self.headers, 'i_work_here', zero_based=True))

    def test_match_column_identifier_numeric(self):
        self.assertEqual(2, match_column_identifier(self.headers, 3))
        self.assertEqual(3, match_column_identifier(self.headers, 3, zero_based=True))

    def test_match_column_which_could_be_integer_name_is_treated_as_positional_id(self):
        self.assertEqual(0, match_column_identifier(self.headers, '1'))
        self.assertEqual(1, match_column_identifier(self.headers, '1', zero_based=True))

    def test_parse_column_identifiers(self):
        self.assertEqual([2, 0, 1], parse_column_identifiers(' i_work_here, 1,name  ', self.headers))
        self.assertEqual([2, 1, 1], parse_column_identifiers(' i_work_here, 1,name  ', self.headers, zero_based=True))

    def test_range_notation(self):
        self.assertEqual([0,1,2], parse_column_identifiers('1:3', self.headers))
        self.assertEqual([1,2,3], parse_column_identifiers('1:3', self.headers, zero_based=True))
        self.assertEqual([1,2,3], parse_column_identifiers('2-4', self.headers))        
        self.assertEqual([2,3,4], parse_column_identifiers('2-4', self.headers, zero_based=True))
        self.assertEqual([0,1,2,3], parse_column_identifiers('1,2:4', self.headers))        
        self.assertEqual([1,2,3,4], parse_column_identifiers('1,2:4', self.headers, zero_based=True))
        self.assertEqual([4,2,5], parse_column_identifiers('more-header-values,3,stuff', self.headers))
        self.assertEqual([4,3,5], parse_column_identifiers('more-header-values,3,stuff', self.headers,zero_based=True))

    def test_range_notation_open_ended(self):
        self.assertEqual([0,1,2], parse_column_identifiers(':3', self.headers))

        target = range(3,len(self.headers)) # protect against devs adding to self.headers
        target.insert(0,0)
        self.assertEqual(target, parse_column_identifiers('1,4:', self.headers))        

        self.assertEqual(range(0,len(self.headers)), parse_column_identifiers('1:', self.headers))
{'fraction_non_alphanumeric': 0.1388386578259996, 'fraction_numerical': 0.0287321679726743, 'mean_word_length': 4.940334128878281, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4965606', 'n_tokens_mistral': 1633, 'n_tokens_neox': 1587, 'n_words': 235}