<commit_before># Natural Language Toolkit: Indian Language POS-Tagged Corpus Reader
#
# Copyright (C) 2001-2014 NLTK Project
# Author: <NAME> <<EMAIL>>
#         <NAME> <<EMAIL>>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

"""
Indian Language POS-Tagged Corpus
Collected by A Kumaran, Microsoft Research, India
Distributed with permission

Contents:
  - Bangla: IIT Kharagpur
  - Hindi: Microsoft Research India
  - Marathi: IIT Bombay
  - Telugu: IIIT Hyderabad
"""

import codecs

from nltk import compat
from nltk.tag import str2tuple, map_tag

from nltk.corpus.reader.util import *
from nltk.corpus.reader.api import *

class IndianCorpusReader(CorpusReader):
    """
    List of words, one per line.  Blank lines are ignored.
    """
    def words(self, fileids=None):
        return concat([IndianCorpusView(fileid, enc,
                                        False, False)
                       for (fileid, enc) in self.abspaths(fileids, True)])

    def tagged_words(self, fileids=None, tagset=None):
        if tagset and tagset != self._tagset:
            tag_mapping_function = lambda t: map_tag(self._tagset, tagset, t)
        else:
            tag_mapping_function = None
        return concat([IndianCorpusView(fileid, enc,
                                        True, False, tag_mapping_function)
                       for (fileid, enc) in self.abspaths(fileids, True)])

    def sents(self, fileids=None):
        return concat([IndianCorpusView(fileid, enc,
                                        False, True)
                       for (fileid, enc) in self.abspaths(fileids, True)])

    def tagged_sents(self, fileids=None, tagset=None):
        if tagset and tagset != self._tagset:
            tag_mapping_function = lambda t: map_tag(self._tagset, tagset, t)
        else:
            tag_mapping_function = None
        return concat([IndianCorpusView(fileid, enc,
                                        True, True, tag_mapping_function)
                       for (fileid, enc) in self.abspaths(fileids, True)])

    def raw(self, fileids=None):
        if fileids is None: fileids = self._fileids
        elif isinstance(fileids, compat.string_types): fileids = [fileids]
        return concat([self.open(f).read() for f in fileids])


class IndianCorpusView(StreamBackedCorpusView):
    def __init__(self, corpus_file, encoding, tagged,
                 group_by_sent, tag_mapping_function=None):
        self._tagged = tagged
        self._group_by_sent = group_by_sent
        self._tag_mapping_function = tag_mapping_function
        StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)

    def read_block(self, stream):
        line = stream.readline()
        if line.startswith('<'):
            return []
        sent = [str2tuple(word, sep='_') for word in line.split()]
        if self._tag_mapping_function:
            sent = [(w, self._tag_mapping_function(t)) for (w,t) in sent]
        if not self._tagged: sent = [w for (w,t) in sent]
        if self._group_by_sent:
            return [sent]
        else:
            return sent


<commit_msg>removed unnecessary import of codecs, cf https://github.com/robwahl/nltk/commit/13815be63d6243308d585245bd5041d56ed460e7<commit_after># Natural Language Toolkit: Indian Language POS-Tagged Corpus Reader
#
# Copyright (C) 2001-2014 NLTK Project
# Author: <NAME> <<EMAIL>>
#         <NAME> <<EMAIL>>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

"""
Indian Language POS-Tagged Corpus
Collected by <NAME>, Microsoft Research, India
Distributed with permission

Contents:
  - Bangla: IIT Kharagpur
  - Hindi: Microsoft Research India
  - Marathi: IIT Bombay
  - Telugu: IIIT Hyderabad
"""

from nltk import compat
from nltk.tag import str2tuple, map_tag

from nltk.corpus.reader.util import *
from nltk.corpus.reader.api import *

class IndianCorpusReader(CorpusReader):
    """
    List of words, one per line.  Blank lines are ignored.
    """
    def words(self, fileids=None):
        return concat([IndianCorpusView(fileid, enc,
                                        False, False)
                       for (fileid, enc) in self.abspaths(fileids, True)])

    def tagged_words(self, fileids=None, tagset=None):
        if tagset and tagset != self._tagset:
            tag_mapping_function = lambda t: map_tag(self._tagset, tagset, t)
        else:
            tag_mapping_function = None
        return concat([IndianCorpusView(fileid, enc,
                                        True, False, tag_mapping_function)
                       for (fileid, enc) in self.abspaths(fileids, True)])

    def sents(self, fileids=None):
        return concat([IndianCorpusView(fileid, enc,
                                        False, True)
                       for (fileid, enc) in self.abspaths(fileids, True)])

    def tagged_sents(self, fileids=None, tagset=None):
        if tagset and tagset != self._tagset:
            tag_mapping_function = lambda t: map_tag(self._tagset, tagset, t)
        else:
            tag_mapping_function = None
        return concat([IndianCorpusView(fileid, enc,
                                        True, True, tag_mapping_function)
                       for (fileid, enc) in self.abspaths(fileids, True)])

    def raw(self, fileids=None):
        if fileids is None: fileids = self._fileids
        elif isinstance(fileids, compat.string_types): fileids = [fileids]
        return concat([self.open(f).read() for f in fileids])


class IndianCorpusView(StreamBackedCorpusView):
    def __init__(self, corpus_file, encoding, tagged,
                 group_by_sent, tag_mapping_function=None):
        self._tagged = tagged
        self._group_by_sent = group_by_sent
        self._tag_mapping_function = tag_mapping_function
        StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)

    def read_block(self, stream):
        line = stream.readline()
        if line.startswith('<'):
            return []
        sent = [str2tuple(word, sep='_') for word in line.split()]
        if self._tag_mapping_function:
            sent = [(w, self._tag_mapping_function(t)) for (w,t) in sent]
        if not self._tagged: sent = [w for (w,t) in sent]
        if self._group_by_sent:
            return [sent]
        else:
            return sent



{'fraction_non_alphanumeric': 0.09593342753964515, 'fraction_numerical': 0.00785052598524101, 'mean_word_length': 2.3775185577942737, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1853449', 'n_tokens_mistral': 1972, 'n_tokens_neox': 1875, 'n_words': 501}