# Path: pyramids/categorization.py
# CATEGORY_WILDCARD = _CATEGORY_WILDCARD
# 
# Path: pyramids/rules/leaf.py
# class LeafRule(ParseRule):
#     """Used by Parser to identify base-level (atomic) tokens, which are the
#     leaves in a parse tree."""
# 
#     def __init__(self, category: Category):
#         super().__init__()
#         self._category = category
# 
#     @property
#     def category(self) -> Category:
#         return self._category
# 
#     def __contains__(self, token: str) -> bool:
#         raise NotImplementedError()
# 
#     def __call__(self, parser_state: 'parsing.ParserState', new_token: str, index: int) -> bool:
#         if new_token in self:
#             positive, negative = self.discover_case_properties(new_token)
#             category = self._category.promote_properties(positive, negative)
#             category = extend_properties(parser_state.model, category)
#             node = trees.ParseTreeUtils.make_leaf_parse_tree_node(parser_state.tokens, self, index,
#                                                                   category)
#             parser_state.add_node(node)
#             return True
#         else:
#             return False
# 
#     @classmethod
#     def discover_case_properties(cls, token: str) -> Tuple[Set[Property], Set[Property]]:
#         token_uppered = token.upper()
#         token_lowered = token.lower()
#         positive = set()
#         if token_uppered == token_lowered:
#             positive.add(CASE_FREE)
#         elif token == token_lowered:
#             positive.add(LOWER_CASE)
#         else:
#             if token == token_uppered:
#                 positive.add(UPPER_CASE)
#             if token == token.title():
#                 positive.add(TITLE_CASE)
#                 positive.add(MIXED_CASE)
#         if not positive:
#             positive.add(MIXED_CASE)
#         negative = {CASE_FREE, LOWER_CASE, UPPER_CASE, TITLE_CASE, MIXED_CASE} - positive
#         return positive, negative
# 
#     def iter_scoring_features(self, parse_node: trees.TreeNodeInterface) \
#             -> Iterator[scoring.ScoringFeature]:
#         # CAREFUL!!! Scoring features must be perfectly recoverable via
#         # ast.literal_eval(repr(feature))
#         head_cat = str(parse_node.payload.category.name)
#         head_token = parse_node.payload.head_spelling
#         yield scoring.ScoringFeature(('head spelling', (head_cat, head_token)))
#         for prop in parse_node.payload.category.positive_properties:
#             yield scoring.ScoringFeature(('head properties', (head_cat, str(prop))))
# 
# Path: pyramids/word_sets.py
# class WordSetUtils:
# 
#     @staticmethod
#     def load_word_set(file_path: str) -> SortedSet:
#         """Load a word set and return it as a set rule."""
#         with open(file_path, encoding='utf-8') as file:
#             return SortedSet(line.strip() for line in file)
# 
#     @staticmethod
#     def save_word_set(file_path: str, words: Iterable[str]) -> None:
#         """Load a word set and return it as a set rule."""
#         with open(file_path, 'w', encoding='utf-8') as file:
#             for word in SortedSet(words):
#                 file.write(word)
#                 file.write('\n')


import os
from sys import intern
from typing import Iterable, FrozenSet, Optional
from pyramids.categorization import Category
from pyramids.rules.leaf import LeafRule
from pyramids.word_sets import WordSetUtils
        from pyramids.grammar import GrammarSyntaxError, GrammarParser



class SetRule(LeafRule):

    @classmethod
    def from_word_set(cls, file_path: str, verbose: bool = False) -> 'SetRule':
        """Load a word set and return it as a set rule."""
        folder, filename = os.path.split(file_path)
        category_definition = os.path.splitext(filename)[0]
        try:
            category = GrammarParser.parse_category(category_definition)
        except GrammarSyntaxError as error:
            raise IOError("Badly named word set file: " + file_path) from error
        if verbose:
            print("Loading category", str(category), "from", file_path, "...")

# Path: pyramids/categorization.py
# CATEGORY_WILDCARD = _CATEGORY_WILDCARD
# 
# Path: pyramids/rules/leaf.py
# class LeafRule(ParseRule):
#     """Used by Parser to identify base-level (atomic) tokens, which are the
#     leaves in a parse tree."""
# 
#     def __init__(self, category: Category):
#         super().__init__()
#         self._category = category
# 
#     @property
#     def category(self) -> Category:
#         return self._category
# 
#     def __contains__(self, token: str) -> bool:
#         raise NotImplementedError()
# 
#     def __call__(self, parser_state: 'parsing.ParserState', new_token: str, index: int) -> bool:
#         if new_token in self:
#             positive, negative = self.discover_case_properties(new_token)
#             category = self._category.promote_properties(positive, negative)
#             category = extend_properties(parser_state.model, category)
#             node = trees.ParseTreeUtils.make_leaf_parse_tree_node(parser_state.tokens, self, index,
#                                                                   category)
#             parser_state.add_node(node)
#             return True
#         else:
#             return False
# 
#     @classmethod
#     def discover_case_properties(cls, token: str) -> Tuple[Set[Property], Set[Property]]:
#         token_uppered = token.upper()
#         token_lowered = token.lower()
#         positive = set()
#         if token_uppered == token_lowered:
#             positive.add(CASE_FREE)
#         elif token == token_lowered:
#             positive.add(LOWER_CASE)
#         else:
#             if token == token_uppered:
#                 positive.add(UPPER_CASE)
#             if token == token.title():
#                 positive.add(TITLE_CASE)
#                 positive.add(MIXED_CASE)
#         if not positive:
#             positive.add(MIXED_CASE)
#         negative = {CASE_FREE, LOWER_CASE, UPPER_CASE, TITLE_CASE, MIXED_CASE} - positive
#         return positive, negative
# 
#     def iter_scoring_features(self, parse_node: trees.TreeNodeInterface) \
#             -> Iterator[scoring.ScoringFeature]:
#         # CAREFUL!!! Scoring features must be perfectly recoverable via
#         # ast.literal_eval(repr(feature))
#         head_cat = str(parse_node.payload.category.name)
#         head_token = parse_node.payload.head_spelling
#         yield scoring.ScoringFeature(('head spelling', (head_cat, head_token)))
#         for prop in parse_node.payload.category.positive_properties:
#             yield scoring.ScoringFeature(('head properties', (head_cat, str(prop))))
# 
# Path: pyramids/word_sets.py
# class WordSetUtils:
# 
#     @staticmethod
#     def load_word_set(file_path: str) -> SortedSet:
#         """Load a word set and return it as a set rule."""
#         with open(file_path, encoding='utf-8') as file:
#             return SortedSet(line.strip() for line in file)
# 
#     @staticmethod
#     def save_word_set(file_path: str, words: Iterable[str]) -> None:
#         """Load a word set and return it as a set rule."""
#         with open(file_path, 'w', encoding='utf-8') as file:
#             for word in SortedSet(words):
#                 file.write(word)
#                 file.write('\n')


# Path: pyramids/rules/token_set.py
import os
from sys import intern
from typing import Iterable, FrozenSet, Optional
from pyramids.categorization import Category
from pyramids.rules.leaf import LeafRule
from pyramids.word_sets import WordSetUtils
        from pyramids.grammar import GrammarSyntaxError, GrammarParser




class SetRule(LeafRule):

    @classmethod
    def from_word_set(cls, file_path: str, verbose: bool = False) -> 'SetRule':
        """Load a word set and return it as a set rule."""
        folder, filename = os.path.split(file_path)
        category_definition = os.path.splitext(filename)[0]
        try:
            category = GrammarParser.parse_category(category_definition)
        except GrammarSyntaxError as error:
            raise IOError("Badly named word set file: " + file_path) from error
        if verbose:
            print("Loading category", str(category), "from", file_path, "...")

        return SetRule(category, WordSetUtils.load_word_set(file_path), _word_set_path=file_path)