<commit_before>from . import models
from pagemodel import (Node, StrictNode, Text, List, Opt,
                       Html, StrictHtml, ThisClass)
from pagemodel.bs import PageModel


class Example(PageModel):
    model_class = models.Example

    page_tree = StrictHtml(
        Node("strong")(
            key=Text()
        ),
        Node("div.SEP")(),
        Node("p.EXAMPLE")(
            content=Text()
        ),
    )


class Sense(PageModel):
    model_class = models.Sense

    page_tree = StrictHtml(
        Node("div.SENSE-NUM", "span.SYNTAX-CODING", opt=True),
        Node("span.STYLE-LEVEL", opt=True)(
            style_level=Text()
        ),
        Node("span.DEFINITION", "span.QUICK-DEFINITION")(
            definition=Text()
        ),
        List(Node("strong", "span.SENSE-VARIANT span.BASE",
                  "span.MULTIWORD span.BASE")(
            keys=Text()
        )),
        List(Node("div.EXAMPLES")(
            examples=Example()
        )),
        Node("div.THES")(),
        Node("ol.SUB-SENSES", opt=True)(
            sub_senses=List(Node("div.SUB-SENSE-CONTENT")(
                ThisClass()
            ))
        ),
    )


class RelatedLink(PageModel):
    model_class = models.RelatedLink

    page_tree = None # TODO


class PhraseLink(PageModel):
    model_class = models.LazyDictEntry

    page_tree = None # TODO


class DictEntry(PageModel):
    model_class = models.DictEntry

    page_tree = Html(
        Node("div#headword div#headwordleft span.BASE")(
            word=Text()
        ),
        
        Node("div#headbar")(
            Node("span.STYLE-LEVEL", opt=True)(
                style_level=Text()
            ),
            Node("span.PRON", opt=True)(
                pron=Text()
            )
            Node("span.PART-OF-SPEECH", opt=True)
        )
        Node("ol.SENSES")(
            senses=List(Node("div.SENSE-BODY")(
                Sense()
            ))
        ),
        Node("div#phrases_container > ul")(
            phrases=List(Node("li")(
                # here code from macm_parser_css is outdated,
                # and now they are links to separate dictionary definitions
                # so this is TODO
                PhraseLink()
            ))
        ),
        Node("div#phrasal_verbs_container > ul")(
            phrasal_verbs=List(Node("li")(
                PhraseLink()
            ))
        ),
        Node("div.entrylist > ul")(
            related=List(Node("li")(
                RelatedLink()
            ))
        )
        # TODO
    )<commit_msg>little changes<commit_after>from . import models
from pagemodel import (Node, StrictNode, Text, NodeList, Opt,
                       Html, StrictHtml, ThisClass)
# ShallowText, 
# important: Text should have some subset of string methods available
from pagemodel.bs import PageModel
# na razie robimy tylko to, co jest w tym przykładzie użycia:
# nie robimy jeszcze żadnego Reduce ani join, bo mogą się nie przydać
# natomiast przydadzą się Text.{various string methods} oraz ShallowText
# zmiana: List(Node( .. )) -> NodeList(..) - łatwiejsza w implementacji i bardziej
# naturalne to jest i czytelne. Ponadto, keyword args to mogą być albo Text, 
# albo NestedModel. NIE MOŻNA: keyword=NodeList(Node(..)), etc.

class Example(PageModel):
    model_class = models.Example

    page_tree = StrictHtml(
        Node("strong")(
            key=Text()
        ),
        Node("div.SEP")(),
        Node("p.EXAMPLE")(
            content=Text()
        ),
    )


class Sense(PageModel):
    model_class = models.Sense

    page_tree = StrictHtml(
        Node("div.SENSE-NUM", "span.SYNTAX-CODING", opt=True),
        Node("span.STYLE-LEVEL", opt=True)(
            style_level=Text.replace("$", "").lower()
        ),
        Node("span.DEFINITION", "span.QUICK-DEFINITION")(
            definition=Text()
        ),
        NodeList("strong", "span.SENSE-VARIANT span.BASE",
                  "span.MULTIWORD span.BASE")(
            keys=Text.strip()
        ),
        NodeList("div.EXAMPLES")(
            examples=Example()
        ),
        Node("div.THES")(),
        Node("ol.SUB-SENSES", opt=True)(
            NodeList("div.SUB-SENSE-CONTENT")(
                sub_senses=ThisClass()
            )
        ),
    )


class RelatedLink(PageModel):
    model_class = models.RelatedLink

    page_tree = None # TODO


class PhraseLink(PageModel):
    model_class = models.LazyDictEntry

    page_tree = None # TODO


class DictEntry(PageModel):
    model_class = models.DictEntry

    page_tree = Html(
        Node("div#headword div#headwordleft span.BASE")(
            word=Text.strip()
        ),
        
        Node("div#headbar")(
            Node("span.STYLE-LEVEL", opt=True)(
                style_level=Text.lower()
            ),
            Node("span.PRON", opt=True)(
                pron=Text()
            )
            Node("span.PART-OF-SPEECH", opt=True)
        )
        Node("ol.SENSES")(
            NodeList("div.SENSE-BODY")(
                senses=Sense()
            )
        ),
        Node("div#phrases_container > ul")(
            NodeList("li")(
                # here code from macm_parser_css is outdated,
                # and now they are links to separate dictionary definitions
                # so this is TODO
                phrases=PhraseLink()
            ))
        ),
        Node("div#phrasal_verbs_container > ul")(
            NodeList("li")(
                phrasal_verbs=PhraseLink()
            ))
        ),
        Node("div.entrylist > ul")(
            NodeList("li")(
                related=RelatedLink()
            ))
        )
        # TODO
    )
{'fraction_non_alphanumeric': 0.12561576354679804, 'fraction_numerical': 0.0, 'mean_word_length': 1.7331730769230769, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4030885', 'n_tokens_mistral': 1778, 'n_tokens_neox': 1720, 'n_words': 321}