compliance_checks/general_limitations.py

from compliance_checks.base import ComplianceResult, ComplianceCheck, walk_to_next_heading
from bs4 import BeautifulSoup


class GeneralLimitationsResult(ComplianceResult):
    name = "General Limitations"

    def __init__(
            self,
            limitations: str = None,
            *args,
            **kwargs,
    ):
        super().__init__(*args, **kwargs)
        self.limitations = limitations

    def __eq__(self, other):
        if isinstance(other, GeneralLimitationsResult):
            if super().__eq__(other):
                try:
                    assert self.limitations == other.limitations
                    return True
                except AssertionError:
                    return False
        else:
            return False

    def to_string(self):
        if self.status:
            return """\
            It's important for model cards to document the model's general limitations! We found some documentation \
            for this in this model card. We look for this by searching for headings that say things like:
            - Bias, Risks, and Limitations
            - Intended uses & limitations
            - Limitations
            """
        else:
            return """\
            We weren't able to find a section in this model card for the model's limitations, but it's easy to \
            add one! You can add the following section to the model card and, once you fill in the \
            `[More Information Needed]` sections, the "General Limitations" check should pass 🤗

            ```md
            ## Bias, Risks, and Limitations
            
            <!-- This section is meant to convey both technical and sociotechnical limitations. -->
            
            [More Information Needed]
            
            ### Recommendations
            
            <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
            
            Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
            ```
            """


class GeneralLimitationsCheck(ComplianceCheck):
    name = "General Limitations"

    def run_check(self, card: BeautifulSoup):
        combos = [
            ("h1", "Bias, Risks, and Limitations"), ("h2", "Bias, Risks, and Limitations"),
            ("h2", "Intended uses & limitations"),
            ("h1", "Risks and Limitations"),
            ("h2", "Risks, Limitations and Biases"),
            ("h2", "Limitations and Bias"),
            ("h3", "Limitations and bias"),
            ("h1", "Limitations"), ("h2", "Limitations"),
            ("h2", "Performance and Limitations"),
        ]

        for hX, heading in combos:
            purpose_check = walk_to_next_heading(card, hX, heading)
            if purpose_check:
                return GeneralLimitationsResult(
                    status=True,
                )

        return GeneralLimitationsResult()