<commit_before>"""Functions for extracting document attributes from its plaintext
contents.
"""

from collections import OrderedDict
from datetime import datetime
from dateutil.parser import parse as date_parse
import os
import re


empty_line = re.compile(r"\s*\n$")

def properties(lines):
    property_pattern = re.compile(r"^([a-z]+(\s+[a-z]+)*): (.*)(\n|$)", re.I)
    properties = {}
    last_property = None

    for line in lines:
        m = property_pattern.match(line)

        if m:
            properties[m.group(1)] = m.group(3)
            last_property = m.group(1).strip()
        elif empty_line.match(line):
            last_property = None
        elif last_property:
            properties[last_property] += " " + line.strip()

    return properties

def paragraphize(lines):
    ps = []
    current_p = []

    for line in lines:
        if empty_line.match(line):
            ps.append(current_p)
            current_p = []
        else:
            current_p.append(line.strip())

    ps.append(current_p)

    return [" ".join(p) for p in ps if p]

def make_matcher(patt, group=None, value=None, fn=None):
    assert group or value, "Matcher must have a group or value"

    if isinstance(patt, str):
        patt = re.compile(patt)

    def matcher(line):
        m = patt.search(line)

        if m:
            v = m.group(group) if group else value

            return fn(v) if fn else v

        return None

    return matcher

def footer_matcher(line):
    if re.match(r"CITY HALL", line):
        def skip_lines(infile):
            next(infile)
            next(infile)
        return skip_lines



def generate_sections(lines, matchers):
    """

    :returns: A generator that produces 2-tuples containing each section
    name and its contents as a list of strings

    """
    section_name = "header"
    section = []

    for line in lines:
        line = line.strip()
        new_section_name = None

        for matcher in matchers:
            name = matcher(line)

            if callable(name):
                name = name(lines)

            if isinstance(name, str):
                new_section_name = name
                break

        if new_section_name:
            yield section_name, section
            section_name = new_section_name
            section = []
        else:
            section.append(line)

    yield section_name, section


def make_sections(lines, matchers):
    """Partition the contents of a file into sections using the given list
    of matches.

    :param lines: An iterator or generator that produces lines of text
    input

    :param matchers: A list of callables that, when called with a line
    of text, should return either None, a section name, or a callable.

    :returns: An OrderedDict mapping section names to section contents
    (a list of string)

    """
    found = OrderedDict()

    for section_name, section in generate_sections(lines, matchers):
        found[section_name] = section

    return found

def get_lines(doc):
    """Returns a generator that successively produces lines from the
    document."""
    enc = doc.encoding
    lines = (line.decode(enc) for line in doc.fulltext)

    return lines

staff_report_section_matchers = [
    footer_matcher,
    make_matcher(r"PLANNING STAFF REPORT",
                 value="planning staff report"),
    make_matcher(r"^[IVX]+\. ([^a-z]+)(\n|$)",
                 group=1,
                 fn=str.lower),
]

def staff_report_properties(doc):
    """Extract a dictionary of properties from the plaintext contents of a
    Planning Staff Report.
    """
    lines = get_lines(doc)
    sections = make_sections(lines, staff_report_section_matchers)

    props = {}

    props.update(properties(sections["header"]))
    props.update(properties(sections["planning staff report"]))

    return props

# Decision Documents
def find_vote(decision):
    """
    From the decision text,
    """
    patt = re.compile(r"voted (\d+-\d+)", re.I)
    m = re.search(patt, decision)

    if m:
        return m.group(1)

    return None, None

decision_section_matchers = [
    footer_matcher,
    make_matcher(r"(ZBA DECISION|DESCRIPTION):?", group=1, fn=str.lower),
    make_matcher(r"DECISION:", value="decision")
]

def decision_properties(doc):
    """Extract a dictionary of properties from the contents of a Decision
    Document."""
    lines = get_lines(doc)
    sections = make_sections(lines, decision_section_matchers)

    props = {}
    props.update(properties(sections["zba decision"]))

    vote = find_vote(" ".join(sections["decision"]))
    if vote:
        props["Vote"] = vote

    return props

    #props.update(properties(sections[

def doc_type(doc):
    if doc.field == "reports" and re.match(r"staff report", doc.title, re.I):
        return "report"
    elif re.match(r"decision", doc.title, re.I):
        return "decision"

    return ""

def get_properties(doc):<commit_msg>Extract more properties from staff reports<commit_after>"""Functions for extracting document attributes from its plaintext
contents.
"""

from collections import OrderedDict
from datetime import datetime
from dateutil.parser import parse as date_parse
from utils import pushback_iter

import os
import re


empty_line = re.compile(r"\s*\n$")
property_pattern = re.compile(r"^([a-z]+(\s+[a-z]+)*): (.*)(\n|$)", re.I)

def properties(lines):
    properties = {}
    last_property = None

    for line in lines:
        m = property_pattern.match(line)

        if m:
            properties[m.group(1)] = m.group(3)
            last_property = m.group(1).strip()
        elif empty_line.match(line):
            last_property = None
        elif last_property:
            properties[last_property] += " " + line.strip()

    return properties


def paragraphize(lines):
    ps = []
    current_p = []

    for line in lines:
        if empty_line.match(line):
            ps.append(current_p)
            current_p = []
        else:
            current_p.append(line.strip())

    ps.append(current_p)

    return [" ".join(p) for p in ps if p]


def make_matcher(patt, group=0, value=None, fn=None):
    if isinstance(patt, str):
        patt = re.compile(patt)

    def matcher(line):
        m = patt.search(line)

        if m:
            v = m.group(group) if group else value

            return fn(v) if (fn and v) else v

        return None

    return matcher


def skip_match(patt, n=1):
    if isinstance(patt, str):
        patt = re.compile(patt)

    def skipper(line):
        if re.match(patt, line):
            def skip(inlines):
                for i in range(n):
                    next(inlines)
            return skip
    return skipper

def subsection_matcher(line):
    if re.match(r"^[0-9]+\.$", line):
        subsection_patt = re.compile(r"^([a-z]+(\s+[a-z]+)*):", re.I)
        def get_subsection_name(inlines):
            for line in inlines:
                m = subsection_patt.match(line)
                if m:
                    if isinstance(inlines, pushback_iter):
                        inlines.pushback(line[m.end():])
                    return m.group(1)
        return get_subsection_name


top_section_matcher = make_matcher(r"^([^a-z]{2,}):$", group=1,
                                   fn=str.lower)


def generate_sections(lines, matchers):
    """
    :param lines: An iterable of strings
    :param matchers: An iterable of functions

    :returns: A generator that produces 2-tuples containing each section
    name and its contents as a list of strings

    """
    section_name = "header"
    section = []

    for line in lines:
        line = line.strip()
        new_section_name = None

        for matcher in matchers:
            name = matcher(line)

            if callable(name):
                name = name(lines)

            if isinstance(name, str):
                new_section_name = name
                break

        if new_section_name:
            if section:
                yield section_name, section
            section_name = new_section_name
            section = []
        else:
            section.append(line)

    yield section_name, section

def make_sections(lines, matchers):
    """Partition the contents of a file into sections using the given list
    of matchers.

    :param lines: An iterator or generator that produces lines of text
    input

    :param matchers: A list of callables that, when called with a line
    of text, should return either None, a section name, or a callable.

    :returns: An OrderedDict mapping section names to section contents
    (a list of string)

    """
    return OrderedDict(generate_sections(lines, matchers))


def get_lines(doc):
    """Returns a generator that successively produces lines from the
    document."""
    enc = doc.encoding
    lines = (line.decode(enc) for line in doc.fulltext)

    return lines

staff_report_section_matchers = [
    skip_match(r"CITY HALL", 2),
    skip_match(r"Page \d+ of \d+"),
    make_matcher(r"PLANNING STAFF REPORT",
                 fn=str.lower),
    make_matcher(r"^[IVX]+\. ([^a-z]+)(\n|$)",
                 group=1,
                 fn=str.lower)
]

def staff_report_sections(doc):
    lines = get_lines(doc)
    return make_sections(lines, staff_report_section_matchers)


def staff_report_properties(doc):
    """Extract a dictionary of properties from the plaintext contents of a
    Planning Staff Report.
    """
    sections = staff_report_sections(doc)
    props = {}

    props.update(properties(sections["header"]))
    props.update(properties(sections["planning staff report"]))

    pd = sections.get("project description")
    if pd:
        subsections = make_sections(pushback_iter(pd),
                                    [subsection_matcher])

        for pname in ["Proposal", "Subject Property",
                      "Green Building Practices"]:
            v = subsections.get(pname)
            if v:
                props[pname] = "\n".join(paragraphize(v))


    return props

# Decision Documents
def find_vote(decision):
    """
    From the decision text,
    """
    patt = re.compile(r"voted (\d+-\d+)", re.I)
    m = re.search(patt, decision)

    if m:
        return m.group(1)

    return None, None

decision_section_matchers = [
    skip_match(r"CITY HALL", 2),
    make_matcher(r"(ZBA DECISION|DESCRIPTION):?", group=1, fn=str.lower),
    make_matcher(r"DECISION:", value="decision"),
    top_section_matcher
]

def decision_sections(doc):
    lines = get_lines(doc)
    return make_sections(lines, decision_section_matchers)

def decision_properties(doc):
    """Extract a dictionary of properties from the contents of a Decision
    Document."""
    sections = decision_sections(doc)
    props = {}
    props.update(properties(sections["zba decision"]))

    vote = find_vote(" ".join(sections["decision"]))
    if vote:
        props["Vote"] = vote

    return props

    #props.update(properties(sections[

def doc_type(doc):
    if doc.field == "reports" and re.match(r"staff report", doc.title, re.I):
        return "report"
    elif re.match(r"decision", doc.title, re.I):
        return "decision"

    return ""

def get_properties(doc):
{'fraction_non_alphanumeric': 0.10882169298010883, 'fraction_numerical': 0.0020515565070020514, 'mean_word_length': 2.4224664224664223, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7430492', 'n_tokens_mistral': 3437, 'n_tokens_neox': 3292, 'n_words': 928}