<commit_before># Copyright © 2013, 2014 <NAME>
# This file is part of Red Wind.
#
# Red Wind is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Red Wind is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Red Wind.  If not, see <http://www.gnu.org/licenses/>.

from . import app
from collections import deque
import mf2py
import datetime
import re


class Author:
    def __init__(self, name, url, photo):
        self.name = name
        self.url = url
        self.photo = photo

    def __repr__(self):
        return "Author[name={}, url={}, photo={}]".format(
            self.name, self.url, self.photo)


class Reference:
    def __init__(self, url, reftype):
        self.url = url
        self.reftype = reftype

    def __repr__(self):
        return "Reference[url={}, reftype={}]".format(self.url, self.reftype)


class Entry:
    def __init__(self, author, permalink, pub_date,
                 references, title, content):
        self.author = author
        self.permalink = permalink
        self.pub_date = pub_date
        self.references = references
        self.title = title
        self.content = content

    def __repr__(self):
        return "Entry[author={}, permalink={}, pub_date={}, references={}, title={}, content={}]".format(
            self.author, self.permalink, self.pub_date,
            self.references, self.title, self.content[:100].replace('\n', ''))


def parse_html(txt, source):
    p = mf2py.Parser(doc=txt, url=source)
    return parse_json(p.to_dict())


def parse_json(d, source):
    def parse_references(objs, reftype):
        refs = []
        for obj in objs:
            if isinstance(obj, str):
                refs.append(Reference(obj, reftype))
            else:
                refs += [Reference(url, reftype) for url
                         in obj.get('properties', {}).get('url', [])]
        return refs

    def parse_author(obj):
        if isinstance(obj, str):
            return Author(obj, None, None)
        else:
            names = obj['properties'].get('name')
            photos = obj['properties'].get('photo')
            urls = obj['properties'].get('url')
            return Author(names and names[0],
                          urls and urls[0],
                          photos and photos[0])

    references = []

    for rel, rel_url in d['rels'].items():
        if rel in ('like', 'like-of'):
            references.append(Reference(rel_url, 'like'))
        elif rel in ('reply', 'reply-to', 'in-reply-to'):
            references.append(Reference(rel_url, 'reply'))
        elif rel in ('repost', 'repost-of'):
            references.append(Reference(rel_url, 'repost'))

    entry = None
    queue = deque(item for item in d['items'])
    while queue:
        item = queue.popleft()
        if 'h-entry' in item['type']:
            hentry = item
            permalink = next((perma for perma
                              in hentry['properties'].get('url', [])), source)

            for prop in ('in-reply-to', 'reply-to', 'reply'):
                references += parse_references(
                    hentry['properties'].get(prop, []), 'reply')

            for prop in ('like-of', 'like'):
                references += parse_references(
                    hentry['properties'].get(prop, []), 'like')

            for prop in ('repost-of', 'repost'):
                references += parse_references(
                    hentry['properties'].get(prop, []), 'repost')

            date_strs = hentry['properties'].get('published')
            pub_date = date_strs and parse_datetime(' '.join(date_strs))
            if pub_date and pub_date.tzinfo:
                pub_date = pub_date.astimezone(datetime.timezone.utc)

            content_html = ''.join(content['html'].strip() for content
                                   in hentry['properties'].get('content', []))
            content_value = ''.join(content['value'].strip() for content
                                    in hentry['properties'].get('content', []))

            title = ''.join(part.strip() for part
                            in hentry['properties'].get('name', ''))

            if title == content_value:
                title = None

            author = None
            for obj in hentry['properties'].get('author', []):
                author = parse_author(obj)<commit_msg>use first e-content entry instead of joining them together<commit_after># Copyright © 2013, 2014 <NAME>
# This file is part of Red Wind.
#
# Red Wind is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Red Wind is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Red Wind.  If not, see <http://www.gnu.org/licenses/>.

from . import app
from collections import deque
import mf2py
import datetime
import re


class Author:
    def __init__(self, name, url, photo):
        self.name = name
        self.url = url
        self.photo = photo

    def __repr__(self):
        return "Author[name={}, url={}, photo={}]".format(
            self.name, self.url, self.photo)


class Reference:
    def __init__(self, url, reftype):
        self.url = url
        self.reftype = reftype

    def __repr__(self):
        return "Reference[url={}, reftype={}]".format(self.url, self.reftype)


class Entry:
    def __init__(self, author, permalink, pub_date,
                 references, title, content):
        self.author = author
        self.permalink = permalink
        self.pub_date = pub_date
        self.references = references
        self.title = title
        self.content = content

    def __repr__(self):
        return "Entry[author={}, permalink={}, pub_date={}, references={}, title={}, content={}]".format(
            self.author, self.permalink, self.pub_date,
            self.references, self.title, self.content[:100].replace('\n', ''))


def parse_html(txt, source):
    p = mf2py.Parser(doc=txt, url=source)
    return parse_json(p.to_dict())


def parse_json(d, source):
    def parse_references(objs, reftype):
        refs = []
        for obj in objs:
            if isinstance(obj, str):
                refs.append(Reference(obj, reftype))
            else:
                refs += [Reference(url, reftype) for url
                         in obj.get('properties', {}).get('url', [])]
        return refs

    def parse_author(obj):
        if isinstance(obj, str):
            return Author(obj, None, None)
        else:
            names = obj['properties'].get('name')
            photos = obj['properties'].get('photo')
            urls = obj['properties'].get('url')
            return Author(names and names[0],
                          urls and urls[0],
                          photos and photos[0])

    references = []

    for rel, rel_url in d['rels'].items():
        if rel in ('like', 'like-of'):
            references.append(Reference(rel_url, 'like'))
        elif rel in ('reply', 'reply-to', 'in-reply-to'):
            references.append(Reference(rel_url, 'reply'))
        elif rel in ('repost', 'repost-of'):
            references.append(Reference(rel_url, 'repost'))

    entry = None
    queue = deque(item for item in d['items'])
    while queue:
        item = queue.popleft()
        if 'h-entry' in item['type']:
            hentry = item
            permalink = next((perma for perma
                              in hentry['properties'].get('url', [])), source)

            for prop in ('in-reply-to', 'reply-to', 'reply'):
                references += parse_references(
                    hentry['properties'].get(prop, []), 'reply')

            for prop in ('like-of', 'like'):
                references += parse_references(
                    hentry['properties'].get(prop, []), 'like')

            for prop in ('repost-of', 'repost'):
                references += parse_references(
                    hentry['properties'].get(prop, []), 'repost')

            date_strs = hentry['properties'].get('published')
            pub_date = date_strs and parse_datetime(' '.join(date_strs))
            if pub_date and pub_date.tzinfo:
                pub_date = pub_date.astimezone(datetime.timezone.utc)

            content = hentry['properties'].get('content')
            if content:
                content_html = content[0]['html'].strip()
                content_value = content[0]['value'].strip()

            title = ''.join(part.strip() for part
                            in hentry['properties'].get('name', ''))

            if title == content_value:
                title = None

            author = None
            for obj in hentry['properties'].get('author', []):
                author = parse_author(obj)
{'fraction_non_alphanumeric': 0.12762384550797648, 'fraction_numerical': 0.003778337531486146, 'mean_word_length': 2.1150702844066687, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4193555', 'n_tokens_mistral': 2703, 'n_tokens_neox': 2614, 'n_words': 775}