<commit_before>import datetime
import json
import os
import random
import sys
import time

if sys.version[0] == '2':
    from HTMLParser import HTMLParser
else:
    from html.parser import HTMLParser

import dateutil.parser
import email.utils

# Python 2.x compabitlity
if sys.version[0] == '2':
    FileNotFoundError = IOError

class HTMLStripper(HTMLParser):
    """Strips HTML off an string"""
    def __init__(self):
        self.reset()
        self.strict = False
        self.fed = []
        self.convert_charrefs = True
        self.numlinks = 0
        self.links = {}

    def handle_data(self, d):
        self.fed.append(d)

    def handle_starttag(self, tag, attrs):
        if tag == 'img':
            for attr in attrs:
                if attr[0] == 'src':
                    link = attr[1]
                    break;
            self.fed.append('[Image]: {}\n'.format(link))
        elif tag == 'a':
            for attr in attrs:
                if attr[0] == 'href':
                    self.links[self.numlinks] = attr[1]
        elif tag == 'li':
            self.fed.append('- ')

    def handle_endtag(self, tag):
        if tag == 'a':
            self.fed.append(' [{}]'.format(self.numlinks))
            self.numlinks += 1

    def get_data(self):
        out = ''.join(self.fed)
        if self.numlinks:
            out += '\n'
            for l in range(self.numlinks):
                out += '  [{}]: {}\n'.format(l, self.links[l])
        return out

class Converter:
    """Compares the already parsed feeds and converts new ones to maildir"""

    TEMPLATE = u"""MIME-Version: 1.0
Date: {}
Subject: {}
From: {}
Content-Type: text/plain

[Feed2Maildir] Read the update here:
{}

{}
"""

    def __init__(self, db='~/.f2mdb', maildir='~/mail/feeds', strip=False,
                 links=False, silent=False):
        self.silent = silent
        self.maildir = os.path.expanduser(maildir)
        self.db = os.path.expanduser(db)
        self.links = links
        self.strip = strip
        if self.strip:
            self.stripper = HTMLStripper()

        try: # to read the database
            with open(self.db, 'r') as f:
                self.dbdata = json.loads(f.read())
        except FileNotFoundError:
            self.dbdata = None
        except ValueError:
            self.output('WARNING: database is malformed and will be ignored')
            self.dbdata = None

    def run(self):
        """Do a full run"""
        if self.feeds:
            self.check_maildir(self.maildir)
            self.news = self.find_new(self.feeds, self.dbdata)
            for newfeed, posts in self.news.items():
                for newpost in posts:
                    self.write(self.compose(newfeed, newpost))

    def load(self, feeds):
        """Load a list of feeds in feedparser-dict form"""
        self.feeds = feeds

    def find_new(self, feeds, db, writedb=True, dbfile=None):
        """Find the new posts by comparing them to the db, by default
        refreshing the db"""
        new = {}
        newtimes = {}
        for feed in feeds:
            feedname = feed.feed.title
            feedaliasname = feed.feed_alias_name
            try: # to get the update time from the feed itself
                feedup = self.mktime(feed.feed.updated)
            except: # there is no info, then find it in the posts
                feedup = self.find_update_time(feed)
            try: # to localize the timezone
                feedup = feedup.astimezone(dateutil.tz.tzutc())
            except: # it is naive, force UTC
                feedup = feedup.replace(tzinfo=dateutil.tz.tzutc())
            try: # to find the old update time in the db
                oldtime = self.mktime(db[feedname]).astimezone(
                            dateutil.tz.tzutc())
            except: # it is not there
                oldtime = None
            if oldtime and not oldtime.tzinfo: # force UTC
                oldtime = oldtime.replace(tzinfo=dateutil.tz.tzutc())
            # print(feedname, feedup.tzinfo)
            if not oldtime or oldtime < feedup:
                for post in feed.entries:
                    feedtime = self.post_update_time(post)
                    try: # to localize the timezone
                        feedtime = feedtime.astimezone(dateutil.tz.tzutc())
                    except: # it is naive
                        feedtime = feedtime.replace(tzinfo=dateutil.tz.tzutc())
                    if not oldtime or oldtime < feedtime:
                        try: # to append the post the the feed-list
                            new[feedaliasname].append(post)
                        except: # it is the first one, make a new list
                            new[feedaliasname] = [post, ]
            if writedb:
                newtimes[feedname] = feedup.strftime('%Y-%m-%d %H:%M:%S %Z')

        if writedb:
            if not dbfile: # use own dbfile as default
                dbfile = self.db
            try: # to write the new database
                with open(dbfile, 'w') as f:
                    f.write(json.dumps(newtimes))
            except:
                self.output('WARNING: failed to write the new database')

        return new

    def post_update_time(self, post):
        """Try to get the post time"""
        try:
            return self.mktime(post.updated)
        except AttributeError:
            try:
                return self.mktime(post.published)
            except AttributeError: # last resort
                return datetime.datetime.now()

    def find_update_time(self, feed):
        """Find the last updated post in a feed"""
        times = []
        for post in feed.entries:
            times.append(self.post_update_time(post))
        return sorted(times)[-1]

    def check_maildir(self, maildir):
        """Check access to the maildir and try to create it if not present"""
        mdirs = ('', '/tmp', '/new', '/cur')
        for mdir in mdirs:
            fullname = maildir + mdir
            if not os.access(fullname, os.W_OK):<commit_msg>Feed update time is max(newest post time, feed update time)<commit_after>import datetime
import json
import os
import random
import sys
import time

if sys.version[0] == '2':
    from HTMLParser import HTMLParser
else:
    from html.parser import HTMLParser

import dateutil.parser
import email.utils

# Python 2.x compabitlity
if sys.version[0] == '2':
    FileNotFoundError = IOError

class HTMLStripper(HTMLParser):
    """Strips HTML off an string"""
    def __init__(self):
        self.reset()
        self.strict = False
        self.fed = []
        self.convert_charrefs = True
        self.numlinks = 0
        self.links = {}

    def handle_data(self, d):
        self.fed.append(d)

    def handle_starttag(self, tag, attrs):
        if tag == 'img':
            for attr in attrs:
                if attr[0] == 'src':
                    link = attr[1]
                    break;
            self.fed.append('[Image]: {}\n'.format(link))
        elif tag == 'a':
            for attr in attrs:
                if attr[0] == 'href':
                    self.links[self.numlinks] = attr[1]
        elif tag == 'li':
            self.fed.append('- ')

    def handle_endtag(self, tag):
        if tag == 'a':
            self.fed.append(' [{}]'.format(self.numlinks))
            self.numlinks += 1

    def get_data(self):
        out = ''.join(self.fed)
        if self.numlinks:
            out += '\n'
            for l in range(self.numlinks):
                out += '  [{}]: {}\n'.format(l, self.links[l])
        return out

class Converter:
    """Compares the already parsed feeds and converts new ones to maildir"""

    TEMPLATE = u"""MIME-Version: 1.0
Date: {}
Subject: {}
From: {}
Content-Type: text/plain

[Feed2Maildir] Read the update here:
{}

{}
"""

    def __init__(self, db='~/.f2mdb', maildir='~/mail/feeds', strip=False,
                 links=False, silent=False):
        self.silent = silent
        self.maildir = os.path.expanduser(maildir)
        self.db = os.path.expanduser(db)
        self.links = links
        self.strip = strip
        if self.strip:
            self.stripper = HTMLStripper()

        try: # to read the database
            with open(self.db, 'r') as f:
                self.dbdata = json.loads(f.read())
        except FileNotFoundError:
            self.dbdata = None
        except ValueError:
            self.output('WARNING: database is malformed and will be ignored')
            self.dbdata = None

    def run(self):
        """Do a full run"""
        if self.feeds:
            self.check_maildir(self.maildir)
            self.news = self.find_new(self.feeds, self.dbdata)
            for newfeed, posts in self.news.items():
                for newpost in posts:
                    self.write(self.compose(newfeed, newpost))

    def load(self, feeds):
        """Load a list of feeds in feedparser-dict form"""
        self.feeds = feeds

    def find_new(self, feeds, db, writedb=True, dbfile=None):
        """Find the new posts by comparing them to the db, by default
        refreshing the db"""
        new = {}
        newtimes = {}
        for feed in feeds:
            feedname = feed.feed.title
            feedaliasname = feed.feed_alias_name
            feedup = self.feed_update_time(feed)
            try: # to localize the timezone
                feedup = feedup.astimezone(dateutil.tz.tzutc())
            except: # it is naive, force UTC
                feedup = feedup.replace(tzinfo=dateutil.tz.tzutc())
            try: # to find the old update time in the db
                oldtime = self.mktime(db[feedname]).astimezone(
                            dateutil.tz.tzutc())
            except: # it is not there
                oldtime = None
            if oldtime and not oldtime.tzinfo: # force UTC
                oldtime = oldtime.replace(tzinfo=dateutil.tz.tzutc())
            # print(feedname, feedup.tzinfo)
            if not oldtime or oldtime < feedup:
                for post in feed.entries:
                    feedtime = self.post_update_time(post)
                    try: # to localize the timezone
                        feedtime = feedtime.astimezone(dateutil.tz.tzutc())
                    except: # it is naive
                        feedtime = feedtime.replace(tzinfo=dateutil.tz.tzutc())
                    if not oldtime or oldtime < feedtime:
                        try: # to append the post the the feed-list
                            new[feedaliasname].append(post)
                        except: # it is the first one, make a new list
                            new[feedaliasname] = [post, ]
            if writedb:
                newtimes[feedname] = feedup.strftime('%Y-%m-%d %H:%M:%S %Z')

        if writedb:
            if not dbfile: # use own dbfile as default
                dbfile = self.db
            try: # to write the new database
                with open(dbfile, 'w') as f:
                    f.write(json.dumps(newtimes))
            except:
                self.output('WARNING: failed to write the new database')

        return new

    def post_update_time(self, post):
        """Try to get the post time"""
        try:
            return self.mktime(post.updated)
        except AttributeError:
            try:
                return self.mktime(post.published)
            except AttributeError: # last resort
                return datetime.datetime.now()

    def find_update_time(self, feed):
        """Find the last updated post in a feed"""
        times = []
        for post in feed.entries:
            times.append(self.post_update_time(post))
        return sorted(times)[-1]

    def feed_update_time(self, feed):
        # find the newest post and get its time
        newest_post_time = self.find_update_time(feed)
        try: # to get the update time from the feed itself
            feed_time = self.mktime(feed.feed.updated)
        except:
            return newest_post_time
        # Some feeds like Youtube do not update the feed's update time.
        # The value 'feed_time' is a valid date, but outdated so
        # we have to compare it with the posts' times and get the latest.
        return max(newest_post_time, feed_time)

    def check_maildir(self, maildir):
        """Check access to the maildir and try to create it if not present"""
        mdirs = ('', '/tmp', '/new', '/cur')
        for mdir in mdirs:
            fullname = maildir + mdir
            if not os.access(fullname, os.W_OK):
{'fraction_non_alphanumeric': 0.10412026726057906, 'fraction_numerical': 0.002545338848234171, 'mean_word_length': 1.6865384615384615, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '207589', 'n_tokens_mistral': 3666, 'n_tokens_neox': 3541, 'n_words': 1046}