<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -

# newspipe - A Web based news aggregator.
# Copyright (C) 2010-2019  <NAME> - https://www.cedricbonhomme.org
#
# For more information : https://gitlab.com/newspipe/newspipe
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

__author__ = "<NAME>"
__version__ = "$Revision: 4.0 $"
__date__ = "$Date: 2010/09/02 $"
__revision__ = "$Date: 2010/05/21 $"
__copyright__ = "Copyright (c) <NAME>"
__license__ = "AGPLv3"

import asyncio
import logging
import feedparser
import dateutil.parser
from datetime import datetime, timezone, timedelta
from sqlalchemy import or_

import conf
from bootstrap import db
from web.models import User
from web.controllers import FeedController, ArticleController
from lib.feed_utils import construct_feed_from, is_parsing_ok
from lib.article_utils import construct_article, extract_id, \
                                    get_article_content

logger = logging.getLogger(__name__)

sem = asyncio.Semaphore(5)


def get(*args, **kwargs):
    #kwargs["connector"] = aiohttp.TCPConnector(verify_ssl=False)
    try:
        logger.info('Retrieving feed {}'.format(args[0]))
        data = feedparser.parse(args[0])
        return data
    except Exception as e:
        raise e


def parse_feed(user, feed):
    """
    Fetch a feed.
    Update the feed and return the articles.
    """
    parsed_feed = None
    up_feed = {}
    articles = []
    #with (await sem):
    try:
        parsed_feed = get(feed.link)
    except Exception as e:
        up_feed['last_error'] = str(e)
        up_feed['error_count'] = feed.error_count + 1
        logger.exception("error when parsing feed: " + str(e))
    finally:
        up_feed['last_retrieved'] = datetime.now(dateutil.tz.tzlocal())
        if parsed_feed is None:
            try:
                FeedController().update({'id': feed.id}, up_feed)
            except Exception as e:
                logger.exception('something bad here: ' + str(e))
            return

    if not is_parsing_ok(parsed_feed):
        up_feed['last_error'] = str(parsed_feed['bozo_exception'])
        up_feed['error_count'] = feed.error_count + 1
        FeedController().update({'id': feed.id}, up_feed)
        return
    if parsed_feed['entries'] != []:
        articles = parsed_feed['entries']

    up_feed['error_count'] = 0
    up_feed['last_error'] = ""

    # Feed information
    try:
        construct_feed_from(feed.link, parsed_feed).update(up_feed)
    except:
         logger.exception('error when constructing feed: {}'.format(feed.link))
    if feed.title and 'title' in up_feed:
        # do not override the title set by the user
        del up_feed['title']
    FeedController().update({'id': feed.id}, up_feed)

    return articles


async def insert_articles(queue, nḅ_producers=1):
    """Consumer coroutines.
    """
    nb_producers_done = 0
    while True:
        item = await queue.get()
        if item is None:
            nb_producers_done += 1
            if nb_producers_done == nḅ_producers:
                print('All producers done.')
                print('Process finished.')
                break
            continue

        user, feed, articles = item
    
        
        if None is articles:
            logger.info('None')
            articles = []

        logger.info('Inserting articles for {}'.format(feed.title))

        new_articles = []
        art_contr = ArticleController(user.id)
        for article in articles:
            new_article = await construct_article(article, feed)

            try:
                existing_article_req = art_contr.read(feed_id=feed.id,
                                entry_id=extract_id(article))
            except Exception as e:
                logger.exception("existing_article_req: " + str(e))
                continue
            exist = existing_article_req.count() != 0
            if exist:
                continue
                # if the article has been already retrieved, we only update
                # the content or the title
                logger.info('Article already in the database: {}'. \
                                format(article['link']))
                existing_article = existing_article_req.first()

                if new_article['date'].replace(tzinfo=None) != \
                                                            existing_article.date:
                    existing_article.date = new_article['date']
                    existing_article.updated_date = new_article['date']
                    if existing_article.title != new_article['title']:
                        existing_article.title = new_article['title']
                    content = get_article_content(article)
                    if existing_article.content != content:
                        existing_article.content = content
                        existing_article.readed = False
                    art_contr.update({'entry_id': existing_article.entry_id},
                                                            existing_article.dump())
                    logger.info('Article updated: {}'.format(article['link']))
                continue<commit_msg>Do not update existing articles.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -

# newspipe - A Web based news aggregator.
# Copyright (C) 2010-2019  <NAME> - https://www.cedricbonhomme.org
#
# For more information : https://gitlab.com/newspipe/newspipe
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

__author__ = "<NAME>"
__version__ = "$Revision: 4.0 $"
__date__ = "$Date: 2010/09/02 $"
__revision__ = "$Date: 2010/05/21 $"
__copyright__ = "Copyright (c) <NAME>"
__license__ = "AGPLv3"

import asyncio
import logging
import feedparser
import dateutil.parser
from datetime import datetime, timezone, timedelta
from sqlalchemy import or_

import conf
from bootstrap import db
from web.models import User
from web.controllers import FeedController, ArticleController
from lib.feed_utils import construct_feed_from, is_parsing_ok
from lib.article_utils import construct_article, extract_id, \
                                    get_article_content

logger = logging.getLogger(__name__)

sem = asyncio.Semaphore(5)


def get(*args, **kwargs):
    #kwargs["connector"] = aiohttp.TCPConnector(verify_ssl=False)
    try:
        logger.info('Retrieving feed {}'.format(args[0]))
        data = feedparser.parse(args[0])
        return data
    except Exception as e:
        raise e


def parse_feed(user, feed):
    """
    Fetch a feed.
    Update the feed and return the articles.
    """
    parsed_feed = None
    up_feed = {}
    articles = []
    #with (await sem):
    try:
        parsed_feed = get(feed.link)
    except Exception as e:
        up_feed['last_error'] = str(e)
        up_feed['error_count'] = feed.error_count + 1
        logger.exception("error when parsing feed: " + str(e))
    finally:
        up_feed['last_retrieved'] = datetime.now(dateutil.tz.tzlocal())
        if parsed_feed is None:
            try:
                FeedController().update({'id': feed.id}, up_feed)
            except Exception as e:
                logger.exception('something bad here: ' + str(e))
            return

    if not is_parsing_ok(parsed_feed):
        up_feed['last_error'] = str(parsed_feed['bozo_exception'])
        up_feed['error_count'] = feed.error_count + 1
        FeedController().update({'id': feed.id}, up_feed)
        return
    if parsed_feed['entries'] != []:
        articles = parsed_feed['entries']

    up_feed['error_count'] = 0
    up_feed['last_error'] = ""

    # Feed information
    try:
        construct_feed_from(feed.link, parsed_feed).update(up_feed)
    except:
         logger.exception('error when constructing feed: {}'.format(feed.link))
    if feed.title and 'title' in up_feed:
        # do not override the title set by the user
        del up_feed['title']
    FeedController().update({'id': feed.id}, up_feed)

    return articles


async def insert_articles(queue, nḅ_producers=1):
    """Consumer coroutines.
    """
    nb_producers_done = 0
    while True:
        item = await queue.get()
        if item is None:
            nb_producers_done += 1
            if nb_producers_done == nḅ_producers:
                print('All producers done.')
                print('Process finished.')
                break
            continue

        user, feed, articles = item
    
        
        if None is articles:
            logger.info('None')
            articles = []

        logger.info('Inserting articles for {}'.format(feed.title))

        new_articles = []
        art_contr = ArticleController(user.id)
        for article in articles:
            new_article = await construct_article(article, feed)

            try:
                existing_article_req = art_contr.read(feed_id=feed.id,
                                entry_id=extract_id(article))
            except Exception as e:
                logger.exception("existing_article_req: " + str(e))
                continue
            exist = existing_article_req.count() != 0
            if exist:
                continue
{'fraction_non_alphanumeric': 0.09867722313411219, 'fraction_numerical': 0.007531138360529111, 'mean_word_length': 2.319871794871795, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2861848', 'n_tokens_mistral': 2967, 'n_tokens_neox': 2815, 'n_words': 826}