<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""This is a little python package that groups all the functions needed by other scripts."""

import logging
import json
import configparser
import feedparser

def get_configuration():
    """Get global configuration from service.cfg"""

    config = configparser.ConfigParser()
    config.read("service.cfg")

    return config

def get_logger(debug):
    """Get logger object"""

    logging.basicConfig(
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
        level=logging.INFO
        )
    logger = logging.getLogger(__name__)

    if debug is False:
        logging.disable(logging.CRITICAL)

    return logger

def write_json(data, json_file):
    """General function used everywhere to write data into a json file"""

    with open(json_file, "w") as json_file:
        json.dump(data, json_file)

def read_json(json_file):
    """General function used everywhere to read a json file"""

    with open(json_file, "r") as json_file:
        return json.load(json_file)

def pull_news():
    """This function is built to pull 10 news from the rss endpoint"""

    document = feedparser.parse(
        "http://www.disim.univaq.it/didattica/content.php?fid=rss&pid=114&did=8&lid=it"
        )
    news = [
        {"title": item.title, "description": item.description.replace("&amp;#39;", "'"),
         "id": item.id}
        for item in document["entries"][:10]
        ]

    return news

def check_news():
    """This function check if there are some unread news from the website"""

    pulled_news = pull_news()
    stored_news = read_json("json/news.json")
    unread_news = []

    for i in range(0, 10):
        if pulled_news[i]["id"] != stored_news[i]["id"]:
            unread_news.append({"title": pulled_news[i]["title"],
                                "description": pulled_news[i].description})

    return unread_news<commit_msg>Extended news command to retrieve an arbitrary number of news<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""This is a little python package that groups all the functions needed by other scripts."""

import logging
import json
import configparser
import feedparser

def get_configuration():
    """Get global configuration from service.cfg"""

    config = configparser.ConfigParser()
    config.read("service.cfg")

    return config

def get_logger(debug):
    """Get logger object"""

    logging.basicConfig(
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
        level=logging.INFO
        )
    logger = logging.getLogger(__name__)

    if debug is False:
        logging.disable(logging.CRITICAL)

    return logger

def write_json(data, json_file):
    """General function used everywhere to write data into a json file"""

    with open(json_file, "w") as json_file:
        json.dump(data, json_file)

def read_json(json_file):
    """General function used everywhere to read a json file"""

    with open(json_file, "r") as json_file:
        return json.load(json_file)

def pull_news(num):
    """This function is built to pull 10 (or an arbitrary number) news from the rss endpoint"""

    document = feedparser.parse(
        "http://www.disim.univaq.it/didattica/content.php?fid=rss&pid=114&did=8&lid=it"
        )
    news = [
        {"title": item.title, "description": item.description.replace("&amp;#39;", "'"),
         "id": item.id}
        for item in document["entries"][:int(num)]
        ]

    return news

def check_news():
    """This function check if there are some unread news from the website"""

    pulled_news = pull_news(10)
    stored_news = read_json("json/news.json")
    unread_news = []

    for i in range(0, 10):
        if pulled_news[i]["id"] != stored_news[i]["id"]:
            unread_news.append({"title": pulled_news[i]["title"],
                                "description": pulled_news[i].description})

    return unread_news
{'fraction_non_alphanumeric': 0.13087417887822134, 'fraction_numerical': 0.007074279939363315, 'mean_word_length': 3.0563524590163933, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5850767', 'n_tokens_mistral': 1227, 'n_tokens_neox': 1185, 'n_words': 334}