<commit_before>#!/usr/bin/env python

"""

ownCloud - News

@author <NAME>
@copyright 2012 <NAME> <EMAIL>

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
License as published by the Free Software Foundation; either
version 3 of the License, or any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU AFFERO GENERAL PUBLIC LICENSE for more details.

You should have received a copy of the GNU Affero General Public
License along with this library.  If not, see <http://www.gnu.org/licenses/>.

"""

import sys
import time
import json
import argparse
import threading
import socket
import urllib.request
import urllib.error

def get_basic_auth_opener(url, user, password):
    auth = urllib.request.HTTPPasswordMgrWithDefaultRealm()
    auth.add_password(None, url, user, password)
    auth_handler = urllib.request.HTTPBasicAuthHandler(auth)
    return urllib.request.build_opener(auth_handler)


class UpdateThread(threading.Thread):

    lock = threading.Lock()

    def __init__(self, feeds, update_url, user, password, timeout):
        super().__init__()
        self.feeds = feeds
        self.update_url = update_url
        self.user = user
        self.password = password
        self.timeout = timeout

    def run(self):
        with UpdateThread.lock:
            if len(self.feeds) > 0:
                feed = self.feeds.pop()
            else:
                return

        feed['feedId'] = feed['id']
        del feed['id']

        # call the update method of one feed
        data = urllib.parse.urlencode(feed)
        headers = {
            'Content-type': 'application/json',
            'Accept': 'text/plain'
        }
        url = '%s?%s' % (self.update_url, data)

        try:
            opener = get_basic_auth_opener(url, self.user, self.password)
            opener.open(url, timeout=self.timeout)
        except (urllib.error.HTTPError, socket.timeout) as e:
            print('%s: %s' % (url, e))

        self.run()


class Updater:

    def __init__(self, base_url, thread_num, interval, user, password, timeout, runonce):
        self.thread_num = thread_num
        self.interval = interval
        self.base_url = base_url
        self.user = user
        self.password = password
        self.timeout = timeout
        self.runonce = runonce

        if self.base_url[-1] != '/':
            self.base_url += '/'
        self.base_url += 'index.php/apps/news/api/v1-2'

        self.cleanup_url = '%s/cleanup' % self.base_url
        self.all_feeds_url = '%s/feeds/all' % self.base_url
        self.update_url = '%s/feeds/update' % self.base_url


    def run(self):
        try:
            opener = get_basic_auth_opener(self.base_url, self.user,
                self.password)
            # run the cleanup request and get all the feeds to update
            opener.open(self.cleanup_url)
            feeds_response = opener.open(self.all_feeds_url)
            feeds_json = feeds_response.read().decode('utf-8')
            feeds = json.loads(feeds_json)['feeds']

            # start thread_num for feeds
            threads = []
            for num in range(0, self.thread_num):
                thread = UpdateThread(feeds, self.update_url, self.user,
                    self.password, self.timeout)
                thread.start()
                threads.append(thread)

            for thread in threads:
                thread.join()

            if self.runonce != True:
                # wait until the interval finished to run again
                time.sleep(self.interval)
                self.run()

        except (ValueError, urllib.error.HTTPError) as e:
            print('%s: %s' % (self.base_url, e))
            print('Trying again in 30 seconds')
            time.sleep(30)
            self.run()

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--runonce',
        help='Run update only once, only recommended for testing', action='store_true')
    parser.add_argument('--threads',
        help='How many feeds should be fetched in paralell, defaults to 10',
        default=10,
        type=int)
    parser.add_argument('--timeout',
        help='Maximum number of seconds for updating a feed, defaults to 5 minutes',
        default=5*60,
        type=int)
    parser.add_argument('--interval',
        help='Update interval between fetching the next round of \
            updates in minutes, defaults to 30 minutes',
        default=30,
        type=int)
    parser.add_argument('--user',
        help='Admin username to log into ownCloud', required=True)<commit_msg>Update updater.py<commit_after>#!/usr/bin/env python

"""

ownCloud - News

@author <NAME>
@copyright 2012 <NAME> <EMAIL>

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
License as published by the Free Software Foundation; either
version 3 of the License, or any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU AFFERO GENERAL PUBLIC LICENSE for more details.

You should have received a copy of the GNU Affero General Public
License along with this library.  If not, see <http://www.gnu.org/licenses/>.

"""

import sys
import time
import json
import argparse
import threading
import socket
import urllib.request
import urllib.error

def get_basic_auth_opener(url, user, password):
    auth = urllib.request.HTTPPasswordMgrWithDefaultRealm()
    auth.add_password(None, url, user, password)
    auth_handler = urllib.request.HTTPBasicAuthHandler(auth)
    return urllib.request.build_opener(auth_handler)


class UpdateThread(threading.Thread):

    lock = threading.Lock()

    def __init__(self, feeds, update_url, user, password, timeout):
        super().__init__()
        self.feeds = feeds
        self.update_url = update_url
        self.user = user
        self.password = password
        self.timeout = timeout

    def run(self):
        with UpdateThread.lock:
            if len(self.feeds) > 0:
                feed = self.feeds.pop()
            else:
                return

        feed['feedId'] = feed['id']
        del feed['id']

        # call the update method of one feed
        data = urllib.parse.urlencode(feed)
        headers = {
            'Content-type': 'application/json',
            'Accept': 'text/plain'
        }
        url = '%s?%s' % (self.update_url, data)

        try:
            opener = get_basic_auth_opener(url, self.user, self.password)
            opener.open(url, timeout=self.timeout)
        except (urllib.error.HTTPError, socket.timeout) as e:
            print('%s: %s' % (url, e))

        self.run()


class Updater:

    def __init__(self, base_url, thread_num, interval, user, password, timeout, runonce):
        self.thread_num = thread_num
        self.interval = interval
        self.base_url = base_url
        self.user = user
        self.password = password
        self.timeout = timeout
        self.runonce = runonce

        if self.base_url[-1] != '/':
            self.base_url += '/'
        self.base_url += 'index.php/apps/news/api/v1-2'

        self.cleanup_url = '%s/cleanup' % self.base_url
        self.all_feeds_url = '%s/feeds/all' % self.base_url
        self.update_url = '%s/feeds/update' % self.base_url


    def run(self):
        try:
            opener = get_basic_auth_opener(self.base_url, self.user,
                self.password)
            # run the cleanup request and get all the feeds to update
            opener.open(self.cleanup_url)
            feeds_response = opener.open(self.all_feeds_url)
            feeds_json = feeds_response.read().decode('utf-8')
            feeds = json.loads(feeds_json)['feeds']

            # start thread_num for feeds
            threads = []
            for num in range(0, self.thread_num):
                thread = UpdateThread(feeds, self.update_url, self.user,
                    self.password, self.timeout)
                thread.start()
                threads.append(thread)

            for thread in threads:
                thread.join()

            if self.runonce != True:
                # wait until the interval finished to run again
                time.sleep(self.interval)
                self.run()

        except (ValueError, urllib.error.HTTPError) as e:
            print('%s: %s' % (self.base_url, e))
            print('Trying again in 30 seconds')
            time.sleep(30)
            self.run()

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--runonce',
        help='Run update only once, DO NOT use this in a cron job, only recommended for testing', action='store_true')
    parser.add_argument('--threads',
        help='How many feeds should be fetched in paralell, defaults to 10',
        default=10,
        type=int)
    parser.add_argument('--timeout',
        help='Maximum number of seconds for updating a feed, defaults to 5 minutes',
        default=5*60,
        type=int)
    parser.add_argument('--interval',
        help='Update interval between fetching the next round of \
            updates in minutes, defaults to 30 minutes',
        default=30,
        type=int)
    parser.add_argument('--user',
        help='Admin username to log into ownCloud', required=True)
{'fraction_non_alphanumeric': 0.098537496110362, 'fraction_numerical': 0.005601078726273208, 'mean_word_length': 2.2607372336827867, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4574775', 'n_tokens_mistral': 2851, 'n_tokens_neox': 2742, 'n_words': 802}