<commit_before># -*- coding: utf-8 -*-
"""Mechanics to slow down wiki read and/or write rate."""
#
# (C) Pywikibot team, 2008-2020
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, division, unicode_literals

import math
import threading
import time

import pywikibot
from pywikibot import config
from pywikibot.tools import deprecated

_logger = 'wiki.throttle'

# global process identifier
#
# When the first Throttle is instantiated, it will set this variable to a
# positive integer, which will apply to all throttle objects created by this
# process.
pid = False


class Throttle(object):

    """Control rate of access to wiki server.

    Calling this object blocks the calling thread until at least 'delay'
    seconds have passed since the previous call.

    Each Site initiates one Throttle object (site.throttle) to control the
    rate of access.

    """

    def __init__(self, site, mindelay=None, maxdelay=None, writedelay=None,
                 multiplydelay=True):
        """Initializer."""
        self.lock = threading.RLock()
        self.lock_write = threading.RLock()
        self.lock_read = threading.RLock()
        self.mysite = str(site)
        self.ctrlfilename = config.datafilepath('throttle.ctrl')
        self.mindelay = mindelay
        if self.mindelay is None:
            self.mindelay = config.minthrottle
        self.maxdelay = maxdelay
        if self.maxdelay is None:
            self.maxdelay = config.maxthrottle
        self.writedelay = writedelay
        if self.writedelay is None:
            self.writedelay = config.put_throttle
        self.last_read = 0
        self.last_write = 0
        self.next_multiplicity = 1.0

        # Check logfile again after this many seconds:
        self.checkdelay = 300

        # Ignore processes that have not made a check in this many seconds:
        self.dropdelay = 600

        # Free the process id after this many seconds:
        self.releasepid = 1200

        self.retry_after = 0  # set by http.request
        self.delay = 0
        self.checktime = 0
        self.multiplydelay = multiplydelay
        if self.multiplydelay:
            self.checkMultiplicity()
        self.setDelays()

    @property
    @deprecated(since='20180423')
    def lastwait(self):
        """DEPRECATED property."""
        return 0.0

    def checkMultiplicity(self):
        """Count running processes for site and set process_multiplicity."""
        global pid
        mysite = self.mysite
        pywikibot.debug('Checking multiplicity: pid = %(pid)s' % globals(),
                        _logger)
        with self.lock:
            processes = []
            my_pid = pid or 1  # start at 1 if global pid not yet set
            count = 1
            # open throttle.log
            try:
                f = open(self.ctrlfilename, 'r')
            except IOError:
                if pid:
                    raise
            else:
                now = time.time()
                for line in f.readlines():
                    # parse line; format is "pid timestamp site"
                    try:
                        line = line.split(' ')
                        this_pid = int(line[0])
                        ptime = int(line[1].split('.')[0])
                        this_site = line[2].rstrip()
                    except (IndexError, ValueError):
                        # Sometimes the file gets corrupted ignore that line
                        continue
                    if now - ptime > self.releasepid:
                        continue    # process has expired, drop from file
                    if now - ptime <= self.dropdelay \
                       and this_site == mysite \
                       and this_pid != pid:
                        count += 1
                    if this_site != self.mysite or this_pid != pid:
                        processes.append({'pid': this_pid,
                                          'time': ptime,
                                          'site': this_site})
                    if not pid and this_pid >= my_pid:
                        my_pid = this_pid + 1  # next unused process id
            finally:
                f.close()

            if not pid:
                pid = my_pid
            self.checktime = time.time()
            processes.append({'pid': pid,
                              'time': self.checktime,
                              'site': mysite})
            processes.sort(key=lambda p: (p['pid'], p['site']))
            try:
                with open(self.ctrlfilename, 'w') as f:
                    for p in processes:
                        f.write('{pid} {time} {site}\n'.format(**p))
            except IOError:
                pass

            self.process_multiplicity = count
            pywikibot.log(
                'Found {0} {1} processes running, including this one.'.format(
                    count, mysite))

    def setDelays(self, delay=None, writedelay=None, absolute=False):<commit_msg>Revert "[IMPR] Always close files finally in throttle.py"<commit_after># -*- coding: utf-8 -*-
"""Mechanics to slow down wiki read and/or write rate."""
#
# (C) Pywikibot team, 2008-2020
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, division, unicode_literals

import math
import threading
import time

import pywikibot
from pywikibot import config
from pywikibot.tools import deprecated

_logger = 'wiki.throttle'

# global process identifier
#
# When the first Throttle is instantiated, it will set this variable to a
# positive integer, which will apply to all throttle objects created by this
# process.
pid = False


class Throttle(object):

    """Control rate of access to wiki server.

    Calling this object blocks the calling thread until at least 'delay'
    seconds have passed since the previous call.

    Each Site initiates one Throttle object (site.throttle) to control the
    rate of access.

    """

    def __init__(self, site, mindelay=None, maxdelay=None, writedelay=None,
                 multiplydelay=True):
        """Initializer."""
        self.lock = threading.RLock()
        self.lock_write = threading.RLock()
        self.lock_read = threading.RLock()
        self.mysite = str(site)
        self.ctrlfilename = config.datafilepath('throttle.ctrl')
        self.mindelay = mindelay
        if self.mindelay is None:
            self.mindelay = config.minthrottle
        self.maxdelay = maxdelay
        if self.maxdelay is None:
            self.maxdelay = config.maxthrottle
        self.writedelay = writedelay
        if self.writedelay is None:
            self.writedelay = config.put_throttle
        self.last_read = 0
        self.last_write = 0
        self.next_multiplicity = 1.0

        # Check logfile again after this many seconds:
        self.checkdelay = 300

        # Ignore processes that have not made a check in this many seconds:
        self.dropdelay = 600

        # Free the process id after this many seconds:
        self.releasepid = 1200

        self.retry_after = 0  # set by http.request
        self.delay = 0
        self.checktime = 0
        self.multiplydelay = multiplydelay
        if self.multiplydelay:
            self.checkMultiplicity()
        self.setDelays()

    @property
    @deprecated(since='20180423')
    def lastwait(self):
        """DEPRECATED property."""
        return 0.0

    def checkMultiplicity(self):
        """Count running processes for site and set process_multiplicity."""
        global pid
        mysite = self.mysite
        pywikibot.debug('Checking multiplicity: pid = %(pid)s' % globals(),
                        _logger)
        with self.lock:
            processes = []
            my_pid = pid or 1  # start at 1 if global pid not yet set
            count = 1
            # open throttle.log
            try:
                f = open(self.ctrlfilename, 'r')
            except IOError:
                if pid:
                    raise
            else:
                now = time.time()
                for line in f.readlines():
                    # parse line; format is "pid timestamp site"
                    try:
                        line = line.split(' ')
                        this_pid = int(line[0])
                        ptime = int(line[1].split('.')[0])
                        this_site = line[2].rstrip()
                    except (IndexError, ValueError):
                        # Sometimes the file gets corrupted ignore that line
                        continue
                    if now - ptime > self.releasepid:
                        continue    # process has expired, drop from file
                    if now - ptime <= self.dropdelay \
                       and this_site == mysite \
                       and this_pid != pid:
                        count += 1
                    if this_site != self.mysite or this_pid != pid:
                        processes.append({'pid': this_pid,
                                          'time': ptime,
                                          'site': this_site})
                    if not pid and this_pid >= my_pid:
                        my_pid = this_pid + 1  # next unused process id
                f.close()

            if not pid:
                pid = my_pid
            self.checktime = time.time()
            processes.append({'pid': pid,
                              'time': self.checktime,
                              'site': mysite})
            processes.sort(key=lambda p: (p['pid'], p['site']))
            try:
                f = open(self.ctrlfilename, 'w')
                for p in processes:
                    f.write('%(pid)s %(time)s %(site)s\n' % p)
            except IOError:
                pass
            else:
                f.close()
            self.process_multiplicity = count
            pywikibot.log(
                'Found {0} {1} processes running, including this one.'.format(
                    count, mysite))

    def setDelays(self, delay=None, writedelay=None, absolute=False):
{'fraction_non_alphanumeric': 0.08437777558472319, 'fraction_numerical': 0.009276620941478337, 'mean_word_length': 1.5234063745019921, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2810395', 'n_tokens_mistral': 2781, 'n_tokens_neox': 2605, 'n_words': 822}