<commit_before># -*- coding: utf-8 -*-
from commands import Command
import config
import data.clients as clients
from datetime import datetime
import json
import os.path
import packetFactory
import plugins
from pprint import pformat
from twisted.internet import defer
from twisted.internet import protocol
from twisted.internet import reactor
from twisted.internet import task
from twisted.python import log
from twisted.web.client import Agent
from twisted.web.client import RedirectAgent
from twisted.web.http_headers import Headers


try:
    import GlobalChat
    useGlobalChat = True
except ImportError:
    useGlobalChat = False

try:
    from twisted.web.client import readBody
except ImportError:
    class SimpleBodyProtocol(protocol.Protocol):
        def __init__(self, status, message, deferred):
            self.deferred = deferred
            self.dataBuffer = []

        def dataReceived(self, data):
            self.dataBuffer.append(data)

        def connectionLost(self, reason):
            self.deferred.callback(b''.join(self.dataBuffer))

    def readBody(response):
        d = defer.Deferred()
        response.deliverBody(SimpleBodyProtocol(response.code, response.phrase, d))
        return d

try:
    from twisted.web.client import HTTPConnectionPool
    pool = HTTPConnectionPool(reactor)
    pool._factory.noisy = False
    agent = RedirectAgent(Agent(reactor, pool=pool))
except ImportError:
    agent = RedirectAgent(Agent(reactor))

eqnotice_config = config.YAMLConfig("cfg/EQ_Notice.config.yml", {'enabled': True, 'timer': 60, 'debug': False, '1': "http://acf.me.uk/Public/PSO2EQ/pso2eq.txt"}, True)

#HTTP Headers
ETag_Headers     = ['', '', '', '', '', '', '', '', '', '']
Modified_Headers = ['', '', '', '', '', '', '', '', '', '']
#HTTP Modified in time
Modified_time    = ['', '', '', '', '', '', '', '', '', '']
#HTTP Data
HTTP_Data        = ['', '', '', '', '', '', '', '', '', '']
#was "【1時間前】" in the data?
ishour_eq        = [False, False, False, False, False, False, False, False, False, False]
#Hour of EQ
hour_eq          = ['', '', '', '', '', '', '', '', '', '']
#Mins of EQ
mins_eq          = ['', '', '', '', '', '', '', '', '', '']
#EQ Data
data_eq          = ['', '', '', '', '', '', '', '', '', '']
#EQ Data
msg_eq           = ['', '', '', '', '', '', '', '', '', '']

eqJP             = []
taskrun          = []

eq_mode = eqnotice_config.get_key('enabled')
tasksec = eqnotice_config.get_key('timer')
debug = eqnotice_config.get_key('debug')


def logdebug(message):
    if debug:
        print ("[EQ Notice Debug] %s" % message)


def load_eqJP_names():
    global eqJP
    RESloaded = False
    if os.path.exists("cfg/eqJP_custom.resources.json"):
        RESloaded = True
        f = open("cfg/eqJP_custom.resources.json", 'r')
        try:
            eqJP = json.load(f, "utf-8")
        except ValueError:
            print ("[EQ Notice] Bad custom resource file, falling back")
            RESloaded = False
        f.close()
    if not RESloaded and os.path.exists("cfg/eqJP.resources.json"):
        f = open("cfg/eqJP.resources.json", 'r')
        try:
            eqJP = json.load(f, "utf-8")<commit_msg>Turning it up to 11<commit_after># -*- coding: utf-8 -*-
from commands import Command
import config
import data.clients as clients
from datetime import datetime
import json
import os.path
import packetFactory
import plugins
from pprint import pformat
from twisted.internet import defer
from twisted.internet import protocol
from twisted.internet import reactor
from twisted.internet import task
from twisted.python import log
from twisted.web.client import Agent
from twisted.web.client import RedirectAgent
from twisted.web.http_headers import Headers


try:
    import GlobalChat
    useGlobalChat = True
except ImportError:
    useGlobalChat = False

try:
    from twisted.web.client import readBody
except ImportError:
    class SimpleBodyProtocol(protocol.Protocol):
        def __init__(self, status, message, deferred):
            self.deferred = deferred
            self.dataBuffer = []

        def dataReceived(self, data):
            self.dataBuffer.append(data)

        def connectionLost(self, reason):
            self.deferred.callback(b''.join(self.dataBuffer))

    def readBody(response):
        d = defer.Deferred()
        response.deliverBody(SimpleBodyProtocol(response.code, response.phrase, d))
        return d

try:
    from twisted.web.client import HTTPConnectionPool
    pool = HTTPConnectionPool(reactor)
    pool._factory.noisy = False
    agent = RedirectAgent(Agent(reactor, pool=pool))
except ImportError:
    agent = RedirectAgent(Agent(reactor))

eqnotice_config = config.YAMLConfig("cfg/EQ_Notice.config.yml", {'enabled': True, 'timer': 60, 'debug': False, '1': "http://acf.me.uk/Public/PSO2EQ/pso2eq.txt"}, True)

#HTTP Headers
ETag_Headers     = ['', '', '', '', '', '', '', '', '', '', '']
Modified_Headers = ['', '', '', '', '', '', '', '', '', '', '']
#HTTP Modified in time
Modified_time    = ['', '', '', '', '', '', '', '', '', '', '']
#HTTP Data
HTTP_Data        = ['', '', '', '', '', '', '', '', '', '', '']
#was "【1時間前】" in the data?
ishour_eq        = [False, False, False, False, False, False, False, False, False, False, False]
#Hour of EQ
hour_eq          = ['', '', '', '', '', '', '', '', '', '', '']
#Mins of EQ
mins_eq          = ['', '', '', '', '', '', '', '', '', '', '']
#EQ Data
data_eq          = ['', '', '', '', '', '', '', '', '', '', '']
#EQ Data
msg_eq           = ['', '', '', '', '', '', '', '', '', '', '']

eqJP             = []
taskrun          = []

eq_mode = eqnotice_config.get_key('enabled')
tasksec = eqnotice_config.get_key('timer')
debug = eqnotice_config.get_key('debug')


def logdebug(message):
    if debug:
        print ("[EQ Notice Debug] %s" % message)


def load_eqJP_names():
    global eqJP
    RESloaded = False
    if os.path.exists("cfg/eqJP_custom.resources.json"):
        RESloaded = True
        f = open("cfg/eqJP_custom.resources.json", 'r')
        try:
            eqJP = json.load(f, "utf-8")
        except ValueError:
            print ("[EQ Notice] Bad custom resource file, falling back")
            RESloaded = False
        f.close()
    if not RESloaded and os.path.exists("cfg/eqJP.resources.json"):
        f = open("cfg/eqJP.resources.json", 'r')
        try:
            eqJP = json.load(f, "utf-8")
{'fraction_non_alphanumeric': 0.1734709838886282, 'fraction_numerical': 0.003128421711246676, 'mean_word_length': 3.0163316582914574, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1991192', 'n_tokens_mistral': 1960, 'n_tokens_neox': 1905, 'n_words': 472}