<commit_before># ---------------------------------------------------------------------------
# $Id: HeadHoncho.py,v 1.104 2004/10/09 08:27:01 freddie Exp $
# ---------------------------------------------------------------------------
# Main class used to control everything
import binascii
import bisect
import nntplib
import os
import re
import select
import socket
import sys
import time

import Hellanzb
from Hellanzb.Logging import *
from Hellanzb.Util import *

from zlib import crc32

from NZBParser import ParseNZB
from WrapPost import WrapPost
from WrapServer import WrapServer

# We need the useful yenc module
sys.path.append(os.path.expanduser('~/lib/python'))
try:
        import _yenc
        HAVE_YENC = 1
except ImportError:
        HAVE_YENC = 0

# ---------------------------------------------------------------------------

CHECK_FAIL = 0
CHECK_PASS = 1
CHECK_NEED_MORE = 2
CHECK_IGNORE = 3

TYPE_UUENCODE = 'uuencode'
TYPE_YENC = 'yenc'

# ---------------------------------------------------------------------------

def ShowError(text, *args):
        if args:
                text = text % args
        print 'ERROR: %s' % text
        sys.exit(-1)

class HeadHoncho:
        def __init__(self, jobs):
                self.jobs = jobs
                
                # Set up our connections
                self.FDs = {}
                self.Servers = []
                
                # Groups we've updated already
                self.Updated = {}

                self._incomplete_threshold = int(Hellanzb.Newsleecher.INCOMPLETE_THRESHOLD)
                
                # Work out our servers. First we sort by priority, then name.
                servers = []
                for id in Hellanzb.SERVERS:
                        serverInfo = Hellanzb.SERVERS[id]

                        priority = int(serverInfo['priority'])
                        servers.append((priority, serverInfo))
                        
                servers.sort()
                
                # Now actually start them up
                for priority, serverInfo in servers:
                        # Build our wrap
                        swrap = WrapServer(serverInfo)
                        self.Servers.append(swrap)
                        
                        # Connect
                        swrap.connect()
                        if swrap.Conns:
                                for fd in swrap.Conns.keys():
                                        self.FDs[fd] = swrap
                        else:
                                self.Servers.pop()
                
                # Did we connect?
                if not self.FDs:
                        ShowError('failed to open any server connections!')

        def super_anti_idle(self):
                _select = select.select
                ready = []
                for swrap in self.Servers:
                        for fd, nwrap in swrap.Conns.items():
                                nwrap.setblocking(0)
                                ready.append(fd)
                can_read = _select(ready, [], [], 0)[0]                
                for fd in ready:
                        nwrap = self.FDs[fd].Conns[fd]
                        nwrap.anti_idle()
                                
        # ---------------------------------------------------------------------------
        # Our main loop, obviously
        def main_loop(self):
                for job in self.jobs:
                        print
                        
                        # Is it a nzb job?
                        if job.endswith('.nzb'):
                                self.nzb_job(job)

                        # Wtf is it then?
                        else:
                                ShowError("unknown job type '%s'", job)
                                #scroll('Unknown file type: ' + job)
        
        # ---------------------------------------------------------------------------
        # Does all the stuff we need for a 'nzb' job.
        def nzb_job(self, job):
                # Make sure the file exists first
                if not os.access(job, os.R_OK):
                        print "File '%s' does not exist or is not readable!"
                        #scroll('File ' + job + ' does not exist or is not readable!')
                        return
                
                print "Attempting to parse '%s'..." % job,
                sys.stdout.flush()
                
                try:
                        newsgroups, posts = ParseNZB(job, self.Servers)
                
                except Exception, msg:<commit_msg>Put the nzb_job under main_loop into exception catching, so it can respawn the connections.<commit_after># ---------------------------------------------------------------------------
# $Id: HeadHoncho.py,v 1.104 2004/10/09 08:27:01 freddie Exp $
# ---------------------------------------------------------------------------
# Main class used to control everything
import binascii
import bisect
import nntplib
import os
import re
import select
import socket
import sys
import time

import Hellanzb
from Hellanzb.Logging import *
from Hellanzb.Util import *

from zlib import crc32

from NZBParser import ParseNZB
from WrapPost import WrapPost
from WrapServer import WrapServer

# We need the useful yenc module
sys.path.append(os.path.expanduser('~/lib/python'))
try:
        import _yenc
        HAVE_YENC = 1
except ImportError:
        HAVE_YENC = 0

# ---------------------------------------------------------------------------

CHECK_FAIL = 0
CHECK_PASS = 1
CHECK_NEED_MORE = 2
CHECK_IGNORE = 3

TYPE_UUENCODE = 'uuencode'
TYPE_YENC = 'yenc'

# ---------------------------------------------------------------------------

def ShowError(text, *args):
        if args:
                text = text % args
        print 'ERROR: %s' % text
        sys.exit(-1)

class HeadHoncho:
        def __init__(self, jobs):
                self.jobs = jobs
                
                # Set up our connections
                self.FDs = {}
                self.Servers = []
                
                # Groups we've updated already
                self.Updated = {}

                self._incomplete_threshold = int(Hellanzb.Newsleecher.INCOMPLETE_THRESHOLD)
                
                # Work out our servers. First we sort by priority, then name.
                servers = []
                for id in Hellanzb.SERVERS:
                        serverInfo = Hellanzb.SERVERS[id]

                        priority = int(serverInfo['priority'])
                        servers.append((priority, serverInfo))
                        
                servers.sort()
                
                # Now actually start them up
                for priority, serverInfo in servers:
                        # Build our wrap
                        swrap = WrapServer(serverInfo)
                        self.Servers.append(swrap)
                        
                        # Connect
                        swrap.connect()
                        if swrap.Conns:
                                for fd in swrap.Conns.keys():
                                        self.FDs[fd] = swrap
                        else:
                                self.Servers.pop()
                
                # Did we connect?
                if not self.FDs:
                        ShowError('failed to open any server connections!')

        def super_anti_idle(self):
                _select = select.select
                ready = []
                for swrap in self.Servers:
                        for fd, nwrap in swrap.Conns.items():
                                nwrap.setblocking(0)
                                ready.append(fd)
                can_read = _select(ready, [], [], 0)[0]                
                for fd in ready:
                        nwrap = self.FDs[fd].Conns[fd]
                        nwrap.anti_idle()
                                
        # ---------------------------------------------------------------------------
        # Our main loop, obviously
        def main_loop(self):
                for job in self.jobs:
                        print

                        while 1:
                                # Is it a nzb job?
                                if job.endswith('.nzb'):
                                        try:
                                                self.nzb_job(job)
                                        except nntplib.NNTPPermanentError, e:
                                                print '\r* Lost a server connection, restarting all connections'
                                                self.__init__(self.jobs)
                                        else:
                                                break

                                        # Wtf is it then?
                                else:
                                        ShowError("unknown job type '%s'", job)
        
        # ---------------------------------------------------------------------------
        # Does all the stuff we need for a 'nzb' job.
        def nzb_job(self, job):
                # Make sure the file exists first
                if not os.access(job, os.R_OK):
                        print "File '%s' does not exist or is not readable!"
                        #scroll('File ' + job + ' does not exist or is not readable!')
                        return
                
                print "Attempting to parse '%s'..." % job,
                sys.stdout.flush()
                
                try:
                        newsgroups, posts = ParseNZB(job, self.Servers)
                
                except Exception, msg:
{'fraction_non_alphanumeric': 0.1548677086525692, 'fraction_numerical': 0.006231484319133722, 'mean_word_length': 0.815316150565548, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1988590', 'n_tokens_mistral': 2295, 'n_tokens_neox': 2098, 'n_words': 620}