<commit_before>"""

NZBModel - Representations of the NZB file format in memory

(c) Copyright 2005 <NAME>
[See end of file]
"""
import Hellanzb, os, re, time
from sets import Set
from threading import Lock, RLock
from twisted.internet import reactor
from xml.sax import make_parser
from xml.sax.handler import ContentHandler, feature_external_ges, feature_namespaces
from Hellanzb.Log import *
from Hellanzb.NZBLeecher.ArticleDecoder import parseArticleData, setRealFileName, tryFinishNZB
from Hellanzb.Util import archiveName, getFileExtension, PriorityQueue

__id__ = '$Id$'

segmentEndRe = re.compile(r'^segment\d{4}$')
def needsDownload(object, threadRealNameWork = False):
    """ Whether or not this object needs to be downloaded (isn't on the file system). This
    function is generalized to support both NZBFile and NZBSegment objects. A NZBFile
    needs to be downloaded when it's file does not exist on the filesystem. An NZBSegment
    needs to be downloaded when either it's segment file, or it's parent NZBFile's file
    does not exist on the filesystem. This function does some magic to handle
    tempFileNames """
    # We need to ensure that we're not in the process of renaming from a temp file
    # name, so we have to lock.
    isSegment = isinstance(object, NZBSegment)
    if isSegment:
        filename = object.nzbFile.filename
        subject = object.nzbFile.subject
        tempFileNameLock = object.nzbFile.tempFileNameLock
    else:
        filename = object.filename
        subject = object.subject
        tempFileNameLock = object.tempFileNameLock

    tempFileNameLock.acquire()

    if os.path.isfile(object.getDestination()):
        tempFileNameLock.release()
        return False

    elif filename == None:
        # We only know about the temp filename. In that case, fall back to matching
        # filenames in our subject line
        for file in os.listdir(Hellanzb.WORKING_DIR):
            ext = getFileExtension(file)

            # Segment Match
            if isSegment and ext != None and segmentEndRe.match(ext):

                # Quickest/easiest way to determine this file is not this segment's
                # file is by checking it's segment number
                segmentNumber = int(file[-4:])
                if segmentNumber != object.number:
                    continue

                # Strip the segment suffix, and if that filename is in our subject,
                # we've found a match
                prefix = file[0:-len('.segmentXXXX')]
                if object.nzbFile.subject.find(prefix) > -1:
                    # HACK: filename is None. so we only have the temporary name in
                    # memory. since we didnt see the temporary name on the filesystem, but
                    # we found a subject match, that means we have the real name on the
                    # filesystem. In the case where this happens, and we are segment #1,
                    # we've figured out the real filename (hopefully!)
                    if object.number == 1:
                        #debug('needsDownload: GOT real file name from PREFIX! ')
                        if threadRealNameWork:
                            reactor.callInThread(setRealFileName, object, prefix)
                        else:
                            setRealFileName(object, prefix)

                    tempFileNameLock.release()
                    return False

            # Whole file match
            elif subject.find(file) > -1:
                tempFileNameLock.release()
                return False

    tempFileNameLock.release()
    return True

class NZB:
    """ Representation of an nzb file -- the root <nzb> tag """
    def __init__(self, nzbFileName):
        self.nzbFileName = nzbFileName
        self.archiveName = archiveName(self.nzbFileName)
        self.nzbFileElements = []
        
class NZBFile:
    """ <nzb><file/><nzb> """
    needsDownload = needsDownload

    def __init__(self, subject, date = None, poster = None, nzb = None):
        # from xml attributes
        self.subject = subject
        self.date = date
        self.poster = poster
        
        # Parent NZB
        self.nzb = nzb
        # FIXME: thread safety?
        self.nzb.nzbFileElements.append(self)
        
        self.groups = []<commit_msg>timing needsDownload because apparently it's O(1) as hell and shows with huge nzb archives<commit_after>"""

NZBModel - Representations of the NZB file format in memory

(c) Copyright 2005 <NAME>
[See end of file]
"""
import Hellanzb, os, re, time
from sets import Set
from threading import Lock, RLock
from twisted.internet import reactor
from xml.sax import make_parser
from xml.sax.handler import ContentHandler, feature_external_ges, feature_namespaces
from Hellanzb.Log import *
from Hellanzb.NZBLeecher.ArticleDecoder import parseArticleData, setRealFileName, tryFinishNZB
from Hellanzb.Util import archiveName, getFileExtension, PriorityQueue

__id__ = '$Id$'

segmentEndRe = re.compile(r'^segment\d{4}$')
def needsDownload(object, threadRealNameWork = False):
    """ Whether or not this object needs to be downloaded (isn't on the file system). This
    function is generalized to support both NZBFile and NZBSegment objects. A NZBFile
    needs to be downloaded when it's file does not exist on the filesystem. An NZBSegment
    needs to be downloaded when either it's segment file, or it's parent NZBFile's file
    does not exist on the filesystem. This function does some magic to handle
    tempFileNames """
    start = time.time()
    # We need to ensure that we're not in the process of renaming from a temp file
    # name, so we have to lock.
    isSegment = isinstance(object, NZBSegment)
    if isSegment:
        filename = object.nzbFile.filename
        subject = object.nzbFile.subject
        tempFileNameLock = object.nzbFile.tempFileNameLock
    else:
        filename = object.filename
        subject = object.subject
        tempFileNameLock = object.tempFileNameLock

    tempFileNameLock.acquire()

    if os.path.isfile(object.getDestination()):
        tempFileNameLock.release()
        end = time.time() - start
        debug('needsDownload took: ' + str(end))
        return False

    elif filename == None:
        # We only know about the temp filename. In that case, fall back to matching
        # filenames in our subject line
        for file in os.listdir(Hellanzb.WORKING_DIR):
            ext = getFileExtension(file)

            # Segment Match
            if isSegment and ext != None and segmentEndRe.match(ext):

                # Quickest/easiest way to determine this file is not this segment's
                # file is by checking it's segment number
                segmentNumber = int(file[-4:])
                if segmentNumber != object.number:
                    continue

                # Strip the segment suffix, and if that filename is in our subject,
                # we've found a match
                prefix = file[0:-len('.segmentXXXX')]
                if object.nzbFile.subject.find(prefix) > -1:
                    # HACK: filename is None. so we only have the temporary name in
                    # memory. since we didnt see the temporary name on the filesystem, but
                    # we found a subject match, that means we have the real name on the
                    # filesystem. In the case where this happens, and we are segment #1,
                    # we've figured out the real filename (hopefully!)
                    if object.number == 1:
                        #debug('needsDownload: GOT real file name from PREFIX! ')
                        if threadRealNameWork:
                            reactor.callInThread(setRealFileName, object, prefix)
                        else:
                            setRealFileName(object, prefix)

                    tempFileNameLock.release()
                    end = time.time() - start
                    debug('needsDownload took: ' + str(end))
                    return False

            # Whole file match
            elif subject.find(file) > -1:
                tempFileNameLock.release()
                end = time.time() - start
                debug('needsDownload took: ' + str(end))
                return False

    tempFileNameLock.release()
    end = time.time() - start
    debug('needsDownload took: ' + str(end))
    return True

class NZB:
    """ Representation of an nzb file -- the root <nzb> tag """
    def __init__(self, nzbFileName):
        self.nzbFileName = nzbFileName
        self.archiveName = archiveName(self.nzbFileName)
        self.nzbFileElements = []
        
class NZBFile:
    """ <nzb><file/><nzb> """
    needsDownload = needsDownload

    def __init__(self, subject, date = None, poster = None, nzb = None):
        # from xml attributes
        self.subject = subject
        self.date = date
        self.poster = poster
        
        # Parent NZB
        self.nzb = nzb
        # FIXME: thread safety?
        self.nzb.nzbFileElements.append(self)
        
        self.groups = []
{'fraction_non_alphanumeric': 0.0731170973062122, 'fraction_numerical': 0.0025288620120945573, 'mean_word_length': 2.1214824982841454, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4764250', 'n_tokens_mistral': 2465, 'n_tokens_neox': 2302, 'n_words': 853}