<commit_before>from binascii import crc32
import os, json

from twisted.python import log
from twisted.protocols import basic
from twisted.internet.protocol import ServerFactory, ClientFactory
from twisted.protocols.basic import FileSender, LineReceiver
from twisted.internet.defer import Deferred
from twisted.internet import reactor

from utils import getFilenameFromPath

class FileReceiverProtocol(LineReceiver):
    """protocol that will be used to transfer files/raw data."""

    def __init__(self, downloadPath):
        self.outfile = None
        self.remain = 0
        # self.crc = 0
        self.downloadPath = downloadPath
        
    def lineReceived(self, line):
        # maybe you need to explicitly check for the keys before running
        log.msg("lineReceived: " + line)
        self.instruction = json.loads(line)
        #self.instruction.update(dict(client=self.transport.getPeer().host))
        self.size = self.instruction['file_size']
        self.original_fname = self.instruction.get('original_file_path',
                                                   'not given by client')
        self.outfilename = os.path.join(self.downloadPath, 
                                        self.original_fname)
        #getFilenameFromPath(self.original_fname))
        log.msg("* Receiving into file @" + self.outfilename)
        try:
            self.outfile = open(self.outfilename,'wb')
        except Exception, value:
            log.msg("! Unable to open file {0} {1}".format(self.outfilename, 
                                                           value))
            # might be a good place for an errback
            self.transport.loseConnection()
            return
        self.remain = int(self.size)
        log.msg("Entering raw mode. {0} {1}".format(self.outfile, 
                                                      self.remain))
        self.setRawMode()


    def rawDataReceived(self, data):
        if self.remain % 10000 == 0:
            log.msg("remaining {0}/{1}".format(self.remain, self.size))
        self.remain -= len(data)
        # worry about this later, it should be sent as well so you can
        # compare the hashes before and after
        #self.crc = crc32(data, self.crc)
        self.outfile.write(data)
        # does this drop off once all data has been received? 
        # you could probably switch back to message mode after this is finished
        if self.remain == 0:
            self.outfile.close()
            self.setLineMode()
        
    def connectionMade(self):
        basic.LineReceiver.connectionMade(self)

    def connectionLost(self, reason):
        basic.LineReceiver.connectionLost(self, reason)
        if self.outfile:
            self.outfile.close()
        # Problem uploading - tmpfile will be discarded
        if self.remain != 0:
            print str(self.remain) + ')!=0'
            remove_base = '--> removing tmpfile@'
            if self.remain < 0:
                reason = ' .. file moved too much'
            if self.remain > 0:
                reason = ' .. file moved too little'
            print remove_base + self.outfilename + reason
            os.remove(self.outfilename)

        # Success uploading - tmpfile will be saved to disk.
        else:
            print '\n--> finished saving upload@ ' + self.outfilename
            client = self.instruction.get('client', 'anonymous')

class FileReceiverFactory(ServerFactory):

    protocol = FileReceiverProtocol

    def __init__(self, downloadPath):<commit_msg>working on using message. Cleaing up line received<commit_after>from binascii import crc32
import os, json

from twisted.python import log
from twisted.protocols import basic
from twisted.internet.protocol import ServerFactory, ClientFactory
from twisted.protocols.basic import FileSender, LineReceiver
from twisted.internet.defer import Deferred
from twisted.internet import reactor

from utils import getFilenameFromPath

class FileReceiverMessage(object):
    """
    This contains all of the information that will be exchanged to send a file.
    """
    def __init__(self, 
                 file_size, 
                 read_from,
                 write_to
                 ):
        self.file_size = file_size
        self.read_from = read_from
        self.write_to = write_to

    def serialize(self):
        return json.dumps({
            "file_size" : self.file_size,
            "read_from" : self.read_from,
            "write_to" : self.write_to
            })

    @classmethod
    def from_str(self, line):
        """alternate construct for a message, makes properties a 
        bit simpler to read
        """
        # possible use attr.get instead of direct access!
        from_msg = json.loads(line)
        self.file_size = from_msg.file_size
        self.read_from = from_msg.read_from
        self.write_to = from_msg.write_to


class FileReceiverProtocol(LineReceiver):
    """protocol that will be used to transfer files/raw data."""

    def __init__(self, downloadPath):
        self.outfile = None
        self.remain = 0
        # self.crc = 0
        self.downloadPath = downloadPath
        
    def lineReceived(self, line):
        # maybe you need to explicitly check for the keys before running
        log.msg("lineReceived: " + line)
        msg = Message(line)
        #self.instruction.update(dict(client=self.transport.getPeer().host))
        ## parse the instruction
        self.size = msg.file_size
        self.original_fname = msg.read_from
        self.outfilename = os.path.join(self.downloadPath, self.original_fname)
        log.msg("* Receiving into file @" + self.outfilename)
        try:
            self.outfile = open(self.outfilename,'wb')
        except Exception, value:
            log.msg("! Unable to open file {0} {1}".format(self.outfilename, 
                                                           value))
            # might be a good place for an errback
            self.transport.loseConnection()
            return
        self.remain = int(self.size)
        log.msg("Entering raw mode. {0} {1}".format(self.outfile, 
                                                      self.remain))
        self.setRawMode()


    def rawDataReceived(self, data):
        if self.remain % 10000 == 0:
            log.msg("remaining {0}/{1}".format(self.remain, self.size))
        self.remain -= len(data)
        # worry about this later, it should be sent as well so you can
        # compare the hashes before and after
        #self.crc = crc32(data, self.crc)
        self.outfile.write(data)
        # does this drop off once all data has been received? 
        # you could probably switch back to message mode after this is finished
        if self.remain == 0:
            # close the file handle as you are no longer using it
            self.outfile.close()
            self.setLineMode()
        
    def connectionMade(self):
        basic.LineReceiver.connectionMade(self)

    def connectionLost(self, reason):
        basic.LineReceiver.connectionLost(self, reason)
        if self.outfile:
            self.outfile.close()
        # Problem uploading - tmpfile will be discarded
        if self.remain != 0:
            print str(self.remain) + ')!=0'
            remove_base = '--> removing tmpfile@'
            if self.remain < 0:
                reason = ' .. file moved too much'
            if self.remain > 0:
                reason = ' .. file moved too little'
            print remove_base + self.outfilename + reason
            os.remove(self.outfilename)

        # Success uploading - tmpfile will be saved to disk.
        else:
            print '\n--> finished saving upload@ ' + self.outfilename
            client = self.instruction.get('client', 'anonymous')

class FileReceiverFactory(ServerFactory):

    protocol = FileReceiverProtocol

    def __init__(self, downloadPath):
{'fraction_non_alphanumeric': 0.08825757575757576, 'fraction_numerical': 0.005808080808080808, 'mean_word_length': 1.9924442765394788, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1506141', 'n_tokens_mistral': 2140, 'n_tokens_neox': 2054, 'n_words': 607}