<commit_before>import socket
import os, sys, time, threading, shutil, time
import Image, ImageDraw
import io
from packages.rmmedia import mediaplayer
from constants import *


def readInt(data):
    intBytes = data[:4]
    if len(data) > 4:
        remainingData = data[4:]
    else:
        remainingData = []
    num = (intBytes[0] << 24) + (intBytes[1] << 16) + (intBytes[2] << 8) + intBytes[3]
    return num, remainingData

# added size!
def readString(data, size):
    #size, data = readInt(data)
    strBytes = data[:size]
    if len(data) > size:
        remainingData = data[size:]
    else:
        remainingData = []
    inStr = str(strBytes)
    return inStr, remainingData


def interpret(tmpFilePath):
    # open temp file in binary mode
    with open(tmpFilePath, 'rb') as f:
        data = bytearray(f.read(4))
        numFiles, data = readInt(data)

        # check thumbnails path
        thumbsPath = os.path.join(os.getcwd(), '/media/thumbs/')
        if not os.path.isdir(thumbsPath):
            os.mkdir(thumbsPath)

        isImage = False
        print "READING %d FILES" % numFiles
        for i in range(numFiles):
            # read file type
            data = bytearray(f.read(4))
            fileType, data = readInt(data)
            if fileType == FILE_TYPE_IMAGE:
                isImage = True
            else:
                isImage = False
            # read file name
            data = bytearray(f.read(4))
            size, data = readInt(data)
            data = bytearray(f.read(size))
            name, data = readString(data, size)
            openPathDir = os.getcwd() + '/media/'
            openPath = os.path.join(openPathDir, name)
            data = bytearray(f.read(4))
            fileSize, data = readInt(data)
            if not os.path.isdir(openPath):
                #f = open(openPath, 'w+') #open in binary
                l = bytearray(f.read(fileSize))
                if isImage:
                    stream = io.BytesIO(l)
                    img = Image.open(stream)
                    draw = ImageDraw.Draw(img)
                    img.save(openPath)
                    #l = data[:fileSize]
                    #data = data[fileSize:]
                    #f.write(l)
                    #f.close()

                    # save thumbnail
                    img = Image.open(openPath)
                    w = img.size[0]
                    h = img.size[1]
                    newW = 200
                    newH = newW * h / w
                    img.thumbnail((newW,newH))
                    img.save(os.path.join(thumbsPath, name))
                else:
                    with open(openPath, 'w+') as newFile:
                        newFile.write(l)
    # remove temp file
    os.remove(tmpFilePath)

def _openSocket():
    # create temp directory for received data
    if os.path.isdir(TCP_TEMP):
        shutil.rmtree(TCP_TEMP)
    os.mkdir(TCP_TEMP)

    global s
    s = socket.socket()
    s.bind(('',60020))
    s.listen(10) # Accept max. 10 connections
    print "File socket ready, listening for incoming file connections..."
    while True:
        sc, address = s.accept()
        print "TCP CLIENT CONNECTED: ", address

        dataSizeBytes = sc.recv(4)
        dataSize, remaining = readInt(bytearray(dataSizeBytes))
        print "Receiving %d Bytes" % (dataSize)
        buff = ''
        bytesRead = 0
        tmpFile = "tmp_" + str(int(round(time.time())))
        with open(TCP_TEMP + "/" + tmpFile, 'w') as tmp:
            while bytesRead < dataSize:
                data = sc.recv(1024)
                bytesRead += len(data)
                tmp.write(data)
                #buff += sc.recv(1024)

        print "Closing TCP Client connection..."
        sc.close()

        print "Data read to buffer, processing..."
        interpret(TCP_TEMP + "/" + tmpFile)

        data = bytearray(buff)
        print "FILES SAVED!"

        if mediaplayer.playerState == PLAYER_STARTED:
            mediaplayer.stop()
            time.sleep(5)
            mediaplayer.play()

def openFileSocket():
    global server_thread
    # Start a thread with the server
    server_thread = threading.Thread(target=_openSocket)
    # Exit the server thread when the main thread terminates
    server_thread.daemon = True
    server_thread.start()
    print "File socket loop running in thread:", server_thread.name


def closeFileSocket():
    global s
    s.close()


server_thread = None
s = None
<commit_msg>Thumbnail creation after TCP transfer.<commit_after>import socket
import os, sys, time, threading, shutil, time
import Image, ImageDraw
import io
from packages.rmmedia import mediaplayer
from constants import *


def readInt(data):
    intBytes = data[:4]
    if len(data) > 4:
        remainingData = data[4:]
    else:
        remainingData = []
    num = (intBytes[0] << 24) + (intBytes[1] << 16) + (intBytes[2] << 8) + intBytes[3]
    return num, remainingData

# added size!
def readString(data, size):
    #size, data = readInt(data)
    strBytes = data[:size]
    if len(data) > size:
        remainingData = data[size:]
    else:
        remainingData = []
    inStr = str(strBytes)
    return inStr, remainingData


def interpret(tmpFilePath):
    # open temp file in binary mode
    with open(tmpFilePath, 'rb') as f:
        data = bytearray(f.read(4))
        numFiles, data = readInt(data)

        # check thumbnails path
        thumbsPath = os.path.join(os.getcwd(), '/media/thumbs/')
        if not os.path.isdir(thumbsPath):
            os.mkdir(thumbsPath)

        isImage = False
        print "READING %d FILES" % numFiles
        imgsIncluded = False
        for i in range(numFiles):
            # read file type
            data = bytearray(f.read(4))
            fileType, data = readInt(data)
            if fileType == FILE_TYPE_IMAGE:
                isImage = True
                imgsIncluded = True
            else:
                isImage = False
            # read file name
            data = bytearray(f.read(4))
            size, data = readInt(data)
            data = bytearray(f.read(size))
            name, data = readString(data, size)
            openPathDir = os.getcwd() + '/media/'
            openPath = os.path.join(openPathDir, name)
            data = bytearray(f.read(4))
            fileSize, data = readInt(data)
            if not os.path.isdir(openPath):
                #f = open(openPath, 'w+') #open in binary
                l = bytearray(f.read(fileSize))
                if isImage:
                    stream = io.BytesIO(l)
                    img = Image.open(stream)
                    draw = ImageDraw.Draw(img)
                    img.save(openPath)
                    #l = data[:fileSize]
                    #data = data[fileSize:]
                    #f.write(l)
                    #f.close()
                    '''
                    # save thumbnail
                    #img = Image.open(openPath)
                    thumb = Image.open()
                    w = img.size[0]
                    h = img.size[1]
                    newW = 200
                    newH = newW * h / w
                    img.thumbnail((newW,newH))
                    img.save(os.path.join(thumbsPath, name))
                    '''
                else:
                    with open(openPath, 'w+') as newFile:
                        newFile.write(l)
    # remove temp file
    os.remove(tmpFilePath)
    if imgsIncluded:
        checkThumbnails()

def checkThumbnails():
    print "Checking thumbnails..."
    mediaPath = os.getcwd() + '/media/'
    thumbPath = mediaPath + 'thumbs/'

    if not os.path.isdir(thumbPath):
        os.mkdir(thumbPath)
    cnt = 0
    files = rmmedia.mediaplayer.getImageFilelist()
    for name in files:
        oPath = os.path.join(mediaPath, name)
        tPath = os.path.join(thumbPath, name)
        if not os.path.isfile(tPath):
            # no thumbnail for image present -> create and save thumbnail
            img = Image.open(oPath)
            w = img.size[0]
            h = img.size[1]
            newW = 200
            newH = newW * h / w
            img.thumbnail((newW,newH))
            img.save(os.path.join(thumbPath, name))
            cnt += 1
    print "%d missing thumbnails created and saved." % cnt

def _openSocket():
    # create temp directory for received data
    if os.path.isdir(TCP_TEMP):
        shutil.rmtree(TCP_TEMP)
    os.mkdir(TCP_TEMP)

    global s
    s = socket.socket()
    s.bind(('',60020))
    s.listen(10) # Accept max. 10 connections
    print "File socket ready, listening for incoming file connections..."
    while True:
        sc, address = s.accept()
        print "TCP CLIENT CONNECTED: ", address

        dataSizeBytes = sc.recv(4)
        dataSize, remaining = readInt(bytearray(dataSizeBytes))
        print "Receiving %d Bytes" % (dataSize)
        buff = ''
        bytesRead = 0
        tmpFile = "tmp_" + str(int(round(time.time())))
        with open(TCP_TEMP + "/" + tmpFile, 'w') as tmp:
            while bytesRead < dataSize:
                data = sc.recv(1024)
                bytesRead += len(data)
                tmp.write(data)
                #buff += sc.recv(1024)

        print "Closing TCP Client connection..."
        sc.close()

        print "Data read to buffer, processing..."
        interpret(TCP_TEMP + "/" + tmpFile)

        data = bytearray(buff)
        print "FILES SAVED!"

        if mediaplayer.playerState == PLAYER_STARTED:
            mediaplayer.stop()
            time.sleep(5)
            mediaplayer.play()

def openFileSocket():
    global server_thread
    # Start a thread with the server
    server_thread = threading.Thread(target=_openSocket)
    # Exit the server thread when the main thread terminates
    server_thread.daemon = True
    server_thread.start()
    print "File socket loop running in thread:", server_thread.name


def closeFileSocket():
    global s
    s.close()


server_thread = None
s = None

{'fraction_non_alphanumeric': 0.10700369667299431, 'fraction_numerical': 0.008891997202517734, 'mean_word_length': 1.7017543859649122, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5661321', 'n_tokens_mistral': 3044, 'n_tokens_neox': 2893, 'n_words': 755}