<commit_before>import os
import socket

from consts import CHUNKSIZE, CUTOCS_READ, CSTOCU_READ_DATA, CSTOCU_READ_STATUS
from utils import uint64, pack, unpack

mfsdirs = []
def _scan():
    for conf in ('/etc/mfshdd.cfg', '/usr/local/etc/mfshdd.cfg'):
        if not os.path.exists(conf):
            continue
        f = open(conf)
        for line in f:
            path = line.strip('#* \n')
            if os.path.exists(path):
                mfsdirs.append(path)
        f.close()
_scan()

CHUNKHDRSIZE = 1024 * 5

def read_chunk_from_local(chunkid, version, size, offset=0):
    if offset + size > CHUNKSIZE:
        raise ValueError("size too large %s > %s" % 
            (size, CHUNKSIZE-offset))
    
    from dpark.accumulator import ReadBytes, LocalReadBytes
    name = '%02X/chunk_%016X_%08X.mfs' % (chunkid & 0xFF, chunkid, version)
    for d in mfsdirs:
        p = os.path.join(d, name)
        if os.path.exists(p):
            if os.path.getsize(p) < CHUNKHDRSIZE + offset + size:
                raise ValueError("size too large")
            f = open(p)
            f.seek(CHUNKHDRSIZE + offset)
            while size > 0:
                to_read = min(size, 1024*1024*4)<commit_msg>bugfix: local chunk is not completed, ignore it<commit_after>import os
import socket

from consts import CHUNKSIZE, CUTOCS_READ, CSTOCU_READ_DATA, CSTOCU_READ_STATUS
from utils import uint64, pack, unpack

mfsdirs = []
def _scan():
    for conf in ('/etc/mfshdd.cfg', '/usr/local/etc/mfshdd.cfg'):
        if not os.path.exists(conf):
            continue
        f = open(conf)
        for line in f:
            path = line.strip('#* \n')
            if os.path.exists(path):
                mfsdirs.append(path)
        f.close()
_scan()

CHUNKHDRSIZE = 1024 * 5

def read_chunk_from_local(chunkid, version, size, offset=0):
    if offset + size > CHUNKSIZE:
        raise ValueError("size too large %s > %s" % 
            (size, CHUNKSIZE-offset))
    
    from dpark.accumulator import ReadBytes, LocalReadBytes
    name = '%02X/chunk_%016X_%08X.mfs' % (chunkid & 0xFF, chunkid, version)
    for d in mfsdirs:
        p = os.path.join(d, name)
        if os.path.exists(p):
            if os.path.getsize(p) < CHUNKHDRSIZE + offset + size:
                print p, 'is not completed', os.path.getsize(p), '<', CHUNKHDRSIZE + offset + size
                #raise ValueError("size too large")
            f = open(p)
            f.seek(CHUNKHDRSIZE + offset)
            while size > 0:
                to_read = min(size, 1024*1024*4)
{'fraction_non_alphanumeric': 0.11713836477987422, 'fraction_numerical': 0.020440251572327043, 'mean_word_length': 2.1972361809045227, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5679514', 'n_tokens_mistral': 903, 'n_tokens_neox': 862, 'n_words': 219}