<commit_before>from IPython.display import display, Markdown
import glob
import h5py

def dispattrs(files,*args):
    """Display attributes of a group of H5 files.
    ex : dispattrs('scan*.h5','power','rbw')"""
    allf = glob.glob(files)
    allf.sort()
    s = f"|File|{'|'.join(args)}|\n"
    s = s + f"| ---- | {'|'.join(['----' for a in args])} |\n"
    for f in allf:
        with h5py.File(f,'r') as h5f:
            d0 = h5f['data0000'].attrs
            s = s + f"| {f} | {'|'.join([ str(d0[a]) for a in args])} |\n"
    return display(Markdown(s))
    
def fixattr(file,attr,value):
    """Fix or add the value of an attribute for all items in the file.
    ex : fixattr('scan0000.h5','rbw',10)"""
    with h5py.File(f,'a') as h5f:
        for d in h5f.items():
            d.attrs[attr]=value

    
def copyh5(fin,fout,range):
    """Copy datasets from one file to another. Datasets are supposed to be named dataXXXX.
    ex : copyh5('scan0000.h5','scan0001.h5',range(100,200))"""
    with h5py.File(fout,'a') as h5fout:
        with h5py.File(fin,'r') as h5fin:
            for i in range:
                h5fin.copy(h5fin,h5fout,f'data{i:04d}')


def delh5(file,range):
    """Delete datasets from one. Datasets are supposed to be named dataXXXX.
    ex : delh5('scan0000.h5',range(100,200))"""
    with h5py.File(file,'a') as h5f:
        for i in range:
            del h5f[f'data{i:04d}']


<commit_msg>Bug fix in pydata.utilities<commit_after>from IPython.display import display, Markdown
import glob
import h5py

def dispattrs(files,*args):
    """Display attributes of a group of H5 files.
    ex : dispattrs('scan*.h5','power','rbw')"""
    allf = glob.glob(files)
    allf.sort()
    s = f"|File|{'|'.join(args)}|\n"
    s = s + f"| ---- | {'|'.join(['----' for a in args])} |\n"
    for f in allf:
        with h5py.File(f,'r') as h5f:
            d0 = h5f['data0000'].attrs
            s = s + f"| {f} | {'|'.join([ str(d0[a]) for a in args])} |\n"
    return display(Markdown(s))
    
def fixattr(file,attr,value):
    """Fix or add the value of an attribute for all items in the file.
    ex : fixattr('scan0000.h5','rbw',10)"""
    with h5py.File(file,'a') as h5f:
        for d in h5f.values():
            d.attrs[attr]=value
    
def copyh5(fin,fout,range):
    """Copy datasets from one file to another. Datasets are supposed to be named dataXXXX.
    ex : copyh5('scan0000.h5','scan0001.h5',range(100,200))"""
    with h5py.File(fout,'a') as h5fout:
        with h5py.File(fin,'r') as h5fin:
            for i in range:
                h5fin.copy(h5fin[f'data{i:04d}'],h5fout)

def delh5(file,range):
    """Delete datasets from one. Datasets are supposed to be named dataXXXX.
    ex : delh5('scan0000.h5',range(100,200))"""
    with h5py.File(file,'a') as h5f:
        for i in range:
            del h5f[f'data{i:04d}']



{'fraction_non_alphanumeric': 0.17430871543577178, 'fraction_numerical': 0.04690234511725586, 'mean_word_length': 2.706874189364462, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4184323', 'n_tokens_mistral': 1133, 'n_tokens_neox': 1072, 'n_words': 285}