<commit_before># Class for reading .emd file by Velox.
# This code was written based on FormatSER.py from https://github.com/cctbx/dxtbx/blob/master/format/FormatSER.py
from __future__ import absolute_import, division, print_function

import struct
import h5py
import numpy
import os
import json

from scitbx.array_family import flex
from dxtbx.format.Format import Format
from dxtbx.format.FormatMultiImage import FormatMultiImage

def get_metadata(metadata):
    mds = []
    for i in xrange(metadata.shape[1]):
        metadata_array = metadata[:, i].T
        mdata_string = metadata_array.tostring().decode("utf-8")
        mds.append(json.loads(mdata_string.rstrip('\x00')))

    return mds
# get_metadata()

def analyse_angle(metadata):
    alphas = []
    
    for i, md in enumerate(metadata):
        alpha = numpy.rad2deg(float(md["Stage"]["AlphaTilt"]))
        alphas.append(alpha)

    if len(alphas) < 2:
        return [0,0], 0.

    d_alphas = numpy.diff(alphas)
    q25, q50, q75 = numpy.percentile(d_alphas, [25, 50, 75])
    iqr = q75-q25
    iqrc = 1.5
    lowlim, highlim = q25 - iqrc*iqr, q75 + iqrc*iqr
    d_alphas2 = d_alphas[numpy.where(numpy.logical_and(d_alphas>lowlim, d_alphas<highlim))] # outlier rejected
    d_alpha_z = abs(d_alphas-numpy.mean(d_alphas2))/numpy.std(d_alphas2)

    valid_range = [0, len(metadata)-1]
    for i in xrange(len(metadata)-1):
        if d_alpha_z[i] < 3: break
        valid_range[0] = i+1

    for i in reversed(xrange(len(metadata)-1)):
        if d_alpha_z[i] < 3: break
        valid_range[1] = i

    if valid_range[0] > valid_range[1]:
        valid_range = [0, len(metadata)-1] # reset
        
    mean_alpha_step = (alphas[valid_range[1]] - alphas[valid_range[0]])/(valid_range[1]-valid_range[0])

    return valid_range, mean_alpha_step
# analyse_angle()


class FormatEMD(FormatMultiImage, Format):
    def __init__(self, image_file, **kwargs):

        from dxtbx import IncorrectFormatError

        if not self.understand(image_file):
            raise IncorrectFormatError(self, image_file)
        FormatMultiImage.__init__(self, **kwargs)
        Format.__init__(self, image_file, **kwargs)

    @staticmethod
    def understand(image_file):

        try:
            h = h5py.File(image_file, "r")
        except IOError:
            return False

        if not "/Data/Image" in h:
            return False

        keys = h["/Data/Image"].keys()
        if len(keys) > 1: return False
        d = h["/Data/Image"][keys[0]]
        if "Data" in d and "Metadata" in d and len(d["Data"].shape) == 3:
            return True

        return False

    @staticmethod
    def _read_metadata(image_file):
        h = h5py.File(image_file, "r")
        ret = {}

        image_path = h["/Data/Image"]
        assert len(image_path.keys()) == 1
        k = image_path.keys()[0]

        ret["image_file"] = image_file
        ret["file_handle"] = h<commit_msg>FormatEMD: changes for python3<commit_after># Class for reading .emd file by Velox.
# This code was written based on FormatSER.py from https://github.com/cctbx/dxtbx/blob/master/format/FormatSER.py
from __future__ import absolute_import, division, print_function

import struct
import h5py
import numpy
import os
import json

from scitbx.array_family import flex
from dxtbx.format.Format import Format
from dxtbx.format.FormatMultiImage import FormatMultiImage

def get_metadata(metadata):
    mds = []
    for i in range(metadata.shape[1]):
        metadata_array = metadata[:, i].T
        mdata_string = metadata_array.tostring().decode("utf-8")
        mds.append(json.loads(mdata_string.rstrip('\x00')))

    return mds
# get_metadata()

def analyse_angle(metadata):
    alphas = []
    
    for i, md in enumerate(metadata):
        alpha = numpy.rad2deg(float(md["Stage"]["AlphaTilt"]))
        alphas.append(alpha)

    if len(alphas) < 2:
        return [0,0], 0.

    d_alphas = numpy.diff(alphas)
    q25, q50, q75 = numpy.percentile(d_alphas, [25, 50, 75])
    iqr = q75-q25
    iqrc = 1.5
    lowlim, highlim = q25 - iqrc*iqr, q75 + iqrc*iqr
    d_alphas2 = d_alphas[numpy.where(numpy.logical_and(d_alphas>lowlim, d_alphas<highlim))] # outlier rejected
    d_alpha_z = abs(d_alphas-numpy.mean(d_alphas2))/numpy.std(d_alphas2)

    valid_range = [0, len(metadata)-1]
    for i in range(len(metadata)-1):
        if d_alpha_z[i] < 3: break
        valid_range[0] = i+1

    for i in reversed(range(len(metadata)-1)):
        if d_alpha_z[i] < 3: break
        valid_range[1] = i

    if valid_range[0] > valid_range[1]:
        valid_range = [0, len(metadata)-1] # reset
        
    mean_alpha_step = (alphas[valid_range[1]] - alphas[valid_range[0]])/(valid_range[1]-valid_range[0])

    return valid_range, mean_alpha_step
# analyse_angle()


class FormatEMD(FormatMultiImage, Format):
    def __init__(self, image_file, **kwargs):

        from dxtbx import IncorrectFormatError

        if not self.understand(image_file):
            raise IncorrectFormatError(self, image_file)
        FormatMultiImage.__init__(self, **kwargs)
        Format.__init__(self, image_file, **kwargs)

    @staticmethod
    def understand(image_file):

        try:
            h = h5py.File(image_file, "r")
        except IOError:
            return False

        if not "/Data/Image" in h:
            return False

        keys = list(h["/Data/Image"].keys())
        if len(keys) > 1: return False
        d = h["/Data/Image"][keys[0]]
        if "Data" in d and "Metadata" in d and len(d["Data"].shape) == 3:
            return True

        return False

    @staticmethod
    def _read_metadata(image_file):
        h = h5py.File(image_file, "r")
        ret = {}

        image_path = h["/Data/Image"]
        assert len(image_path.keys()) == 1
        k = list(image_path.keys())[0]

        ret["image_file"] = image_file
        ret["file_handle"] = h
{'fraction_non_alphanumeric': 0.12334577536477774, 'fraction_numerical': 0.020190023752969122, 'mean_word_length': 2.847911227154047, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5186016', 'n_tokens_mistral': 2106, 'n_tokens_neox': 2013, 'n_words': 448}