<commit_before>from iotbx.xds.read_ascii import reader
import sys
for n, argv in enumerate(sys.argv[1:]):
    r = reader(open(argv))
    mas = r.as_miller_arrays(merge_equivalents = False)
    assert(len(mas) == 1)
    ma = mas[0].apply_scaling(target_max = 9.99e5)

    i = ma.data()
    s = ma.sigmas()
    hkl = ma.indices()

    for j, h in enumerate(hkl):

        _i = ('%f' % i[j])[:7]
        assert('.' in _i)
        _s = ('%f' % s[j])[:7]
        assert('.' in _s)

        print '%4d%4d%4d%8s%8s%4d' % (h[0], h[1], h[2], 
                                      _i, _s, n + 1)
<commit_msg>Exclude reflections with -ve sigma (needed when reading XDS_ASCII.HKL as input)<commit_after>from iotbx.xds.read_ascii import reader
import sys
for n, argv in enumerate(sys.argv[1:]):
    r = reader(open(argv))
    mas = r.as_miller_arrays(merge_equivalents = False)
    assert(len(mas) == 1)
    ma = mas[0].apply_scaling(target_max = 9.99e5)

    i = ma.data()
    s = ma.sigmas()
    hkl = ma.indices()

    for j, h in enumerate(hkl):

        _i = ('%f' % i[j])[:7]
        assert('.' in _i)
        _s = ('%f' % s[j])[:7]
        assert('.' in _s)

        if s[j] >= 0.0:
            print '%4d%4d%4d%8s%8s%4d' % (h[0], h[1], h[2], 
                                          _i, _s, n + 1)

{'fraction_non_alphanumeric': 0.17746913580246915, 'fraction_numerical': 0.030864197530864196, 'mean_word_length': 2.0734597156398102, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4127914', 'n_tokens_mistral': 556, 'n_tokens_neox': 516, 'n_words': 119}