<commit_before>-- lua-npge, Nucleotide PanGenome explorer (Lua module)
-- Copyright (C) 2014-2015 <NAME>
-- See the LICENSE file for terms of use.

return function(bs, silent)
    -- input a blockset of good blocks only
    -- output: pangenome (partition, blocks are good or unique,
    --      no new good blocks can be found neither in blast
    --      hits, not in results of joining neighbour blocks)
    local algo = require 'npge.algo'
    assert(not algo.HasOverlap(bs))
    bs = algo.GoodSubblocks(bs)
    local bs1
    while not bs1 or bs1 ~= bs do
        if bs1 then
            bs = bs1
        end
        local bs_covered = algo.Cover(bs)
        -- blast
        local hits = algo.AddGoodBlast(bs_covered, bs_covered)
        bs1 = algo.BlocksWithoutOverlaps(bs, hits)
        -- join
        local joined = algo.Join(bs1)
        joined = algo.BetterSubblocks(joined, bs1)
        bs1 = algo.BlocksWithoutOverlaps(bs1, joined)
        -- extend
        local extended = algo.Extend(bs1)
        extended = algo.BetterSubblocks(extended, bs1)
        bs1 = algo.BlocksWithoutOverlaps(bs1, extended)
        -- print '.'
        if not silent then
            io.stderr:write('.')
            io.stderr:flush()
        end
    end
    -- prettify
    bs = algo.Cover(bs)
    local minor = algo.Align(algo.JoinMinor(bs))
    bs = algo.BlocksWithoutOverlaps(bs, minor)
    assert(bs:isPartition())
    bs = algo.Orient(bs)
    bs = algo.GiveNames(bs)
    return bs
end<commit_msg>PangenomeMaker: exclude self-overlaps after extend<commit_after>-- lua-npge, Nucleotide PanGenome explorer (Lua module)
-- Copyright (C) 2014-2015 <NAME>
-- See the LICENSE file for terms of use.

return function(bs, silent)
    -- input a blockset of good blocks only
    -- output: pangenome (partition, blocks are good or unique,
    --      no new good blocks can be found neither in blast
    --      hits, not in results of joining neighbour blocks)
    local algo = require 'npge.algo'
    assert(not algo.HasOverlap(bs))
    bs = algo.GoodSubblocks(bs)
    local bs1
    while not bs1 or bs1 ~= bs do
        if bs1 then
            bs = bs1
        end
        local bs_covered = algo.Cover(bs)
        -- blast
        local hits = algo.AddGoodBlast(bs_covered, bs_covered)
        bs1 = algo.BlocksWithoutOverlaps(bs, hits)
        -- join
        local joined = algo.Join(bs1)
        joined = algo.BetterSubblocks(joined, bs1)
        bs1 = algo.BlocksWithoutOverlaps(bs1, joined)
        -- extend
        local extended = algo.Extend(bs1)
        extended = algo.ExcludeSelfOverlap(extended)
        extended = algo.BetterSubblocks(extended, bs1)
        bs1 = algo.BlocksWithoutOverlaps(bs1, extended)
        -- print '.'
        if not silent then
            io.stderr:write('.')
            io.stderr:flush()
        end
    end
    -- prettify
    bs = algo.Cover(bs)
    local minor = algo.Align(algo.JoinMinor(bs))
    bs = algo.BlocksWithoutOverlaps(bs, minor)
    assert(bs:isPartition())
    bs = algo.Orient(bs)
    bs = algo.GiveNames(bs)
    return bs
end
{'fraction_non_alphanumeric': 0.09746588693957114, 'fraction_numerical': 0.014294996751137101, 'mean_word_length': 2.2274633123689727, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5878665', 'n_tokens_mistral': 1021, 'n_tokens_neox': 956, 'n_words': 275}