<commit_before>-module(etorrent_chunkset).
-include("types.hrl").

-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-endif.

-export([new/2,
         size/1,
         min/1,
         delete/3,
         insert/3,
         from_list/3]).

-record(chunkset, {
    piece_len :: pos_integer(),
    chunk_len :: pos_integer(),
    chunks :: list({non_neg_integer(), pos_integer()})}).

-opaque chunkset() :: #chunkset{}.
-export_type([chunkset/0]).

%% @doc
%% Create a set of chunks for a piece.
%% @end
-spec new(pos_integer(), pos_integer()) -> chunkset().
new(PieceLen, ChunkLen) ->
    #chunkset{
        piece_len=PieceLen,
        chunk_len=ChunkLen,
        chunks=[{0, PieceLen - 1}]}.

from_list(PieceLen, ChunkLen, Chunks) ->
    #chunkset{
        piece_len=PieceLen,
        chunk_len=ChunkLen,
        chunks=Chunks}.

%% @doc
%% Get sum of the size of all chunks in the chunkset.
%% @end
-spec size(chunkset()) -> non_neg_integer().
size(Chunkset) ->
    #chunkset{chunks=Chunks} = Chunkset,
    Lengths = [1 + End - Start || {Start, End} <- Chunks],
    lists:sum(Lengths).

%% @doc
%% Get the offset and length of the chunk that is the closest
%% to the beginning of the piece. The chunk that is returned may
%% be shorter than the chunk length of the set.
%% @end
-spec min(chunkset()) -> {pos_integer(), pos_integer()}.
min(Chunkset) ->
    #chunkset{chunk_len=ChunkLen, chunks=Chunks} = Chunkset,
    case Chunks of
        [] ->
            error(badarg);
        [{Start, End}|_] when (1 + End - Start) > ChunkLen ->
            {Start, ChunkLen};
        [{Start, End}|_] when (1 + End - Start) =< ChunkLen ->
            {Start, (End - Start) + 1}
    end.

%% @doc
%% 
%% @end
delete(_, Length, _) when Length < 1 ->
    error(badarg);
delete(Offset, _, _) when Offset < 0 ->
    error(badarg);
delete(Offset, Length, Chunkset) ->
    #chunkset{chunks=Chunks} = Chunkset,
    NewChunks = delete_(Offset, Offset + Length - 1, Chunks),
    Chunkset#chunkset{chunks=NewChunks}.
<commit_msg>Add min and delete for multiple chunks to chunkset<commit_after>-module(etorrent_chunkset).
-include("types.hrl").

-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-endif.

-export([new/2,
         size/1,
         min/1,
         min/2,
         delete/2,
         delete/3,
         insert/3,
         from_list/3]).

-record(chunkset, {
    piece_len :: pos_integer(),
    chunk_len :: pos_integer(),
    chunks :: list({non_neg_integer(), pos_integer()})}).

-opaque chunkset() :: #chunkset{}.
-export_type([chunkset/0]).

%% @doc
%% Create a set of chunks for a piece.
%% @end
-spec new(pos_integer(), pos_integer()) -> chunkset().
new(PieceLen, ChunkLen) ->
    #chunkset{
        piece_len=PieceLen,
        chunk_len=ChunkLen,
        chunks=[{0, PieceLen - 1}]}.

from_list(PieceLen, ChunkLen, Chunks) ->
    #chunkset{
        piece_len=PieceLen,
        chunk_len=ChunkLen,
        chunks=Chunks}.

%% @doc
%% Get sum of the size of all chunks in the chunkset.
%% @end
-spec size(chunkset()) -> non_neg_integer().
size(Chunkset) ->
    #chunkset{chunks=Chunks} = Chunkset,
    Lengths = [1 + End - Start || {Start, End} <- Chunks],
    lists:sum(Lengths).

%% @doc
%% Get the offset and length of the chunk that is the closest
%% to the beginning of the piece. The chunk that is returned may
%% be shorter than the chunk length of the set.
%% @end
-spec min(chunkset()) -> {pos_integer(), pos_integer()}.
min(Chunkset) ->
    case min_(Chunkset) of
        none  -> error(badarg);
        Other -> Other
    end.

min_(Chunkset) ->
    #chunkset{chunk_len=ChunkLen, chunks=Chunks} = Chunkset,
    case Chunks of
        [] ->
            none;
        [{Start, End}|_] when (1 + End - Start) > ChunkLen ->
            {Start, ChunkLen};
        [{Start, End}|_] when (1 + End - Start) =< ChunkLen ->
            {Start, (End - Start) + 1}
    end.

%% @doc
%% Get at most N chunks from the beginning of the chunkset.
%% @end
min(_, Numchunks) when Numchunks < 1 ->
    error(badarg);
min(Chunkset, Numchunks) ->
    case min_(Chunkset, Numchunks) of
        [] -> error(badarg);
        List -> List
    end.

min_(_, 0) ->
    [];
min_(Chunkset, Numchunks) ->
    case min_(Chunkset) of
        none -> [];
        {Start, End}=Chunk ->
            Without = delete(Start, End, Chunkset),
            [Chunk|min_(Without, Numchunks - 1)]
    end.

%% @doc
%%
%% @end
delete([], Chunkset) ->
    Chunkset;
delete([{Offset, Length}|T], Chunkset) ->
    delete(T, delete(Offset, Length, Chunkset)).


%% @doc
%% 
%% @end
delete(_, Length, _) when Length < 1 ->
    error(badarg);
delete(Offset, _, _) when Offset < 0 ->
    error(badarg);
delete(Offset, Length, Chunkset) ->
    #chunkset{chunks=Chunks} = Chunkset,
    NewChunks = delete_(Offset, Offset + Length - 1, Chunks),
    Chunkset#chunkset{chunks=NewChunks}.

{'fraction_non_alphanumeric': 0.1636063544460491, 'fraction_numerical': 0.007633587786259542, 'mean_word_length': 2.7206446661550268, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7135815', 'n_tokens_mistral': 1713, 'n_tokens_neox': 1722, 'n_words': 422}