<commit_before>=head1 NAME

Bio::JBrowse::HashStore - on-disk 2-level hash table

=head1 DESCRIPTION

Makes an on-disk hash table, designed to be easily read as static files over HTTP.

Makes a set of JSON-encoded files at paths based on a hash of the key.
For example:

  path/to/dir/29c/c14/fc.json
  path/to/dir/23f/5ad/11.json
  path/to/dir/711/7c1/29.json
  path/to/dir/5ec/b24/6a.json
  path/to/dir/4de/9ac/c6.json
  path/to/dir/41b/c43/27.json
  path/to/dir/28c/d86/e9.json

Where each file contains a JSON object containing data items like:

  { foo: "bar", ... }

Where "foo" is the original key, and "bar" is the JSON-encoded data.

=cut

package Bio::JBrowse::HashStore;
use strict;
use warnings;

use JSON 2;

use File::Spec ();
use File::Path ();

use Cache::Ref::FIFO ();

use ExternalSorter;

my $bucket_class = 'Bio::JBrowse::HashStore::Bucket';


=head2 open( dir => "/path/to/dir", hash_bits => 16 )

=cut

sub open {
    my $class = shift;

    # source of data: defaults, overridden by open args, overridden by meta.json contents
    my $self = bless { @_ }, $class;

    $self->empty if $self->{empty};

    %$self = (
        %$self,
        %{$self->_read_meta}
    );

    $self->{hash_bits} ||= 16;
    $self->{hash_characters} = int( $self->{hash_bits}/4 );
    $self->{file_extension} = '.json';

    $self->{bucket_cache} = Cache::Ref::FIFO->new( size => 200 );
    $self->{bucket_path_cache} = Cache::Ref::FIFO->new( size => 1000 );

    return bless $self, $class;
}

# write out meta.json file when the store itself is destroyed
sub DESTROY {
    my ( $self ) = @_;
    File::Path::mkpath( $self->{dir} );
    my $meta_path = $self->_meta_path;
    CORE::open my $out, '>', $meta_path or die "$! writing $meta_path";
    $out->print( JSON::to_json(
        {
            hash_bits => $self->{hash_bits},
            %{ $self->{meta} || {} }
        }
        )) or die "$! writing $meta_path";
}
sub _meta_path {
    File::Spec->catfile( shift->{dir}, 'meta.json' );
}
sub _read_meta {
    my ( $self ) = @_;
    my $meta_path = $self->_meta_path;
    return {} unless -r $meta_path;
    CORE::open my $meta, '<', $meta_path or die "$! reading $meta_path";
    local $/;
    my $d = eval { JSON::from_json( scalar <$meta> ) };
    warn $@ if $@;
    return $d || {};
}

=head2 get( $key )

=cut

sub get {
    my ( $self, $key ) = @_;
    my $bucket = $self->_getBucket( $key );
    return $bucket->{data}{$key};
}


=head2 set( $key, $value )

=cut

sub set {
    my ( $self, $key, $value ) = @_;

    my $bucket = $self->_getBucket( $key );
    $bucket->{data}{$key} = $value;
    $bucket->{dirty} = 1;

    return $value;
}

=head2 sort_stream( $data_stream )

Given a data stream (sub that returns arrayrefs of [ key, ... ] when
called repeatedly), returns another data stream that returns the
arrayrefs in order of their hash bucket numbers.

Using this can greatly speed up bulk operations on the hash store,
because it allows the internal caches of the HashStore to operate at
maximum efficiency.<commit_msg>make the perl HashStore record the key of the last entry written to, can be used to have a handle on at least one entry in the hash<commit_after>=head1 NAME

Bio::JBrowse::HashStore - on-disk 2-level hash table

=head1 DESCRIPTION

Makes an on-disk hash table, designed to be easily read as static files over HTTP.

Makes a set of JSON-encoded files at paths based on a hash of the key.
For example:

  path/to/dir/29c/c14/fc.json
  path/to/dir/23f/5ad/11.json
  path/to/dir/711/7c1/29.json
  path/to/dir/5ec/b24/6a.json
  path/to/dir/4de/9ac/c6.json
  path/to/dir/41b/c43/27.json
  path/to/dir/28c/d86/e9.json

Where each file contains a JSON object containing data items like:

  { foo: "bar", ... }

Where "foo" is the original key, and "bar" is the JSON-encoded data.

=cut

package Bio::JBrowse::HashStore;
use strict;
use warnings;

use JSON 2;

use File::Spec ();
use File::Path ();

use Cache::Ref::FIFO ();

use ExternalSorter;

my $bucket_class = 'Bio::JBrowse::HashStore::Bucket';


=head2 open( dir => "/path/to/dir", hash_bits => 16 )

=cut

sub open {
    my $class = shift;

    # source of data: defaults, overridden by open args, overridden by meta.json contents
    my $self = bless { @_ }, $class;

    $self->empty if $self->{empty};

    %$self = (
        %$self,
        %{$self->_read_meta}
    );

    $self->{hash_bits} ||= 16;
    $self->{hash_characters} = int( $self->{hash_bits}/4 );
    $self->{file_extension} = '.json';

    $self->{bucket_cache} = Cache::Ref::FIFO->new( size => 200 );
    $self->{bucket_path_cache} = Cache::Ref::FIFO->new( size => 1000 );

    return bless $self, $class;
}

# write out meta.json file when the store itself is destroyed
sub DESTROY {
    my ( $self ) = @_;
    File::Path::mkpath( $self->{dir} );
    my $meta_path = $self->_meta_path;
    CORE::open my $out, '>', $meta_path or die "$! writing $meta_path";
    $out->print( JSON::to_json(
        {
            hash_bits => $self->{hash_bits},
            %{ $self->{meta} || {} }
        }
        )) or die "$! writing $meta_path";
}
sub _meta_path {
    File::Spec->catfile( shift->{dir}, 'meta.json' );
}
sub _read_meta {
    my ( $self ) = @_;
    my $meta_path = $self->_meta_path;
    return {} unless -r $meta_path;
    CORE::open my $meta, '<', $meta_path or die "$! reading $meta_path";
    local $/;
    my $d = eval { JSON::from_json( scalar <$meta> ) };
    warn $@ if $@;
    return $d || {};
}

=head2 get( $key )

=cut

sub get {
    my ( $self, $key ) = @_;
    my $bucket = $self->_getBucket( $key );
    return $bucket->{data}{$key};
}


=head2 set( $key, $value )

=cut

sub set {
    my ( $self, $key, $value ) = @_;

    my $bucket = $self->_getBucket( $key );
    $bucket->{data}{$key} = $value;
    $bucket->{dirty} = 1;
    $self->{meta}{last_changed_entry} = $key;

    return $value;
}

=head2 sort_stream( $data_stream )

Given a data stream (sub that returns arrayrefs of [ key, ... ] when
called repeatedly), returns another data stream that returns the
arrayrefs in order of their hash bucket numbers.

Using this can greatly speed up bulk operations on the hash store,
because it allows the internal caches of the HashStore to operate at
maximum efficiency.
{'fraction_non_alphanumeric': 0.1606457800511509, 'fraction_numerical': 0.01758312020460358, 'mean_word_length': 3.0472186287192757, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7229847', 'n_tokens_mistral': 2338, 'n_tokens_neox': 2217, 'n_words': 650}