<commit_before>package RSP::Extension::Datastore;

use strict;
use warnings;

use JSON::XS;
use Cache::Memcached::Fast;
use RSP::ObjectStore;
use Scalar::Util qw( reftype );

my $encoder = JSON::XS->new->utf8->allow_nonref;
my $mdservers = [ {address => '127.0.0.1:11211'} ];

sub getrd {
  my $class = shift;
  my $tx    = shift;
  my $rd  = $tx->{dbh} ||= RSP::ObjectStore->new( $tx->dbfile );
  return $rd;
}

sub provide {
  my $class = shift;
  my $tx    = shift;
  return (
    'datastore' => {
    
      'write' => sub {
        my $type = shift;
        my $obj = shift;
        my $trans = shift;
	my $md = Cache::Memcached::Fast->new( { servers => $mdservers } );
                
        if (!$obj) { die "no object" }
        if (reftype($obj) ne 'HASH') { die "not an Object" }
  
        my $id    = delete $obj->{id};
        if (!$id) { die "object has no id" }

        ## if the object is transient, don't even bother going 
        ## to the database with, just put it in memcache and be done.
        ## primarily used for sessions.
        if ( $trans ) {
	  if ($md->set( $id, $encoder->encode( $obj ) )) {
	    return 1;
	  } else { $tx->log("can't find memcache, falling back to db on transient store"); }
	}

        my @parts = ( [ $id, 'type', $encoder->encode( $type ) ] );
        foreach my $key (keys %$obj) {
          push @parts, [ $id, $key, $encoder->encode( $obj->{$key} ) ];
        }
        my $rd = $class->getrd( $tx );
        if ( $rd->save( @parts ) ) {
          $md->set( $id, $encoder->encode( $obj ) );
          return 1;        
        }
        return 0;
      },
      
      'remove' => sub {
        my $type = shift;
        my $id   = shift;
	my $md = Cache::Memcached::Fast->new( { servers => $mdservers } );
        $md->delete( $id );
        $class->getrd( $tx )->delete( $id );
      },

      'search' => sub {
        my $type  = shift;
        my $query = shift;
	my $md = Cache::Memcached::Fast->new( { servers => $mdservers } );
        
        my $set;
        foreach my $key (keys %$query) {
          my $val = $query->{$key};
          my $encval = $encoder->encode( $val );
          my $nset = $class->getrd( $tx )->query( $key, '=', $encval );
          if ( $set ) {
            $set = $set->intersection( $nset );
          } else {
            $set = $nset;
          }
        }
        return [ ] if !$set; ## empty array
               
        
        my @objects;
        foreach my $member ( $set->members ) {
          my $parts = $class->getrd( $tx )->get( $member );
          push @objects, $class->parts2object( $member, $parts );
        }
       
        return \@objects;
      },
      
      'get'    => sub {
        my $type = shift;
        my $id   = shift;
	my $md = Cache::Memcached::Fast->new( { servers => $mdservers } );
        my $cached = $md->get( $id );
        if ($cached) { return $encoder->decode( $cached ); }
        my $parts = $class->getrd( $tx )->get( $id );        
        return $class->parts2object( $id, $parts );
      }
    }
  );
}

sub parts2object {
  my $class = shift;
  my $id    = shift;
  my $parts = shift;
  my $object = {};
  foreach my $part ( @$parts ) {
    my $key = $part->[0];
    my $val = $part->[1];
    $object->{ $key } = $encoder->decode( $part->[1] );
  }  
  $object->{id} = $id;
  return $object;
}

1;
<commit_msg>whoops, Set::Scalar overrides truthiness, which causes an intersection on empty sets to fail...<commit_after>package RSP::Extension::Datastore;

use strict;
use warnings;

use JSON::XS;
use Cache::Memcached::Fast;
use RSP::ObjectStore;
use Scalar::Util qw( reftype );

my $encoder = JSON::XS->new->utf8->allow_nonref;
my $mdservers = [ {address => '127.0.0.1:11211'} ];

sub getrd {
  my $class = shift;
  my $tx    = shift;
  my $rd  = $tx->{dbh} ||= RSP::ObjectStore->new( $tx->dbfile );
  return $rd;
}

sub provide {
  my $class = shift;
  my $tx    = shift;
  return (
    'datastore' => {
    
      'write' => sub {
        my $type = shift;
        my $obj = shift;
        my $trans = shift;
	my $md = Cache::Memcached::Fast->new( { servers => $mdservers } );
                
        if (!$obj) { die "no object" }
        if (reftype($obj) ne 'HASH') { die "not an Object" }
  
        my $id    = delete $obj->{id};
        if (!$id) { die "object has no id" }

        ## if the object is transient, don't even bother going 
        ## to the database with, just put it in memcache and be done.
        ## primarily used for sessions.
        if ( $trans ) {
	  if ($md->set( $id, $encoder->encode( $obj ) )) {
	    return 1;
	  } else { $tx->log("can't find memcache, falling back to db on transient store"); }
	}

        my @parts = ( [ $id, 'type', $encoder->encode( $type ) ] );
        foreach my $key (keys %$obj) {
          push @parts, [ $id, $key, $encoder->encode( $obj->{$key} ) ];
        }
        my $rd = $class->getrd( $tx );
        if ( $rd->save( @parts ) ) {
          $md->set( $id, $encoder->encode( $obj ) );
          return 1;        
        }
        return 0;
      },
      
      'remove' => sub {
        my $type = shift;
        my $id   = shift;
	my $md = Cache::Memcached::Fast->new( { servers => $mdservers } );
        $md->delete( $id );
        $class->getrd( $tx )->delete( $id );
      },

      'search' => sub {
        my $type  = shift;
        my $query = shift;
	my $md = Cache::Memcached::Fast->new( { servers => $mdservers } );
        
        my $set;
        foreach my $key (keys %$query) {
          my $val = $query->{$key};
          my $encval = $encoder->encode( $val );
          my $nset = $class->getrd( $tx )->query( $key, '=', $encval );
          if ( ref($set) ) {
            $set = $set->intersection( $nset );
          } else {
            $set = $nset;
          }
        }
        return [ ] if !$set; ## empty array

        my @objects;
        foreach my $member ( $set->members ) {
          my $parts = $class->getrd( $tx )->get( $member );
          push @objects, $class->parts2object( $member, $parts );
        }
       
        return \@objects;
      },
      
      'get'    => sub {
        my $type = shift;
        my $id   = shift;
	my $md = Cache::Memcached::Fast->new( { servers => $mdservers } );
        my $cached = $md->get( $id );
        if ($cached) { return $encoder->decode( $cached ); }
        my $parts = $class->getrd( $tx )->get( $id );        
        return $class->parts2object( $id, $parts );
      }
    }
  );
}

sub parts2object {
  my $class = shift;
  my $id    = shift;
  my $parts = shift;
  my $object = {};
  foreach my $part ( @$parts ) {
    my $key = $part->[0];
    my $val = $part->[1];
    $object->{ $key } = $encoder->decode( $part->[1] );
  }  
  $object->{id} = $id;
  return $object;
}

1;

{'fraction_non_alphanumeric': 0.1838894605321182, 'fraction_numerical': 0.006467734822872262, 'mean_word_length': 1.8302828618968385, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1439911', 'n_tokens_mistral': 2379, 'n_tokens_neox': 2354, 'n_words': 640}