<commit_before>// ObjectId.java

package ed.db;

import java.util.*;

import ed.util.*;

/**
 * 12 bytes
 * ---
 *  0 time
 *  1
 *  2
 *  3 
 *  4 machine
 *  5 
 *  6 
 *  7 pid
 *  8 
 *  9 inc
 * 10
 * 11
 */
public class ObjectId {

    static final boolean D = false;
    
    public static ObjectId get(){
        return new ObjectId();
    }
    
    public ObjectId( String s ){
        if ( s.length() < 18 )
            throw new RuntimeException( "invalid ObjectId : bad length : " + s.length() );

        for ( int i=0; i<s.length(); i++ ){
            char c = s.charAt( i );
            if ( c >= '0' && c <= '9' )
                continue;
            if ( c >= 'a' && c <= 'f' )
                continue;
            if ( c >= 'A' && c <= 'F' )
                continue;
            throw new RuntimeException( "invalid ObjectId : bad character : " + c );
        }
        String baseString = s.substring( 0 , 16 );
        String incString = s.substring( 16 );

        _base = Long.parseLong( baseString , 16 );
        _inc = Integer.parseInt( incString , 16 );

        _new = false;
    }
    
    ObjectId( long base , int inc ){
        _base = base;
        _inc = inc;
        
        _new = false;
    }
    
    private ObjectId(){
        _base = ( ((long)_time) << 32) | _machine;
        
        if ( D ) System.out.println( "base : " + Long.toHexString( _base ) );
        
        synchronized ( _incLock ){
            if ( _nextShort == Short.MAX_VALUE )
                _nextByte++;

            int myb = ( ((int)_nextByte) << 16 ) & 0xFF0000;
            int myi = ( _nextShort++ ) & 0xFFFF;
            
            _inc = myb | myi;
        }
        
        _new = true;
    }

    public int hashCode(){
        return _inc;
    }

    public boolean equals( Object o ){
        
        if ( this == o )
            return true;

        if  ( ! ( o instanceof ObjectId ) )
            return false;
        
        ObjectId other = (ObjectId)o;
        return 
            _base == other._base && 
            _inc == other._inc;
    }

    public String toString(){
        String a = Long.toHexString( _base );
        String b = Integer.toHexString( _inc );
        
        StringBuilder buf = new StringBuilder( 16 );

        for ( int i=0; i<(16-a.length()); i++ )
            buf.append( "0" );
        buf.append( a );

        for ( int i=0; i<(8-b.length()); i++ )
            buf.append( "0" );
        buf.append( b );
        
        return buf.toString();
    }

    final long _base;
    final int _inc;
    
    boolean _new;
    
    private static byte _nextByte = (byte)(new java.util.Random()).nextInt();
    private static short _nextShort = (short)(new java.util.Random()).nextInt();
    private static final String _incLock = new String( "ObjectId._incLock" );

    private static int _time = (int)(System.currentTimeMillis()/1000);
    
    static final Thread _timeFixer;
    private static final long _machine;
    private static final int _bottomTop;
    static {
        try {
            int startTime = (int)( java.lang.management.ManagementFactory.getRuntimeMXBean().getStartTime() & 0xFFFF );
            _bottomTop = ( startTime & 0xFF ) << 24;
            if ( D ) System.out.println( "top of last piece : " + Integer.toHexString( _bottomTop ) );
            int machinePiece = ( java.net.InetAddress.getLocalHost().getHostName().hashCode() & 0xFFFFFF ) << 8;
            _machine = ( ( startTime >> 8 ) | machinePiece ) & 0x7FFFFFFF;
            if ( D ) System.out.println( "machine piece : " + Long.toHexString( _machine ) );
        }
        catch ( java.io.IOException ioe ){
            throw new RuntimeException( ioe );
        }

        _timeFixer = new Thread("ObjectId-TimeFixer"){
                public void run(){
                    while ( true ){
                        ThreadUtil.sleep( 999 );
                        _time = (int)(System.currentTimeMillis()/1000);
                    }
                }
            };
        _timeFixer.setDaemon( true );
        _timeFixer.start();
    }

    public static void main( String args[] ){
        Set<ObjectId> s = new HashSet<ObjectId>();
        while ( true ){
            ObjectId i = get();
            if ( s.contains( i ) )
                throw new RuntimeException( "fuck" );
            s.add( i );

            ObjectId o = new ObjectId( i.toString() );
            if ( ! i.equals( o ) )
                throw new RuntimeException( o.toString() + " != " + i.toString() );
        }

    }

}
<commit_msg>exposing some nice functions<commit_after>// ObjectId.java

package ed.db;

import java.util.*;

import ed.util.*;

/**
 * 12 bytes
 * ---
 *  0 time
 *  1
 *  2
 *  3 
 *  4 machine
 *  5 
 *  6 
 *  7 pid
 *  8 
 *  9 inc
 * 10
 * 11
 */
public class ObjectId {

    static final boolean D = false;
    
    public static ObjectId get(){
        return new ObjectId();
    }
    
    public static boolean isValid( String s ){
        if ( s == null )
            return false;
        
        if ( s.length() < 18 )
            return false;

        for ( int i=0; i<s.length(); i++ ){
            char c = s.charAt( i );
            if ( c >= '0' && c <= '9' )
                continue;
            if ( c >= 'a' && c <= 'f' )
                continue;
            if ( c >= 'A' && c <= 'F' )
                continue;

            return false;
        }        

        return true;
    }

    public ObjectId( String s ){

        if ( ! isValid( s ) )
            throw new IllegalArgumentException( "invalid ObjectId [" + s + "]" );

        String baseString = s.substring( 0 , 16 );
        String incString = s.substring( 16 );

        _base = Long.parseLong( baseString , 16 );
        _inc = Integer.parseInt( incString , 16 );

        _new = false;
    }
    
    ObjectId( long base , int inc ){
        _base = base;
        _inc = inc;
        
        _new = false;
    }
    
    private ObjectId(){
        _base = ( ((long)_time) << 32) | _machine;
        
        if ( D ) System.out.println( "base : " + Long.toHexString( _base ) );
        
        synchronized ( _incLock ){
            if ( _nextShort == Short.MAX_VALUE )
                _nextByte++;

            int myb = ( ((int)_nextByte) << 16 ) & 0xFF0000;
            int myi = ( _nextShort++ ) & 0xFFFF;
            
            _inc = myb | myi;
        }
        
        _new = true;
    }

    public int hashCode(){
        return _inc;
    }

    public boolean equals( Object o ){
        
        if ( this == o )
            return true;

        if  ( ! ( o instanceof ObjectId ) )
            return false;
        
        ObjectId other = (ObjectId)o;
        return 
            _base == other._base && 
            _inc == other._inc;
    }

    public String toString(){
        String a = Long.toHexString( _base );
        String b = Integer.toHexString( _inc );
        
        StringBuilder buf = new StringBuilder( 16 );

        for ( int i=0; i<(16-a.length()); i++ )
            buf.append( "0" );
        buf.append( a );

        for ( int i=0; i<(8-b.length()); i++ )
            buf.append( "0" );
        buf.append( b );
        
        return buf.toString();
    }

    final long _base;
    final int _inc;
    
    boolean _new;
    
    private static byte _nextByte = (byte)(new java.util.Random()).nextInt();
    private static short _nextShort = (short)(new java.util.Random()).nextInt();
    private static final String _incLock = new String( "ObjectId._incLock" );

    private static int _time = (int)(System.currentTimeMillis()/1000);
    
    static final Thread _timeFixer;
    private static final long _machine;
    private static final int _bottomTop;
    static {
        try {
            int startTime = (int)( java.lang.management.ManagementFactory.getRuntimeMXBean().getStartTime() & 0xFFFF );
            _bottomTop = ( startTime & 0xFF ) << 24;
            if ( D ) System.out.println( "top of last piece : " + Integer.toHexString( _bottomTop ) );
            int machinePiece = ( java.net.InetAddress.getLocalHost().getHostName().hashCode() & 0xFFFFFF ) << 8;
            _machine = ( ( startTime >> 8 ) | machinePiece ) & 0x7FFFFFFF;
            if ( D ) System.out.println( "machine piece : " + Long.toHexString( _machine ) );
        }
        catch ( java.io.IOException ioe ){
            throw new RuntimeException( ioe );
        }

        _timeFixer = new Thread("ObjectId-TimeFixer"){
                public void run(){
                    while ( true ){
                        ThreadUtil.sleep( 999 );
                        _time = (int)(System.currentTimeMillis()/1000);
                    }
                }
            };
        _timeFixer.setDaemon( true );
        _timeFixer.start();
    }

    public static void main( String args[] ){
        Set<ObjectId> s = new HashSet<ObjectId>();
        while ( true ){
            ObjectId i = get();
            if ( s.contains( i ) )
                throw new RuntimeException( "fuck" );
            s.add( i );

            ObjectId o = new ObjectId( i.toString() );
            if ( ! i.equals( o ) )
                throw new RuntimeException( o.toString() + " != " + i.toString() );
        }

    }

}

{'fraction_non_alphanumeric': 0.12615616261561627, 'fraction_numerical': 0.01484190148419015, 'mean_word_length': 1.4803947719391837, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2462028', 'n_tokens_mistral': 2880, 'n_tokens_neox': 2723, 'n_words': 735}