<commit_before>package com.sigpwned.jsonification.parser;

import java.io.IOException;
import java.io.PushbackReader;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;

import com.sigpwned.jsonification.exception.ParseJsonException;

/**
 * Copyright 2015 <NAME>
 *     
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
public class JsonParser implements AutoCloseable {
    public static interface Handler {
        public void openObject(String name);
        
        public void closeObject();
        
        public void openArray(String name);
        
        public void closeArray();
        
        public void nil(String name);
        
        public void scalar(String name, long value);
        
        public void scalar(String name, double value);
        
        public void scalar(String name, boolean value);
        
        public void scalar(String name, String value);
        
        public void eof();
    }
    
    private static class Token {
        public static enum Type {
            OPEN_OBJECT, CLOSE_OBJECT,
            OPEN_ARRAY, CLOSE_ARRAY,
            SYMBOL, COLON, COMMA, LONG, DOUBLE,
            STRING, TRUE, FALSE, NULL, EOF;
        }
        
        public final Token.Type type;
        public final String text;

        public Token(Token.Type type, String text) {
            this.type = type;
            this.text = text;
        }
    }
    
    
    private static class Scope {
        public static enum Type {
            ROOT, OBJECT, ARRAY;
        }
        
        public final Scope.Type type;
        public int count;
        
        public Scope(Scope.Type type) {
            this.type = type;
            this.count = 0;
        }
        
        public String toString() {
            return type.name();
        }
    }
    
    private final PushbackReader reader;
    private final char[] cpbuf;
    private final List<Scope> scopes;
    private Token peek;
    
    public JsonParser(String text) {
        this(new StringReader(text));
    }
    
    public JsonParser(Reader reader) {
        this(new PushbackReader(reader, 2));
    }
    
    public JsonParser(PushbackReader reader) {
        this.reader = reader;
        this.cpbuf = new char[2];
        this.scopes = new ArrayList<>();
        this.scopes.add(new Scope(Scope.Type.ROOT));
    }
    
    public void parse(final JsonParser.Handler delegate) throws IOException {
        final boolean[] eof=new boolean[1];
        final JsonParser.Handler handler=new JsonParser.Handler() {
            @Override
            public void scalar(String name, String value) {
                delegate.scalar(name, value);
            }
            
            @Override
            public void scalar(String name, boolean value) {
                delegate.scalar(name, value);
            }
            
            @Override
            public void scalar(String name, double value) {
                delegate.scalar(name, value);
            }
            
            @Override
            public void scalar(String name, long value) {
                delegate.scalar(name, value);
            }
            
            @Override
            public void openObject(String name) {
                delegate.openObject(name);
            }
            
            @Override
            public void openArray(String name) {
                delegate.openArray(name);
            }
            
            @Override
            public void nil(String name) {
                delegate.nil(name);
            }
            
            @Override
            public void eof() {
                eof[0] = true;
                delegate.eof();
            }
            
            @Override
            public void closeObject() {
                delegate.closeObject();
            }
            
            @Override
            public void closeArray() {
                delegate.closeArray();
            }
        };
        do {
            next(handler);
        } while(eof[0] == false);
    }

    public void next(final JsonParser.Handler handler) throws IOException {
        Scope scope=scopes.get(scopes.size()-1);
        
        switch(scope.type) {
        case ARRAY:
        {
            Token token=read();
            if(token.type == Token.Type.EOF)
                throw new ParseJsonException("Unexpected EOF in array");
            else
            if(token.type == Token.Type.CLOSE_ARRAY) {
                handler.closeArray();
                scopes.remove(scopes.size()-1);
            }
            else {
                if(token.type == Token.Type.COMMA) {
                    if(scope.count != 0)<commit_msg>JsonParser#parse() now only parses a single JSON value from its input<commit_after>package com.sigpwned.jsonification.parser;

import java.io.IOException;
import java.io.PushbackReader;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;

import com.sigpwned.jsonification.exception.ParseJsonException;

/**
 * Copyright 2015 <NAME>
 *     
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
public class JsonParser implements AutoCloseable {
    public static interface Handler {
        public void openObject(String name);
        
        public void closeObject();
        
        public void openArray(String name);
        
        public void closeArray();
        
        public void nil(String name);
        
        public void scalar(String name, long value);
        
        public void scalar(String name, double value);
        
        public void scalar(String name, boolean value);
        
        public void scalar(String name, String value);
        
        public void eof();
    }
    
    private static class Token {
        public static enum Type {
            OPEN_OBJECT, CLOSE_OBJECT,
            OPEN_ARRAY, CLOSE_ARRAY,
            SYMBOL, COLON, COMMA, LONG, DOUBLE,
            STRING, TRUE, FALSE, NULL, EOF;
        }
        
        public final Token.Type type;
        public final String text;

        public Token(Token.Type type, String text) {
            this.type = type;
            this.text = text;
        }
    }
    
    
    private static class Scope {
        public static enum Type {
            ROOT, OBJECT, ARRAY;
        }
        
        public final Scope.Type type;
        public int count;
        
        public Scope(Scope.Type type) {
            this.type = type;
            this.count = 0;
        }
        
        public String toString() {
            return type.name();
        }
    }
    
    private final PushbackReader reader;
    private final char[] cpbuf;
    private final List<Scope> scopes;
    private Token peek;
    
    public JsonParser(String text) {
        this(new StringReader(text));
    }
    
    public JsonParser(Reader reader) {
        this(new PushbackReader(reader, 2));
    }
    
    public JsonParser(PushbackReader reader) {
        this.reader = reader;
        this.cpbuf = new char[2];
        this.scopes = new ArrayList<>();
        this.scopes.add(new Scope(Scope.Type.ROOT));
    }
    
    /**
     * Handle JSON events until one complete JSON value has been parsed. A
     * JSON value is one complete scalar, object, array, or nil. If the given
     * input contains more than one complete JSON value, only the first is
     * parsed.
     */
    public boolean parse(final JsonParser.Handler delegate) throws IOException {
        final boolean[] eof=new boolean[1];
        final boolean[] completed=new boolean[1];
        final int[] depth=new int[1];
        final JsonParser.Handler handler=new JsonParser.Handler() {
            @Override
            public void scalar(String name, String value) {
                delegate.scalar(name, value);
                completed[0] = true;
            }
            
            @Override
            public void scalar(String name, boolean value) {
                delegate.scalar(name, value);
                completed[0] = true;
            }
            
            @Override
            public void scalar(String name, double value) {
                delegate.scalar(name, value);
                completed[0] = true;
            }
            
            @Override
            public void scalar(String name, long value) {
                delegate.scalar(name, value);
                completed[0] = true;
            }
            
            @Override
            public void openObject(String name) {
                delegate.openObject(name);
                depth[0] = depth[0]+1;
            }
            
            @Override
            public void openArray(String name) {
                delegate.openArray(name);
                depth[0] = depth[0]+1;
            }
            
            @Override
            public void nil(String name) {
                delegate.nil(name);
                completed[0] = true;
            }
            
            @Override
            public void eof() {
                eof[0] = true;
                delegate.eof();
            }
            
            @Override
            public void closeObject() {
                depth[0] = depth[0]-1;
                delegate.closeObject();
                completed[0] = true;
            }
            
            @Override
            public void closeArray() {
                depth[0] = depth[0]-1;
                delegate.closeArray();
                completed[0] = true;
            }
        };
        
        int count=0;
        do {
            completed[0] = false;
            next(handler);
            count = count+1;
        } while(eof[0]==false && (completed[0]==false || depth[0]!=0));
        
        if(completed[0]==false && count!=0)
            throw new ParseJsonException("Unexpect EOF in value");
        
        return completed[0];
    }

    public void next(final JsonParser.Handler handler) throws IOException {
        Scope scope=scopes.get(scopes.size()-1);
        
        switch(scope.type) {
        case ARRAY:
        {
            Token token=read();
            if(token.type == Token.Type.EOF)
                throw new ParseJsonException("Unexpected EOF in array");
            else
            if(token.type == Token.Type.CLOSE_ARRAY) {
                handler.closeArray();
                scopes.remove(scopes.size()-1);
            }
            else {
                if(token.type == Token.Type.COMMA) {
                    if(scope.count != 0)
{'fraction_non_alphanumeric': 0.0853448275862069, 'fraction_numerical': 0.005517241379310344, 'mean_word_length': 1.5374015748031495, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2275599', 'n_tokens_mistral': 2978, 'n_tokens_neox': 2885, 'n_words': 856}