<commit_before>// (c) 2012 <NAME> -- Distributed under the Eclipse Public License
package bpsm.edn.parser;

import static bpsm.edn.TaggedValue.newTaggedValue;
import static bpsm.edn.parser.Token.END_LIST;
import static bpsm.edn.parser.Token.END_MAP_OR_SET;
import static bpsm.edn.parser.Token.END_VECTOR;

import java.io.IOException;

import bpsm.edn.EdnException;
import bpsm.edn.EdnIOException;
import bpsm.edn.Tag;

class ParserImpl implements Parser {
    private Config cfg;
    private Scanner scanner;
    private Object curr;
    private int discard;

    ParserImpl(Config cfg, Scanner scanner) throws IOException {
        this.scanner = scanner;
        this.curr = scanner.nextToken();
        this.cfg = cfg;
        this.discard = 0;
    }

    public void close() {
        if (scanner != null) {
            try {
                scanner.close();
            } catch (IOException e) {
                throw new EdnIOException(e);
            } finally {
                scanner = null;
            }
        }
    }

    public Object nextValue() {
        try {
            return nextValue0();
        } catch (IOException e) {
            try {
                close();
            } catch (EdnIOException _) {
                // suppress _ in favor of e
            }
            throw new EdnIOException(e);
        }
    }

    private Object nextValue0() throws IOException {
        assert discard >= 0;
        if (curr instanceof Token) {
            switch ((Token) curr) {
            case BEGIN_LIST:
                nextToken();
                return parseIntoCollection(cfg.getListFactory(), END_LIST);
            case BEGIN_VECTOR:
                nextToken();
                return parseIntoCollection(cfg.getVectorFactory(), END_VECTOR);
            case BEGIN_SET:
                nextToken();
                return parseIntoCollection(cfg.getSetFactory(), END_MAP_OR_SET);
            case BEGIN_MAP:
                nextToken();
                return parseIntoMap(cfg.getMapFactory());
            case DISCARD:
                nextToken();
                discardValue();
                return nextValue();
            case END_LIST:
            case END_MAP_OR_SET:
            case END_VECTOR:
                throw new EdnException();
            case END_OF_INPUT:
                return END_OF_INPUT;
            case NIL:
                nextToken();
                return null;
            default:
                throw new EdnException();
            }
        } else if (curr instanceof Tag) {
            Tag t = (Tag) curr;
            nextToken();
            return nextValue(t);
        } else {
            Object value = curr;
            nextToken();
            return value;
        }
    }

    private Object nextToken() throws IOException {
        try {
            return curr = scanner.nextToken();
        } catch (IOException e) {
            curr = Token.END_OF_INPUT;
            try {
                scanner.close();
            } catch (IOException _) {
                // suppress _ in favor of e
            }
            throw e;
        }
    }

    private Object nextValue(Tag t) throws IOException {
        Object v = nextValue();
        if (discard == 0) {
            TagHandler x = cfg.getTagHandler(t);
            if (x != null) {
                return x.transform(t, v);
            } else {
                return newTaggedValue(t, v);
            }
        } else {
            return null;
        }
    }

    private void discardValue() throws IOException {
        try {
            discard ++;
            nextValue();
        } finally {
            discard --;
        }
    }

    private Object parseIntoMap(CollectionBuilder.Factory f) throws IOException {
        CollectionBuilder b = (discard == 0) ? f.builder() : null;
        while (curr != END_MAP_OR_SET) {
            Object o = nextValue();
            if (o == END_OF_INPUT) {
                throw new EdnException("Expected '}', but found end of input.\n" +
                        String.valueOf(b.build()));
            }
            if (discard == 0) {
                b.add(o);
            }
        }
        nextToken();
        return (discard == 0) ? b.build() : null;
    }

    private Object parseIntoCollection(CollectionBuilder.Factory f, Token end) throws IOException {
        CollectionBuilder b = (discard == 0) ? f.builder() : null;
        while (curr != end) {
            Object value = nextValue();
            if (value == END_OF_INPUT) {
                throw new EdnException();
            }
            if (discard == 0) {
                b.add(value);
            }
        }
        nextToken();
        return (discard == 0) ? b.build() : null;
    }


}<commit_msg>ParserImpl free of mutable state, except for Scanner<commit_after>// (c) 2012 <NAME> -- Distributed under the Eclipse Public License
package bpsm.edn.parser;

import static bpsm.edn.TaggedValue.newTaggedValue;
import static bpsm.edn.parser.Token.END_LIST;
import static bpsm.edn.parser.Token.END_MAP_OR_SET;
import static bpsm.edn.parser.Token.END_VECTOR;

import java.io.IOException;

import bpsm.edn.EdnException;
import bpsm.edn.EdnIOException;
import bpsm.edn.Tag;

class ParserImpl implements Parser {
    
    private static final Object DISCARDED_VALUE = new Object() {
        @Override
        public String toString() { return "##discarded value##"; }
    };
    
    private Config cfg;
    private Scanner scanner;

    ParserImpl(Config cfg, Scanner scanner) throws IOException {
        this.scanner = scanner;
        this.cfg = cfg;
    }

    public void close() {
        if (scanner != null) {
            try {
                scanner.close();
            } catch (IOException e) {
                throw new EdnIOException(e);
            } finally {
                scanner = null;
            }
        }
    }

    public Object nextValue() {
        return nextValue(nextToken(), false);
    }

    private Object nextToken() {
        try {
            return scanner.nextToken();
        } catch (IOException e) {
            try {
                scanner.close();
            } catch (IOException _) {
                // suppress _ in favor of e
            }
            throw new EdnIOException(e);
        }
    }

    private Object nextValue(Object curr, boolean discard) {
        if (curr instanceof Token) {
            switch ((Token) curr) {
            case BEGIN_LIST:
                return parseIntoCollection(cfg.getListFactory(), END_LIST, nextToken(), discard);
            case BEGIN_VECTOR:
                return parseIntoCollection(cfg.getVectorFactory(), END_VECTOR, nextToken(), discard);
            case BEGIN_SET:
                return parseIntoCollection(cfg.getSetFactory(), END_MAP_OR_SET, nextToken(), discard);
            case BEGIN_MAP:
                return parseIntoMap(cfg.getMapFactory(), nextToken(), discard);
            case DISCARD:
                nextValue(nextToken(), true);
                return nextValue(nextToken(), discard);
            case NIL:
                return null;
            case END_OF_INPUT:
                return END_OF_INPUT;
            case END_LIST:
            case END_MAP_OR_SET:
            case END_VECTOR:
                throw new EdnException("Unexpected Token: " + curr);
            default:
                throw new EdnException("Unrecognized Token: " + curr);
            }
        } else if (curr instanceof Tag) {
            return nextValue((Tag)curr, nextToken(), discard);
        } else {
            return curr;
        }
    }

    private Object nextValue(Tag t, Object curr, boolean discard) {
        Object v = nextValue(curr, discard);
        if (discard) {
            // It doesn't matter what we return here, as it will be discarded.
            return DISCARDED_VALUE;
        }
        TagHandler x = cfg.getTagHandler(t);
        return x != null ? x.transform(t, v) : newTaggedValue(t, v);
    }

    private Object parseIntoMap(CollectionBuilder.Factory f, Object curr, boolean discard) {
        CollectionBuilder b = !discard ? f.builder() : null;
        while (curr != END_MAP_OR_SET) {
            Object o = nextValue(curr, discard);
            if (o == END_OF_INPUT) {
                throw new EdnException("Expected '}', but found end of input.\n" +
                        String.valueOf(b.build()));
            }
            if (!discard) {
                b.add(o);
            }
            curr = nextToken();
        }
        return (!discard) ? b.build() : null;
    }

    private Object parseIntoCollection(CollectionBuilder.Factory f, Token end, Object curr, boolean discard) {
        CollectionBuilder b = !discard ? f.builder() : null;
        while (curr != end) {
            Object value = nextValue(curr, discard);
            if (value == END_OF_INPUT) {
                throw new EdnException();
            }
            if (!discard) {
                b.add(value);
            }
            curr = nextToken();
        }
        return !discard ? b.build() : null;
    }

}
{'fraction_non_alphanumeric': 0.09625727143013939, 'fraction_numerical': 0.002085391285259576, 'mean_word_length': 1.5509518477043673, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2641950', 'n_tokens_mistral': 2520, 'n_tokens_neox': 2422, 'n_words': 641}