<commit_before>/*******************************************************************************
 * Copyright (c) 2008 itemis AG (http://www.itemis.eu) and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *******************************************************************************/
package org.eclipse.xtext.parser.antlr;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;
import org.eclipse.emf.common.util.WrappedException;
import org.eclipse.xtext.util.Strings;

import com.google.inject.Inject;

/**
 * @author <NAME> - Initial contribution and API
 */
public class AntlrTokenDefProvider implements ITokenDefProvider {
	
	private static final Logger log = Logger.getLogger(AntlrTokenDefProvider.class);
	
	@Inject
	private IAntlrTokenFileProvider antlrTokenFileProvider;
	
	protected Map<Integer, String> tokenDefMap;
	
	private static final String TOKEN_COUNT = "Tokens";
	
	public Map<Integer, String> getTokenDefMap() {
		if (antlrTokenFileProvider == null)
			return Collections.emptyMap();
		if (tokenDefMap == null) {
			InputStream tokenFile = antlrTokenFileProvider.getAntlrTokenFile();
			try {
				BufferedReader br = new BufferedReader(new InputStreamReader(tokenFile));
				tokenDefMap = new HashMap<Integer, String>();
				String line = br.readLine();
				Pattern pattern = Pattern.compile("(.*)=(\\d+)");
				while (line != null) {
					Matcher m = pattern.matcher(line);
					if (!m.matches()) {
						throw new IllegalStateException("Couldn't match line : '" + line + "'");
					}

					int antlrTokenType = Integer.parseInt(m.group(2));
					String antlrTokenDef = m.group(1);
					if (!TOKEN_COUNT.equals(antlrTokenDef)) {
						if (antlrTokenDef.startsWith("'")) {
							antlrTokenDef = antlrTokenDef.substring(1, antlrTokenDef.length() - 1);
							antlrTokenDef = Strings.convertFromJavaString(antlrTokenDef, true);
							antlrTokenDef = "'" + antlrTokenDef + "'";
						}
						tokenDefMap.put(antlrTokenType, antlrTokenDef);
					}
					line = br.readLine();
				}
			} catch (Exception e) {
				log.error(e, e);
				tokenDefMap = null;
				throw new WrappedException(e);
			} finally {
				try {
					tokenFile.close();
				} catch (IOException e) {
					//ignore
				}
			}
		}
		return tokenDefMap;
	}
	
	public void setAntlrTokenFileProvider(IAntlrTokenFileProvider antlrTokenFileProvider) {
		this.antlrTokenFileProvider = antlrTokenFileProvider;
	}

	public IAntlrTokenFileProvider getAntlrTokenFileProvider() {
		return antlrTokenFileProvider;
	}
}
<commit_msg>Fix: AntlrTokenDefProvider may be marked as singleton<commit_after>/*******************************************************************************
 * Copyright (c) 2008 itemis AG (http://www.itemis.eu) and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *******************************************************************************/
package org.eclipse.xtext.parser.antlr;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;
import org.eclipse.emf.common.util.WrappedException;
import org.eclipse.xtext.util.Strings;

import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;
import com.google.inject.Singleton;

/**
 * @author <NAME> - Initial contribution and API
 */
@Singleton
public class AntlrTokenDefProvider implements ITokenDefProvider {
	
	private static final Logger log = Logger.getLogger(AntlrTokenDefProvider.class);
	
	@Inject
	private IAntlrTokenFileProvider antlrTokenFileProvider;
	
	protected volatile Map<Integer, String> tokenDefMap;
	
	private static final String TOKEN_COUNT = "Tokens";
	
	public Map<Integer, String> getTokenDefMap() {
		if (antlrTokenFileProvider == null)
			return Collections.emptyMap();
		if (tokenDefMap == null) {
			InputStream tokenFile = antlrTokenFileProvider.getAntlrTokenFile();
			try {
				BufferedReader br = new BufferedReader(new InputStreamReader(tokenFile));
				Map<Integer, String> tokenDefMap = new HashMap<Integer, String>();
				String line = br.readLine();
				Pattern pattern = Pattern.compile("(.*)=(\\d+)");
				while (line != null) {
					Matcher m = pattern.matcher(line);
					if (!m.matches()) {
						throw new IllegalStateException("Couldn't match line : '" + line + "'");
					}

					int antlrTokenType = Integer.parseInt(m.group(2));
					String antlrTokenDef = m.group(1);
					if (!TOKEN_COUNT.equals(antlrTokenDef)) {
						if (antlrTokenDef.startsWith("'")) {
							antlrTokenDef = antlrTokenDef.substring(1, antlrTokenDef.length() - 1);
							antlrTokenDef = Strings.convertFromJavaString(antlrTokenDef, true);
							antlrTokenDef = "'" + antlrTokenDef + "'";
						}
						tokenDefMap.put(antlrTokenType, antlrTokenDef);
					}
					line = br.readLine();
				}
				this.tokenDefMap = ImmutableMap.copyOf(tokenDefMap);
			} catch (Exception e) {
				log.error(e, e);
				tokenDefMap = null;
				throw new WrappedException(e);
			} finally {
				try {
					tokenFile.close();
				} catch (IOException e) {
					//ignore
				}
			}
		}
		return tokenDefMap;
	}
	
	public void setAntlrTokenFileProvider(IAntlrTokenFileProvider antlrTokenFileProvider) {
		this.antlrTokenFileProvider = antlrTokenFileProvider;
	}

	public IAntlrTokenFileProvider getAntlrTokenFileProvider() {
		return antlrTokenFileProvider;
	}
}

{'fraction_non_alphanumeric': 0.1580408163265306, 'fraction_numerical': 0.0042448979591836735, 'mean_word_length': 4.331592689295039, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3025156', 'n_tokens_mistral': 2095, 'n_tokens_neox': 1790, 'n_words': 408}