<commit_before>/*******************************************************************************
 * Copyright (c) 2010 itemis AG (http://www.itemis.eu) and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *******************************************************************************/
package org.eclipse.xtext.parser.encoding;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.CharConversionException;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;

import org.eclipse.emf.common.util.URI;
import org.eclipse.emf.common.util.WrappedException;
import org.eclipse.emf.ecore.resource.Resource;
import org.eclipse.emf.ecore.resource.Resource.Factory;
import org.eclipse.xtext.resource.XtextResource;
import org.eclipse.xtext.tests.AbstractXtextTests;
import org.junit.Test;

/**
 * @author <NAME> - Initial contribution and API
 */
public class EncodingTest extends AbstractXtextTests {

	private static final String ISO_8859_1 = "ISO-8859-1";
	private static final String UTF_8 = "UTF-8";
	private String model;
	private byte[] utfBytes;
	private byte[] isoBytes;
	private Map<String, String> isoOptions;
	private Map<String, String> utfOptions;

	@Override
	public void setUp() throws Exception {
		super.setUp();
		with(EncodingTestLanguageStandaloneSetup.class);
		model = "clp Mdelng Frmwrk";
		utfBytes = model.getBytes(UTF_8);
		isoBytes = model.getBytes(ISO_8859_1);
		isoOptions = Collections.singletonMap(XtextResource.OPTION_ENCODING, ISO_8859_1);
		utfOptions = Collections.singletonMap(XtextResource.OPTION_ENCODING, UTF_8);
	}
	
	@Override
	public void tearDown() throws Exception {
		model = null;
		utfBytes = null;
		isoBytes = null;
		isoOptions = null;
		utfOptions = null;
		super.tearDown();
	}
	
	@Test public void testDefaultEncoding() throws Exception {
		XtextResource resource = createXtextResource();
		assertEquals(Charset.defaultCharset().name(), resource.getEncoding());
	}
	
	@Test public void testSetup() {
		assertFalse(Arrays.equals(utfBytes, isoBytes));
		assertTrue(Charset.defaultCharset().name().equals(UTF_8) || Charset.defaultCharset().name().equals(ISO_8859_1));
	}
	
	@Test public void testUtfBytesWithIsoOptions() throws Exception {
		XtextResource resource = createXtextResource();
		try {
			resource.load(new ByteArrayInputStream(utfBytes), isoOptions);
			assertFalse(resource.getErrors().toString(), resource.getErrors().isEmpty());
		} catch (WrappedException e) {
			if (e.getCause() instanceof CharConversionException) {
				// ok
			} else {
				throw e;
			}
		}
		resource.reparse(model);<commit_msg>removed EncodingTest.testSetup #348<commit_after>/*******************************************************************************
 * Copyright (c) 2010 itemis AG (http://www.itemis.eu) and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *******************************************************************************/
package org.eclipse.xtext.parser.encoding;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.CharConversionException;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;

import org.eclipse.emf.common.util.URI;
import org.eclipse.emf.common.util.WrappedException;
import org.eclipse.emf.ecore.resource.Resource;
import org.eclipse.emf.ecore.resource.Resource.Factory;
import org.eclipse.xtext.resource.XtextResource;
import org.eclipse.xtext.tests.AbstractXtextTests;
import org.junit.Test;

/**
 * @author <NAME> - Initial contribution and API
 */
public class EncodingTest extends AbstractXtextTests {

	private static final String ISO_8859_1 = "ISO-8859-1";
	private static final String UTF_8 = "UTF-8";
	private String model;
	private byte[] utfBytes;
	private byte[] isoBytes;
	private Map<String, String> isoOptions;
	private Map<String, String> utfOptions;

	@Override
	public void setUp() throws Exception {
		super.setUp();
		with(EncodingTestLanguageStandaloneSetup.class);
		model = "clp Mdelng Frmwrk";
		utfBytes = model.getBytes(UTF_8);
		isoBytes = model.getBytes(ISO_8859_1);
		isoOptions = Collections.singletonMap(XtextResource.OPTION_ENCODING, ISO_8859_1);
		utfOptions = Collections.singletonMap(XtextResource.OPTION_ENCODING, UTF_8);
	}
	
	@Override
	public void tearDown() throws Exception {
		model = null;
		utfBytes = null;
		isoBytes = null;
		isoOptions = null;
		utfOptions = null;
		super.tearDown();
	}
	
	@Test public void testDefaultEncoding() throws Exception {
		XtextResource resource = createXtextResource();
		assertEquals(Charset.defaultCharset().name(), resource.getEncoding());
	}
	
	@Test public void testUtfBytesWithIsoOptions() throws Exception {
		XtextResource resource = createXtextResource();
		try {
			resource.load(new ByteArrayInputStream(utfBytes), isoOptions);
			assertFalse(resource.getErrors().toString(), resource.getErrors().isEmpty());
		} catch (WrappedException e) {
			if (e.getCause() instanceof CharConversionException) {
				// ok
			} else {
				throw e;
			}
		}
		resource.reparse(model);
{'fraction_non_alphanumeric': 0.16080129940443963, 'fraction_numerical': 0.013174517235156108, 'mean_word_length': 5.399538106235566, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5454804', 'n_tokens_mistral': 1754, 'n_tokens_neox': 1595, 'n_words': 381}