<commit_before>/*******************************************************************************
 * Copyright (c) 2016-2017 Pivotal, Inc.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     Pivotal, Inc. - initial API and implementation
 *******************************************************************************/

package org.springframework.ide.vscode.commons.languageserver.util;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.springframework.ide.vscode.commons.util.Assert;
import org.springframework.ide.vscode.commons.util.BadLocationException;
import org.springframework.ide.vscode.commons.util.text.IDocument;
import org.springframework.ide.vscode.commons.util.text.IRegion;
import org.springframework.ide.vscode.commons.util.text.Region;

/**
 * A non-sucky alternative to {@link IRegion}. Represents a region of text in a document.
 * <p>
 * Caution: assumes the underlying document is not mutated during the lifetime of the
 * region object (otherwise start/end positions may no longer be valid).
 * <p>
 * Implements {@link CharSequence} for convenience (e.g you can use {@link DocumentRegion} as
 * input to a {@link Pattern} and other standard JRE functions which expect a {@link CharSequence}.
 *
 * @author <NAME>
 */
public class DocumentRegion implements CharSequence {
	final IDocument doc;
	final int start;
	final int end;
	public DocumentRegion(IDocument doc, IRegion r) {
		this(doc,
			r.getOffset(),
			r.getOffset()+r.getLength()
		);
	}

	/**
	 * Constructs a {@link DocumentRegion} on a given document. Tries its
	 * best to behave sensibly when passed 'strange' coordinates by
	 * adjusting them logically rather than throw an Exception.
	 * <p>
	 * A position before the start of the document is moved to be the start
	 * of the document.
	 * <p>
	 * A position after the end of the document is moved to the end
	 * of the document.
	 * <p>
	 * If 'end' position is before the start position it is moved be
	 * exactly at the start position (this avoids region with
	 * negative length).
	 */
	public DocumentRegion(IDocument doc, int start, int end) {
		this.doc = doc;
		this.start = limitRange(start, 0, doc.getLength());
		this.end = limitRange(end, start, doc.getLength());
	}

	/**
	 * Create {@link DocumentRegion} covering the whole document.
	 */
	public DocumentRegion(IDocument doc) {
		this(doc, 0, doc.getLength());
	}

	private int limitRange(int offset, int min, int max) {
		if (offset<min) {
			return min;
		}
		if (offset>max) {
			return max;
		}
		return offset;
	}

	@Override
	public String toString() {
		return DocumentUtil.textBetween(doc, start, end);
	}

	public DocumentRegion trim() {
		return trimEnd().trimStart();
	}

	public DocumentRegion trimStart() {
		int howMany = 0;
		int len = length();
		while (howMany<len && Character.isWhitespace(charAt(howMany))) {
			howMany++;
		}
		return subSequence(howMany, len);
	}

	public DocumentRegion trimEnd() {
		int howMany = 0; //how many chars to remove from the end
		int len = length();
		int lastChar = len-1;
		while (howMany<len && Character.isWhitespace(charAt(lastChar-howMany))) {
			howMany++;
		}
		if (howMany>0) {
			return subSequence(0, len-howMany);
		}
		return this;
	}

	/**
	 * Gets character from the region, offset from the start of the region
	 * @return the character from the document (char)0 if the offset is outside the region.
	 */
	@Override
	public char charAt(int offset) {
		if (offset<0 || offset>=length()) {
			throw new IndexOutOfBoundsException(""+offset);
		}
		try {
			return doc.getChar(start+offset);
		} catch (BadLocationException e) {
			throw new IndexOutOfBoundsException(""+offset);
		}
	}

	@Override
	public int length() {
		return end-start;
	}

	@Override
	public DocumentRegion subSequence(int start, int end) {
		int len = length();
		Assert.isLegal(start>=0);
		Assert.isLegal(end<=len);
		if (start==0 && end==len) {
			return this;
		}
		return new DocumentRegion(doc, this.start+start, this.start+end);
	}

	public boolean isEmpty() {
		return length()==0;
	}

	public DocumentRegion subSequence(int start) {
		return subSequence(start, length());
	}

	public IRegion asRegion() {
		return new Region(start, end-start);
	}

	public int indexOf(char ch, int fromIndex) {
		while (fromIndex < length()) {
			if (charAt(fromIndex)==ch) {
				return fromIndex;
			}
			fromIndex++;
		}
		return -1;
	}

	public DocumentRegion[] split(char c) {
		List<DocumentRegion> pieces = new ArrayList<>();
		int start = 0;
		int end;
		while ((end=indexOf(c, start))>=0) {
			pieces.add(subSequence(start, end));
			start = end+1;
		}
		// Do not forget the last piece!
		pieces.add(subSequence(start, length()));
		return pieces.toArray(new DocumentRegion[pieces.size()]);
	}

	public DocumentRegion[] split(Pattern delimiter) {
		List<DocumentRegion> pieces = new ArrayList<>();
		int start = 0;
		Matcher matcher = delimiter.matcher(this);
		while (matcher.find(start)) {
			int end = matcher.start();
			pieces.add(subSequence(start, end));
			start = matcher.end();
		}
		// Do not forget the last piece!
		pieces.add(subSequence(start, length()));
		return pieces.toArray(new DocumentRegion[pieces.size()]);
	}

	/**
	 * Removes a single occurrence of pat from the start of this region.
	 */
	public DocumentRegion trimStart(Pattern pat) {
		pat = Pattern.compile("^("+pat.pattern()+")");
		Matcher matcher = pat.matcher(this);
		if (matcher.find()) {
			return subSequence(matcher.end());
		}
		return this;
	}

	/**
	 * Removes a single occurrence of pat from the end of this region.
	 */
	public DocumentRegion trimEnd(Pattern pat) {
		pat = Pattern.compile("("+pat.pattern()+")$");
		Matcher matcher = pat.matcher(this);
		if (matcher.find()) {
			return subSequence(0, matcher.start());
		}
		return this;
	}

	/**
	 * Get the region after this one with a given  lenght.
	 * <p>
	 * If the document is too short to provide the requested lenght
	 * then the region is truncated to end of the document.
	 */
	public DocumentRegion textAfter(int len) {
		Assert.isLegal(len>=0);
		return new DocumentRegion(doc, end, end+len);
	}

	/**
	 * Get the region before this one with a given lenght.
	 * <p>
	 * If the requested region extends before the start of the document,
	 * then the region is shortened so its start coincides with document start.
	 */
	public DocumentRegion textBefore(int len) {
		Assert.isLegal(len>=0);
		return new DocumentRegion(doc, start-len, start);
	}

	public IDocument getDocument() {
		return doc;
	}

	/**
	 * Get the start of this region in 'absolute' terms (i.e. relative to the document).
	 */
	public int getStart() {
		return start;
	}

	/**
	 * Get the end of this region in 'absolute' terms (i.e. relative to the document).
	 */
	public int getEnd() {
		return end;
	}

	/**
	 * Convert the given document offset into an offset relative to this region.
	 */
	public int toRelative(int offset) {
		return offset-start;
	}

	public int getLength() {
		return length();
	}

	public boolean endsWith(CharSequence string) {
		int myLen = length();
		int strLen = string.length();
		if (myLen>=strLen) {
			for (int i = 0; i < strLen; i++) {
				if (charAt(myLen-strLen+i)!=string.charAt(i)) {
					return false;
				}
			}
			return true;
		}
		return false;
	}

	/**
	 * Create a document region of lenght 0 at the every end of the document.
	 */
	public static DocumentRegion endOf(IDocument doc) {
		return new DocumentRegion(doc, doc.getLength(), doc.getLength());<commit_msg>Tweak: try to make problem marker at end of document more visible.<commit_after>/*******************************************************************************
 * Copyright (c) 2016-2017 Pivotal, Inc.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     Pivotal, Inc. - initial API and implementation
 *******************************************************************************/

package org.springframework.ide.vscode.commons.languageserver.util;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.springframework.ide.vscode.commons.util.Assert;
import org.springframework.ide.vscode.commons.util.BadLocationException;
import org.springframework.ide.vscode.commons.util.text.IDocument;
import org.springframework.ide.vscode.commons.util.text.IRegion;
import org.springframework.ide.vscode.commons.util.text.Region;

/**
 * A non-sucky alternative to {@link IRegion}. Represents a region of text in a document.
 * <p>
 * Caution: assumes the underlying document is not mutated during the lifetime of the
 * region object (otherwise start/end positions may no longer be valid).
 * <p>
 * Implements {@link CharSequence} for convenience (e.g you can use {@link DocumentRegion} as
 * input to a {@link Pattern} and other standard JRE functions which expect a {@link CharSequence}.
 *
 * @author <NAME>
 */
public class DocumentRegion implements CharSequence {
	final IDocument doc;
	final int start;
	final int end;
	public DocumentRegion(IDocument doc, IRegion r) {
		this(doc,
			r.getOffset(),
			r.getOffset()+r.getLength()
		);
	}

	/**
	 * Constructs a {@link DocumentRegion} on a given document. Tries its
	 * best to behave sensibly when passed 'strange' coordinates by
	 * adjusting them logically rather than throw an Exception.
	 * <p>
	 * A position before the start of the document is moved to be the start
	 * of the document.
	 * <p>
	 * A position after the end of the document is moved to the end
	 * of the document.
	 * <p>
	 * If 'end' position is before the start position it is moved be
	 * exactly at the start position (this avoids region with
	 * negative length).
	 */
	public DocumentRegion(IDocument doc, int start, int end) {
		this.doc = doc;
		this.start = limitRange(start, 0, doc.getLength());
		this.end = limitRange(end, start, doc.getLength());
	}

	/**
	 * Create {@link DocumentRegion} covering the whole document.
	 */
	public DocumentRegion(IDocument doc) {
		this(doc, 0, doc.getLength());
	}

	private int limitRange(int offset, int min, int max) {
		if (offset<min) {
			return min;
		}
		if (offset>max) {
			return max;
		}
		return offset;
	}

	@Override
	public String toString() {
		return DocumentUtil.textBetween(doc, start, end);
	}

	public DocumentRegion trim() {
		return trimEnd().trimStart();
	}

	public DocumentRegion trimStart() {
		int howMany = 0;
		int len = length();
		while (howMany<len && Character.isWhitespace(charAt(howMany))) {
			howMany++;
		}
		return subSequence(howMany, len);
	}

	public DocumentRegion trimEnd() {
		int howMany = 0; //how many chars to remove from the end
		int len = length();
		int lastChar = len-1;
		while (howMany<len && Character.isWhitespace(charAt(lastChar-howMany))) {
			howMany++;
		}
		if (howMany>0) {
			return subSequence(0, len-howMany);
		}
		return this;
	}

	/**
	 * Gets character from the region, offset from the start of the region
	 * @return the character from the document (char)0 if the offset is outside the region.
	 */
	@Override
	public char charAt(int offset) {
		if (offset<0 || offset>=length()) {
			throw new IndexOutOfBoundsException(""+offset);
		}
		try {
			return doc.getChar(start+offset);
		} catch (BadLocationException e) {
			throw new IndexOutOfBoundsException(""+offset);
		}
	}

	@Override
	public int length() {
		return end-start;
	}

	@Override
	public DocumentRegion subSequence(int start, int end) {
		int len = length();
		Assert.isLegal(start>=0);
		Assert.isLegal(end<=len);
		if (start==0 && end==len) {
			return this;
		}
		return new DocumentRegion(doc, this.start+start, this.start+end);
	}

	public boolean isEmpty() {
		return length()==0;
	}

	public DocumentRegion subSequence(int start) {
		return subSequence(start, length());
	}

	public IRegion asRegion() {
		return new Region(start, end-start);
	}

	public int indexOf(char ch, int fromIndex) {
		while (fromIndex < length()) {
			if (charAt(fromIndex)==ch) {
				return fromIndex;
			}
			fromIndex++;
		}
		return -1;
	}

	public DocumentRegion[] split(char c) {
		List<DocumentRegion> pieces = new ArrayList<>();
		int start = 0;
		int end;
		while ((end=indexOf(c, start))>=0) {
			pieces.add(subSequence(start, end));
			start = end+1;
		}
		// Do not forget the last piece!
		pieces.add(subSequence(start, length()));
		return pieces.toArray(new DocumentRegion[pieces.size()]);
	}

	public DocumentRegion[] split(Pattern delimiter) {
		List<DocumentRegion> pieces = new ArrayList<>();
		int start = 0;
		Matcher matcher = delimiter.matcher(this);
		while (matcher.find(start)) {
			int end = matcher.start();
			pieces.add(subSequence(start, end));
			start = matcher.end();
		}
		// Do not forget the last piece!
		pieces.add(subSequence(start, length()));
		return pieces.toArray(new DocumentRegion[pieces.size()]);
	}

	/**
	 * Removes a single occurrence of pat from the start of this region.
	 */
	public DocumentRegion trimStart(Pattern pat) {
		pat = Pattern.compile("^("+pat.pattern()+")");
		Matcher matcher = pat.matcher(this);
		if (matcher.find()) {
			return subSequence(matcher.end());
		}
		return this;
	}

	/**
	 * Removes a single occurrence of pat from the end of this region.
	 */
	public DocumentRegion trimEnd(Pattern pat) {
		pat = Pattern.compile("("+pat.pattern()+")$");
		Matcher matcher = pat.matcher(this);
		if (matcher.find()) {
			return subSequence(0, matcher.start());
		}
		return this;
	}

	/**
	 * Get the region after this one with a given  lenght.
	 * <p>
	 * If the document is too short to provide the requested lenght
	 * then the region is truncated to end of the document.
	 */
	public DocumentRegion textAfter(int len) {
		Assert.isLegal(len>=0);
		return new DocumentRegion(doc, end, end+len);
	}

	/**
	 * Get the region before this one with a given lenght.
	 * <p>
	 * If the requested region extends before the start of the document,
	 * then the region is shortened so its start coincides with document start.
	 */
	public DocumentRegion textBefore(int len) {
		Assert.isLegal(len>=0);
		return new DocumentRegion(doc, start-len, start);
	}

	public IDocument getDocument() {
		return doc;
	}

	/**
	 * Get the start of this region in 'absolute' terms (i.e. relative to the document).
	 */
	public int getStart() {
		return start;
	}

	/**
	 * Get the end of this region in 'absolute' terms (i.e. relative to the document).
	 */
	public int getEnd() {
		return end;
	}

	/**
	 * Convert the given document offset into an offset relative to this region.
	 */
	public int toRelative(int offset) {
		return offset-start;
	}

	public int getLength() {
		return length();
	}

	public boolean endsWith(CharSequence string) {
		int myLen = length();
		int strLen = string.length();
		if (myLen>=strLen) {
			for (int i = 0; i < strLen; i++) {
				if (charAt(myLen-strLen+i)!=string.charAt(i)) {
					return false;
				}
			}
			return true;
		}
		return false;
	}

	/**
	 * Create a document region of lenght 0 at the every end of the document.
	 */
	public static DocumentRegion endOf(IDocument doc) {
		DocumentRegion trimmedDoc = new DocumentRegion(doc).trimEnd();
		return new DocumentRegion(doc, trimmedDoc.getLength(), doc.getLength());
{'fraction_non_alphanumeric': 0.1386852005356118, 'fraction_numerical': 0.00433590512019384, 'mean_word_length': 3.800734618916437, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6967510', 'n_tokens_mistral': 5268, 'n_tokens_neox': 4734, 'n_words': 1572}