<commit_before>package helpers;

import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import models.openfire.User;

import org.owasp.html.HtmlPolicyBuilder;
import org.owasp.html.PolicyFactory;

import com.typesafe.config.ConfigFactory;

import play.Logger;
import play.libs.F.Promise;
import play.libs.ws.WS;
import play.libs.ws.WSResponse;

public class ContentHelper {

	private String content;
	private final static Pattern URL_PATTERN = Pattern.compile("\\b(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]");
	private final static int MAX_GET_TIMEOUT = 2500;
	private final static String USER_URL_TEMPLATE = ConfigFactory.load().getString("user.url.template");
	public ContentHelper(String content) {
		this.content = content;
	}

	public static String prepare(String content) {
		if (content == null) {
			return content;
		}
		ContentHelper ch = new ContentHelper(content.trim());
		return ch.detectUsers().sanitize().detectLinks().normalize().toString();
	}

	public ContentHelper normalize() {
		content = content.replace("\n", "<br />\n");
		return this;
	}

	public ContentHelper sanitize() {
		PolicyFactory policy = new HtmlPolicyBuilder()
				.allowUrlProtocols("http")
				.allowUrlProtocols("https")
				.allowElements("a")
				.allowElements("img")			
				.allowElements("br")
				.allowAttributes("href")
				.onElements("a")
				.allowAttributes("target")
				.onElements("a")
				.allowAttributes("src")
				.onElements("img")
				.requireRelNofollowOnLinks()
				.toFactory();
		content = policy.sanitize(content);
		return this;
	}

	public ContentHelper detectLinks() {
		String urlStr;
		StringBuilder contentBuilder = new StringBuilder();
		for (String line : content.split("\n")) {
			Matcher m = URL_PATTERN.matcher(line);
			while (m.find()) {
				urlStr = m.group();
				Logger.debug("found link |" + urlStr + "|");
				line = line.replace(urlStr, embedLink(urlStr.replace("&#61;", "=")));		
				if(urlStr.startsWith("http://www.youtube.com") || urlStr.startsWith("https://www.youtube.com")) {
					Logger.debug("YT: "+line);
					line += "<br />"+embedYT(urlStr.replace("&#61;", "="));
				}
			}
			contentBuilder.append(line.trim());
			contentBuilder.append("\n");
		}
		content = contentBuilder.toString();
		return this;
	}<commit_msg>activate image, if only image url is in message<commit_after>package helpers;

import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import models.openfire.User;

import org.owasp.html.HtmlPolicyBuilder;
import org.owasp.html.PolicyFactory;

import com.typesafe.config.ConfigFactory;

import play.Logger;
import play.libs.F.Promise;
import play.libs.ws.WS;
import play.libs.ws.WSResponse;

public class ContentHelper {

	private String content;
	private final static Pattern URL_PATTERN = Pattern.compile("\\b(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]");
	private final static int MAX_GET_TIMEOUT = 2500;
	private final static String USER_URL_TEMPLATE = ConfigFactory.load().getString("user.url.template");
	public ContentHelper(String content) {
		this.content = content;
	}

	public static String prepare(String content) {
		if (content == null) {
			return content;
		}
		ContentHelper ch = new ContentHelper(content.trim());
		return ch.detectUsers().sanitize().detectLinks().normalize().toString();
	}

	public ContentHelper normalize() {
		content = content.replace("\n", "<br />\n");
		return this;
	}

	public ContentHelper sanitize() {
		PolicyFactory policy = new HtmlPolicyBuilder()
				.allowUrlProtocols("http")
				.allowUrlProtocols("https")
				.allowElements("a")
				.allowElements("img")			
				.allowElements("br")
				.allowAttributes("href")
				.onElements("a")
				.allowAttributes("target")
				.onElements("a")
				.allowAttributes("src")
				.onElements("img")
				.requireRelNofollowOnLinks()
				.toFactory();
		content = policy.sanitize(content);
		return this;
	}

	public ContentHelper detectLinks() {
		String urlStr;
		StringBuilder contentBuilder = new StringBuilder();
		for (String line : content.split("\n")) {
			Matcher m = URL_PATTERN.matcher(line);
			while (m.find()) {
				urlStr = m.group();
				Logger.debug("found link |" + urlStr + "|");
				line = line.replace(urlStr, embedLink(urlStr.replace("&#61;", "=")));		
				if(urlStr.startsWith("http://www.youtube.com") || urlStr.startsWith("https://www.youtube.com")) {
					Logger.debug("YT: "+line);
					line += "<br />"+embedYT(urlStr.replace("&#61;", "="));
				}
				if(line.trim().equals(urlStr)) {
					line += "<br />"+embedImage(urlStr.replace("&#61;", "="));
				}
			}
			contentBuilder.append(line.trim());
			contentBuilder.append("\n");
		}
		content = contentBuilder.toString();
		return this;
	}
{'fraction_non_alphanumeric': 0.1716525685991335, 'fraction_numerical': 0.005364142768722922, 'mean_word_length': 4.212903225806452, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '336935', 'n_tokens_mistral': 1809, 'n_tokens_neox': 1580, 'n_words': 298}