<commit_before>package com.devmpv.service;

import static com.devmpv.config.Const.BOARD;
import static com.devmpv.config.Const.TEXT;
import static com.devmpv.config.Const.THREAD;
import static com.devmpv.config.Const.TITLE;
import static com.devmpv.config.WebSocketConfig.MESSAGE_PREFIX;

import java.io.IOException;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

import javax.annotation.PostConstruct;
import javax.transaction.Transactional;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.safety.Whitelist;
import org.nibor.autolink.Autolink;
import org.nibor.autolink.LinkExtractor;
import org.nibor.autolink.LinkSpan;
import org.nibor.autolink.LinkType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.hateoas.EntityLinks;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;

import com.devmpv.exceptions.CRException;
import com.devmpv.model.Attachment;
import com.devmpv.model.Board;
import com.devmpv.model.Message;
import com.devmpv.model.Thread;
import com.devmpv.repositories.BoardRepository;
import com.devmpv.repositories.MessageRepository;
import com.devmpv.repositories.ThreadRepository;

import ac.simons.oembed.OembedResponse;
import ac.simons.oembed.OembedService;

/**
 * Service to manage basic message operations.
 * 
 * @author devmpv
 *
 */
@Service
public class MessageService {

    @Value("${chan.message.maxCount}")
    private int messageMaxCount;

    @Value("${chan.message.bumpLimit}")
    private int messageBumpLimit;

    @Value("${chan.thread.maxCount}")
    private int threadMaxCount;

    @Autowired
    private ThreadRepository threadRepo;
    @Autowired
    private MessageRepository messageRepo;
    @Autowired
    private BoardRepository boardRepo;
    @Autowired
    private AttachmentService attachmentService;
    @Autowired
    private SimpMessagingTemplate template;
    @Autowired
    private EntityLinks entityLinks;
    @Autowired
    private OembedService oembedService;

    private LinkExtractor linkExtractor = LinkExtractor.builder().linkTypes(EnumSet.of(LinkType.URL)).build();

    private OutputSettings textSettings = new OutputSettings();

    private String getPath(Message message) {
	return entityLinks.linkForSingleResource(message.getClass(), message.getId()).toUri().getPath();
    }

    private void notify(Message message) {
	Map<String, Object> headers = new HashMap<>();
	headers.put("thread", message.getThread().getId());
	template.convertAndSend(MESSAGE_PREFIX + "/newMessage", getPath(message), headers);
    }

    @PostConstruct
    private void postConstruct() {
	textSettings.prettyPrint(false);
    }

    private String prepareText(String input) {
	String result = Jsoup.clean(input, "", Whitelist.basic(), textSettings);
	result = input.replaceAll("&gt;&gt;([0-9]{1,8})", "<a id='reply-link' key='$1'>$1</a>");
	Iterable<LinkSpan> links = linkExtractor.extractLinks(result);
	result = Autolink.renderLinks(result, links, (link, text, sb) -> {
	    String url = text.subSequence(link.getBeginIndex(), link.getEndIndex()).toString();
	    Optional<OembedResponse> response = oembedService.getOembedResponseFor(url);
	    sb.append("<a href=\"").append(url).append("\">").append(url).append("</a>");
	    if (response.isPresent()) {
		sb.append("<div>").append(response.get().getHtml()).append("</div>");
	    }
	});
	return result;<commit_msg>Fixing bug with text cleanup<commit_after>package com.devmpv.service;

import static com.devmpv.config.Const.BOARD;
import static com.devmpv.config.Const.TEXT;
import static com.devmpv.config.Const.THREAD;
import static com.devmpv.config.Const.TITLE;
import static com.devmpv.config.WebSocketConfig.MESSAGE_PREFIX;

import java.io.IOException;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

import javax.annotation.PostConstruct;
import javax.transaction.Transactional;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.safety.Whitelist;
import org.nibor.autolink.Autolink;
import org.nibor.autolink.LinkExtractor;
import org.nibor.autolink.LinkSpan;
import org.nibor.autolink.LinkType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.hateoas.EntityLinks;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;

import com.devmpv.exceptions.CRException;
import com.devmpv.model.Attachment;
import com.devmpv.model.Board;
import com.devmpv.model.Message;
import com.devmpv.model.Thread;
import com.devmpv.repositories.BoardRepository;
import com.devmpv.repositories.MessageRepository;
import com.devmpv.repositories.ThreadRepository;

import ac.simons.oembed.OembedResponse;
import ac.simons.oembed.OembedService;

/**
 * Service to manage basic message operations.
 * 
 * @author devmpv
 *
 */
@Service
public class MessageService {

    private static final String REPLY_STRING = "&gt;&gt;([0-9]{1,8})";
    private static final String REPLY_REPLACE = "<a id='reply-link' key='$1'>$1</a>";

    @Value("${chan.message.maxCount}")
    private int messageMaxCount;

    @Value("${chan.message.bumpLimit}")
    private int messageBumpLimit;

    @Value("${chan.thread.maxCount}")
    private int threadMaxCount;

    @Autowired
    private ThreadRepository threadRepo;
    @Autowired
    private MessageRepository messageRepo;
    @Autowired
    private BoardRepository boardRepo;
    @Autowired
    private AttachmentService attachmentService;
    @Autowired
    private SimpMessagingTemplate template;
    @Autowired
    private EntityLinks entityLinks;
    @Autowired
    private OembedService oembedService;

    private LinkExtractor linkExtractor = LinkExtractor.builder().linkTypes(EnumSet.of(LinkType.URL)).build();

    private OutputSettings textSettings = new OutputSettings();

    private String getPath(Message message) {
	return entityLinks.linkForSingleResource(message.getClass(), message.getId()).toUri().getPath();
    }

    private void notify(Message message) {
	Map<String, Object> headers = new HashMap<>();
	headers.put("thread", message.getThread().getId());
	template.convertAndSend(MESSAGE_PREFIX + "/newMessage", getPath(message), headers);
    }

    @PostConstruct
    private void postConstruct() {
	textSettings.prettyPrint(false);
    }

    private String prepareText(String input) {
	String result = Jsoup.clean(input, "", Whitelist.basic(), textSettings).replaceAll(REPLY_STRING, REPLY_REPLACE);
	Iterable<LinkSpan> links = linkExtractor.extractLinks(result);
	result = Autolink.renderLinks(result, links, (link, text, sb) -> {
	    String url = text.subSequence(link.getBeginIndex(), link.getEndIndex()).toString();
	    Optional<OembedResponse> response = oembedService.getOembedResponseFor(url);
	    sb.append("<a href=\"").append(url).append("\">").append(url).append("</a>");
	    if (response.isPresent()) {
		sb.append("<div>").append(response.get().getHtml()).append("</div>");
	    }
	});
	return result;
{'fraction_non_alphanumeric': 0.12422030524220305, 'fraction_numerical': 0.0015925680159256802, 'mean_word_length': 5.26955074875208, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '367668', 'n_tokens_mistral': 2305, 'n_tokens_neox': 2300, 'n_words': 429}