<commit_before>package org.sotorrent.posthistoryextractor.history;

import org.sotorrent.posthistoryextractor.blocks.CodeBlockVersion;
import org.sotorrent.posthistoryextractor.blocks.PostBlockVersion;
import org.sotorrent.posthistoryextractor.blocks.TextBlockVersion;
import org.sotorrent.posthistoryextractor.version.PostVersion;
import org.sotorrent.posthistoryextractor.version.TitleVersion;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.csv.QuoteMode;
import org.sotorrent.util.FileUtils;
import org.sotorrent.util.LogUtils;

import javax.persistence.*;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Timestamp;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Entity
@Table(name="PostHistory")
public class PostHistory {
    private static Logger logger = null;

    static {
        // configure logger
        try {
            logger = LogUtils.getClassLogger(PostHistory.class, false);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /*
     * 2: Initial Body - The first raw body text a post is submitted with.
     *  => Frequency: 34,112,377 (2017-01-27)
     * 5: Edit Body - A post's body has been changed, the raw text is stored here as markdown.
     *  => rows with PostHistoryType 2 and 5 won't have same RevisionGUID (checked using:
     *     select RevisionGUID, PostHistoryTypeId from PostHistoryEntry_ where PostHistoryTypeId=2 or PostHistoryTypeId=5 group by RevisionGUID, PostHistoryTypeId having count(RevisionGUID)>1;)
     *  => Frequency: 17,673,423 (2017-01-27)
     * 8: Rollback Body - A post's body has reverted to a previous version - the raw text is stored here.
     *  => Comment contains RevisionGUID of old version (e.g., "Rollback to [cca4acef-243d-4c7c-ad1d-5476c94323d2] - Cleaner formatting.")
     *     (see http://meta.stackexchange.com/a/2678)
     *  => Frequency: 77,957 (2017-01-27)
     */
    public static final Set<Byte> contentPostHistoryTypes = new HashSet<>();
    static {
        // see https://meta.stackexchange.com/a/2678
        contentPostHistoryTypes.add((byte)2); // Initial Body - initial post raw body text
        contentPostHistoryTypes.add((byte)5); // Edit Body - modified post body (raw markdown)
        contentPostHistoryTypes.add((byte)8); // Rollback Body - reverted body (raw markdown)
    }

    public static final Set<Byte> titlePostHistoryTypes = new HashSet<>();
    static {
        // see https://meta.stackexchange.com/a/2678
        titlePostHistoryTypes.add((byte)1); // Initial Title - initial title (questions only)
        titlePostHistoryTypes.add((byte)4); // Edit Title - modified title (questions only)
        titlePostHistoryTypes.add((byte)7); // Rollback Title - reverted title (questions only)
    }

    public static final Pattern fileNamePattern = Pattern.compile("(\\d+)\\.csv");

    public static final CSVFormat csvFormatPostHistory;
    static {
        // configure CSV format for ground truth
        csvFormatPostHistory = CSVFormat.DEFAULT
                .withHeader("Id", "PostId", "UserId", "PostHistoryTypeId", "RevisionGUID", "CreationDate", "Text", "UserDisplayName", "Comment")
                .withDelimiter(';')
                .withQuote('"')
                .withQuoteMode(QuoteMode.MINIMAL)
                .withEscape('\\')
                .withFirstRecordAsHeader();
    }

    // regex for escaped newline characters
    public static final String newLineRegex = "((?:&#xD;|&#xA;)?&#xA;)";

    // a code block is indented by four spaces or a tab (which can be preceded by spaces)
    private static final Pattern codeBlockPattern = Pattern.compile("^( {4}|[ ]*\\t)");
    private static final Pattern whiteSpaceLinePattern = Pattern.compile("^\\s+$");
    private static final Pattern containsLetterOrDigitPattern = Pattern.compile("[a-zA-Z0-9]");
    // see https://stackoverflow.blog/2014/09/16/introducing-runnable-javascript-css-and-html-code-snippets/
    private static final Pattern stackSnippetBeginPattern = Pattern.compile("<!--\\s+begin\\s+snippet[^>]+>", Pattern.CASE_INSENSITIVE);
    private static final Pattern stackSnippetEndPattern = Pattern.compile("<!--\\s+end\\s+snippet\\s+-->", Pattern.CASE_INSENSITIVE);
    private static final Pattern snippetDividerPattern = Pattern.compile("<([!?])--\\s+-->\\s*");
    // see https://stackoverflow.com/editing-help#syntax-highlighting
    private static final Pattern snippetLanguagePattern = Pattern.compile("<!--\\s+language:[^>]+>", Pattern.CASE_INSENSITIVE);
    // see https://meta.stackexchange.com/q/125148; example: https://stackoverflow.com/posts/32342082/revisions
    private static final Pattern alternativeCodeBlockBeginPattern = Pattern.compile("^\\s*(```)");
    private static final Pattern alternativeCodeBlockEndPattern = Pattern.compile("(```)\\s*$");
    // see, e.g., source of question 19175014 (<pre><code> ... </pre></code> instead of correct indention)
    private static final Pattern codeTagBeginPattern = Pattern.compile("^\\s*<pre>(<code>)?", Pattern.CASE_INSENSITIVE);
    private static final Pattern codeTagEndPattern = Pattern.compile("</pre>(</code>)?\\s*$", Pattern.CASE_INSENSITIVE);
    // see, e.g., source of question 3381751 version 1 (<script type="text/javascript"> ... </script> instead of correct indention)
    private static final Pattern scriptTagBeginPattern = Pattern.compile("^\\s*<script[^>]+>", Pattern.CASE_INSENSITIVE);
    private static final Pattern scriptTagEndPattern = Pattern.compile("</script>\\s*$", Pattern.CASE_INSENSITIVE);
    // see, e.g., source of question 17158055, version 6 (line containing only `mydomain.com/bn/products/1`)
    // also consider possible HTML newline character, as in question 49311849, version 4
    private static final Pattern inlineCodeLinePattern = Pattern.compile("\\s*`([^`]+)`\\s*(?:<br\\s*/?\\s*>)?", Pattern.CASE_INSENSITIVE);

    // database
    private int id;
    private int postId;
    private String userId;
    private byte postHistoryTypeId;
    private String revisionGuid;
    private Timestamp creationDate;
    private String text;
    private String userDisplayName;
    private String comment;
    // internal
    private List<PostBlockVersion> postBlocks;
    private int localIdCount;

    public PostHistory() {}

    public PostHistory(int id, int postId, String userId, byte postHistoryTypeId, String revisionGuid,
                       Timestamp creationDate, String text, String userDisplayName, String comment) {
        this.id = id;
        this.postId = postId;
        this.userId = userId;
        this.postHistoryTypeId = postHistoryTypeId;<commit_msg>Fix order of tags in codeTagEndPattern<commit_after>package org.sotorrent.posthistoryextractor.history;

import org.sotorrent.posthistoryextractor.blocks.CodeBlockVersion;
import org.sotorrent.posthistoryextractor.blocks.PostBlockVersion;
import org.sotorrent.posthistoryextractor.blocks.TextBlockVersion;
import org.sotorrent.posthistoryextractor.version.PostVersion;
import org.sotorrent.posthistoryextractor.version.TitleVersion;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.csv.QuoteMode;
import org.sotorrent.util.FileUtils;
import org.sotorrent.util.LogUtils;

import javax.persistence.*;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Timestamp;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Entity
@Table(name="PostHistory")
public class PostHistory {
    private static Logger logger = null;

    static {
        // configure logger
        try {
            logger = LogUtils.getClassLogger(PostHistory.class, false);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /*
     * 2: Initial Body - The first raw body text a post is submitted with.
     *  => Frequency: 34,112,377 (2017-01-27)
     * 5: Edit Body - A post's body has been changed, the raw text is stored here as markdown.
     *  => rows with PostHistoryType 2 and 5 won't have same RevisionGUID (checked using:
     *     select RevisionGUID, PostHistoryTypeId from PostHistoryEntry_ where PostHistoryTypeId=2 or PostHistoryTypeId=5 group by RevisionGUID, PostHistoryTypeId having count(RevisionGUID)>1;)
     *  => Frequency: 17,673,423 (2017-01-27)
     * 8: Rollback Body - A post's body has reverted to a previous version - the raw text is stored here.
     *  => Comment contains RevisionGUID of old version (e.g., "Rollback to [cca4acef-243d-4c7c-ad1d-5476c94323d2] - Cleaner formatting.")
     *     (see http://meta.stackexchange.com/a/2678)
     *  => Frequency: 77,957 (2017-01-27)
     */
    public static final Set<Byte> contentPostHistoryTypes = new HashSet<>();
    static {
        // see https://meta.stackexchange.com/a/2678
        contentPostHistoryTypes.add((byte)2); // Initial Body - initial post raw body text
        contentPostHistoryTypes.add((byte)5); // Edit Body - modified post body (raw markdown)
        contentPostHistoryTypes.add((byte)8); // Rollback Body - reverted body (raw markdown)
    }

    public static final Set<Byte> titlePostHistoryTypes = new HashSet<>();
    static {
        // see https://meta.stackexchange.com/a/2678
        titlePostHistoryTypes.add((byte)1); // Initial Title - initial title (questions only)
        titlePostHistoryTypes.add((byte)4); // Edit Title - modified title (questions only)
        titlePostHistoryTypes.add((byte)7); // Rollback Title - reverted title (questions only)
    }

    public static final Pattern fileNamePattern = Pattern.compile("(\\d+)\\.csv");

    public static final CSVFormat csvFormatPostHistory;
    static {
        // configure CSV format for ground truth
        csvFormatPostHistory = CSVFormat.DEFAULT
                .withHeader("Id", "PostId", "UserId", "PostHistoryTypeId", "RevisionGUID", "CreationDate", "Text", "UserDisplayName", "Comment")
                .withDelimiter(';')
                .withQuote('"')
                .withQuoteMode(QuoteMode.MINIMAL)
                .withEscape('\\')
                .withFirstRecordAsHeader();
    }

    // regex for escaped newline characters
    public static final String newLineRegex = "((?:&#xD;|&#xA;)?&#xA;)";

    // a code block is indented by four spaces or a tab (which can be preceded by spaces)
    private static final Pattern codeBlockPattern = Pattern.compile("^( {4}|[ ]*\\t)");
    private static final Pattern whiteSpaceLinePattern = Pattern.compile("^\\s+$");
    private static final Pattern containsLetterOrDigitPattern = Pattern.compile("[a-zA-Z0-9]");
    // see https://stackoverflow.blog/2014/09/16/introducing-runnable-javascript-css-and-html-code-snippets/
    private static final Pattern stackSnippetBeginPattern = Pattern.compile("<!--\\s+begin\\s+snippet[^>]+>", Pattern.CASE_INSENSITIVE);
    private static final Pattern stackSnippetEndPattern = Pattern.compile("<!--\\s+end\\s+snippet\\s+-->", Pattern.CASE_INSENSITIVE);
    private static final Pattern snippetDividerPattern = Pattern.compile("<([!?])--\\s+-->\\s*");
    // see https://stackoverflow.com/editing-help#syntax-highlighting
    private static final Pattern snippetLanguagePattern = Pattern.compile("<!--\\s+language:[^>]+>", Pattern.CASE_INSENSITIVE);
    // see https://meta.stackexchange.com/q/125148; example: https://stackoverflow.com/posts/32342082/revisions
    private static final Pattern alternativeCodeBlockBeginPattern = Pattern.compile("^\\s*(```)");
    private static final Pattern alternativeCodeBlockEndPattern = Pattern.compile("(```)\\s*$");
    // see, e.g., source of question 19175014 (<pre><code> ... </pre></code> instead of correct indention)
    private static final Pattern codeTagBeginPattern = Pattern.compile("^\\s*<pre>(<code>)?", Pattern.CASE_INSENSITIVE);
    private static final Pattern codeTagEndPattern = Pattern.compile("(</code>)?</pre>\\s*$", Pattern.CASE_INSENSITIVE);
    // see, e.g., source of question 3381751 version 1 (<script type="text/javascript"> ... </script> instead of correct indention)
    private static final Pattern scriptTagBeginPattern = Pattern.compile("^\\s*<script[^>]+>", Pattern.CASE_INSENSITIVE);
    private static final Pattern scriptTagEndPattern = Pattern.compile("</script>\\s*$", Pattern.CASE_INSENSITIVE);
    // see, e.g., source of question 17158055, version 6 (line containing only `mydomain.com/bn/products/1`)
    // also consider possible HTML newline character, as in question 49311849, version 4
    private static final Pattern inlineCodeLinePattern = Pattern.compile("\\s*`([^`]+)`\\s*(?:<br\\s*/?\\s*>)?", Pattern.CASE_INSENSITIVE);

    // database
    private int id;
    private int postId;
    private String userId;
    private byte postHistoryTypeId;
    private String revisionGuid;
    private Timestamp creationDate;
    private String text;
    private String userDisplayName;
    private String comment;
    // internal
    private List<PostBlockVersion> postBlocks;
    private int localIdCount;

    public PostHistory() {}

    public PostHistory(int id, int postId, String userId, byte postHistoryTypeId, String revisionGuid,
                       Timestamp creationDate, String text, String userDisplayName, String comment) {
        this.id = id;
        this.postId = postId;
        this.userId = userId;
        this.postHistoryTypeId = postHistoryTypeId;
{'fraction_non_alphanumeric': 0.12563642882753676, 'fraction_numerical': 0.021226245966296162, 'mean_word_length': 3.6056803170409513, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 65, 'https://': 12, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3019241', 'n_tokens_mistral': 4244, 'n_tokens_neox': 3966, 'n_words': 1028}