<commit_before>/*
 * Copyright 2016 <NAME> AS
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package no.dusken.momus.service.drive;

import org.apache.commons.lang3.StringEscapeUtils;
import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Converts content from a Google Drive Document
 * to our representation
 */
@Service
public class GoogleDocsTextConverter {

    Pattern body = Pattern.compile("<body.*?>(.*)</body>");
    Pattern css = Pattern.compile("<style type=\"text/css\">(.*)</style>");
    Pattern italicStyleName = Pattern.compile("\\.([A-Za-z][^{]*?)\\{[^}]*font-style:italic[^}]*\\}");
    Pattern boldStyleName = Pattern.compile("\\.([A-Za-z][^{]*?)\\{[^}]*font-weight:700[^}]*\\}");


    Pattern aTags = Pattern.compile("<a[^>]*?></a>");
    Pattern classes = Pattern.compile(" class=\".*?\"");
    Pattern spans = Pattern.compile("</?span.*?>");
    Pattern emptyP = Pattern.compile("<p>\\s?</p>");

    ArrayList<Pattern> hTags = initTitles();

    Pattern dashes = Pattern.compile("--");
    Pattern inlineComments = Pattern.compile("<sup>.*?</sup>");
    Pattern spaces = Pattern.compile("&nbsp;");
    Pattern comments = Pattern.compile("<div><p>.*?</p></div>");

    Pattern lists = Pattern.compile(" start=\".*?\"");

    Pattern table = Pattern.compile("<table[^>]*?>.*?</table>");
    Pattern img = Pattern.compile("<img.*?>");


    String ltUnicode = Character.toString((char) 44000);
    String gtUnicode = Character.toString((char) 44001);
    Pattern ltToUnicode = Pattern.compile("&lt;");
    Pattern gtToUnicode = Pattern.compile("&gt;");
    Pattern unicodeToLt = Pattern.compile(ltUnicode);
    Pattern unicodeToGt = Pattern.compile(gtUnicode);

    public String convert(String input) {
        String body = extractBody(input);
        String css = extractCss(input);

        String out;

        out = findItalicsAndBold(body, css);

        out = removeEmptyATags(out);
        out = removeClasses(out);
        out = removeSpans(out);
        out = removeComments(out);
        out = removeInvalidContent(out);
        out = removeListAttributes(out);
        out = removeEmptyPTags(out);
        out = unescapeHtml(out);
        out = replaceTitles(out);
        out = replaceDashes(out);

        return out;
    }

    /**
     * Only interested in the stuff inside <body></body>
     */
    private String extractBody(String in) {
        Matcher m = body.matcher(in);

        if (m.find()) {
            return m.group(1);
        }
        return in;
    }

    private String extractCss(String in) {
        Matcher m = css.matcher(in);

        if (m.find()) {
            return m.group(1);
        }
        return  in;
    }

    private ArrayList<Pattern> initTitles(){
        ArrayList<Pattern> hTags = new ArrayList<Pattern>();
        for(int i=1; i<5; i++){
            hTags.add(Pattern.compile("<h" + i + " [^>]*>"));
        }
        return hTags;
    }

    /**
     * Bold and italics are not marked with tags in GDocs, instead it is applied with CSS.
     * For instance:
     * .c1{font-weight:bold}
     * lalala <span class="c1">bold</span>
     *
     * The classnames change each time, so need to dynamicall find it and change the span to <i> or <b>
     */
    private String findItalicsAndBold(String body, String css) {
        Matcher italicsMatcher = italicStyleName.matcher(css);
        Matcher boldMatcher = boldStyleName.matcher(css);

        int start = 0;
        while (italicsMatcher.find(start)) {
            String italicSelectorName = italicsMatcher.group(1);
            Pattern italicClasses = Pattern.compile("<span class=\"[^\"]*" + italicSelectorName + "[^\"]*\">(.*?)</span>");
            Matcher spanMatcherItalics = italicClasses.matcher(body);

            body = spanMatcherItalics.replaceAll("<i>$1</i>"); // $1 means what is matched inside the parentheses in the pattern
            start = italicsMatcher.start() + 1;
        }

        start = 0;
        while (boldMatcher.find(start)) {
            String boldSelectorName = boldMatcher.group(1);
            Pattern boldClasses = Pattern.compile("<span class=\"[^\"]*" + boldSelectorName + "[^\"]*\">(.*?)</span>");
            Matcher spanMatcherBold = boldClasses.matcher(body);

            body = spanMatcherBold.replaceAll("<b>$1</b>");
            start = boldMatcher.start() + 1;
        }


        return body;
    }

    /**
     * Remove <a name=*></a> stuff google inserts everywhere
     */<commit_msg>Fix bold and italic when importing from google docs Code is very ugly<commit_after>/*
 * Copyright 2016 Studentmediene i Trondheim AS
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package no.dusken.momus.service.drive;

import org.apache.commons.lang3.StringEscapeUtils;
import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Converts content from a Google Drive Document
 * to our representation
 */
@Service
public class GoogleDocsTextConverter {

    Pattern body = Pattern.compile("<body.*?>(.*)</body>");
    Pattern css = Pattern.compile("<style type=\"text/css\">(.*)</style>");
    Pattern italicStyleName = Pattern.compile("\\.([A-Za-z][^{]*?)\\{[^}]*font-style:italic[^}]*\\}");
    Pattern boldStyleName = Pattern.compile("\\.([A-Za-z][^{]*?)\\{[^}]*font-weight:700[^}]*\\}");


    Pattern aTags = Pattern.compile("<a[^>]*?></a>");
    Pattern classes = Pattern.compile(" class=\".*?\"");
    Pattern spans = Pattern.compile("</?span.*?>");
    Pattern emptyP = Pattern.compile("<p>\\s?</p>");

    ArrayList<Pattern> hTags = initTitles();

    Pattern dashes = Pattern.compile("--");
    Pattern inlineComments = Pattern.compile("<sup>.*?</sup>");
    Pattern spaces = Pattern.compile("&nbsp;");
    Pattern comments = Pattern.compile("<div><p>.*?</p></div>");

    Pattern lists = Pattern.compile(" start=\".*?\"");

    Pattern table = Pattern.compile("<table[^>]*?>.*?</table>");
    Pattern img = Pattern.compile("<img.*?>");


    String ltUnicode = Character.toString((char) 44000);
    String gtUnicode = Character.toString((char) 44001);
    Pattern ltToUnicode = Pattern.compile("&lt;");
    Pattern gtToUnicode = Pattern.compile("&gt;");
    Pattern unicodeToLt = Pattern.compile(ltUnicode);
    Pattern unicodeToGt = Pattern.compile(gtUnicode);

    public String convert(String input) {
        String body = extractBody(input);
        String css = extractCss(input);

        String out;

        out = findItalicsAndBold(body, css);

        out = removeEmptyATags(out);
        out = removeClasses(out);
        out = removeSpans(out);
        out = removeComments(out);
        out = removeInvalidContent(out);
        out = removeListAttributes(out);
        out = removeEmptyPTags(out);
        out = unescapeHtml(out);
        out = replaceTitles(out);
        out = replaceDashes(out);

        return out;
    }

    /**
     * Only interested in the stuff inside <body></body>
     */
    private String extractBody(String in) {
        Matcher m = body.matcher(in);

        if (m.find()) {
            return m.group(1);
        }
        return in;
    }

    private String extractCss(String in) {
        Matcher m = css.matcher(in);

        if (m.find()) {
            return m.group(1);
        }
        return  in;
    }

    private ArrayList<Pattern> initTitles(){
        ArrayList<Pattern> hTags = new ArrayList<Pattern>();
        for(int i=1; i<5; i++){
            hTags.add(Pattern.compile("<h" + i + " [^>]*>"));
        }
        return hTags;
    }

    /**
     * Bold and italics are not marked with tags in GDocs, instead it is applied with CSS.
     * For instance:
     * .c1{font-weight:bold}
     * lalala <span class="c1">bold</span>
     *
     * The classnames change each time, so need to dynamicall find it and change the span to <i> or <b>
     */
    private String findItalicsAndBold(String body, String css) {
        Matcher italicsMatcher = italicStyleName.matcher(css);
        Matcher boldMatcher = boldStyleName.matcher(css);

        int start = 0;
        while (italicsMatcher.find(start)) {
            String italicSelectorName = italicsMatcher.group(1);
            Pattern italicClasses = Pattern.compile("<span class=\"[^\"]*" + italicSelectorName + "[^\"]*\">(.*?)</span>");
            Matcher spanMatcherItalics = italicClasses.matcher(body);
            
            System.out.println("stuff");
            System.out.println(spanMatcherItalics);
            StringBuffer sb = new StringBuffer();
            while(spanMatcherItalics.find()) {
              boolean italicbold = false;
              int boldStart = 0;
              while(boldMatcher.find(boldStart)) {
                String boldSelectorName = boldMatcher.group(1);
                Pattern boldClasses = Pattern.compile("<span class=\"[^\"]*" + boldSelectorName + "[^\"]*\">(.*?)</span>");
                Matcher spanMatcherBold = boldClasses.matcher(spanMatcherItalics.group());
                if(spanMatcherBold.matches()) {
                  italicbold = true;
                }
                boldStart = boldMatcher.start() + 1;
              }
              String replaceString = "<i>$1</i>";
              if(italicbold){
                replaceString = "<i><b>$1</b></i>";
              }
              spanMatcherItalics.appendReplacement(sb, replaceString);
            }
            body = spanMatcherItalics.appendTail(sb).toString();
            //body = spanMatcherItalics.replaceAll("<i>$1</i>"); // $1 means what is matched inside the parentheses in the pattern
            start = italicsMatcher.start() + 1;
        }

        start = 0;
        while (boldMatcher.find(start)) {
            String boldSelectorName = boldMatcher.group(1);
            Pattern boldClasses = Pattern.compile("<span class=\"[^\"]*" + boldSelectorName + "[^\"]*\">(.*?)</span>");
            Matcher spanMatcherBold = boldClasses.matcher(body);

            body = spanMatcherBold.replaceAll("<b>$1</b>");
            start = boldMatcher.start() + 1;
        }


        return body;
    }

    /**
     * Remove <a name=*></a> stuff google inserts everywhere
     */
{'fraction_non_alphanumeric': 0.1512501104337839, 'fraction_numerical': 0.006979415142680449, 'mean_word_length': 2.6375321336760926, 'pattern_counts': {'":': 0, '<': 92, '<?xml version=': 0, '>': 96, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '18433', 'n_tokens_mistral': 3374, 'n_tokens_neox': 3220, 'n_words': 818}