<commit_before>package com.redhat.ceylon.compiler.typechecker.analyzer;

import static com.redhat.ceylon.compiler.typechecker.parser.CeylonLexer.AVERBATIM_STRING;
import static com.redhat.ceylon.compiler.typechecker.parser.CeylonLexer.STRING_END;
import static com.redhat.ceylon.compiler.typechecker.parser.CeylonLexer.STRING_MID;
import static com.redhat.ceylon.compiler.typechecker.parser.CeylonLexer.STRING_START;
import static com.redhat.ceylon.compiler.typechecker.parser.CeylonLexer.VERBATIM_STRING;
import static java.lang.Character.toChars;
import static java.lang.Integer.parseInt;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.antlr.runtime.Token;

import com.redhat.ceylon.compiler.typechecker.tree.Node;
import com.redhat.ceylon.compiler.typechecker.tree.Tree.CharLiteral;
import com.redhat.ceylon.compiler.typechecker.tree.Tree.FloatLiteral;
import com.redhat.ceylon.compiler.typechecker.tree.Tree.Literal;
import com.redhat.ceylon.compiler.typechecker.tree.Tree.NaturalLiteral;
import com.redhat.ceylon.compiler.typechecker.tree.Tree.QuotedLiteral;
import com.redhat.ceylon.compiler.typechecker.tree.Tree.StringLiteral;
import com.redhat.ceylon.compiler.typechecker.tree.Visitor;

public class LiteralVisitor extends Visitor {
	
    @Override
    public void visit(StringLiteral that) {
        int type = that.getToken().getType();
        String text = that.getText();
        if (type==VERBATIM_STRING || type==AVERBATIM_STRING) {
            that.setText(text.substring(3,text.length()-3));
        }
        else {
            StringBuilder result = new StringBuilder();
            stripIndent(text, getIndentPosition(that), result);
            interpolateEscapes(result, that);
            if (type==STRING_END || type==STRING_MID) {
                result.deleteCharAt(0);
            }
            if (type==STRING_START || type==STRING_MID) {
                result.deleteCharAt(result.length()-1);
            }
            that.setText(result.substring(1, result.length()-1));
        }
    }

    @Override
    public void visit(QuotedLiteral that) {
        StringBuilder result = new StringBuilder();
        stripIndent(that.getText(), getIndentPosition(that), result);
        //interpolateEscapes(result, that);
        that.setText(result.toString());
    }
    
	private int getIndentPosition(Literal that) {
		Token token = that.getToken();
		return token==null ? 0 : token.getCharPositionInLine()+1;
	}
    
    @Override
    public void visit(CharLiteral that) {
        StringBuilder result = new StringBuilder(that.getText());
        interpolateEscapes(result, that);
        that.setText(result.toString());
    }
    
    @Override
    public void visit(FloatLiteral that) {
    	that.setText(that.getText()
    			.replace("_", "")
                .replace("k", "e+3")
                .replace("M", "e+6")
                .replace("G", "e+9")
                .replace("T", "e+12")
                .replace("P", "e+15")
                .replace("m", "e-3")
                .replace("u", "e-6")
                .replace("n", "e-9")
                .replace("p", "e-12")
                .replace("f", "e-15"));
    }
    
    @Override
    public void visit(NaturalLiteral that) {
    	that.setText(that.getText()
    			.replace("_", "")
                .replace("k", "000")
                .replace("M", "000000")
                .replace("G", "000000000")
                .replace("T", "000000000000")
                .replace("P", "000000000000000"));
    }
    
    private static void stripIndent(final String text, final int start, 
            final StringBuilder result) {
        int num = 0;
        for (String line: text.split("\n|\r\n?")) {
            if (num++==0 || line.length()<start) {
                result.append(line);
            }
            else {
                boolean trimIndent = true;
                for (int i=0; i<start; i++) {
                    if (line.charAt(i)!=' ') {
                        trimIndent = false;
                        break;
                    }
                }
                if (trimIndent) {
                    result.append(line.substring(start));
                }
                else {
                    result.append(line);
                }
            }
            result.append("\n");
        }
        result.setLength(result.length()-1);
    }
    
    private static Pattern re = Pattern.compile("\\\\(\\{#([^}]*)\\}|(.))");
    
    private static void interpolateEscapes(final StringBuilder result, Node node) {
        Matcher m;
        int start=0;
        while ((m = re.matcher(result)).find(start)) {
            String hex = m.group(2);
            if (hex!=null) {<commit_msg>strategy for dealing with leading whitespace suggested in #577<commit_after>package com.redhat.ceylon.compiler.typechecker.analyzer;

import static com.redhat.ceylon.compiler.typechecker.parser.CeylonLexer.AVERBATIM_STRING;
import static com.redhat.ceylon.compiler.typechecker.parser.CeylonLexer.STRING_END;
import static com.redhat.ceylon.compiler.typechecker.parser.CeylonLexer.STRING_MID;
import static com.redhat.ceylon.compiler.typechecker.parser.CeylonLexer.STRING_START;
import static com.redhat.ceylon.compiler.typechecker.parser.CeylonLexer.VERBATIM_STRING;
import static java.lang.Character.isWhitespace;
import static java.lang.Character.toChars;
import static java.lang.Integer.parseInt;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.antlr.runtime.Token;

import com.redhat.ceylon.compiler.typechecker.tree.Node;
import com.redhat.ceylon.compiler.typechecker.tree.Tree.CharLiteral;
import com.redhat.ceylon.compiler.typechecker.tree.Tree.FloatLiteral;
import com.redhat.ceylon.compiler.typechecker.tree.Tree.Literal;
import com.redhat.ceylon.compiler.typechecker.tree.Tree.NaturalLiteral;
import com.redhat.ceylon.compiler.typechecker.tree.Tree.QuotedLiteral;
import com.redhat.ceylon.compiler.typechecker.tree.Tree.StringLiteral;
import com.redhat.ceylon.compiler.typechecker.tree.Visitor;

public class LiteralVisitor extends Visitor {
	
    @Override
    public void visit(StringLiteral that) {
        int type = that.getToken().getType();
        String text = that.getText();
        StringBuilder result = new StringBuilder();
        int indent = getIndentPosition(that);
        boolean allTrimmed = stripIndent(text, indent, result);
        if (!allTrimmed) {
            that.addError("multiline string content should begin at character position " + indent);
        }
        if (type==VERBATIM_STRING || type==AVERBATIM_STRING) {
            that.setText(result.substring(3,text.length()-3));
        }
        else {
            interpolateEscapes(result, that);
            if (type==STRING_END || type==STRING_MID) {
                result.deleteCharAt(0);
            }
            if (type==STRING_START || type==STRING_MID) {
                result.deleteCharAt(result.length()-1);
            }
            that.setText(result.substring(1, result.length()-1));
        }
    }

    @Override
    public void visit(QuotedLiteral that) {
        StringBuilder result = new StringBuilder();
        stripIndent(that.getText(), getIndentPosition(that), result);
        //interpolateEscapes(result, that);
        that.setText(result.toString());
    }
    
	private int getIndentPosition(Literal that) {
		Token token = that.getToken();
		return token==null ? 0 : token.getCharPositionInLine()+1;
	}
    
    @Override
    public void visit(CharLiteral that) {
        StringBuilder result = new StringBuilder(that.getText());
        interpolateEscapes(result, that);
        that.setText(result.toString());
    }
    
    @Override
    public void visit(FloatLiteral that) {
    	that.setText(that.getText()
    			.replace("_", "")
                .replace("k", "e+3")
                .replace("M", "e+6")
                .replace("G", "e+9")
                .replace("T", "e+12")
                .replace("P", "e+15")
                .replace("m", "e-3")
                .replace("u", "e-6")
                .replace("n", "e-9")
                .replace("p", "e-12")
                .replace("f", "e-15"));
    }
    
    @Override
    public void visit(NaturalLiteral that) {
    	that.setText(that.getText()
    			.replace("_", "")
                .replace("k", "000")
                .replace("M", "000000")
                .replace("G", "000000000")
                .replace("T", "000000000000")
                .replace("P", "000000000000000"));
    }
    
    private static boolean stripIndent(final String text, final int start, 
            final StringBuilder result) {
        boolean allTrimmed = true;
        int num = 0;
        for (String line: text.split("\n|\r\n?")) {
            if (num++==0) {
                result.append(line);
            }
            else {
                boolean trimIndent = true;
                if (line.length()<start) {
                    trimIndent = false;
                }
                else {
                    for (int i=0; i<start; i++) {
                        if (!isWhitespace(line.charAt(i))) {
                            trimIndent = false;
                            break;
                        }
                    }
                }
                if (trimIndent) {
                    result.append(line.substring(start));
                }
                else {
                    allTrimmed = false;
                    result.append(line);
                }
            }
            result.append("\n");
        }
        result.setLength(result.length()-1);
        return allTrimmed;
    }
    
    private static Pattern re = Pattern.compile("\\\\(\\{#([^}]*)\\}|(.))");
    
    private static void interpolateEscapes(final StringBuilder result, Node node) {
        Matcher m;
        int start=0;
        while ((m = re.matcher(result)).find(start)) {
            String hex = m.group(2);
            if (hex!=null) {
{'fraction_non_alphanumeric': 0.13013972055888223, 'fraction_numerical': 0.014870259481037923, 'mean_word_length': 2.057046979865772, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6745240', 'n_tokens_mistral': 3083, 'n_tokens_neox': 2869, 'n_words': 510}