<commit_before>package org.yinwang.pysonar.demos;

import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.yinwang.pysonar.*;
import org.yinwang.pysonar.types.ModuleType;

import java.io.File;
import java.util.*;
import java.util.Map.Entry;
import java.util.regex.Pattern;

/**
 * Collects per-file hyperlinks, as well as styles that require the
 * symbol table to resolve properly.
 */
class Linker {

    private static final Pattern CONSTANT = Pattern.compile("[A-Z_][A-Z0-9_]*");

    // Map of file-path to semantic styles & links for that path.
    @NotNull
    private Map<String, List<StyleRun>> fileStyles = new HashMap<String, List<StyleRun>>();

    private File outDir;  // where we're generating the output html
    private String rootPath;

    /**
     * Constructor.
     * @param root the root of the directory tree being indexed
     * @param outdir the html output directory
     */
    public Linker(String root, File outdir) {
        rootPath = root;
        outDir = outdir;
    }

    /**
     * Process all bindings across all files and record per-file semantic styles.
     * Should be called once per index.
     */

    int nDef = 0;

    public void findLinks(@NotNull Indexer indexer) {
        Util.msg("Adding xref links for " + indexer.getAllBindings().size() + " bindings");

        for (List<Binding> bindings : indexer.getAllBindings().values()) {
            for (Binding b : bindings) {
                addSemanticStyles(b);
                for (Def def : b.getDefs()) {
                    processDef(def, b);
                    nDef++;
                }
            }
        }

        // highlight definitions
        Util.msg("\nAdding ref links");
        for (Entry<Ref,List<Binding>> e : indexer.getReferences().entrySet()) {
            processRef(e.getKey(), e.getValue());
        }

        Util.msg("\n#xrefs: " + nXref);
        Util.msg("#defs: " + nDef);
        Util.msg("#refs: " + nRef);

//        for (List<Diagnostic> ld: indexer.semanticErrors.values()) {
//            for (Diagnostic d: ld) {
//                processDiagnostic(d);
//            }
//        }

//        for (List<Diagnostic> ld: indexer.parseErrors.values()) {
//            for (Diagnostic d: ld) {
//                processDiagnostic(d);
//            }
//        }
    }


    int nXref = 0;
    Progress progressXRef = new Progress(10000, 50);

    private void processDef(@Nullable Def def, @NotNull Binding binding) {
        if (def == null || def.isURL() || def.getStart() < 0) {
            return;
        }
        StyleRun style = new StyleRun(StyleRun.Type.ANCHOR, def.getStart(), def.getLength());
        style.message = binding.getQname() + " : " + binding.getType();
        style.url = binding.getQname();
        style.id = "" + Math.abs(def.hashCode());

        Set<Ref> refs = binding.getRefs();
        style.highlight = new ArrayList<>();


        for (Ref r : refs) {
            progressXRef.tick();
            nXref++;
//            style.highlight.add(Integer.toString(Math.abs(r.hashCode())));
        }
//        addFileStyle(def.getFile(), style);
    }



    int nRef = 0;
    Progress progressRef = new Progress(10000, 50);

    void processRef(@NotNull Ref ref, @NotNull List<Binding> bindings) {
        StyleRun link = new StyleRun(StyleRun.Type.LINK, ref.start(), ref.length());
        link.id = Integer.toString(Math.abs(ref.hashCode()));

        List<String> typings = new ArrayList<>();
        for (Binding b : bindings) {
            typings.add(b.getQname() + " : " + b.getType());
        }
        link.message = Util.joinWithSep(typings, " | ", "{", "}");

        link.highlight = new ArrayList<>();

        for (Binding b : bindings) {
            for (Def d : b.getDefs()) {<commit_msg>forgot to add back highlighting<commit_after>package org.yinwang.pysonar.demos;

import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.yinwang.pysonar.*;
import org.yinwang.pysonar.types.ModuleType;

import java.io.File;
import java.util.*;
import java.util.Map.Entry;
import java.util.regex.Pattern;

/**
 * Collects per-file hyperlinks, as well as styles that require the
 * symbol table to resolve properly.
 */
class Linker {

    private static final Pattern CONSTANT = Pattern.compile("[A-Z_][A-Z0-9_]*");

    // Map of file-path to semantic styles & links for that path.
    @NotNull
    private Map<String, List<StyleRun>> fileStyles = new HashMap<String, List<StyleRun>>();

    private File outDir;  // where we're generating the output html
    private String rootPath;

    /**
     * Constructor.
     * @param root the root of the directory tree being indexed
     * @param outdir the html output directory
     */
    public Linker(String root, File outdir) {
        rootPath = root;
        outDir = outdir;
    }

    /**
     * Process all bindings across all files and record per-file semantic styles.
     * Should be called once per index.
     */

    int nDef = 0;

    public void findLinks(@NotNull Indexer indexer) {
        Util.msg("Adding xref links for " + indexer.getAllBindings().size() + " bindings");

        for (List<Binding> bindings : indexer.getAllBindings().values()) {
            for (Binding b : bindings) {
                addSemanticStyles(b);
                for (Def def : b.getDefs()) {
                    processDef(def, b);
                    nDef++;
                }
            }
        }

        // highlight definitions
        Util.msg("\nAdding ref links");
        for (Entry<Ref,List<Binding>> e : indexer.getReferences().entrySet()) {
            processRef(e.getKey(), e.getValue());
        }

        Util.msg("\n#xrefs: " + nXref);
        Util.msg("#defs: " + nDef);
        Util.msg("#refs: " + nRef);

//        for (List<Diagnostic> ld: indexer.semanticErrors.values()) {
//            for (Diagnostic d: ld) {
//                processDiagnostic(d);
//            }
//        }

//        for (List<Diagnostic> ld: indexer.parseErrors.values()) {
//            for (Diagnostic d: ld) {
//                processDiagnostic(d);
//            }
//        }
    }


    int nXref = 0;
    Progress progressXRef = new Progress(10000, 50);

    private void processDef(@Nullable Def def, @NotNull Binding binding) {
        if (def == null || def.isURL() || def.getStart() < 0) {
            return;
        }
        StyleRun style = new StyleRun(StyleRun.Type.ANCHOR, def.getStart(), def.getLength());
        style.message = binding.getQname() + " : " + binding.getType();
        style.url = binding.getQname();
        style.id = "" + Math.abs(def.hashCode());

        Set<Ref> refs = binding.getRefs();
        style.highlight = new ArrayList<>();


        for (Ref r : refs) {
            progressXRef.tick();
            nXref++;
            style.highlight.add(Integer.toString(Math.abs(r.hashCode())));
        }
        addFileStyle(def.getFile(), style);
    }


    int nRef = 0;
    Progress progressRef = new Progress(10000, 50);

    void processRef(@NotNull Ref ref, @NotNull List<Binding> bindings) {
        StyleRun link = new StyleRun(StyleRun.Type.LINK, ref.start(), ref.length());
        link.id = Integer.toString(Math.abs(ref.hashCode()));

        List<String> typings = new ArrayList<>();
        for (Binding b : bindings) {
            typings.add(b.getQname() + " : " + b.getType());
        }
        link.message = Util.joinWithSep(typings, " | ", "{", "}");

        link.highlight = new ArrayList<>();

        for (Binding b : bindings) {
            for (Def d : b.getDefs()) {
{'fraction_non_alphanumeric': 0.13947402852283136, 'fraction_numerical': 0.005233547036503991, 'mean_word_length': 2.1379310344827585, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5758679', 'n_tokens_mistral': 2368, 'n_tokens_neox': 2223, 'n_words': 556}