<commit_before>/*
 * Copyright 2000-2009 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.intellij.internal;

import com.intellij.codeInsight.lookup.LookupElement;
import com.intellij.codeInsight.lookup.LookupManager;
import com.intellij.codeInsight.lookup.impl.LookupImpl;
import com.intellij.codeInsight.completion.CompletionLookupArranger;
import com.intellij.openapi.actionSystem.*;
import com.intellij.openapi.editor.Editor;
import com.intellij.openapi.project.DumbAware;

import javax.swing.*;
import java.util.Arrays;

/**
 * @author peter
 */
public class DumpLookupElementWeights extends AnAction implements DumbAware {

  public void actionPerformed(final AnActionEvent e) {
    final Editor editor = e.getData(PlatformDataKeys.EDITOR);
    dumpLookupElementWeights((LookupImpl)LookupManager.getActiveLookup(editor));
  }

  @Override
  public void update(final AnActionEvent e) {
    final Presentation presentation = e.getPresentation();
    final Editor editor = e.getData(PlatformDataKeys.EDITOR);
    presentation.setEnabled(editor != null && LookupManager.getActiveLookup(editor) != null);
  }

  public static void dumpLookupElementWeights(final LookupImpl lookup) {
    final ListModel model = lookup.getList().getModel();
    final int count = lookup.getPreferredItemsCount();
    for (int i = 0; i < model.getSize(); i++) {
      final LookupElement item = (LookupElement)model.getElementAt(i);
      System.out.println(item.getLookupString() + Arrays.toString(item.getUserData(CompletionLookupArranger.WEIGHT)));
      if (i == count - 1) {
        System.out.println("------------");
      }
    }<commit_msg>more human-readable 'dump lookup item weights' output<commit_after>/*
 * Copyright 2000-2009 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.intellij.internal;

import com.intellij.codeInsight.completion.CompletionLookupArranger;
import com.intellij.codeInsight.lookup.LookupElement;
import com.intellij.codeInsight.lookup.LookupManager;
import com.intellij.codeInsight.lookup.impl.LookupImpl;
import com.intellij.openapi.actionSystem.AnAction;
import com.intellij.openapi.actionSystem.AnActionEvent;
import com.intellij.openapi.actionSystem.PlatformDataKeys;
import com.intellij.openapi.actionSystem.Presentation;
import com.intellij.openapi.editor.Editor;
import com.intellij.openapi.project.DumbAware;
import com.intellij.openapi.util.text.StringUtil;

import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author peter
 */
public class DumpLookupElementWeights extends AnAction implements DumbAware {

  public void actionPerformed(final AnActionEvent e) {
    final Editor editor = e.getData(PlatformDataKeys.EDITOR);
    dumpLookupElementWeights((LookupImpl)LookupManager.getActiveLookup(editor));
  }

  @Override
  public void update(final AnActionEvent e) {
    final Presentation presentation = e.getPresentation();
    final Editor editor = e.getData(PlatformDataKeys.EDITOR);
    presentation.setEnabled(editor != null && LookupManager.getActiveLookup(editor) != null);
  }

  public static void dumpLookupElementWeights(final LookupImpl lookup) {
    final List<LookupElement> items = lookup.getItems();
    final int count = lookup.getPreferredItemsCount();
    final Pattern pattern = Pattern.compile("[\\[ ](([a-zA-Z0-9.@])+)=(([a-zA-Z0-9.@])+)[,\\]]");
    Map<String, String> values = new HashMap<String, String>();
    Set<String> toExclude = new HashSet<String>();
    for (LookupElement item : items) {
      final String weight = Arrays.toString(item.getUserData(CompletionLookupArranger.WEIGHT));
      final Matcher matcher = pattern.matcher(weight);
      int start = 0;
      while (matcher.find(start)) {
        start = matcher.end();
        final String name = matcher.group(1);
        final String value = matcher.group(3);
        values.put(name, values.containsKey(name) && !value.equals(values.get(name)) ? null : value);
      }
    }

    for (String name : values.keySet()) {
      final String value = values.get(name);
      if (value != null) {
        toExclude.add(name + "=" + value);
      }
    }

    for (int i = 0; i < items.size(); i++) {
      LookupElement item = items.get(i);
      String weight = Arrays.toString(item.getUserData(CompletionLookupArranger.WEIGHT));
      for (String s : toExclude) {
        weight = StringUtil.replace(weight, s, "", false);
      }
      System.out.println(item.getLookupString() + weight);
      if (i == count - 1) {
        System.out.println("------------");
      }
    }
{'fraction_non_alphanumeric': 0.11556267806267806, 'fraction_numerical': 0.006232193732193732, 'mean_word_length': 4.006238859180035, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3142634', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1619, 'n_words': 430}