<commit_before>package org.python.indexer;

import java.io.*;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.util.*;

public class Util {

    public static final Charset UTF_8 = Charset.forName("UTF-8");

    private static int gensymCount = -1;

    public static String gensym(String base) {
        gensymCount++;
        return base + gensymCount;
    }

    public static String getSystemTempDir() {
        String tmp = System.getProperty("java.io.tmpdir");
        String sep = System.getProperty("file.separator");
        if (tmp.endsWith(sep)) {
            return tmp;
        }
        return tmp + sep;
    }

    /**
     * Returns the parent qname of {@code qname} -- everything up to the
     * last dot (exclusive), or if there are no dots, the empty string.
     */
    public static String getQnameParent(String qname) {
        if (qname == null || qname.isEmpty()) {
            return "";
        }
        int index = qname.lastIndexOf(".");
        if (index == -1) {
            return "";
        }
        return qname.substring(0, index);
    }

    /**
     * Determines the fully-qualified module name for the specified file.  A
     * module's qname is a function of the module's absolute path and the sys
     * path; it does not depend on how the module name may have been specified
     * in import statements. The module qname is the relative path of the module
     * under the load path, with slashes converted to dots.
     *
     * @param file absolute canonical path to a file (__init__.py for dirs)
     * @return null if {@code file} is not somewhere under the load path
     */
    public static String moduleQname(String file) {
        boolean initpy = file.endsWith("/__init__.py");
        if (initpy) {
            file = file.substring(0, file.length() - "/__init__.py".length());
        } else if (file.endsWith(".py")) {
            file = file.substring(0, file.length() - ".py".length());
        }
        for (String root : Indexer.idx.getLoadPath()) {
            if (file.startsWith(root)) {
                return file.substring(root.length()).replace('/', '.');
            }
        }
        return null;
    }
<commit_msg>another piece toward fixing Issue #1<commit_after>package org.python.indexer;

import java.io.*;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.util.*;

public class Util {

    public static final Charset UTF_8 = Charset.forName("UTF-8");

    private static int gensymCount = -1;

    public static String gensym(String base) {
        gensymCount++;
        return base + gensymCount;
    }

    public static String getSystemTempDir() {
        String tmp = System.getProperty("java.io.tmpdir");
        String sep = System.getProperty("file.separator");
        if (tmp.endsWith(sep)) {
            return tmp;
        }
        return tmp + sep;
    }

    /**
     * Returns the parent qname of {@code qname} -- everything up to the
     * last dot (exclusive), or if there are no dots, the empty string.
     */
    public static String getQnameParent(String qname) {
        if (qname == null || qname.isEmpty()) {
            return "";
        }
        int index = qname.lastIndexOf(".");
        if (index == -1) {
            return "";
        }
        return qname.substring(0, index);
    }

    /**
     * Determines the fully-qualified module name for the specified file.  A
     * module's qname is a function of the module's absolute path and the sys
     * path; it does not depend on how the module name may have been specified
     * in import statements. The module qname is the relative path of the module
     * under the load path, with slashes converted to dots.
     *
     * @param file absolute canonical path to a file (__init__.py for dirs)
     * @return null if {@code file} is not somewhere under the load path
     */
    public static String moduleQname(String file) {
        File f = new File(file);
        boolean initpy = f.getName().equals("__init__.py");

        if (initpy) {
            file = f.getAbsolutePath();
        } else if (file.endsWith(".py")) {
            file = file.substring(0, file.length() - ".py".length());
        }

        for (String root : Indexer.idx.getLoadPath()) {
            if (file.startsWith(root)) {
                if (file.contains("/")) {
                    return file.substring(root.length()).replace('/', '.');
                } else if (file.contains("\\")) {
                    return file.substring(root.length()).replace('\\', '.');
                }
            }
        }
        return null;
    }

{'fraction_non_alphanumeric': 0.11645896003437903, 'fraction_numerical': 0.0030081650193382035, 'mean_word_length': 2.0827814569536423, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6828334', 'n_tokens_mistral': 1311, 'n_tokens_neox': 1282, 'n_words': 426}