<commit_before> /*
 *
 * Autopsy Forensic Browser
 * 
 * Copyright 2012-2018 Basis Technology Corp.
 * 
 * Copyright 2012 42six Solutions.
 * Contact: aebadirad <at> 42six <dot> com
 * Project Contact/Architect: carrier <at> sleuthkit <dot> org
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.recentactivity;

import org.sleuthkit.autopsy.coreutils.SQLiteDBConnect;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.sql.ResultSet;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.StringTokenizer;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;

/**
 *
 * @author Alex
 */
class Util {

    private static Logger logger = Logger.getLogger(Util.class.getName());

    private Util() {
    }

    public static boolean pathexists(String path) {
        File file = new File(path);
        boolean exists = file.exists();
        return exists;
    }

    public static String utcConvert(String utc) {
        SimpleDateFormat formatter = new SimpleDateFormat("MM-dd-yyyy HH:mm");
        String tempconvert = formatter.format(new Date(Long.parseLong(utc)));
        return tempconvert;
    }

    public static String readFile(String path) throws IOException {
        FileInputStream stream = new FileInputStream(new File(path));
        try {
            FileChannel fc = stream.getChannel();
            MappedByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
            /*
             * Instead of using default, pass in a decoder.
             */
            return Charset.defaultCharset().decode(bb).toString();
        } finally {
            stream.close();
        }
    }

    public static String getBaseDomain(String url) {
        String host = null;
        
        //strip protocol
        String cleanUrl = url.replaceFirst(".*:\\/\\/", "");

        //strip after slashes
        String dirToks[] = cleanUrl.split("\\/");
        if (dirToks.length > 0) {
            host = dirToks[0];
        } else {
            host = cleanUrl;
        }

        //get the domain part from host (last 2)
        StringTokenizer tok = new StringTokenizer(host, ".");
        StringBuilder hostB = new StringBuilder();
        int toks = tok.countTokens();

        for (int count = 0; count < toks; ++count) {
            String part = tok.nextToken();
            int diff = toks - count;
            if (diff < 3) {
                hostB.append(part);
            }
            if (diff == 2) {
                hostB.append(".");
            }
        }

        return hostB.toString();
    }

    public static String extractDomain(String value) {
        if (value == null) {
            return "";

        }
        String result = "";
        // String domainPattern = "(\\w+)\\.(AC|AD|AE|AERO|AF|AG|AI|AL|AM|AN|AO|AQ|AR|ARPA|AS|ASIA|AT|AU|AW|AX|AZ|BA|BB|BD|BE|BF|BG|BH|BI|BIZ|BJ|BM|BN|BO|BR|BS|BT|BV|BW|BY|BZ|CA|CAT|CC|CD|CF|CG|CH|CI|CK|CL|CM|CN|CO|COM|COOP|CR|CU|CV|CW|CX|CY|CZ|DE|DJ|DK|DM|DO|DZ|EC|EDU|EE|EG|ER|ES|ET|EU|FI|FJ|FK|FM|FO|FR|GA|GB|GD|GE|GF|GG|GH|GI|GL|GM|GN|GOV|GP|GQ|GR|GS|GT|GU|GW|GY|HK|HM|HN|HR|HT|HU|ID|IE|IL|IM|IN|INFO|INT|IO|IQ|IR|IS|IT|JE|JM|JO|JOBS|JP|KE|KG|KH|KI|KM|KN|KP|KR|KW|KY|KZ|LA|LB|LC|LI|LK|LR|LS|LT|LU|LV|LY|MA|MC|MD|ME|MG|MH|MIL|MK|ML|MM|MN|MO|MOBI|MP|MQ|MR|MS|MT|MU|MUSEUM|MV|MW|MX|MY|MZ|NA|NAME|NC|NE|NET|NF|NG|NI|NL|NO|NP|NR|NU|NZ|OM|ORG|PA|PE|PF|PG|PH|PK|PL|PM|PN|PR|PRO|PS|PT|PW|PY|QA|RE|RO|RS|RU|RW|SA|SB|SC|SD|SE|SG|SH|SI|SJ|SK|SL|SM|SN|SO|SR|ST|SU|SV|SX|SY|SZ|TC|TD|TEL|TF|TG|TH|TJ|TK|TL|TM|TN|TO|TP|TR|TRAVEL|TT|TV|TW|TZ|UA|UG|UK|US|UY|UZ|VA|VC|VE|VG|VI|VN|VU|WF|WS|XXX|YE|YT|ZA|ZM|ZW(co\\.[a-z].))";
        //  Pattern p = Pattern.compile(domainPattern,Pattern.CASE_INSENSITIVE);
        //  Matcher m = p.matcher(value);
        //  while (m.find()) {
        //  result = value.substring(m.start(0),m.end(0));
        //  }

        try {
            URL url = new URL(value);
            result = url.getHost();
        } catch (MalformedURLException ex) {
            //do not log if not a valid URL, and handle later
            //Logger.getLogger(Util.class.getName()).log(Level.SEVERE, null, ex);
        }

        //was not a valid URL, try a less picky method
        if (result == null || result.trim().isEmpty()) {
            System.out.println("getBaseDomain result: " + getBaseDomain(value));
            return getBaseDomain(value);
        }
        System.out.println("extractDomain result: " + result);
        return result;
    }

    public static String getFileName(String value) {
        String filename = "";<commit_msg>Cleanup<commit_after> /*
 *
 * Autopsy Forensic Browser
 * 
 * Copyright 2012-2018 Basis Technology Corp.
 * 
 * Copyright 2012 42six Solutions.
 * Contact: aebadirad <at> 42six <dot> com
 * Project Contact/Architect: carrier <at> sleuthkit <dot> org
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.recentactivity;

import org.sleuthkit.autopsy.coreutils.SQLiteDBConnect;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.sql.ResultSet;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.StringTokenizer;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;

/**
 *
 * @author Alex
 */
class Util {

    private static Logger logger = Logger.getLogger(Util.class.getName());

    private Util() {
    }

    public static boolean pathexists(String path) {
        File file = new File(path);
        boolean exists = file.exists();
        return exists;
    }

    public static String utcConvert(String utc) {
        SimpleDateFormat formatter = new SimpleDateFormat("MM-dd-yyyy HH:mm");
        String tempconvert = formatter.format(new Date(Long.parseLong(utc)));
        return tempconvert;
    }

    public static String readFile(String path) throws IOException {
        FileInputStream stream = new FileInputStream(new File(path));
        try {
            FileChannel fc = stream.getChannel();
            MappedByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
            /*
             * Instead of using default, pass in a decoder.
             */
            return Charset.defaultCharset().decode(bb).toString();
        } finally {
            stream.close();
        }
    }

    public static String getBaseDomain(String url) {
        String host = null;
        
        //strip protocol
        String cleanUrl = url.replaceFirst(".*:\\/\\/", "");

        //strip after slashes
        String dirToks[] = cleanUrl.split("\\/");
        if (dirToks.length > 0) {
            host = dirToks[0];
        } else {
            host = cleanUrl;
        }

        //get the domain part from host (last 2)
        StringTokenizer tok = new StringTokenizer(host, ".");
        StringBuilder hostB = new StringBuilder();
        int toks = tok.countTokens();

        for (int count = 0; count < toks; ++count) {
            String part = tok.nextToken();
            int diff = toks - count;
            if (diff < 3) {
                hostB.append(part);
            }
            if (diff == 2) {
                hostB.append(".");
            }
        }

        return hostB.toString();
    }

    public static String extractDomain(String value) {
        if (value == null) {
            return "";

        }
        String result = "";
        // String domainPattern = "(\\w+)\\.(AC|AD|AE|AERO|AF|AG|AI|AL|AM|AN|AO|AQ|AR|ARPA|AS|ASIA|AT|AU|AW|AX|AZ|BA|BB|BD|BE|BF|BG|BH|BI|BIZ|BJ|BM|BN|BO|BR|BS|BT|BV|BW|BY|BZ|CA|CAT|CC|CD|CF|CG|CH|CI|CK|CL|CM|CN|CO|COM|COOP|CR|CU|CV|CW|CX|CY|CZ|DE|DJ|DK|DM|DO|DZ|EC|EDU|EE|EG|ER|ES|ET|EU|FI|FJ|FK|FM|FO|FR|GA|GB|GD|GE|GF|GG|GH|GI|GL|GM|GN|GOV|GP|GQ|GR|GS|GT|GU|GW|GY|HK|HM|HN|HR|HT|HU|ID|IE|IL|IM|IN|INFO|INT|IO|IQ|IR|IS|IT|JE|JM|JO|JOBS|JP|KE|KG|KH|KI|KM|KN|KP|KR|KW|KY|KZ|LA|LB|LC|LI|LK|LR|LS|LT|LU|LV|LY|MA|MC|MD|ME|MG|MH|MIL|MK|ML|MM|MN|MO|MOBI|MP|MQ|MR|MS|MT|MU|MUSEUM|MV|MW|MX|MY|MZ|NA|NAME|NC|NE|NET|NF|NG|NI|NL|NO|NP|NR|NU|NZ|OM|ORG|PA|PE|PF|PG|PH|PK|PL|PM|PN|PR|PRO|PS|PT|PW|PY|QA|RE|RO|RS|RU|RW|SA|SB|SC|SD|SE|SG|SH|SI|SJ|SK|SL|SM|SN|SO|SR|ST|SU|SV|SX|SY|SZ|TC|TD|TEL|TF|TG|TH|TJ|TK|TL|TM|TN|TO|TP|TR|TRAVEL|TT|TV|TW|TZ|UA|UG|UK|US|UY|UZ|VA|VC|VE|VG|VI|VN|VU|WF|WS|XXX|YE|YT|ZA|ZM|ZW(co\\.[a-z].))";
        //  Pattern p = Pattern.compile(domainPattern,Pattern.CASE_INSENSITIVE);
        //  Matcher m = p.matcher(value);
        //  while (m.find()) {
        //  result = value.substring(m.start(0),m.end(0));
        //  }

        try {
            URL url = new URL(value);
            result = url.getHost();
        } catch (MalformedURLException ex) {
            //do not log if not a valid URL, and handle later
            //Logger.getLogger(Util.class.getName()).log(Level.SEVERE, null, ex);
        }

        //was not a valid URL, try a less picky method
        if (result == null || result.trim().isEmpty()) {
            return getBaseDomain(value);
        }
        return result;
    }

    public static String getFileName(String value) {
        String filename = "";
{'fraction_non_alphanumeric': 0.14444837340876945, 'fraction_numerical': 0.005127298444130127, 'mean_word_length': 2.976449912126538, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5831370', 'n_tokens_mistral': 4107, 'n_tokens_neox': 3872, 'n_words': 748}