<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs.server.namenode;

import com.google.common.net.InetAddresses;
import io.hops.common.INodeUtil;
import io.hops.exception.StorageException;
import io.hops.leader_election.node.ActiveNode;
import io.hops.leader_election.node.SortedActiveNodeList;
import io.hops.merge.HttpConfig2;
import io.hops.metadata.hdfs.entity.INodeIdentifier;
import io.hops.transaction.handler.HDFSOperationType;
import io.hops.transaction.handler.HopsTransactionalRequestHandler;
import io.hops.transaction.lock.LockFactory;
import io.hops.transaction.lock.TransactionLockTypes;
import io.hops.transaction.lock.TransactionLocks;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ServletUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.VersionInfo;
import org.znerd.xmlenc.XMLOutputter;

import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.jsp.JspWriter;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URLEncoder;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import static org.apache.hadoop.hdfs.DFSUtil.percent2String;

class NamenodeJspHelper {
  static String fraction2String(double value) {
    return StringUtils.format("%.2f", value);
  }

  static String fraction2String(long numerator, long denominator) {
    return fraction2String(numerator / (double) denominator);
  }

  static String getSafeModeText(FSNamesystem fsn) throws IOException {
    if (!fsn.isInSafeMode()) {
      return "";
    }
    return "Safe mode is ON. <em>" + fsn.getSafeModeTip() + "</em><br>";
  }

  /**
   * returns security mode of the cluster (namenode)
   *
   * @return "on" if security is on, and "off" otherwise
   */
  static String getSecurityModeText() {
    if (UserGroupInformation.isSecurityEnabled()) {
      return "<div class=\"security\">Security is <em>ON</em></div>";
    } else {
      return "<div class=\"security\">Security is <em>OFF</em></div>";
    }
  }

  static String getInodeLimitText(FSNamesystem fsn) throws IOException {
    long inodes = fsn.dir.totalInodes();
    long blocks = fsn.getBlocksTotal();
    long maxobjects = fsn.getMaxObjects();

    MemoryMXBean mem = ManagementFactory.getMemoryMXBean();
    MemoryUsage heap = mem.getHeapMemoryUsage();
    long totalMemory = heap.getUsed();
    long maxMemory = heap.getMax();
    long commitedMemory = heap.getCommitted();
    
    MemoryUsage nonHeap = mem.getNonHeapMemoryUsage();
    long totalNonHeap = nonHeap.getUsed();
    long maxNonHeap = nonHeap.getMax();
    long commitedNonHeap = nonHeap.getCommitted();

    long used = (totalMemory * 100) / commitedMemory;
    long usedNonHeap = (totalNonHeap * 100) / commitedNonHeap;

    String str =
        "<div>" + inodes + " files and directories, " + blocks + " blocks = " +
            (inodes + blocks) + " total";
    if (maxobjects != 0) {
      long pct = ((inodes + blocks) * 100) / maxobjects;
      str += " / " + maxobjects + " (" + pct + "%)";
    }
    str += ".</div>";
    str +=
        "<div>Heap Memory used " + StringUtils.byteDesc(totalMemory) + " is " +
            " " + used + "% of Commited Heap Memory " +
            StringUtils.byteDesc(commitedMemory) + ". Max Heap Memory is " +
            StringUtils.byteDesc(maxMemory) +
            ". </div>";
    str += "<div>Non Heap Memory used " + StringUtils.byteDesc(totalNonHeap) +
        " is" + " " + usedNonHeap + "% of " + " Commited Non Heap Memory " +
        StringUtils.byteDesc(commitedNonHeap) + ". Max Non Heap Memory is " +
        StringUtils.byteDesc(maxNonHeap) + ".</div>";
    return str;
  }

  /**
   * Return a table containing version information.
   */
  static String getVersionTable(FSNamesystem fsn, NameNode nn) {
    return "<div class='dfstable'><table>" +
        "\n  <tr><td class='col1'>Started:</td><td>" + fsn.getStartTime() +
        "</td></tr>\n" + "\n  <tr><td class='col1'>Version:</td><td>" +
        VersionInfo.getVersion() + ", " + VersionInfo.getRevision() +
        "</td></tr>\n" + "\n  <tr><td class='col1'>Compiled:</td><td>" +
        VersionInfo.getDate() + " by " + VersionInfo.getUser() + " from " +
        VersionInfo.getBranch() +
        "</td></tr>\n  <tr><td class='col1'>Cluster ID:</td><td>" +
        fsn.getClusterId() +
        "</td></tr>\n  <tr><td class='col1'>Block Pool ID:</td><td>" +
        fsn.getBlockPoolId() +
       // "</td></tr>\n  <tr><td class='col1'>Path Ancestor Lock Type:</td><td>" +
       // fsn.getFilePathAncestorLockType() +
        "</td></tr>\n  <tr><td class='col1'>" + nn.getActiveNameNodes().size() +
        " NN(s):</td><td>" +
        getAllActiveNNs(nn.getActiveNameNodes()) +
        "</td></tr>\n</table></div>";
  }
  
  class LexicographicComparator implements Comparator<ActiveNode> {
    @Override<commit_msg>Added Direct Buffer Usage stats and Java Version to HDFS Web Page<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs.server.namenode;

import com.google.common.net.InetAddresses;
import io.hops.common.INodeUtil;
import io.hops.exception.StorageException;
import io.hops.leader_election.node.ActiveNode;
import io.hops.leader_election.node.SortedActiveNodeList;
import io.hops.merge.HttpConfig2;
import io.hops.metadata.hdfs.entity.INodeIdentifier;
import io.hops.transaction.handler.HDFSOperationType;
import io.hops.transaction.handler.HopsTransactionalRequestHandler;
import io.hops.transaction.lock.LockFactory;
import io.hops.transaction.lock.TransactionLockTypes;
import io.hops.transaction.lock.TransactionLocks;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ServletUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.VersionInfo;
import org.znerd.xmlenc.XMLOutputter;

import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.jsp.JspWriter;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URLEncoder;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import static org.apache.hadoop.hdfs.DFSUtil.percent2String;

class NamenodeJspHelper {
  static String fraction2String(double value) {
    return StringUtils.format("%.2f", value);
  }

  static String fraction2String(long numerator, long denominator) {
    return fraction2String(numerator / (double) denominator);
  }

  static String getSafeModeText(FSNamesystem fsn) throws IOException {
    if (!fsn.isInSafeMode()) {
      return "";
    }
    return "Safe mode is ON. <em>" + fsn.getSafeModeTip() + "</em><br>";
  }

  /**
   * returns security mode of the cluster (namenode)
   *
   * @return "on" if security is on, and "off" otherwise
   */
  static String getSecurityModeText() {
    if (UserGroupInformation.isSecurityEnabled()) {
      return "<div class=\"security\">Security is <em>ON</em></div>";
    } else {
      return "<div class=\"security\">Security is <em>OFF</em></div>";
    }
  }

  static String getInodeLimitText(FSNamesystem fsn) throws IOException {
    long inodes = fsn.dir.totalInodes();
    long blocks = fsn.getBlocksTotal();
    long maxobjects = fsn.getMaxObjects();

    MemoryMXBean mem = ManagementFactory.getMemoryMXBean();
    MemoryUsage heap = mem.getHeapMemoryUsage();
    long totalMemory = heap.getUsed();
    long maxMemory = heap.getMax();
    long commitedMemory = heap.getCommitted();
    
    MemoryUsage nonHeap = mem.getNonHeapMemoryUsage();
    long totalNonHeap = nonHeap.getUsed();
    long maxNonHeap = nonHeap.getMax();
    long commitedNonHeap = nonHeap.getCommitted();

    long used = (totalMemory * 100) / commitedMemory;
    long usedNonHeap = (totalNonHeap * 100) / commitedNonHeap;
    long directBuffer = sun.misc.SharedSecrets.getJavaNioAccess().getDirectBufferPool().getMemoryUsed();

    String str =
        "<div>" + inodes + " files and directories, " + blocks + " blocks = " +
            (inodes + blocks) + " total";
    if (maxobjects != 0) {
      long pct = ((inodes + blocks) * 100) / maxobjects;
      str += " / " + maxobjects + " (" + pct + "%)";
    }
    str += ".</div>";
    str +=
        "<div>Heap Memory used " + StringUtils.byteDesc(totalMemory) + " is " +
            " " + used + "% of Commited Heap Memory " +
            StringUtils.byteDesc(commitedMemory) + ". Max Heap Memory is " +
            StringUtils.byteDesc(maxMemory) +
            ". </div>";
    str += "<div>Non Heap Memory used " + StringUtils.byteDesc(totalNonHeap) +
        " is" + " " + usedNonHeap + "% of " + " Commited Non Heap Memory " +
        StringUtils.byteDesc(commitedNonHeap) + ". Max Non Heap Memory is " +
        StringUtils.byteDesc(maxNonHeap) + ".</div>";

    str += "<div>DirectBuffer Memeory used " + StringUtils.byteDesc(directBuffer)+".</div>";
    return str;
  }

  /**
   * Return a table containing version information.
   */
  static String getVersionTable(FSNamesystem fsn, NameNode nn) {
    String javaVersion = System.getProperty("java.version");
    String javaVendor = System.getProperty("java.vendor");
    return "<div class='dfstable'><table>" +
        "\n  <tr><td class='col1'>Started:</td><td>" + fsn.getStartTime() +
        "</td></tr>\n" + "\n  <tr><td class='col1'>Version:</td><td>" +
        VersionInfo.getVersion() + ", " + VersionInfo.getRevision() +
        "</td></tr>\n" + "\n  <tr><td class='col1'>Compiled:</td><td>" +
        VersionInfo.getDate() + " by " + VersionInfo.getUser() + " from " +
        VersionInfo.getBranch() +
        "</td></tr>\n  <tr><td class='col1'>Cluster ID:</td><td>" +
        fsn.getClusterId() +
        "</td></tr>\n  <tr><td class='col1'>Block Pool ID:</td><td>" +
        fsn.getBlockPoolId() +
       // "</td></tr>\n  <tr><td class='col1'>Path Ancestor Lock Type:</td><td>" +
       // fsn.getFilePathAncestorLockType() +
        "</td></tr>\n <tr><td class='col1'>Java Runtime Version :</td><td>" +javaVendor+" "+javaVersion+"</td></tr>"+
        "</td></tr>\n  <tr><td class='col1'>" + nn.getActiveNameNodes().size() +
        " NN(s):</td><td>" +
        getAllActiveNNs(nn.getActiveNameNodes()) +
        "</td></tr>\n"+
        "</table></div>";
  }
  
  class LexicographicComparator implements Comparator<ActiveNode> {
    @Override
{'fraction_non_alphanumeric': 0.13504977686234124, 'fraction_numerical': 0.0037761757638173706, 'mean_word_length': 3.8424202127659575, 'pattern_counts': {'":': 0, '<': 141, '<?xml version=': 0, '>': 141, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1379658', 'n_tokens_mistral': 4469, 'n_tokens_neox': 4446, 'n_words': 999}