<commit_before>/*                     __                                               *\
**     ________ ___   / /  ___     Scala API                            **
**    / __/ __// _ | / /  / _ |    (c) 2003-2017, LAMP/EPFL             **
**  __\ \/ /__/ __ |/ /__/ __ |    http://scala-lang.org/               **
** /____/\___/_/ |_/____/_/ | |                                         **
**                          |/                                          **
\*                                                                      */

package scala
package xml

import Utility.sbToString
import scala.annotation.tailrec
import scala.collection.AbstractIterable

/**
 * Copyright 2008 Google Inc. All Rights Reserved.
 * @author <NAME> <<EMAIL>>
 */
object MetaData {
  /**
   * appends all attributes from new_tail to attribs, without attempting to
   * detect or remove duplicates. The method guarantees that all attributes
   * from attribs come before the attributes in new_tail, but does not
   * guarantee to preserve the relative order of attribs.
   *
   * Duplicates can be removed with `normalize`.
   */
  @tailrec
  def concatenate(attribs: MetaData, new_tail: MetaData): MetaData =
    if (attribs eq Null) new_tail
    else concatenate(attribs.next, attribs copy new_tail)

  /**
   * returns normalized MetaData, with all duplicates removed and namespace prefixes resolved to
   *  namespace URIs via the given scope.
   */
  def normalize(attribs: MetaData, scope: NamespaceBinding): MetaData = {
    def iterate(md: MetaData, normalized_attribs: MetaData, set: Set[String]): MetaData = {
      def key = getUniversalKey(md, scope)
      if (md eq Null) normalized_attribs
      else if ((md.value eq null) || set(key)) iterate(md.next, normalized_attribs, set)
      else md copy iterate(md.next, normalized_attribs, set + key)
    }
    iterate(attribs, Null, Set())
  }

  /**
   * returns key if md is unprefixed, pre+key is md is prefixed
   */
  def getUniversalKey(attrib: MetaData, scope: NamespaceBinding) = attrib match {
    case prefixed: PrefixedAttribute     => scope.getURI(prefixed.pre) + prefixed.key<commit_msg>Avoid duplicate getUniversalKey invocation<commit_after>/*                     __                                               *\
**     ________ ___   / /  ___     Scala API                            **
**    / __/ __// _ | / /  / _ |    (c) 2003-2017, LAMP/EPFL             **
**  __\ \/ /__/ __ |/ /__/ __ |    http://scala-lang.org/               **
** /____/\___/_/ |_/____/_/ | |                                         **
**                          |/                                          **
\*                                                                      */

package scala
package xml

import Utility.sbToString
import scala.annotation.tailrec
import scala.collection.AbstractIterable

/**
 * Copyright 2008 Google Inc. All Rights Reserved.
 * @author <NAME> <<EMAIL>>
 */
object MetaData {
  /**
   * appends all attributes from new_tail to attribs, without attempting to
   * detect or remove duplicates. The method guarantees that all attributes
   * from attribs come before the attributes in new_tail, but does not
   * guarantee to preserve the relative order of attribs.
   *
   * Duplicates can be removed with `normalize`.
   */
  @tailrec
  def concatenate(attribs: MetaData, new_tail: MetaData): MetaData =
    if (attribs eq Null) new_tail
    else concatenate(attribs.next, attribs copy new_tail)

  /**
   * returns normalized MetaData, with all duplicates removed and namespace prefixes resolved to
   *  namespace URIs via the given scope.
   */
  def normalize(attribs: MetaData, scope: NamespaceBinding): MetaData = {
    def iterate(md: MetaData, normalized_attribs: MetaData, set: Set[String]): MetaData = {
      if (md eq Null) {
        normalized_attribs
      } else if (md.value eq null) {
        iterate(md.next, normalized_attribs, set)
      } else {
        val key = getUniversalKey(md, scope)
        if (set(key)) {
          iterate(md.next, normalized_attribs, set)
        } else {
          md copy iterate(md.next, normalized_attribs, set + key)
        }
      }
    }
    iterate(attribs, Null, Set())
  }

  /**
   * returns key if md is unprefixed, pre+key is md is prefixed
   */
  def getUniversalKey(attrib: MetaData, scope: NamespaceBinding) = attrib match {
    case prefixed: PrefixedAttribute     => scope.getURI(prefixed.pre) + prefixed.key
{'fraction_non_alphanumeric': 0.10013440860215053, 'fraction_numerical': 0.005376344086021506, 'mean_word_length': 1.845761631612492, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6263109', 'n_tokens_mistral': 1306, 'n_tokens_neox': 1231, 'n_words': 355}