<commit_before>/*
 * Copyright 2016 <NAME> (http://frugalmechanic.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package fm.common

import java.nio.charset.StandardCharsets.UTF_8
import org.scalatest.{FunSuite, Matchers}

final class TestBase32 extends FunSuite with Matchers {
  private[this] val data: Vector[(String,String)] = Vector(
    "" -> "",
    "Hello World" -> "jbswy3dpeblw64tmmq======",
    "abcdefghijklmnopqrstuvwxyz" -> "mfrggzdfmztwq2lknnwg23tpobyxe43uov3ho6dzpi======",
    "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" -> "mfrggzdfmztwq2lknnwg23tpobyxe43uov3ho6dzpjqwey3emvtgo2djnjvwy3lon5yhc4ttor2xm53ypf5a====",
    """abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^&*()_+-=><,./';":][}{\|""" -> "mfrggzdfmztwq2lknnwg23tpobyxe43uov3ho6dzpjaueq2eivdeoscjjjfuytkoj5ifcustkrkvmv2ylfndcmrtgq2tmnzyheyccqbdeqsv4jrkfauv6kznhu7dylbof4ttwir2lvnx2624pq======",
    new String((0 to 127).map{ _.toByte }.toArray, UTF_8) -> "aaaqeayeaudaocajbifqydiob4ibceqtcqkrmfyydenbwha5dypsaijcemsckjrhfausukzmfuxc6mbrgiztinjwg44dsor3hq6t4p2aifbegrcfizduqskkjnge2tspkbiveu2ukvlfowczljnvyxk6l5qgcytdmrswmz3infvgw3dnnzxxa4lson2hk5txpb4xu634pv7h6==="
  )
  
  test("Basic Encoding and Decoding") {
    data.foreach{ case (original, encoded) => check(original, encoded) }
  }
  
  private def check(original: String, encoded: String): Unit = {
    val bytes: Array[Byte] = original.getBytes(UTF_8)
    val encodedNoPadding: String = stripPadding(encoded)
    
    Base32.encode(bytes) should equal (encoded)
    Base32.encodeUpper(bytes) should equal (encoded.toUpperCase)
    
    Base32.encodeNoPadding(bytes) should equal (encodedNoPadding)
    Base32.encodeUpperNoPadding(bytes) should equal (encodedNoPadding.toUpperCase)
    
    // def decode(data: String)
    Base32.decode(encoded) should equal (bytes)
    Base32.decode(encoded.toCharArray) should equal (bytes)
    Base32.decode(encoded.toCharArray) should equal (bytes)
    
    Base32.decode(encodedNoPadding) should equal (bytes)
    Base32.decode(encodedNoPadding.toCharArray) should equal (bytes)
    Base32.decode(encodedNoPadding.toCharArray) should equal (bytes)
    
    // def decode(data: Array[Char])
    Base32.decode(encoded.toCharArray) should equal (bytes)
    Base32.decode(encoded.toLowerCase.toCharArray) should equal (bytes)
    Base32.decode(encoded.toUpperCase.toCharArray) should equal (bytes)
    
    Base32.decode(encodedNoPadding.toCharArray) should equal (bytes)
    Base32.decode(encodedNoPadding.toLowerCase.toCharArray) should equal (bytes)
    Base32.decode(encodedNoPadding.toUpperCase.toCharArray) should equal (bytes)
  }
  
  private def stripPadding(s: String): String = s.replaceAll("=", "")
}<commit_msg>Remove a few duplcate lines of code<commit_after>/*
 * Copyright 2016 Frugal Mechanic (http://frugalmechanic.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package fm.common

import java.nio.charset.StandardCharsets.UTF_8
import org.scalatest.{FunSuite, Matchers}

final class TestBase32 extends FunSuite with Matchers {
  private[this] val data: Vector[(String,String)] = Vector(
    "" -> "",
    "Hello World" -> "jbswy3dpeblw64tmmq======",
    "abcdefghijklmnopqrstuvwxyz" -> "mfrggzdfmztwq2lknnwg23tpobyxe43uov3ho6dzpi======",
    "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" -> "mfrggzdfmztwq2lknnwg23tpobyxe43uov3ho6dzpjqwey3emvtgo2djnjvwy3lon5yhc4ttor2xm53ypf5a====",
    """abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^&*()_+-=><,./';":][}{\|""" -> "mfrggzdfmztwq2lknnwg23tpobyxe43uov3ho6dzpjaueq2eivdeoscjjjfuytkoj5ifcustkrkvmv2ylfndcmrtgq2tmnzyheyccqbdeqsv4jrkfauv6kznhu7dylbof4ttwir2lvnx2624pq======",
    new String((0 to 127).map{ _.toByte }.toArray, UTF_8) -> "aaaqeayeaudaocajbifqydiob4ibceqtcqkrmfyydenbwha5dypsaijcemsckjrhfausukzmfuxc6mbrgiztinjwg44dsor3hq6t4p2aifbegrcfizduqskkjnge2tspkbiveu2ukvlfowczljnvyxk6l5qgcytdmrswmz3infvgw3dnnzxxa4lson2hk5txpb4xu634pv7h6==="
  )
  
  test("Basic Encoding and Decoding") {
    data.foreach{ case (original, encoded) => check(original, encoded) }
  }
  
  private def check(original: String, encoded: String): Unit = {
    val bytes: Array[Byte] = original.getBytes(UTF_8)
    val encodedNoPadding: String = stripPadding(encoded)
    
    Base32.encode(bytes) should equal (encoded)
    Base32.encodeUpper(bytes) should equal (encoded.toUpperCase)
    
    Base32.encodeNoPadding(bytes) should equal (encodedNoPadding)
    Base32.encodeUpperNoPadding(bytes) should equal (encodedNoPadding.toUpperCase)
    
    // def decode(data: String)
    Base32.decode(encoded) should equal (bytes)
    Base32.decode(encoded.toCharArray) should equal (bytes)
    Base32.decode(encoded.toCharArray) should equal (bytes)
    
    Base32.decode(encodedNoPadding) should equal (bytes)
    Base32.decode(encodedNoPadding.toCharArray) should equal (bytes)

    // def decode(data: Array[Char])
    Base32.decode(encoded.toCharArray) should equal (bytes)
    Base32.decode(encoded.toLowerCase.toCharArray) should equal (bytes)
    Base32.decode(encoded.toUpperCase.toCharArray) should equal (bytes)

    Base32.decode(encodedNoPadding.toLowerCase.toCharArray) should equal (bytes)
    Base32.decode(encodedNoPadding.toUpperCase.toCharArray) should equal (bytes)
  }
  
  private def stripPadding(s: String): String = s.replaceAll("=", "")
}
{'fraction_non_alphanumeric': 0.10437235543018336, 'fraction_numerical': 0.03980567309199185, 'mean_word_length': 5.318811881188119, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6123063', 'n_tokens_mistral': 2424, 'n_tokens_neox': 2280, 'n_words': 439}