<commit_before>package com.yahoo.sketches.frequencies;

import com.yahoo.sketches.memory.Memory;
import com.yahoo.sketches.memory.NativeMemory;

/**
 * Methods of serializing and deserializing arrays of String for use in FrequentItemsSketch
 */
public class ArrayOfStringsSerDe implements ArrayOfItemsSerDe<String> {

  private static final byte TYPE = 2;

  @Override
  public byte[] serializeToByteArray(String[] items) {
    int length = 0;
    byte[][] itemsBytes = new byte[items.length][];
    for (int i = 0; i < items.length; i++) {
      itemsBytes[i] = items[i].getBytes();
      length += itemsBytes[i].length + Integer.BYTES;
    }
    final byte[] bytes = new byte[length];
    final Memory mem = new NativeMemory(bytes);
    long offsetBytes = 0;
    for (int i = 0; i < items.length; i++) {
      mem.putInt(offsetBytes, itemsBytes[i].length);
      offsetBytes += Integer.BYTES;
      mem.putByteArray(offsetBytes, itemsBytes[i], 0, itemsBytes[i].length);
      offsetBytes += itemsBytes[i].length;
    }
    return bytes;
  }

  @Override
  public String[] deserializeFromMemory(Memory mem, int length) {
    final String[] array = new String[length];
    long offsetBytes = 0;
    for (int i = 0; i < length; i++) {
      final int strLength = mem.getInt(offsetBytes);
      offsetBytes += Integer.BYTES;
      final byte[] bytes = new byte[strLength];
      mem.getByteArray(offsetBytes, bytes, 0, strLength);
      offsetBytes += strLength;
      array[i] = new String(bytes);
    }
    return array;
  }<commit_msg>use UTF8 to avoid relying on default platform encoding<commit_after>package com.yahoo.sketches.frequencies;

import java.nio.charset.StandardCharsets;

import com.yahoo.sketches.memory.Memory;
import com.yahoo.sketches.memory.NativeMemory;

/**
 * Methods of serializing and deserializing arrays of String for use in FrequentItemsSketch
 */
public class ArrayOfStringsSerDe implements ArrayOfItemsSerDe<String> {

  private static final byte TYPE = 2;

  @Override
  public byte[] serializeToByteArray(String[] items) {
    int length = 0;
    byte[][] itemsBytes = new byte[items.length][];
    for (int i = 0; i < items.length; i++) {
      itemsBytes[i] = items[i].getBytes(StandardCharsets.UTF_8);
      length += itemsBytes[i].length + Integer.BYTES;
    }
    final byte[] bytes = new byte[length];
    final Memory mem = new NativeMemory(bytes);
    long offsetBytes = 0;
    for (int i = 0; i < items.length; i++) {
      mem.putInt(offsetBytes, itemsBytes[i].length);
      offsetBytes += Integer.BYTES;
      mem.putByteArray(offsetBytes, itemsBytes[i], 0, itemsBytes[i].length);
      offsetBytes += itemsBytes[i].length;
    }
    return bytes;
  }

  @Override
  public String[] deserializeFromMemory(Memory mem, int length) {
    final String[] array = new String[length];
    long offsetBytes = 0;
    for (int i = 0; i < length; i++) {
      final int strLength = mem.getInt(offsetBytes);
      offsetBytes += Integer.BYTES;
      final byte[] bytes = new byte[strLength];
      mem.getByteArray(offsetBytes, bytes, 0, strLength);
      offsetBytes += strLength;
      array[i] = new String(bytes, StandardCharsets.UTF_8);
    }
    return array;
  }
{'fraction_non_alphanumeric': 0.12086196127420362, 'fraction_numerical': 0.00655840099937539, 'mean_word_length': 3.4985955056179776, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5045116', 'n_tokens_mistral': 1012, 'n_tokens_neox': 975, 'n_words': 269}