<commit_before>package converTor;

import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.TorperfResult;
import converTor.encoders.torperf.Torperf;


class ConvertTorperf extends Convert {

  void convert(Descriptor descriptor) {

    TorperfResult desc = (TorperfResult) descriptor;
    Torperf conversion = new Torperf();

    for (String annotation : desc.getAnnotations()) {
      conversion.setDescriptorType(annotation.substring("@type ".length()));
    }
    conversion.setSource(desc.getSource());
    conversion.setFilesize(desc.getFileSize());
    conversion.setStart(desc.getStartMillis());
    conversion.setSocket(desc.getSocketMillis());
    conversion.setConnect(desc.getConnectMillis());
    conversion.setNegotiate(desc.getNegotiateMillis());
    conversion.setRequest(desc.getRequestMillis());
    conversion.setResponse(desc.getResponseMillis());
    conversion.setDatarequest(desc.getDataRequestMillis());
    conversion.setDataresponse(desc.getDataResponseMillis());
    conversion.setDatacomplete(desc.getDataCompleteMillis());
    conversion.setWritebytes(desc.getWriteBytes());
    conversion.setReadbytes(desc.getReadBytes());
    conversion.setDidtimeout(desc.didTimeout());
    if (desc.getDataPercentiles() != null &&
        !desc.getDataPercentiles().isEmpty()) {
      conversion.setDataperc10(desc.getDataPercentiles().get(10));
      conversion.setDataperc20(desc.getDataPercentiles().get(20));
      conversion.setDataperc30(desc.getDataPercentiles().get(30));
      conversion.setDataperc40(desc.getDataPercentiles().get(40));
      conversion.setDataperc50(desc.getDataPercentiles().get(50));
      conversion.setDataperc60(desc.getDataPercentiles().get(60));
      conversion.setDataperc70(desc.getDataPercentiles().get(70));
      conversion.setDataperc80(desc.getDataPercentiles().get(80));
      conversion.setDataperc90(desc.getDataPercentiles().get(90));
    }
    if (desc.getLaunchMillis() >= 0) {
      conversion.setLaunch(desc.getLaunchMillis());
    }
    if (desc.getUsedAtMillis() >= 0) {
      conversion.setUsedAt(desc.getUsedAtMillis());
    }
    if (desc.getPath() != null && !desc.getPath().isEmpty()) {
      conversion.setPath(desc.getPath());
    }
    conversion.setBuildtimes(desc.getBuildTimes());
    if (desc.getTimeout() >= 0) {
      conversion.setTimeout(desc.getTimeout());
    }
    if (desc.getQuantile() >= 0) {
      conversion.setQuantile(desc.getQuantile());
    }
    if (desc.getCircId() >= 0) {
      conversion.setCircId(desc.getCircId());
    }
    if (desc.getUsedBy() >= 0) {
      conversion.setUsedBy(desc.getUsedBy());
    }

    this.type = Types.TORPERF;
    this.date = dateTimeFormat.format(desc.getStartMillis()).substring(0,7);
    this.load = conversion;

  }

}<commit_msg>added logging of Unrecognized Keys to Torperf converter<commit_after>package converTor;

import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.TorperfResult;
import converTor.encoders.torperf.Torperf;


class ConvertTorperf extends Convert {

  void convert(Descriptor descriptor) {

    TorperfResult desc = (TorperfResult) descriptor;
    Torperf conversion = new Torperf();

    for (String annotation : desc.getAnnotations()) {
      conversion.setDescriptorType(annotation.substring("@type ".length()));
    }
    conversion.setSource(desc.getSource());
    conversion.setFilesize(desc.getFileSize());
    conversion.setStart(desc.getStartMillis());
    conversion.setSocket(desc.getSocketMillis());
    conversion.setConnect(desc.getConnectMillis());
    conversion.setNegotiate(desc.getNegotiateMillis());
    conversion.setRequest(desc.getRequestMillis());
    conversion.setResponse(desc.getResponseMillis());
    conversion.setDatarequest(desc.getDataRequestMillis());
    conversion.setDataresponse(desc.getDataResponseMillis());
    conversion.setDatacomplete(desc.getDataCompleteMillis());
    conversion.setWritebytes(desc.getWriteBytes());
    conversion.setReadbytes(desc.getReadBytes());
    conversion.setDidtimeout(desc.didTimeout());
    if (desc.getDataPercentiles() != null &&
        !desc.getDataPercentiles().isEmpty()) {
      conversion.setDataperc10(desc.getDataPercentiles().get(10));
      conversion.setDataperc20(desc.getDataPercentiles().get(20));
      conversion.setDataperc30(desc.getDataPercentiles().get(30));
      conversion.setDataperc40(desc.getDataPercentiles().get(40));
      conversion.setDataperc50(desc.getDataPercentiles().get(50));
      conversion.setDataperc60(desc.getDataPercentiles().get(60));
      conversion.setDataperc70(desc.getDataPercentiles().get(70));
      conversion.setDataperc80(desc.getDataPercentiles().get(80));
      conversion.setDataperc90(desc.getDataPercentiles().get(90));
    }
    if (desc.getLaunchMillis() >= 0) {
      conversion.setLaunch(desc.getLaunchMillis());
    }
    if (desc.getUsedAtMillis() >= 0) {
      conversion.setUsedAt(desc.getUsedAtMillis());
    }
    if (desc.getPath() != null && !desc.getPath().isEmpty()) {
      conversion.setPath(desc.getPath());
    }
    conversion.setBuildtimes(desc.getBuildTimes());
    if (desc.getTimeout() >= 0) {
      conversion.setTimeout(desc.getTimeout());
    }
    if (desc.getQuantile() >= 0) {
      conversion.setQuantile(desc.getQuantile());
    }
    if (desc.getCircId() >= 0) {
      conversion.setCircId(desc.getCircId());
    }
    if (desc.getUsedBy() >= 0) {
      conversion.setUsedBy(desc.getUsedBy());
    }

    this.type = Types.TORPERF;
    this.date = dateTimeFormat.format(desc.getStartMillis()).substring(0,7);
    this.load = conversion;

    if (Args.INSTANCE.isLog()) {
      Base.logger.info("Unrecognized keys: " + desc.getUnrecognizedKeys());
    } else {
      System.out.println("Unrecognized keys: " + desc.getUnrecognizedKeys());
    }
  }

}
{'fraction_non_alphanumeric': 0.14491758241758243, 'fraction_numerical': 0.01510989010989011, 'mean_word_length': 4.438842203548086, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '226831', 'n_tokens_mistral': 1775, 'n_tokens_neox': 1755, 'n_words': 200}