<commit_before>
extern proc memcpy(a:[], b, len);

var barrier : atomic int;

proc string.toBytes() var {
   var b : [1..this.length] uint(8);
   memcpy(b, this, this.length);
   return b;
}

const pairs = "ATCGGCTAUAMKR<KEY>VBHDDHBVNN\n\n".toBytes();
var table : [1..128] uint(8);

proc main() {
  var info : string;
  var temp : string;
  var inFile = openfd(0); // stdin
  const fileLen = inFile.length();
  var data : [1..fileLen] uint(8);
  var r = inFile.reader(kind=ionative, locking=false);

  var numRead = 1;
  const lineSize = 61;
  while numRead <= fileLen && r.readline(data[numRead..fileLen], numRead) do {}

  forall i in 1..pairs.size by 2 {
    table[pairs[i]] = pairs[i+1];      // uppercase
    if pairs[i] != ascii("\n") then
      table[pairs[i] + 32] = pairs[i+1]; // lowercase
  }

  var end = data.size - 1;

  while true {
    var from = end;
    while data[from] != ascii(">") do from -= 1;

    begin process(data, from, end);

    end = from - 2;
    if end <= 0 then break;
  }

  barrier.waitFor(3);

  var f = openfd(1);
  var binout = f.writer(iokind.native, locking=false);
  binout.write(data);
}

proc process(ref data : [],in from : int, end : int) {<commit_msg>Using newest readline<commit_after>
extern proc memcpy(a:[], b, len);

var barrier : atomic int;

proc string.toBytes() var {
   var b : [1..this.length] uint(8);
   memcpy(b, this, this.length);
   return b;
}

const pairs = "ATCGGCTAUAMKRYWWSSYRKMVBHDDHBVNN\n\n".toBytes();
var table : [1..128] uint(8);

proc main() {
  var info : string;
  var temp : string;
  var inFile = openfd(0); // stdin
  const fileLen = inFile.length();
  var data : [1..fileLen] uint(8);
  var r = inFile.reader(kind=ionative, locking=false);

  var numRead = 1;
  const lineSize = 61;
  var idx = 1;
  while numRead <= fileLen && r.readline(data, numRead, idx) do { idx += numRead; }

  forall i in 1..pairs.size by 2 {
    table[pairs[i]] = pairs[i+1];      // uppercase
    if pairs[i] != ascii("\n") then
      table[pairs[i] + 32] = pairs[i+1]; // lowercase
  }

  var end = data.size - 1;

  while true {
    var from = end;
    while data[from] != ascii(">") do from -= 1;

    begin process(data, from, end);

    end = from - 2;
    if end <= 0 then break;
  }

  barrier.waitFor(3);

  var f = openfd(1);
  var binout = f.writer(iokind.native, locking=false);
  binout.write(data);
}

proc process(ref data : [],in from : int, end : int) {
{'fraction_non_alphanumeric': 0.1681087762669963, 'fraction_numerical': 0.021013597033374538, 'mean_word_length': 2.9673202614379086, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7322352', 'n_tokens_mistral': 978, 'n_tokens_neox': 921, 'n_words': 258}