<commit_before>#!/usr/bin/env python
# coding=utf-8

import string
import re


class PercentCodec:

  # TODO this could be SOOO much faster in C, rewrite

  URI_RESERVED = "!*'();:@&=+$,/?#[]"
  URI_UNRESERVED = string.uppercase + string.lowercase + "-_.~"

  def __init__(self, safe=URI_UNRESERVED, unsafe=URI_RESERVED):

    safeset = set([c for c in safe]) - set([c for c in unsafe])
    self.chrtohex = dict((chr(i), '%%%02x' % i) for i in range(256))
    for c in safeset: self.chrtohex[c] = c
    self.chrtohex['%'] = '%%'

    self.hextochr = dict(('%02x' % i, chr(i)) for i in range(256))
    self.hextochr['%'] = '%'

    self.regex = re.compile('%(%|[a-fA-F0-9]{2})')

  def encode(self, s):
    return ''.join([self.chrtohex[c] for c in s])

  def decode(self, s):
    return re.sub(self.regex, lambda m: self.hextochr[m.group(1)], s)


def test_percent_codec():
  codec = PercentCodec(safe=string.printable, unsafe=" \r\n")
  s = "%abc def\nghi§xyz"
  print s
  e = codec.encode(s)
  print e
  d = codec.decode(e)
  print d


<commit_msg>percent / txt codec: fix capital hex decoding<commit_after>#!/usr/bin/env python
# coding=utf-8

import string
import re


class PercentCodec:

  # TODO this could be SOOO much faster in C, rewrite

  URI_RESERVED = "!*'();:@&=+$,/?#[]"
  URI_UNRESERVED = string.uppercase + string.lowercase + "-_.~"

  def __init__(self, safe=URI_UNRESERVED, unsafe=URI_RESERVED):

    safeset = set([c for c in safe]) - set([c for c in unsafe])
    self.chrtohex = dict((chr(i), '%%%02x' % i) for i in range(256))
    for c in safeset: self.chrtohex[c] = c
    self.chrtohex['%'] = '%%'

    h1 = dict( ('%02x' % i, chr(i)) for i in range(256) )
    h2 = dict( ('%02X' % i, chr(i)) for i in range(256) )
    self.hextochr = dict( h1, **h2 )
    self.hextochr['%'] = '%'

    self.regex = re.compile('%(%|[a-fA-F0-9]{2})')

  def encode(self, s):
    return ''.join([self.chrtohex[c] for c in s])

  def decode(self, s):
    return re.sub(self.regex, lambda m: self.hextochr[m.group(1)], s)


def test_percent_codec():
  codec = PercentCodec(safe=string.printable, unsafe=" \r\n")
  s = "%abc def\nghi§xyz"
  print s
  e = codec.encode(s)
  print e
  d = codec.decode(e)
  print d



{'fraction_non_alphanumeric': 0.1884580703336339, 'fraction_numerical': 0.0175834084761046, 'mean_word_length': 3.394059405940594, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2937385', 'n_tokens_mistral': 954, 'n_tokens_neox': 901, 'n_words': 222}