<commit_before># coding: utf-8
import random
from string import ascii_letters, digits


# From http://tools.ietf.org/html/rfc6750#section-2.1
BEARER_TOKEN_CHARSET = ascii_letters + digits + '-._~+/'


def random_hash(length):
  return ''.join(random.sample(BEARER_TOKEN_CHARSET, length))

def random_hash_generator(length):
  return lambda: random_hash(length)
<commit_msg>Add helper for updating URL GET parameters.<commit_after># coding: utf-8
import random
import urlparse
from string import ascii_letters, digits
from urllib2 import urlencode

# From http://tools.ietf.org/html/rfc6750#section-2.1
BEARER_TOKEN_CHARSET = ascii_letters + digits + '-._~+/'


def random_hash(length):
  return ''.join(random.sample(BEARER_TOKEN_CHARSET, length))


def random_hash_generator(length):
  return lambda: random_hash(length)


def update_parameters(url, parameters):
  """ Updates a URL's existing GET parameters.

  @url: a URL string.
  @parameters: a dictionary of parameters, {string:string}.
  """
  parsed_url = urlparse(url)

  query_parameters = urlparse.parse_qsl(parsed_url.query)
  parsed_url.query = urlencode(query_parameters + parameters.items())

  return urlparse.urlunparse(parsed_url)

{'fraction_non_alphanumeric': 0.11166666666666666, 'fraction_numerical': 0.0125, 'mean_word_length': 5.562841530054645, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4582800', 'n_tokens_mistral': 415, 'n_tokens_neox': 397, 'n_words': 90}