<commit_before>#!/usr/bin/env python

import json
import os
import re
from datetime import datetime


__version__ = "0.1"

# Optional: Add bugzilla-to-github mapping here.
GITHUB_MAPPING = {
	# "email": "github_username"
}

EXPORT_DIRECTORY = "export/"
BUGZILLA_JSON = "bugzilla.json"
DEFAULT_MILESTONE_USER = ""
COMMENT_RE = re.compile(r"comment #(\d+)")
COMMENT_SUB = r"```BZ-IMPORT::comment #\1```"
COMMENT_REPLY_RE = re.compile(r"\(In reply to (.+) from comment #(\d+)\)" + "\n")
BUG_NO_HASH_RE = re.compile(r"bug (\d+)")
BUG_NO_HASH_SUB = r"bug #\1"
OP_VERSION_METADATA = "Version: %(version)s\n\n%(body)s"
VERSION_BLACKLIST = ["unspecified", "master"]
CREATED_ATTACHMENT_RE = re.compile(r"Created attachment (\d+)" + "\n")
CREATED_ATTACHMENT_SUB = r"Created [attachment \1](%s)" + "\n\n"
ATTACHMENT_URL = "http://bugs.example.com/attachment.cgi?id=%(attachment_id)i"
MISSING_MAPPING_DISCLAIMER = "Originally posted by %(user)s:\n\n%(text)s"
USER_DELETE_COMMENTS = "<EMAIL>"
CCS_COMMENT_PLACEHOLDER = "This comment is a placeholder to subscribe all extra CCs to this issue. It should be deleted.\n\nCC: %s"
DISPLAY_USER_EMAILS = False

try:
	from local_settings import *
except ImportError:
	pass


class GithubEncoder(json.JSONEncoder):
	def default(self, o):
		if isinstance(o, datetime):
			return o.isoformat()
		elif hasattr(o, "to_github"):
			return o.to_github()
		raise NotImplementedError(o)


def write_json(path, obj):
	dirname = os.path.dirname(path)
	if not os.path.exists(dirname):
		os.makedirs(dirname)

	with open(path, "w") as f:
		print("Writing %r..." % (path))
		json.dump(obj, f, cls=GithubEncoder)


_USERS = {}
class User(object):
	def __init__(self, email):
		self.email = email
		self.name = _USERS.get(email)

	def __bool__(self):
		return self.email != NOBODY_EMAIL

	def __repr__(self):
		return "<%s - %s>" % (self.email, self.name)

	def __str__(self):
		github = self.github_username()
		if github:
			return "@" + github
		if DISPLAY_USER_EMAILS:
			return self.name or self.email
		return self.name or "(unknown user)"

	def __hash__(self):
		return self.email.__hash__()

	def __eq__(self, other):
		if isinstance(other, self.__class__):
			return self.email == other.email
		return False

	def github_username(self):
		return GITHUB_MAPPING.get(self.email)

	def to_github(self):
		if not self:
			return None
		gh_username = self.github_username()
		if gh_username:
			return {"username": gh_username}
		return {"email": self.email}


_MILESTONES = {}
class Milestone(object):
	@classmethod
	def from_bugzilla_xmlrpc(cls, milestone):
		obj = cls()
		obj.id = milestone["id"]
		obj.is_open = milestone["is_active"]
		obj.title = milestone["name"]
		obj.product = milestone["product"]
		if not DEFAULT_MILESTONE_USER:
			raise RuntimeError("DEFAULT_MILESTONE_USER needs to be set")
		obj.creator = User(DEFAULT_MILESTONE_USER)
		obj.created_at = None # We don't have that.
		obj.due_on = None # We don't have that either.
		obj.description = "" # Nor that.
		return obj

	def to_github(self):
		return {
			"number": self.id,
			"state": "open" if self.is_open else "closed",
			"title": self.title,
			"description": self.description,
			"creator": self.creator,
			"created_at": self.created_at,<commit_msg>Properly serialize users with a github username<commit_after>#!/usr/bin/env python

import json
import os
import re
from datetime import datetime


__version__ = "0.1"

# Optional: Add bugzilla-to-github mapping here.
GITHUB_MAPPING = {
	# "email": "github_username"
}

EXPORT_DIRECTORY = "export/"
BUGZILLA_JSON = "bugzilla.json"
DEFAULT_MILESTONE_USER = ""
COMMENT_RE = re.compile(r"comment #(\d+)")
COMMENT_SUB = r"```BZ-IMPORT::comment #\1```"
COMMENT_REPLY_RE = re.compile(r"\(In reply to (.+) from comment #(\d+)\)" + "\n")
BUG_NO_HASH_RE = re.compile(r"bug (\d+)")
BUG_NO_HASH_SUB = r"bug #\1"
OP_VERSION_METADATA = "Version: %(version)s\n\n%(body)s"
VERSION_BLACKLIST = ["unspecified", "master"]
CREATED_ATTACHMENT_RE = re.compile(r"Created attachment (\d+)" + "\n")
CREATED_ATTACHMENT_SUB = r"Created [attachment \1](%s)" + "\n\n"
ATTACHMENT_URL = "http://bugs.example.com/attachment.cgi?id=%(attachment_id)i"
MISSING_MAPPING_DISCLAIMER = "Originally posted by %(user)s:\n\n%(text)s"
USER_DELETE_COMMENTS = "<EMAIL>"
CCS_COMMENT_PLACEHOLDER = "This comment is a placeholder to subscribe all extra CCs to this issue. It should be deleted.\n\nCC: %s"
DISPLAY_USER_EMAILS = False

try:
	from local_settings import *
except ImportError:
	pass


class GithubEncoder(json.JSONEncoder):
	def default(self, o):
		if isinstance(o, datetime):
			return o.isoformat()
		elif hasattr(o, "to_github"):
			return o.to_github()
		raise NotImplementedError(o)


def write_json(path, obj):
	dirname = os.path.dirname(path)
	if not os.path.exists(dirname):
		os.makedirs(dirname)

	with open(path, "w") as f:
		print("Writing %r..." % (path))
		json.dump(obj, f, cls=GithubEncoder)


_USERS = {}
class User(object):
	def __init__(self, email):
		self.email = email
		self.name = _USERS.get(email)

	def __bool__(self):
		return self.email != NOBODY_EMAIL

	def __repr__(self):
		return "<%s - %s>" % (self.email, self.name)

	def __str__(self):
		github = self.github_username()
		if github:
			return "@" + github
		if DISPLAY_USER_EMAILS:
			return self.name or self.email
		return self.name or "(unknown user)"

	def __hash__(self):
		return self.email.__hash__()

	def __eq__(self, other):
		if isinstance(other, self.__class__):
			return self.email == other.email
		return False

	def github_username(self):
		return GITHUB_MAPPING.get(self.email)

	def to_github(self):
		if not self:
			return None
		gh_username = self.github_username()
		if gh_username:
			return {"login": gh_username}
		return {"email": self.email}


_MILESTONES = {}
class Milestone(object):
	@classmethod
	def from_bugzilla_xmlrpc(cls, milestone):
		obj = cls()
		obj.id = milestone["id"]
		obj.is_open = milestone["is_active"]
		obj.title = milestone["name"]
		obj.product = milestone["product"]
		if not DEFAULT_MILESTONE_USER:
			raise RuntimeError("DEFAULT_MILESTONE_USER needs to be set")
		obj.creator = User(DEFAULT_MILESTONE_USER)
		obj.created_at = None # We don't have that.
		obj.due_on = None # We don't have that either.
		obj.description = "" # Nor that.
		return obj

	def to_github(self):
		return {
			"number": self.id,
			"state": "open" if self.is_open else "closed",
			"title": self.title,
			"description": self.description,
			"creator": self.creator,
			"created_at": self.created_at,
{'fraction_non_alphanumeric': 0.14270402694839993, 'fraction_numerical': 0.001531159087429184, 'mean_word_length': 4.6068669527897, 'pattern_counts': {'":': 18, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1955331', 'n_tokens_mistral': 2593, 'n_tokens_neox': 2371, 'n_words': 563}