<commit_before>import os,errno
import ConfigParser
import pymysql.cursors
from dulwich.repo import Repo as git_repo
import click

def load_repo(repo_path):
  """
  Load git repository at the given path, initializing if necessary.

  Inputs:
    * repo_path (str): a path to the output repository
  Output:
    * repo (dulwich.repo.Repo): the git repository
  """
  try:
    os.mkdir(repo_path)
  except OSError as e:
    if e.errno == errno.EEXIST:
      pass
    else:
      raise e
  try:
    repo = git_repo.init(repo_path)
  except OSError as e:
    if e.errno == errno.EEXIST:
      repo = git_repo(repo_path)
    else:
      raise e
  return repo

def load_new_revisions(db_config, web_id, latest_revision_id=0):
  """
  Load all revisions, using the given database configuration to connect.  If a
  `latest_revision_id` is provided, only loads revisions committed after that.
  """
  db_conn = pymysql.connect(host=db_config["host"],
                            user=db_config["user"],
                            password=db_<PASSWORD>["password"],
                            db=db_config["db"],
                            charset=db_config["charset"],
                            cursorclass=pymysql.cursors.DictCursor)
  try:
    with db_conn.cursor() as cursor:
      query = ("SELECT r.id,r.page_id,r.author,r.ip,r.revised_at,r.content,p.name"
        " FROM revisions r INNER JOIN pages p ON (r.page_id=p.id)"
        " WHERE r.web_id=%s AND r.id>%s"
        " ORDER BY r.id") % (web_id, latest_revision_id)
      cursor.execute(query)
      revisions = cursor.fetchall()
  finally:
    db_conn.close()
  return revisions

def commit_revisions_to_repo(repo, revisions, latest_revision_file):
  """
  Commits a set of revisions to the given git repository.

  Inputs:
    * repo (dulwich.repo.Repo): the git repository
    * revisions ([dict]): the revisions, as a list of dictionaries
    * latest_revision_file (file): the file where latest_revision_id is stored
  Output: none
  """
  for rev in revisions:
    with open("%s/%s" % (repo.path, rev["page_id"]), "wb") as f:
      f.write(rev["content"].encode("utf8"))
    repo.stage(["%d" % rev["page_id"]])
    commit_msg = """Revision ID: %s
Revision date: %s
Page name: %s
Author: %s
IP address: %s""" % (rev["id"], rev["revised_at"], rev["name"], rev["author"],
      rev["ip"])
    commit_author = "%s <>" % rev["author"]<commit_msg>put pages into a subdirectory pages/<commit_after>import os,errno
import ConfigParser
import pymysql.cursors
from dulwich.repo import Repo as git_repo
import click

def load_repo(repo_path):
  """
  Load git repository at the given path, initializing if necessary.

  Inputs:
    * repo_path (str): a path to the output repository
  Output:
    * repo (dulwich.repo.Repo): the git repository
  """
  try:
    os.mkdir(repo_path)
  except OSError as e:
    if e.errno == errno.EEXIST:
      pass
    else:
      raise e
  try:
    repo = git_repo.init(repo_path)
  except OSError as e:
    if e.errno == errno.EEXIST:
      repo = git_repo(repo_path)
    else:
      raise e
  return repo

def load_new_revisions(db_config, web_id, latest_revision_id=0):
  """
  Load all revisions, using the given database configuration to connect.  If a
  `latest_revision_id` is provided, only loads revisions committed after that.
  """
  db_conn = pymysql.connect(host=db_config["host"],
                            user=db_config["user"],
                            password=db_config["password"],
                            db=db_config["db"],
                            charset=db_config["charset"],
                            cursorclass=pymysql.cursors.DictCursor)
  try:
    with db_conn.cursor() as cursor:
      query = ("SELECT r.id,r.page_id,r.author,r.ip,r.revised_at,r.content,p.name"
        " FROM revisions r INNER JOIN pages p ON (r.page_id=p.id)"
        " WHERE r.web_id=%s AND r.id>%s"
        " ORDER BY r.id") % (web_id, latest_revision_id)
      cursor.execute(query)
      revisions = cursor.fetchall()
  finally:
    db_conn.close()
  return revisions

def commit_revisions_to_repo(repo, revisions, latest_revision_file):
  """
  Commits a set of revisions to the given git repository.

  Inputs:
    * repo (dulwich.repo.Repo): the git repository
    * revisions ([dict]): the revisions, as a list of dictionaries
    * latest_revision_file (file): the file where latest_revision_id is stored
  Output: none
  """
  for rev in revisions:
    with open("%s/pages/%s" % (repo.path, rev["page_id"]), "wb") as f:
      f.write(rev["content"].encode("utf8"))
    repo.stage(["pages/%d" % rev["page_id"]])
    commit_msg = """Revision ID: %s
Revision date: %s
Page name: %s
Author: %s
IP address: %s""" % (rev["id"], rev["revised_at"], rev["name"], rev["author"],
      rev["ip"])
    commit_author = "%s <>" % rev["author"]
{'fraction_non_alphanumeric': 0.12419921471378384, 'fraction_numerical': 0.0008266170696424882, 'mean_word_length': 2.875100080064051, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '862544', 'n_tokens_mistral': 1654, 'n_tokens_neox': 1583, 'n_words': 428}