<commit_before>import datetime
import hashlib
import logging
import posixpath
import re
import socket
import sys
import time
import traceback
import xmlrpclib

import requests

from celery.exceptions import SoftTimeLimitExceeded
from celery.task import task

from django.core.files.base import ContentFile
from django.db import transaction
from django.utils.timezone import utc

from packages.models import Package, Release, TroveClassifier
from packages.models import ReleaseFile, ReleaseRequire, ReleaseProvide, ReleaseObsolete, ReleaseURI
from pypi.models import ChangeLog, Log, PackageModified, TaskLog


logger = logging.getLogger(__name__)

# @@@ Do We need to mirror The Server Sig's as well? (Our Simple Pages have different markup)
# @@@ We want to log why a particular package is failing,
#       Should We Present this Information to the Average User? The Package Owner?
# @@@ Can we use the has_sig or the server sig (are they the same?) to verify a file download
#       on top of the md5 hash?

_md5_fragment_re = re.compile(r"#md5=([a-zA-Z0-9]{32})")
_disutils2_version_capture = re.compile("^(.*?)(?:\(([^()]+)\))?$")


def split_meta(meta):
    meta_split = meta.split(";", 1)
    meta_name, meta_version = _disutils2_version_capture.search(meta_split[0].strip()).groups()
    meta_env = meta_split[1].strip() if len(meta_split) == 2 else ""

    return {
        "name": meta_name,
        "version": meta_version if meta_version is not None else "",
        "environment": meta_env,
    }


def get_release_data(data, key, default=None):
    if data.get(key) and data[key] != "UNKNOWN":
        return data[key]
    return "" if default is None else default


class PackageHashMismatch(Exception):
    """
    Raised when the hash on the package didn't match what we expected.
    """


def task_log(task_id, status, name, args, kwargs, exception=None):
    defaults = {
        "status": status,
        "name": name,
        "args": str(args),
        "kwargs": str(kwargs),
    }

    if exception is not None:
        exc = "".join(traceback.format_exception(*exception))
        defaults.update({"exception": exc})
    else:
        exc = None

    tlog, c = TaskLog.objects.get_or_create(task_id=task_id.replace("-", ""), defaults=defaults)

    if not c:
        tlog.status = TaskLog.STATUS.retry
        if exc is not None:
            tlog.exception = exc
        tlog.save()

    return tlog


@task(time_limit=120, soft_time_limit=60)
def synchronize_mirror(index=None, since=None):
    if index is None:
        # Default to PyPI
        index = "http://pypi.python.org/"
    index_url = posixpath.join(index, "pypi")

    try:
        if since is None:
            try:
                # Try to Sync from Our Last Time
                dt = Log.objects.filter(type=Log.TYPES.sync, index=index).order_by("-created")[:1][0].created
                since = int(time.mktime(dt.timetuple())) - 1
            except IndexError:
                # Default to the Unix Epoch
                since = 0

        client = xmlrpclib.ServerProxy(index_url)

        with transaction.commit_on_success():
            Log.objects.create(type=Log.TYPES.sync, index=index, message="Synchronizing with %s" % index)

            names = set()

            if since == 0:  # @@@ Should We Do This If since is too far in the past instead of just 0?
                # This is an initial Synchronize and change log will not work
                for name in client.list_packages():
                    names.add(name)
            else:
                changelog = client.changelog(since)

                for item in changelog:
                    name, version, timestamp, action = item

                    handled = False

                    if action.startswith("new") or action.startswith("add") or action.startswith("create"):
                        names.add(name)
                        handled = True
                    elif action.startswith("update"):
                        # This techincally doesn't need to trigger a download, but it's the best way currently to
                        #   get the updated meta data.
                        names.add(name)
                        handled = True
                    elif action == "remove":
                        if version is None:
                            Package.objects.filter(name=name).delete()
                        else:
                            Release.objects.filter(package__name=name, version=version).delete()
                        handled = True
                    elif action.startswith("remove file"):
                        filename = action[12:]
                        ReleaseFile.objects.filter(release__package__name=name, release__version=version, filename=filename).delete()
                        handled = True
                    elif action == "docupdate":
                        handled = True
                    elif action.startswith("remove Owner"):
                        handled = True
                    else:
                        names.add(name)

                    # @@@ Eventually We Probably Want To Get Rid of This?
                    dt = datetime.datetime.fromtimestamp(timestamp).replace(tzinfo=utc)
                    ChangeLog.objects.create(package=name, version=version, timestamp=dt, action=action, handled=handled)

            for name in names:
                process_package.delay(name, index=index)
    except Exception:
        task_log(synchronize_mirror.request.id, TaskLog.STATUS.failed, synchronize_mirror.name, [], {"index": index}, exception=sys.exc_info())


@task(default_retry_delay=30 * 60)
def process_package(name, index=None):
    if index is None:
        # Default to PyPI
        index = "http://pypi.python.org/"
    index_url = posixpath.join(index, "pypi")

    if not Package.objects.filter(name=name).exists():
        Package.objects.create(name=name)

    try:
        client = xmlrpclib.ServerProxy(index_url)<commit_msg>reraise the exception after logging it<commit_after>import datetime
import hashlib
import logging
import posixpath
import re
import socket
import sys
import time
import traceback
import xmlrpclib

import requests

from celery.exceptions import SoftTimeLimitExceeded
from celery.task import task

from django.core.files.base import ContentFile
from django.db import transaction
from django.utils.timezone import utc

from packages.models import Package, Release, TroveClassifier
from packages.models import ReleaseFile, ReleaseRequire, ReleaseProvide, ReleaseObsolete, ReleaseURI
from pypi.models import ChangeLog, Log, PackageModified, TaskLog


logger = logging.getLogger(__name__)

# @@@ Do We need to mirror The Server Sig's as well? (Our Simple Pages have different markup)
# @@@ We want to log why a particular package is failing,
#       Should We Present this Information to the Average User? The Package Owner?
# @@@ Can we use the has_sig or the server sig (are they the same?) to verify a file download
#       on top of the md5 hash?

_md5_fragment_re = re.compile(r"#md5=([a-zA-Z0-9]{32})")
_disutils2_version_capture = re.compile("^(.*?)(?:\(([^()]+)\))?$")


def split_meta(meta):
    meta_split = meta.split(";", 1)
    meta_name, meta_version = _disutils2_version_capture.search(meta_split[0].strip()).groups()
    meta_env = meta_split[1].strip() if len(meta_split) == 2 else ""

    return {
        "name": meta_name,
        "version": meta_version if meta_version is not None else "",
        "environment": meta_env,
    }


def get_release_data(data, key, default=None):
    if data.get(key) and data[key] != "UNKNOWN":
        return data[key]
    return "" if default is None else default


class PackageHashMismatch(Exception):
    """
    Raised when the hash on the package didn't match what we expected.
    """


def task_log(task_id, status, name, args, kwargs, exception=None):
    defaults = {
        "status": status,
        "name": name,
        "args": str(args),
        "kwargs": str(kwargs),
    }

    if exception is not None:
        exc = "".join(traceback.format_exception(*exception))
        defaults.update({"exception": exc})
    else:
        exc = None

    tlog, c = TaskLog.objects.get_or_create(task_id=task_id.replace("-", ""), defaults=defaults)

    if not c:
        tlog.status = TaskLog.STATUS.retry
        if exc is not None:
            tlog.exception = exc
        tlog.save()

    return tlog


@task(time_limit=120, soft_time_limit=60)
def synchronize_mirror(index=None, since=None):
    if index is None:
        # Default to PyPI
        index = "http://pypi.python.org/"
    index_url = posixpath.join(index, "pypi")

    try:
        if since is None:
            try:
                # Try to Sync from Our Last Time
                dt = Log.objects.filter(type=Log.TYPES.sync, index=index).order_by("-created")[:1][0].created
                since = int(time.mktime(dt.timetuple())) - 1
            except IndexError:
                # Default to the Unix Epoch
                since = 0

        client = xmlrpclib.ServerProxy(index_url)

        with transaction.commit_on_success():
            Log.objects.create(type=Log.TYPES.sync, index=index, message="Synchronizing with %s" % index)

            names = set()

            if since == 0:  # @@@ Should We Do This If since is too far in the past instead of just 0?
                # This is an initial Synchronize and change log will not work
                for name in client.list_packages():
                    names.add(name)
            else:
                changelog = client.changelog(since)

                for item in changelog:
                    name, version, timestamp, action = item

                    handled = False

                    if action.startswith("new") or action.startswith("add") or action.startswith("create"):
                        names.add(name)
                        handled = True
                    elif action.startswith("update"):
                        # This techincally doesn't need to trigger a download, but it's the best way currently to
                        #   get the updated meta data.
                        names.add(name)
                        handled = True
                    elif action == "remove":
                        if version is None:
                            Package.objects.filter(name=name).delete()
                        else:
                            Release.objects.filter(package__name=name, version=version).delete()
                        handled = True
                    elif action.startswith("remove file"):
                        filename = action[12:]
                        ReleaseFile.objects.filter(release__package__name=name, release__version=version, filename=filename).delete()
                        handled = True
                    elif action == "docupdate":
                        handled = True
                    elif action.startswith("remove Owner"):
                        handled = True
                    else:
                        names.add(name)

                    # @@@ Eventually We Probably Want To Get Rid of This?
                    dt = datetime.datetime.fromtimestamp(timestamp).replace(tzinfo=utc)
                    ChangeLog.objects.create(package=name, version=version, timestamp=dt, action=action, handled=handled)

            for name in names:
                process_package.delay(name, index=index)
    except Exception:
        task_log(synchronize_mirror.request.id, TaskLog.STATUS.failed, synchronize_mirror.name, [], {"index": index}, exception=sys.exc_info())
        raise


@task(default_retry_delay=30 * 60)
def process_package(name, index=None):
    if index is None:
        # Default to PyPI
        index = "http://pypi.python.org/"
    index_url = posixpath.join(index, "pypi")

    if not Package.objects.filter(name=name).exists():
        Package.objects.create(name=name)

    try:
        client = xmlrpclib.ServerProxy(index_url)
{'fraction_non_alphanumeric': 0.09834703879059722, 'fraction_numerical': 0.0049838026414154, 'mean_word_length': 2.1248377887360497, 'pattern_counts': {'":': 24, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1985563', 'n_tokens_mistral': 3378, 'n_tokens_neox': 3214, 'n_words': 937}