<commit_before>#-*- python -*-
import logging
import json, urllib, re
from datetime import datetime, timedelta
from urllib import urlencode
from webob import exc

# Non-stdlib imports
import pkg_resources
from tg import expose, validate, redirect, flash
from tg.decorators import with_trailing_slash, without_trailing_slash
from pylons import g, c, request, response
from formencode import validators
from pymongo.bson import ObjectId

from ming.orm.ormsession import ThreadLocalORMSession
from ming.orm import session

# Pyforge-specific imports
from allura import model as M
from allura.lib import helpers as h
from allura.app import Application, SitemapEntry, DefaultAdminController
from allura.lib.search import search_artifact
from allura.lib.decorators import audit, react
from allura.lib.security import require, has_artifact_access
from allura.lib import widgets as w
from allura.lib.widgets import form_fields as ffw
from allura.lib.widgets.subscriptions import SubscribeForm
from allura.controllers import AppDiscussionController, AppDiscussionRestController
from allura.controllers import attachments as ac
from allura.controllers import BaseController

# Local imports
from forgetracker import model as TM
from forgetracker import version

from forgetracker.widgets.ticket_form import TicketForm, TicketCustomField
from forgetracker.widgets.bin_form import BinForm
from forgetracker.widgets.ticket_search import TicketSearchResults, MassEdit, MassEditForm
from forgetracker.widgets.admin_custom_fields import TrackerFieldAdmin, TrackerFieldDisplay

log = logging.getLogger(__name__)


class ImportSupport(object):

    def parse_date(self, date_string):
        return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')

    def make_user_placeholder(self, username):
        user = M.User.register(dict(username=username,
                                   display_name=username), False)
        ThreadLocalORMSession.flush_all()
        logging.info('Created user: %s', user._id)
        return user

    def make_artifact(self, ticket_dict):
        FIELD_NAME_MAP = {
          'date': ('created_date', self.parse_date), 
          'date_updated': ('mod_date', self.parse_date), 
          'keywords': ('labels', lambda s: s.split()),
          'version': (None, None),
        }
        remapped = {}
        for f, v in ticket_dict.iteritems():
            f, conv = FIELD_NAME_MAP.get(f, (f, lambda x:x))
            if f:
                remapped[f] = conv(v)

        log.info('==========Calling constr============')
        ticket = TM.Ticket(
            app_config_id=c.app.config._id,
            custom_fields=dict(),
            ticket_num=c.app.globals.next_ticket_num(), mod_date=remapped['mod_date'])
        log.info('==========Calling update============')
        ticket.update(remapped)
        log.info('==========Setting in_migr============')
        log.info('==========Calling save============')
#        session(ticket).save(ticket)
        return ticket

    def make_comment(self, thread, comment_dict):
        ts = self.parse_date(comment_dict['date'])
        comment = thread.post(text=comment_dict['comment'], timestamp=ts)
        user = M.User.by_username(comment_dict['submitter'])
        if not user:
            log.warning('Unknown user %s during comment import', comment_dict['submitter'])
            # XXX: created mostly for debugging
            user = self.make_user_placeholder(comment_dict['submitter'])
            comment.author_id = user._id
        else:
            comment.author_id = user._id

    def collect_users(self, artifacts):
        users = set()
        for a in artifacts:
            users.add(a['submitter'])
            users.add(a['assigned_to'])
            for c in a['comments']:<commit_msg>[#1029] Start handling extra Trac fields as custom in Allura.<commit_after>#-*- python -*-
import logging
import json, urllib, re
from datetime import datetime, timedelta
from urllib import urlencode
from webob import exc

# Non-stdlib imports
import pkg_resources
from tg import expose, validate, redirect, flash
from tg.decorators import with_trailing_slash, without_trailing_slash
from pylons import g, c, request, response
from formencode import validators
from pymongo.bson import ObjectId

from ming.orm.ormsession import ThreadLocalORMSession
from ming.orm import session

# Pyforge-specific imports
from allura import model as M
from allura.lib import helpers as h
from allura.app import Application, SitemapEntry, DefaultAdminController
from allura.lib.search import search_artifact
from allura.lib.decorators import audit, react
from allura.lib.security import require, has_artifact_access
from allura.lib import widgets as w
from allura.lib.widgets import form_fields as ffw
from allura.lib.widgets.subscriptions import SubscribeForm
from allura.controllers import AppDiscussionController, AppDiscussionRestController
from allura.controllers import attachments as ac
from allura.controllers import BaseController

# Local imports
from forgetracker import model as TM
from forgetracker import version

from forgetracker.widgets.ticket_form import TicketForm, TicketCustomField
from forgetracker.widgets.bin_form import BinForm
from forgetracker.widgets.ticket_search import TicketSearchResults, MassEdit, MassEditForm
from forgetracker.widgets.admin_custom_fields import TrackerFieldAdmin, TrackerFieldDisplay

log = logging.getLogger(__name__)


class ImportSupport(object):

    def parse_date(self, date_string):
        return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')

    def custom(self, ticket, field, value):
        if 'custom_fields' not in ticket:
            ticket['custom_fields'] = {}
        ticket['custom_fields']['_' + field] = value

    def make_user_placeholder(self, username):
        user = M.User.register(dict(username=username,
                                   display_name=username), False)
        ThreadLocalORMSession.flush_all()
        logging.info('Created user: %s', user._id)
        return user

    def make_artifact(self, ticket_dict):
        # (new_field_name, value_convertor(val)) or
        # handler(ticket, field, val)
        FIELD_NAME_MAP = {
          'date': ('created_date', self.parse_date), 
          'date_updated': ('mod_date', self.parse_date), 
          'keywords': ('labels', lambda s: s.split()),
          'resolution': self.custom,
          'milestone': self.custom,
          'priority': self.custom,
          'version': (None, None),
        }
        remapped = {}
        for f, v in ticket_dict.iteritems():
            transform = FIELD_NAME_MAP.get(f, (f, lambda x:x))
            if callable(transform):
                transform(remapped, f, v)
            else:
                f, conv = transform
                if f:
                    remapped[f] = conv(v)

        log.info('==========Calling constr============')
        ticket = TM.Ticket(
            app_config_id=c.app.config._id,
            custom_fields=dict(),
            ticket_num=c.app.globals.next_ticket_num())
        log.info('==========Calling update============')
        ticket.update(remapped)
        log.info('==========Setting in_migr============')
        log.info('==========Calling save============')
#        session(ticket).save(ticket)
        return ticket

    def make_comment(self, thread, comment_dict):
        ts = self.parse_date(comment_dict['date'])
        comment = thread.post(text=comment_dict['comment'], timestamp=ts)
        user = M.User.by_username(comment_dict['submitter'])
        if not user:
            log.warning('Unknown user %s during comment import', comment_dict['submitter'])
            # XXX: created mostly for debugging
            user = self.make_user_placeholder(comment_dict['submitter'])
            comment.author_id = user._id
        else:
            comment.author_id = user._id

    def collect_users(self, artifacts):
        users = set()
        for a in artifacts:
            users.add(a['submitter'])
            users.add(a['assigned_to'])
            for c in a['comments']:
{'fraction_non_alphanumeric': 0.11106992834198172, 'fraction_numerical': 0.0004941932295527551, 'mean_word_length': 3.258285113098369, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4543943', 'n_tokens_mistral': 2304, 'n_tokens_neox': 2214, 'n_words': 603}