<commit_before>#-*- python -*-
import logging
import json, urllib, re
from datetime import datetime, timedelta
from urllib import urlencode
from webob import exc

# Non-stdlib imports
import pkg_resources
from tg import expose, validate, redirect, flash
from tg.decorators import with_trailing_slash, without_trailing_slash
from pylons import g, c, request, response
from formencode import validators
from pymongo.bson import ObjectId

from ming.orm.ormsession import ThreadLocalORMSession
from ming.orm import session, state

# Pyforge-specific imports
from allura import model as M
from allura.lib import helpers as h
from allura.app import Application, SitemapEntry, DefaultAdminController
from allura.lib.search import search_artifact
from allura.lib.decorators import audit, react
from allura.lib.security import require, has_artifact_access
from allura.lib import widgets as w
from allura.lib.widgets import form_fields as ffw
from allura.lib.widgets.subscriptions import SubscribeForm
from allura.controllers import AppDiscussionController, AppDiscussionRestController
from allura.controllers import attachments as ac
from allura.controllers import BaseController

# Local imports
from forgetracker import model as TM
from forgetracker import version

from forgetracker.widgets.ticket_form import TicketForm, TicketCustomField
from forgetracker.widgets.bin_form import BinForm
from forgetracker.widgets.ticket_search import TicketSearchResults, MassEdit, MassEditForm
from forgetracker.widgets.admin_custom_fields import TrackerFieldAdmin, TrackerFieldDisplay

log = logging.getLogger(__name__)


class ImportSupport(object):

    @staticmethod
    def parse_date(date_string):
        return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')

    @staticmethod
    def get_user_id(username):
        u = M.User.by_username(username)
        if u:
            return u._id
        return None

    def custom(self, ticket, field, value):
        field = '_' + field
        if not c.app.has_custom_field(field):
            log.warning('Custom field %s is not defined, defining as string', field)
            c.app.add_custom_field(dict(name=field, label=field[1:].capitalize(), type='string'))
            ThreadLocalORMSession.flush_all()
        if 'custom_fields' not in ticket:
            ticket['custom_fields'] = {}
        ticket['custom_fields'][field] = value

    def make_artifact(self, ticket_dict):
        # (new_field_name, value_convertor(val)) or
        # handler(ticket, field, val)
        FIELD_NAME_MAP = {
          'assigned_to': ('assigned_to_id', self.get_user_id),
          'submitter': ('reported_by_id', self.get_user_id),
          'date': ('created_date', self.parse_date), 
          'date_updated': ('mod_date', self.parse_date), 
          'keywords': ('labels', lambda s: s.split()),
          'resolution': self.custom,
          'milestone': self.custom,
          'priority': self.custom,
          'private': self.custom,
          'version': (None, None),
        }
        remapped = {}
        for f, v in ticket_dict.iteritems():
            transform = FIELD_NAME_MAP.get(f, (f, lambda x:x))
            if callable(transform):
                transform(remapped, f, v)
            else:
                f, conv = transform
                if f:
                    remapped[f] = conv(v)

        log.info('==========Calling constr============')
        ticket = TM.Ticket(
            app_config_id=c.app.config._id,
            custom_fields=dict(),
            ticket_num=c.app.globals.next_ticket_num())
        log.info('Ticket schema: %s', ticket.__mongometa__.schema.fields)
        log.info('Ticket state doc: %s', state(ticket).document)
        log.info('==========Calling update============')
        ticket.update(remapped)
        log.info('==========Setting in_migr============')
        log.info('==========Calling save============')
#        session(ticket).save(ticket)
        return ticket

    def make_comment(self, thread, comment_dict):
        ts = self.parse_date(comment_dict['date'])
        comment = thread.post(text=comment_dict['comment'], timestamp=ts)
        comment.author_id = self.get_user_id(comment_dict['submitter'])

    def collect_users(self, artifacts):
        users = set()
        for a in artifacts:
            users.add(a['submitter'])
            users.add(a['assigned_to'])
            for c in a['comments']:
                users.add(c['submitter'])<commit_msg>[#1029] Handle all unknown fields in import as custom.<commit_after>#-*- python -*-
import logging
import json, urllib, re
from datetime import datetime, timedelta
from urllib import urlencode
from webob import exc

# Non-stdlib imports
import pkg_resources
from tg import expose, validate, redirect, flash
from tg.decorators import with_trailing_slash, without_trailing_slash
from pylons import g, c, request, response
from formencode import validators
from pymongo.bson import ObjectId

from ming.orm.ormsession import ThreadLocalORMSession
from ming.orm import session, state

# Pyforge-specific imports
from allura import model as M
from allura.lib import helpers as h
from allura.app import Application, SitemapEntry, DefaultAdminController
from allura.lib.search import search_artifact
from allura.lib.decorators import audit, react
from allura.lib.security import require, has_artifact_access
from allura.lib import widgets as w
from allura.lib.widgets import form_fields as ffw
from allura.lib.widgets.subscriptions import SubscribeForm
from allura.controllers import AppDiscussionController, AppDiscussionRestController
from allura.controllers import attachments as ac
from allura.controllers import BaseController

# Local imports
from forgetracker import model as TM
from forgetracker import version

from forgetracker.widgets.ticket_form import TicketForm, TicketCustomField
from forgetracker.widgets.bin_form import BinForm
from forgetracker.widgets.ticket_search import TicketSearchResults, MassEdit, MassEditForm
from forgetracker.widgets.admin_custom_fields import TrackerFieldAdmin, TrackerFieldDisplay

log = logging.getLogger(__name__)


class ImportSupport(object):

    def __init__(self):
        # At first the idea to use Ticket introspection comes,
        # but it contains various internal fields, so we'd need
        # to define somethig explicitly anyway.
        #
        # Map JSON interchange format fields to Ticket fields
        # key is JSON's field name, value is:
        #   None - drop
        #   True - map as is
        #   (new_field_name, value_convertor(val)) - use new field name and convert JSON's value
        #   handler(ticket, field, val) - arbitrary transform, expected to modify ticket in-place
        ImportSupport.FIELD_MAP = {
            'assigned_to': ('assigned_to_id', self.get_user_id),
            'class': None,
            'date': ('created_date', self.parse_date), 
            'date_updated': ('mod_date', self.parse_date),
            'description': True,
            'id': None,
            'keywords': ('labels', lambda s: s.split()),
            'status': True,
            'submitter': ('reported_by_id', self.get_user_id),
            'summary': True,
        }


    @staticmethod
    def parse_date(date_string):
        return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')

    @staticmethod
    def get_user_id(username):
        u = M.User.by_username(username)
        if u:
            return u._id
        return None

    def custom(self, ticket, field, value):
        field = '_' + field
        if not c.app.has_custom_field(field):
            log.warning('Custom field %s is not defined, defining as string', field)
            c.app.add_custom_field(dict(name=field, label=field[1:].capitalize(), type='string'))
            ThreadLocalORMSession.flush_all()
        if 'custom_fields' not in ticket:
            ticket['custom_fields'] = {}
        ticket['custom_fields'][field] = value

    def make_artifact(self, ticket_dict):
        remapped = {}
        for f, v in ticket_dict.iteritems():
            if f in self.FIELD_MAP:
                transform = self.FIELD_MAP[f]
                if transform is None:
                    continue
                elif transform is True:
                    remapped[f] = v
                elif callable(transform):
                    transform(remapped, f, v)
                else:
                    new_f, conv = transform
                    remapped[new_f] = conv(v)
            else:
                self.custom(remapped, f, v)

        log.info('==========Calling constr============')
        ticket = TM.Ticket(
            app_config_id=c.app.config._id,
            custom_fields=dict(),
            ticket_num=c.app.globals.next_ticket_num())
        log.info('Ticket schema: %s', ticket.__mongometa__.schema.fields)
        log.info('Ticket state doc: %s', state(ticket).document)
        log.info('==========Calling update============')
        ticket.update(remapped)
        log.info('==========Setting in_migr============')
        log.info('==========Calling save============')
#        session(ticket).save(ticket)
        return ticket

    def make_comment(self, thread, comment_dict):
        ts = self.parse_date(comment_dict['date'])
        comment = thread.post(text=comment_dict['comment'], timestamp=ts)
        comment.author_id = self.get_user_id(comment_dict['submitter'])

    def collect_users(self, artifacts):
        users = set()
        for a in artifacts:
            users.add(a['submitter'])
            users.add(a['assigned_to'])
            for c in a['comments']:
                users.add(c['submitter'])
{'fraction_non_alphanumeric': 0.11083540115798181, 'fraction_numerical': 0.0006203473945409429, 'mean_word_length': 2.935313262815297, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5314279', 'n_tokens_mistral': 2779, 'n_tokens_neox': 2675, 'n_words': 732}