<commit_before>import logging
import pprint
import urllib2
import couchdb
import json
import urlparse
import ConfigParser
import os
from pylons import config
from lr.lib import ModelParser, SpecValidationException,helpers as h
from lr.lib.couch_change_monitor import BaseChangeHandler
from lr.model import ResourceDataModel
from couchdb import ResourceConflict
log = logging.getLogger(__name__)

# this doesn't need to be done... should be handled by pylons.config
# scriptPath = os.path.dirname(os.path.abspath(__file__))
# _PYLONS_CONFIG =  os.path.join(scriptPath, '..', '..', '..', 'development.ini')
# _config = ConfigParser.ConfigParser()
# _config.read(_PYLONS_CONFIG)

_RESOURCE_DISTRIBUTABLE_TYPE = "resource_data_distributable"
_RESOURCE_TYPE = "resource_data"
_DOC_TYPE = "doc_type"
_DOC = "doc"


class IncomingCopyHandler(BaseChangeHandler):
	def __init__(self):
		self._serverUrl = config["couchdb.url"]
		self._targetName = config["couchdb.db.resourcedata"]
		
		s = couchdb.Server(self._serverUrl)
		self._db = s[self._targetName]

	def _canHandle(self, change, database):
		if ((_DOC in change) and (change[_DOC].get(_DOC_TYPE) ==_RESOURCE_DISTRIBUTABLE_TYPE or change[_DOC].get(_DOC_TYPE) == _RESOURCE_TYPE)):
			return True
		return False


	def _handle(self, change, database):

		try:
			log.debug('got here')
			newDoc = change[_DOC]
			newDoc['node_timestamp'] = h.nowToISO8601Zformat()
			if newDoc[_DOC_TYPE] == _RESOURCE_DISTRIBUTABLE_TYPE:
				doc['_id'] = doc['doc_ID']
				doc[_DOC_TYPE] = _RESOURCE_TYPE
			rd = ResourceDataModel(newDoc)
			rd.save()
			del database[newDoc['_id']]
		except SpecValidationException:
			log.error(str(newDoc) + " Fails Validation" )
		except ResourceConflict:
			pass #ignore conflicts
		except Exception as ex:
			log.error(ex)
<commit_msg>fixed type of doc -> newDoc<commit_after>import logging
import pprint
import urllib2
import couchdb
import json
import urlparse
import ConfigParser
import os
from pylons import config
from lr.lib import ModelParser, SpecValidationException,helpers as h
from lr.lib.couch_change_monitor import BaseChangeHandler
from lr.model import ResourceDataModel
from couchdb import ResourceConflict
log = logging.getLogger(__name__)

# this doesn't need to be done... should be handled by pylons.config
# scriptPath = os.path.dirname(os.path.abspath(__file__))
# _PYLONS_CONFIG =  os.path.join(scriptPath, '..', '..', '..', 'development.ini')
# _config = ConfigParser.ConfigParser()
# _config.read(_PYLONS_CONFIG)

_RESOURCE_DISTRIBUTABLE_TYPE = "resource_data_distributable"
_RESOURCE_TYPE = "resource_data"
_DOC_TYPE = "doc_type"
_DOC = "doc"


class IncomingCopyHandler(BaseChangeHandler):
	def __init__(self):
		self._serverUrl = config["couchdb.url"]
		self._targetName = config["couchdb.db.resourcedata"]
		
		s = couchdb.Server(self._serverUrl)
		self._db = s[self._targetName]

	def _canHandle(self, change, database):
		if ((_DOC in change) and (change[_DOC].get(_DOC_TYPE) ==_RESOURCE_DISTRIBUTABLE_TYPE or change[_DOC].get(_DOC_TYPE) == _RESOURCE_TYPE)):
			return True
		return False


	def _handle(self, change, database):

		try:
			newDoc = change[_DOC]
			newDoc['node_timestamp'] = h.nowToISO8601Zformat()
			if newDoc[_DOC_TYPEnewDoc_RESOURCE_DISTRIBUTABLE_TYPE:
				newDoc['_id'] = doc['doc_ID']
				newDoc[_DOC_TYPE] = _RESOURCE_TYPE
			rd = ResourceDataModel(newDoc)
			rd.save()
			del database[newDoc['_id']]
		except SpecValidationException:
			log.error(str(newDoc) + " Fails Validation" )
		except ResourceConflict:
			pass #ignore conflicts
		except Exception as ex:
			log.error(ex)

{'fraction_non_alphanumeric': 0.10827318156579678, 'fraction_numerical': 0.00277623542476402, 'mean_word_length': 5.399644760213143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3432898', 'n_tokens_mistral': 1320, 'n_tokens_neox': 1187, 'n_words': 273}