<commit_before>import requests
import urllib
import json
import smtplib
import os
import datetime
from collections import OrderedDict


class curl(object):
	"""Wrapper for requests.request() that will handle Error trapping and try to give JSON for calling.
	If URL is passed on Instantiation, it will automatically run, else, it will wait for you to set 
	properties, then run it with runQuery() command.  Erors should be trapped and put into "errors" array.
	If JSON is returned, it will be put into "data" as per json.loads

	Attributes:
		method: GET, PUT, POST, PATCH, DELETE methods for HTTP call
		url: URL to send the request
		**kwargs:  any other arguments to send to the request
	"""

	def __init__(self, method='GET', url=None, **kwargs):
		self.method = method
		self.url = url
		self.params = None
		self.data = None
		self.headers = None
		self.cookies = None
		self.files = None
		self.auth = None
		self.timeout = None
		self.allow_redirects = True
		self.proxies = None
		self.hooks = None
		self.stream = None
		self.verify = None
		self.cert = None
		self.json = None
		self.request = None
		self.errors = []
		self.jsonData = {}
		self.args = {}
		self.duration = None
		for key, value in kwargs.items():
			self.args[key] = value
			setattr(self, key, value)

		if self.url is not None:
			self.runQuery()



	def setArg(self, key, value):
		if value is not None:
			self.args[key] = value

	def runQuery(self):
		self.setArg('params', self.params)
		self.setArg('data', self.data)
		self.setArg('headers', self.headers)
		self.setArg('cookies', self.cookies)
		self.setArg('files', self.files)
		self.setArg('auth', self.auth)
		self.setArg('timeout', self.timeout)
		self.setArg('allow_redirects', self.allow_redirects)
		self.setArg('proxies', self.proxies)
		self.setArg('hooks', self.hooks)
		self.setArg('stream', self.stream)
		self.setArg('verify', self.verify)
		self.setArg('cert', self.cert)
		self.setArg('json', self.json)

		self.errors = []
		self.jsonData = {}
		before = datetime.datetime.utcnow()
		try:
			self.request = requests.request(self.method, self.url, **self.args)
		except Exception as e:
			self.errors.append(str(e))
		else:
			if self.request.status_code not in range(200,300):
				self.errors.append(str(self.request.status_code)+" = "+self.request.reason+"\n"+str(self.request.text))
			try:
				self.jsonData = json.loads(self.request.text)
			except Exception as err:
				pass
		after = datetime.datetime.utcnow()
		delta = after - before
		self.duration = delta.total_seconds()



class OVImport(object):
	"""Wrapper for calling FireTrackor Imports.  We have the
	following properties:

	Attributes:
		URL: A string representing the website's main URL for instance "trackor.onevizion.com".
		userName: the username used to login to the system
		password: <PASSWORD>
		impSpecId: the numeric identifier for the Import this file is to be applied to
		action: "INSERT_UPDATE", "INSERT", or "UPDATE"
		comments: Comments to add tot the Import
		incemental: Optional value to pass to incremental import parameter
		file: the path and file name of the file to be imported

		errors: array of any errors encounterd
		request: the requests object of call to the web api
		data: the json data converted to python array
		processId: the system processId returned from the API call
	"""

	def __init__(self, URL=None, userName=None, password=<PASSWORD>, impSpecId=None, file=None, action='INSERT_UPDATE', comments=None, incremental=None):
		self.URL = URL
		self.userName = userName
		self.password = password
		self.impSpecId = impSpecId
		self.file = file
		self.action = action
		self.comments = comments
		self.incremental = incremental
		self.errors = []
		self.request = {}
		self.jsonData = {}
		self.processId = None

		# If all info is filled out, go ahead and run the query.
		if URL != None and userName != None and password != None and impSpecId != None and file != None:
			self.makeCall()

	def makeCall(self):
		self.ImportURL = "https://" + self.URL + "/configimport/SubmitUrlImport.do"
		self.ImportParameters = {'impSpecId': self.impSpecId,'action': self.action}
		if self.comments is not None:
			self.ImportParameters['comments'] = self.comments
		if self.incremental is not None:
			self.ImportParameters['isIncremental'] = self.incremental
		self.ImportFile = {'file': open(self.file,'rb')}
		self.curl = curl('POST',self.ImportURL,files=self.ImportFile,data=self.ImportParameters,auth=(self.userName,self.password))
		if len(self.curl.errors) > 0:
			self.errors.append(self.curl.errors)
		elif "userMessages" in self.jsonData and len(self.jsonData["userMessages"]) > 0:
			self.errors.append(self.jsonData["userMessages"])
		else:
			self.processId = self.curl.jsonData["processId"]
		self.request = self.curl.request
		self.jsonData = self.jsonData




class Trackor(object):
	"""Wrapper for calling the FireTrackor API for Trackors.  You can Delete, Read, Update or Create new
		Trackor instances with the like named methods.

	Attributes:
		trackorType: The name of the TrackorType being changed.
		URL: A string representing the website's main URL for instance "trackor.onevizion.com".
		userName: the username used to login to the system
		password: the <PASSWORD> to the system

		errors: array of any errors encounterd
		OVCall: the requests object of call to the web api
		jsonData: the json data converted to python array
	"""

	def __init__(self, trackorType = "", URL = "", userName="", password=""):
		self.TrackorType = trackorType
		self.URL = URL
		self.userName = userName
		self.password = password
		self.errors = []
		self.jsonData = {}
		self.OVCall = curl()
		self.request = None



	def delete(self,trackorId):
		""" Delete a Trackor instance.  Must pass a trackorId, the unique DB number.
		"""
		FilterSection = "trackor_id=" + str(trackorId)

		URL = "https://%s/api/v3/trackor_types/%s/trackors?%s" % (self.URL, self.TrackorType, FilterSection)
		self.errors = []
		self.jsonData = {}
		self.OVCall = curl('DELETE',URL,auth=(self.userName,self.password))
		if len(self.OVCall.errors) > 0:
			self.errors.append(self.OVCall.errors)
		self.jsonData = self.OVCall.jsonData
		self.request = self.OVCall.request

	def read(self, 
		trackorId=None, 
		filterOptions=None, 
		filters={}, 
		viewOptions=None, 
		fields=[],
		sort={},
		page=None,
		perPage=1000
		):
		""" Retrieve some field data from a set of Trackor instances. List of Trackors must be 
			identified either by trackorId or filterOptions, and data fields to be retieved must be
			identified either by viewOptions or a list of fields.
			
			fields is an array of strings that are teh Configured Field Names.
		"""
		
		ViewSection = ""
		if viewOptions is None:
			ViewSection = 'fields=' + ",".join(fields)
		else:
			ViewSection = 'view=' + URLEncode(viewOptions)

		FilterSection = ""
		if trackorId is None:
			if filterOptions is None:
				for key,value in filters.items():
					FilterSection = FilterSection + key + '=' + URLEncode(str(value)) + '&'
				FilterSection = FilterSection.rstrip('?&')
			else:
				FilterSection = "filter="+URLEncode(filterOptions)
		else: 
			FilterSection = "trackor_id=" + str(trackorId)

		SortSection=""
		for key,value in sort.items():
			SortSection=SortSection+","+key+":"+value
		if len(SortSection)>0:
			SortSection="&sort="+URLEncode(SortSection.lstrip(','))

		PageSection=""
		if page is not None:
			PageSection = "&page="+str(page)+"&per_page="+str(perPage)


		URL = "https://%s/api/v3/trackor_types/%s/trackors?%s&%s%s%s" % (
			self.URL, 
			self.TrackorType, 
			FilterSection, 
			ViewSection,
			SortSection,
			PageSection
			)
		self.OVCall = curl('GET',URL,auth=(self.userName,self.password))
		if len(self.OVCall.errors) > 0:
			self.errors.append(self.OVCall.errors)
		self.jsonData = self.OVCall.jsonData


	def update(self, filters={}, fields={}, parents={}, charset=""):
		""" Update data in a list of fields for a Trackor instance.
			"filters" is a list of ConfigFieldName:value pairs that finds the unique 
				Trackor instance to be updated.  Use "TrackorType.ConfigFieldName" to filter
				with parent fields.
			"fields" is a ConfigFieldName:Value pair for what to update.  The Value can either
				be a string, or a dictionary of key:value pairs for parts fo teh field sto be updated
				such as in and EFile field, one can have {"file_name":"name.txt","data":"Base64Encoded Text"}
			"parents" is a list of TrackorType:Filter pairs.
				"Filter" is a list of ConfigFieldName:value exactly like the about "filters"
		"""

		# First build a JSON package from the fields and parents dictionaries given
		JSONObj = {}

		FieldsSection = {}
		for key, value in fields.items():
			if isinstance(value, dict):
				CompundField = {}
				for skey,svalue in value.items():
					CompundField[skey] = JSONEndValue(svalue)
				FieldsSection[key] = CompoundField
			else:
				FieldsSection[key] = JSONEndValue(value)

		ParentsSection = []
		Parentx={}
		for key, value in parents.items():
			Parentx["trackor_type"] = key
			FilterPart = {}
			for fkey,fvalue in value.items():
				FilterPart[fkey]=JSONEndValue(fvalue)
			Parentx["filter"] = FilterPart
			ParentsSection.append(Parentx)

		if len(FieldsSection) > 0:
			JSONObj["fields"] = FieldsSection
		if len(ParentsSection) > 0:
			JSONObj["parents"] = ParentsSection
		JSON = json.dumps(JSONObj)
		print JSON<commit_msg>CompoundField bug fix<commit_after>import requests
import urllib
import json
import smtplib
import os
import datetime
from collections import OrderedDict


class curl(object):
	"""Wrapper for requests.request() that will handle Error trapping and try to give JSON for calling.
	If URL is passed on Instantiation, it will automatically run, else, it will wait for you to set 
	properties, then run it with runQuery() command.  Erors should be trapped and put into "errors" array.
	If JSON is returned, it will be put into "data" as per json.loads

	Attributes:
		method: GET, PUT, POST, PATCH, DELETE methods for HTTP call
		url: URL to send the request
		**kwargs:  any other arguments to send to the request
	"""

	def __init__(self, method='GET', url=None, **kwargs):
		self.method = method
		self.url = url
		self.params = None
		self.data = None
		self.headers = None
		self.cookies = None
		self.files = None
		self.auth = None
		self.timeout = None
		self.allow_redirects = True
		self.proxies = None
		self.hooks = None
		self.stream = None
		self.verify = None
		self.cert = None
		self.json = None
		self.request = None
		self.errors = []
		self.jsonData = {}
		self.args = {}
		self.duration = None
		for key, value in kwargs.items():
			self.args[key] = value
			setattr(self, key, value)

		if self.url is not None:
			self.runQuery()



	def setArg(self, key, value):
		if value is not None:
			self.args[key] = value

	def runQuery(self):
		self.setArg('params', self.params)
		self.setArg('data', self.data)
		self.setArg('headers', self.headers)
		self.setArg('cookies', self.cookies)
		self.setArg('files', self.files)
		self.setArg('auth', self.auth)
		self.setArg('timeout', self.timeout)
		self.setArg('allow_redirects', self.allow_redirects)
		self.setArg('proxies', self.proxies)
		self.setArg('hooks', self.hooks)
		self.setArg('stream', self.stream)
		self.setArg('verify', self.verify)
		self.setArg('cert', self.cert)
		self.setArg('json', self.json)

		self.errors = []
		self.jsonData = {}
		before = datetime.datetime.utcnow()
		try:
			self.request = requests.request(self.method, self.url, **self.args)
		except Exception as e:
			self.errors.append(str(e))
		else:
			if self.request.status_code not in range(200,300):
				self.errors.append(str(self.request.status_code)+" = "+self.request.reason+"\n"+str(self.request.text))
			try:
				self.jsonData = json.loads(self.request.text)
			except Exception as err:
				pass
		after = datetime.datetime.utcnow()
		delta = after - before
		self.duration = delta.total_seconds()



class OVImport(object):
	"""Wrapper for calling FireTrackor Imports.  We have the
	following properties:

	Attributes:
		URL: A string representing the website's main URL for instance "trackor.onevizion.com".
		userName: the username used to login to the system
		password: the <PASSWORD>
		impSpecId: the numeric identifier for the Import this file is to be applied to
		action: "INSERT_UPDATE", "INSERT", or "UPDATE"
		comments: Comments to add tot the Import
		incemental: Optional value to pass to incremental import parameter
		file: the path and file name of the file to be imported

		errors: array of any errors encounterd
		request: the requests object of call to the web api
		data: the json data converted to python array
		processId: the system processId returned from the API call
	"""

	def __init__(self, URL=None, userName=None, password=None, impSpecId=None, file=None, action='INSERT_UPDATE', comments=None, incremental=None):
		self.URL = URL
		self.userName = userName
		self.password = password
		self.impSpecId = impSpecId
		self.file = file
		self.action = action
		self.comments = comments
		self.incremental = incremental
		self.errors = []
		self.request = {}
		self.jsonData = {}
		self.processId = None

		# If all info is filled out, go ahead and run the query.
		if URL != None and userName != None and password != None and impSpecId != None and file != None:
			self.makeCall()

	def makeCall(self):
		self.ImportURL = "https://" + self.URL + "/configimport/SubmitUrlImport.do"
		self.ImportParameters = {'impSpecId': self.impSpecId,'action': self.action}
		if self.comments is not None:
			self.ImportParameters['comments'] = self.comments
		if self.incremental is not None:
			self.ImportParameters['isIncremental'] = self.incremental
		self.ImportFile = {'file': open(self.file,'rb')}
		self.curl = curl('POST',self.ImportURL,files=self.ImportFile,data=self.ImportParameters,auth=(self.userName,self.password))
		if len(self.curl.errors) > 0:
			self.errors.append(self.curl.errors)
		elif "userMessages" in self.jsonData and len(self.jsonData["userMessages"]) > 0:
			self.errors.append(self.jsonData["userMessages"])
		else:
			self.processId = self.curl.jsonData["processId"]
		self.request = self.curl.request
		self.jsonData = self.jsonData




class Trackor(object):
	"""Wrapper for calling the FireTrackor API for Trackors.  You can Delete, Read, Update or Create new
		Trackor instances with the like named methods.

	Attributes:
		trackorType: The name of the TrackorType being changed.
		URL: A string representing the website's main URL for instance "trackor.onevizion.com".
		userName: the username used to login to the system
		password: the <PASSWORD>

		errors: array of any errors encounterd
		OVCall: the requests object of call to the web api
		jsonData: the json data converted to python array
	"""

	def __init__(self, trackorType = "", URL = "", userName="", password=""):
		self.TrackorType = trackorType
		self.URL = URL
		self.userName = userName
		self.password = password
		self.errors = []
		self.jsonData = {}
		self.OVCall = curl()
		self.request = None



	def delete(self,trackorId):
		""" Delete a Trackor instance.  Must pass a trackorId, the unique DB number.
		"""
		FilterSection = "trackor_id=" + str(trackorId)

		URL = "https://%s/api/v3/trackor_types/%s/trackors?%s" % (self.URL, self.TrackorType, FilterSection)
		self.errors = []
		self.jsonData = {}
		self.OVCall = curl('DELETE',URL,auth=(self.userName,self.password))
		if len(self.OVCall.errors) > 0:
			self.errors.append(self.OVCall.errors)
		self.jsonData = self.OVCall.jsonData
		self.request = self.OVCall.request

	def read(self, 
		trackorId=None, 
		filterOptions=None, 
		filters={}, 
		viewOptions=None, 
		fields=[],
		sort={},
		page=None,
		perPage=1000
		):
		""" Retrieve some field data from a set of Trackor instances. List of Trackors must be 
			identified either by trackorId or filterOptions, and data fields to be retieved must be
			identified either by viewOptions or a list of fields.
			
			fields is an array of strings that are teh Configured Field Names.
		"""
		
		ViewSection = ""
		if viewOptions is None:
			ViewSection = 'fields=' + ",".join(fields)
		else:
			ViewSection = 'view=' + URLEncode(viewOptions)

		FilterSection = ""
		if trackorId is None:
			if filterOptions is None:
				for key,value in filters.items():
					FilterSection = FilterSection + key + '=' + URLEncode(str(value)) + '&'
				FilterSection = FilterSection.rstrip('?&')
			else:
				FilterSection = "filter="+URLEncode(filterOptions)
		else: 
			FilterSection = "trackor_id=" + str(trackorId)

		SortSection=""
		for key,value in sort.items():
			SortSection=SortSection+","+key+":"+value
		if len(SortSection)>0:
			SortSection="&sort="+URLEncode(SortSection.lstrip(','))

		PageSection=""
		if page is not None:
			PageSection = "&page="+str(page)+"&per_page="+str(perPage)


		URL = "https://%s/api/v3/trackor_types/%s/trackors?%s&%s%s%s" % (
			self.URL, 
			self.TrackorType, 
			FilterSection, 
			ViewSection,
			SortSection,
			PageSection
			)
		self.OVCall = curl('GET',URL,auth=(self.userName,self.password))
		if len(self.OVCall.errors) > 0:
			self.errors.append(self.OVCall.errors)
		self.jsonData = self.OVCall.jsonData


	def update(self, filters={}, fields={}, parents={}, charset=""):
		""" Update data in a list of fields for a Trackor instance.
			"filters" is a list of ConfigFieldName:value pairs that finds the unique 
				Trackor instance to be updated.  Use "TrackorType.ConfigFieldName" to filter
				with parent fields.
			"fields" is a ConfigFieldName:Value pair for what to update.  The Value can either
				be a string, or a dictionary of key:value pairs for parts fo teh field sto be updated
				such as in and EFile field, one can have {"file_name":"name.txt","data":"Base64Encoded Text"}
			"parents" is a list of TrackorType:Filter pairs.
				"Filter" is a list of ConfigFieldName:value exactly like the about "filters"
		"""

		# First build a JSON package from the fields and parents dictionaries given
		JSONObj = {}

		FieldsSection = {}
		for key, value in fields.items():
			if isinstance(value, dict):
				CompoundField = {}
				for skey,svalue in value.items():
					CompoundField[skey] = JSONEndValue(svalue)
				FieldsSection[key] = CompoundField
			else:
				FieldsSection[key] = JSONEndValue(value)

		ParentsSection = []
		Parentx={}
		for key, value in parents.items():
			Parentx["trackor_type"] = key
			FilterPart = {}
			for fkey,fvalue in value.items():
				FilterPart[fkey]=JSONEndValue(fvalue)
			Parentx["filter"] = FilterPart
			ParentsSection.append(Parentx)

		if len(FieldsSection) > 0:
			JSONObj["fields"] = FieldsSection
		if len(ParentsSection) > 0:
			JSONObj["parents"] = ParentsSection
		JSON = json.dumps(JSONObj)
		print JSON
{'fraction_non_alphanumeric': 0.11644491750931346, 'fraction_numerical': 0.0022352315061202766, 'mean_word_length': 3.9489070318672637, 'pattern_counts': {'":': 6, '<': 8, '<?xml version=': 0, '>': 22, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '48560', 'n_tokens_mistral': 6719, 'n_tokens_neox': 5928, 'n_words': 1799}