<commit_before># django-salesforce
#
# by <NAME>
# (c) 2012-2013 Freelancers Union (http://www.freelancersunion.org)
# See LICENSE.md for details
#

"""
Salesforce introspection code.
"""

import logging
import re

from salesforce.backend import driver
from salesforce.fields import SF_PK
import salesforce.fields

from django.conf import settings
from django.db.backends.base.introspection import BaseDatabaseIntrospection

from salesforce.backend import compiler, query

# require "simplejson" to ensure that it is available to "requests" hook.
import simplejson

from collections import OrderedDict
from django.utils.text import camel_case_to_spaces

log = logging.getLogger(__name__)

# these sObjects are not tables - ignore them
# not queryable, not searchable, not retrieveable, only triggerable
PROBLEMATIC_OBJECTS = [
    'AssetTokenEvent',  # new in API 39.9 Spring '17
    'OrgLifecycleNotification',  # new in API 40.0 Summer '17
    'BatchApexErrorEvent',  # new in API 44.0 Winter '18 
]


class DatabaseIntrospection(BaseDatabaseIntrospection):
    """
    The most comfortable and complete output is by:
        table_list_cache:               property with database and tables attributes
        table_description_cache(name):  method   with table and fields attributes
    Everything is with caching the results for the next use.
    Other methods are very customized with hooks for
    by django.core.management.commands.inspectdb
    """
    data_types_reverse = {
        'base64'                        : 'TextField',
        'boolean'                       : 'BooleanField',
        'byte'                          : 'SmallIntegerField',
        'date'                          : 'DateField',
        'datetime'                      : 'DateTimeField',
        'double'                        : 'DecimalField',
        'int'                           : 'IntegerField',
        'string'                        : 'CharField',
        'time'                          : 'TimeField',
        'anyType'                       : 'CharField',
        'calculated'                    : 'CharField',
        'combobox'                      : 'CharField',
        'currency'                      : 'DecimalField',
        'datacategorygroupreference'    : 'CharField',
        'email'                         : 'EmailField',
        'encryptedstring'               : 'CharField',
        'id'                            : 'AutoField',
        'masterrecord'                  : 'CharField',
        # multipicklist can be implemented by a descendant with a special validator + widget
        'multipicklist'                 : 'CharField',
        'percent'                       : 'DecimalField',
        'phone'                         : 'CharField',
        # picklist is ('CharField', {'choices': (...)})
        'picklist'                      : 'CharField',
        'reference'                     : 'ForeignKey',
        'textarea'                      : 'TextField',
        'url'                           : 'URLField',
    }

    def __init__(self, conn):
        BaseDatabaseIntrospection.__init__(self, conn)
        self._table_list_cache = None
        self._table_description_cache = {}
        self._converted_lead_status = None

    @property
    def table_list_cache(self):
        if self._table_list_cache is None:
            url = query.rest_api_url(self.connection.sf_session, 'sobjects')
            log.debug('Request API URL: %s' % url)
            response = driver.handle_api_exceptions(url, self.connection.sf_session.get)
            # charset is detected from headers by requests package
            self._table_list_cache = response.json(object_pairs_hook=OrderedDict)
            self._table_list_cache['sobjects'] = [
                x for x in self._table_list_cache['sobjects']
                if x['name'] not in PROBLEMATIC_OBJECTS
            ]
        return self._table_list_cache

    def table_description_cache(self, table):
        if table not in self._table_description_cache:
            url = query.rest_api_url(self.connection.sf_session, 'sobjects', table, 'describe/')
            log.debug('Request API URL: %s' % url)
            response = driver.handle_api_exceptions(url, self.connection.sf_session.get)
            self._table_description_cache[table] = response.json(object_pairs_hook=OrderedDict)
            assert self._table_description_cache[table]['fields'][0]['type'] == 'id', (
                "Invalid type of the first field in the table '{}'".format(table))
            assert self._table_description_cache[table]['fields'][0]['name'] == 'Id', (
                "Invalid name of the first field in the table '{}'".format(table))
            del self._table_description_cache[table]['fields'][0]
        return self._table_description_cache[table]

    def table_name_converter(self, name):
        return name

    def get_table_list(self, cursor):
        "Returns a list of table names in the current database."
        result = [SfProtectName(x['name'])
                for x in self.table_list_cache['sobjects']]
        return result

    def get_table_description(self, cursor, table_name):
        "Returns a description of the table, with the DB-API cursor.description interface."
        result = []
        for field in self.table_description_cache(table_name)['fields']:
            params = OrderedDict()
            if field['label'] and field['label'] != camel_case_to_spaces(re.sub('__c$', '', field['name'])).title():
                params['verbose_name'] = field['label']
            if not field['updateable'] or not field['createable']:
                # Fields that are result of a formula or system fields modified
                # by triggers or by other apex code
                sf_read_only = (0 if field['updateable'] else 1) | (0 if field['createable'] else 2)
                # use symbolic names NOT_UPDATEABLE, NON_CREATABLE, READ_ONLY instead of 1, 2, 3
                params['sf_read_only'] = reverse_models_names[sf_read_only]
            if field['defaultValue'] is not None:
                params['default'] = field['defaultValue']
            if field['inlineHelpText']:
                params['help_text'] = field['inlineHelpText']
            if field['picklistValues']:
                params['choices'] = [(x['value'], x['label']) for x in field['picklistValues'] if x['active']]
            if field['defaultedOnCreate'] and field['createable']:
                params['default'] = SymbolicModelsName('DEFAULTED_ON_CREATE')
            if field['type'] == 'reference' and not field['referenceTo']:
                params['ref_comment'] = 'No Reference table'
                field['type'] = 'string'
            # We prefer "length" over "byteLength" for "internal_size".
            # (because strings have usually: byteLength == 3 * length)<commit_msg>Fixed inspectdb for Salesforce version 45.0 Spring '19<commit_after># django-salesforce
#
# by <NAME>
# (c) 2012-2013 Freelancers Union (http://www.freelancersunion.org)
# See LICENSE.md for details
#

"""
Salesforce introspection code.
"""

import logging
import re

from salesforce.backend import driver
from salesforce.fields import SF_PK
import salesforce.fields

from django.conf import settings
from django.db.backends.base.introspection import BaseDatabaseIntrospection

from salesforce.backend import compiler, query

# require "simplejson" to ensure that it is available to "requests" hook.
import simplejson

from collections import OrderedDict
from django.utils.text import camel_case_to_spaces

log = logging.getLogger(__name__)

# these sObjects are not tables - ignore them
# not queryable, not searchable, not retrieveable, only triggerable
PROBLEMATIC_OBJECTS = [
    'AssetTokenEvent',  # new in API 39.9 Spring '17
    'OrgLifecycleNotification',  # new in API 40.0 Summer '17
    'BatchApexErrorEvent',  # new in API 44.0 Winter '19
    'PlatformStatusAlertEvent'  # new in API 45.0 Spring '19
]


class DatabaseIntrospection(BaseDatabaseIntrospection):
    """
    The most comfortable and complete output is by:
        table_list_cache:               property with database and tables attributes
        table_description_cache(name):  method   with table and fields attributes
    Everything is with caching the results for the next use.
    Other methods are very customized with hooks for
    by django.core.management.commands.inspectdb
    """
    data_types_reverse = {
        'base64'                        : 'TextField',
        'boolean'                       : 'BooleanField',
        'byte'                          : 'SmallIntegerField',
        'date'                          : 'DateField',
        'datetime'                      : 'DateTimeField',
        'double'                        : 'DecimalField',
        'int'                           : 'IntegerField',
        'string'                        : 'CharField',
        'time'                          : 'TimeField',
        'anyType'                       : 'CharField',
        'calculated'                    : 'CharField',
        'combobox'                      : 'CharField',
        'currency'                      : 'DecimalField',
        'datacategorygroupreference'    : 'CharField',
        'email'                         : 'EmailField',
        'encryptedstring'               : 'CharField',
        'id'                            : 'AutoField',
        'masterrecord'                  : 'CharField',
        # multipicklist can be implemented by a descendant with a special validator + widget
        'multipicklist'                 : 'CharField',
        'percent'                       : 'DecimalField',
        'phone'                         : 'CharField',
        # picklist is ('CharField', {'choices': (...)})
        'picklist'                      : 'CharField',
        'reference'                     : 'ForeignKey',
        'textarea'                      : 'TextField',
        'url'                           : 'URLField',
    }

    def __init__(self, conn):
        BaseDatabaseIntrospection.__init__(self, conn)
        self._table_list_cache = None
        self._table_description_cache = {}
        self._converted_lead_status = None

    @property
    def table_list_cache(self):
        if self._table_list_cache is None:
            url = query.rest_api_url(self.connection.sf_session, 'sobjects')
            log.debug('Request API URL: %s' % url)
            response = driver.handle_api_exceptions(url, self.connection.sf_session.get)
            # charset is detected from headers by requests package
            self._table_list_cache = response.json(object_pairs_hook=OrderedDict)
            self._table_list_cache['sobjects'] = [
                x for x in self._table_list_cache['sobjects']
                if x['name'] not in PROBLEMATIC_OBJECTS
            ]
        return self._table_list_cache

    def table_description_cache(self, table):
        if table not in self._table_description_cache:
            url = query.rest_api_url(self.connection.sf_session, 'sobjects', table, 'describe/')
            log.debug('Request API URL: %s' % url)
            response = driver.handle_api_exceptions(url, self.connection.sf_session.get)
            self._table_description_cache[table] = response.json(object_pairs_hook=OrderedDict)
            assert self._table_description_cache[table]['fields'][0]['type'] == 'id', (
                "Invalid type of the first field in the table '{}'".format(table))
            assert self._table_description_cache[table]['fields'][0]['name'] == 'Id', (
                "Invalid name of the first field in the table '{}'".format(table))
            del self._table_description_cache[table]['fields'][0]
        return self._table_description_cache[table]

    def table_name_converter(self, name):
        return name

    def get_table_list(self, cursor):
        "Returns a list of table names in the current database."
        result = [SfProtectName(x['name'])
                for x in self.table_list_cache['sobjects']]
        return result

    def get_table_description(self, cursor, table_name):
        "Returns a description of the table, with the DB-API cursor.description interface."
        result = []
        for field in self.table_description_cache(table_name)['fields']:
            params = OrderedDict()
            if field['label'] and field['label'] != camel_case_to_spaces(re.sub('__c$', '', field['name'])).title():
                params['verbose_name'] = field['label']
            if not field['updateable'] or not field['createable']:
                # Fields that are result of a formula or system fields modified
                # by triggers or by other apex code
                sf_read_only = (0 if field['updateable'] else 1) | (0 if field['createable'] else 2)
                # use symbolic names NOT_UPDATEABLE, NON_CREATABLE, READ_ONLY instead of 1, 2, 3
                params['sf_read_only'] = reverse_models_names[sf_read_only]
            if field['defaultValue'] is not None:
                params['default'] = field['defaultValue']
            if field['inlineHelpText']:
                params['help_text'] = field['inlineHelpText']
            if field['picklistValues']:
                params['choices'] = [(x['value'], x['label']) for x in field['picklistValues'] if x['active']]
            if field['defaultedOnCreate'] and field['createable']:
                params['default'] = SymbolicModelsName('DEFAULTED_ON_CREATE')
            if field['type'] == 'reference' and not field['referenceTo']:
                params['ref_comment'] = 'No Reference table'
                field['type'] = 'string'
            if field['calculatedFormula']:
                # calculated formula field are without length in Salesforce 45 Spring '19,
                # but Django requires a length, though the field is read only and never written
                field['length'] = 1300
            # We prefer "length" over "byteLength" for "internal_size".
            # (because strings have usually: byteLength == 3 * length)
{'fraction_non_alphanumeric': 0.10067969413763807, 'fraction_numerical': 0.006372132540356839, 'mean_word_length': 2.126383355467021, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '110697', 'n_tokens_mistral': 3801, 'n_tokens_neox': 3635, 'n_words': 994}