<commit_before>import logging
import requests
import sys

from django.conf import settings
#from django.core.files import File
from django.core.files.base import ContentFile
from django.db.models import FileField
from requests.exceptions import ConnectionError as RequestsConnectionError
from csv import QUOTE_NONNUMERIC
from geo_utils.msg_util import msg, msgt

from shared_dataverse_information.worldmap_api_helper.url_helper import\
    UPLOAD_JOIN_DATATABLE_API_PATH

from apps.worldmap_connect.utils import get_latest_jointarget_information
#from apps.gis_tabular.models import SimpleTabularTest   # for testing

# If a column needs formatting
import pandas as pd
import numpy as np

LOGGER = logging.getLogger('apps.worldmap_connect.join_layer_service')


class TableJoinMapMaker(object):
    """
    Use the WorldMap API to uplodat a datatable and join it to an existing layer

    tj_map_maker = TableJoinMapMaker(tabular_info, dataverse_metadata_dict,
                                    chosen_column_name, chosen_layer_name)
    success = tj_map_maker.run_map_create()
    if success:
        map_info_dict = tj_map_maker.get_map_info()
    else:
        err_msg = tj_map_maker.get_error_msg()
    """
    def __init__(self, datatable_obj, dataverse_metadata_dict,\
        table_attribute_for_join, target_layer_id):
        self.datatable_obj = datatable_obj
        self.dataverse_metadata_dict = dataverse_metadata_dict
        self.table_attribute_for_join = table_attribute_for_join
        self.target_layer_id = target_layer_id

        # If a new join column is needed
        self.formatted_file_created = False
        self.zero_pad_length = None

        # For error messages
        self.err_found = False
        self.err_messages = []

        self.rjson_output = None

        self.sanity_check()
        #self.run_map_create()

    def get_map_info(self):

        if self.rjson_output is None:
            return None

        if not 'data' in self.rjson_output:
            return None

        # In the prep process, was a new column/created formatted?
        if self.was_formatted_file_created():
            # Add a "was_formatted_column_created" attribute
            self.rjson_output['data']['was_formatted_column_created'] = True

            # If used, add a zero_pad_length attribute
            if self.zero_pad_length is not None:
                self.rjson_output['data']['zero_pad_length'] = self.zero_pad_length


        return self.rjson_output

    def add_error(self, err_msg):
        """
        Error detected, store a messsage in the class
        """
        if err_msg is None:
            return
        self.err_found = True
        self.err_messages.append(err_msg)

    def get_error_msg(self):
        return '\n'.join(self.err_messages)

    def sanity_check(self):

        if self.datatable_obj is None:
            self.add_error('The Tabular File object was not specified.')

        if self.dataverse_metadata_dict is None:
            self.add_error('The Dataverse metadata was not specified.')

        if self.table_attribute_for_join is None:
            self.add_error('The join column was not specified.')

        if self.target_layer_id is None:
            self.add_error('The target layer was not specified.')

        if not hasattr(self.datatable_obj, 'name'):
            self.add_error('The target layer column was not specified')

        if not hasattr(self.datatable_obj, 'delimiter'):
            self.add_error('The Tabular File object does not have a "delimiter"')


    def is_formatted_column_needed(self, data_frame, zero_pad_length):
        """
        Column needs formatting if:
            (a) It is type numeric
            (b) The values do not meet the minimal length
        """
        assert isinstance(data_frame, pd.DataFrame),\
            "data_frame must be a pandas DataFrame object"

        df = data_frame

        # --------------------------------------------
        # (1) Is this a numeric column?
        # If so, we need a formatted column b/c we're matching against a string.
        # --------------------------------------------
        if np.isreal(df[self.table_attribute_for_join].dtype):
            return True

        # --------------------------------------------
        # (2) Are the values in our column the correct length for a join?
        # --------------------------------------------
        total_rows = len(df)

        # Count how many columns meet the padding length
        #   - convert our temp column to a string
        # --------------------------------------------

        temp_col = self.table_attribute_for_join + '_temp'
        df[temp_col] = df[self.table_attribute_for_join].astype('str')

        #   - Make a filter to count how many values have
        #       the correct length
        # --------------------------------------------
        filter_criteria = (df[temp_col].str.len() == zero_pad_length)
        num_short_rows = len(df.loc[filter_criteria])

        # Drop the temp column
        df = df.drop(temp_col, axis=1)


        #   - Are all the rows the right length?
        # --------------------------------------------
        if num_short_rows == total_rows:    # Yes
            # Looks ok, no formatting needed
            return False

        # We need a formatted column
        return True



    def format_data_table_for_join(self, zero_pad_length):
        """
        Create a new file and add a formatted column that's zero-padded
        """
        if zero_pad_length is None or zero_pad_length < 1:
            self.add_error('Wrong value for zero_pad_length: %s' % zero_pad_length)
            return False

        if not self.datatable_obj.dv_file or not self.datatable_obj.dv_file.path:
            self.add_error("The file could not be found.")
            return False

        # Cleanup: remove any old join files from the TabularInfo object
        # --------------------------------------------
        if self.datatable_obj.dv_join_file:
            self.datatable_obj.dv_join_file.delete()

        # --------------------------------------------
        # (1) Do we need a formatted column?
        # --------------------------------------------
        df_headers = pd.read_csv(self.datatable_obj.dv_file.path,\
                        sep=self.datatable_obj.delimiter,\
                        nrows=1)
        df_colnames = list(df_headers.columns)


        # (1a) Open the dataverse tabular file in pandas
        # --------------------------------------------
        df = pd.read_csv(self.datatable_obj.dv_file.path,\
                        sep=self.datatable_obj.delimiter,\
                        names=df_colnames)

        # (1b) Is the join column in the data frame?
        # --------------------------------------------
        if not self.table_attribute_for_join in df.columns:
            self.add_error('Failed to find column "%s" for formatting.'\
                % self.table_attribute_for_join)
            return False

        # (1c) Do we need a formatted column?
        #   The data may already be formatted
        # --------------------------------------------
        if not self.is_formatted_column_needed(df, zero_pad_length):
            # The existing column may be used for the join
            return True

        # Looks like we need a formatted column

        # ----------------------------------
        # (2) Add formatted column
        # ----------------------------------
        # new column name = existing name + "_formatted"
        new_column_name = '{0}_formatted'.format(\
                            self.table_attribute_for_join)

        # zero pad formatting
        zero_pad_fmt = '{0:0>%s}' % zero_pad_length

        # make the column
        # ----------------------------------<commit_msg>error message for when a tab file has rows with too many columns<commit_after>import logging
import requests
import sys

from django.conf import settings
#from django.core.files import File
from django.core.files.base import ContentFile
from django.db.models import FileField
from requests.exceptions import ConnectionError as RequestsConnectionError
from csv import QUOTE_NONNUMERIC
from geo_utils.msg_util import msg, msgt

from shared_dataverse_information.worldmap_api_helper.url_helper import\
    UPLOAD_JOIN_DATATABLE_API_PATH

from apps.worldmap_connect.utils import get_latest_jointarget_information
#from apps.gis_tabular.models import SimpleTabularTest   # for testing

# If a column needs formatting
import pandas as pd
import numpy as np

LOGGER = logging.getLogger('apps.worldmap_connect.join_layer_service')


class TableJoinMapMaker(object):
    """
    Use the WorldMap API to uplodat a datatable and join it to an existing layer

    tj_map_maker = TableJoinMapMaker(tabular_info, dataverse_metadata_dict,
                                    chosen_column_name, chosen_layer_name)
    success = tj_map_maker.run_map_create()
    if success:
        map_info_dict = tj_map_maker.get_map_info()
    else:
        err_msg = tj_map_maker.get_error_msg()
    """
    def __init__(self, datatable_obj, dataverse_metadata_dict,\
        table_attribute_for_join, target_layer_id):
        self.datatable_obj = datatable_obj
        self.dataverse_metadata_dict = dataverse_metadata_dict
        self.table_attribute_for_join = table_attribute_for_join
        self.target_layer_id = target_layer_id

        # If a new join column is needed
        self.formatted_file_created = False
        self.zero_pad_length = None

        # For error messages
        self.err_found = False
        self.err_messages = []

        self.rjson_output = None

        self.sanity_check()
        #self.run_map_create()

    def get_map_info(self):

        if self.rjson_output is None:
            return None

        if not 'data' in self.rjson_output:
            return None

        # In the prep process, was a new column/created formatted?
        if self.was_formatted_file_created():
            # Add a "was_formatted_column_created" attribute
            self.rjson_output['data']['was_formatted_column_created'] = True

            # If used, add a zero_pad_length attribute
            if self.zero_pad_length is not None:
                self.rjson_output['data']['zero_pad_length'] = self.zero_pad_length


        return self.rjson_output

    def add_error(self, err_msg):
        """
        Error detected, store a messsage in the class
        """
        if err_msg is None:
            return
        self.err_found = True
        self.err_messages.append(err_msg)

    def get_error_msg(self):
        return '\n'.join(self.err_messages)

    def sanity_check(self):

        if self.datatable_obj is None:
            self.add_error('The Tabular File object was not specified.')

        if self.dataverse_metadata_dict is None:
            self.add_error('The Dataverse metadata was not specified.')

        if self.table_attribute_for_join is None:
            self.add_error('The join column was not specified.')

        if self.target_layer_id is None:
            self.add_error('The target layer was not specified.')

        if not hasattr(self.datatable_obj, 'name'):
            self.add_error('The target layer column was not specified')

        if not hasattr(self.datatable_obj, 'delimiter'):
            self.add_error('The Tabular File object does not have a "delimiter"')


    def is_formatted_column_needed(self, data_frame, zero_pad_length):
        """
        Column needs formatting if:
            (a) It is type numeric
            (b) The values do not meet the minimal length
        """
        assert isinstance(data_frame, pd.DataFrame),\
            "data_frame must be a pandas DataFrame object"

        df = data_frame

        # --------------------------------------------
        # (1) Is this a numeric column?
        # If so, we need a formatted column b/c we're matching against a string.
        # --------------------------------------------
        if np.isreal(df[self.table_attribute_for_join].dtype):
            return True

        # --------------------------------------------
        # (2) Are the values in our column the correct length for a join?
        # --------------------------------------------
        total_rows = len(df)

        # Count how many columns meet the padding length
        #   - convert our temp column to a string
        # --------------------------------------------

        temp_col = self.table_attribute_for_join + '_temp'
        df[temp_col] = df[self.table_attribute_for_join].astype('str')

        #   - Make a filter to count how many values have
        #       the correct length
        # --------------------------------------------
        filter_criteria = (df[temp_col].str.len() == zero_pad_length)
        num_short_rows = len(df.loc[filter_criteria])

        # Drop the temp column
        df = df.drop(temp_col, axis=1)


        #   - Are all the rows the right length?
        # --------------------------------------------
        if num_short_rows == total_rows:    # Yes
            # Looks ok, no formatting needed
            return False

        # We need a formatted column
        return True



    def format_data_table_for_join(self, zero_pad_length):
        """
        Create a new file and add a formatted column that's zero-padded
        """
        if zero_pad_length is None or zero_pad_length < 1:
            self.add_error('Wrong value for zero_pad_length: %s' % zero_pad_length)
            return False

        if not self.datatable_obj.dv_file or not self.datatable_obj.dv_file.path:
            self.add_error("The file could not be found.")
            return False

        # Cleanup: remove any old join files from the TabularInfo object
        # --------------------------------------------
        if self.datatable_obj.dv_join_file:
            self.datatable_obj.dv_join_file.delete()

        # --------------------------------------------
        # (1) Do we need a formatted column?
        # --------------------------------------------

        # (1a) Open the dataverse tabular file in pandas
        # --------------------------------------------
        try:
            df = pd.read_csv(self.datatable_obj.dv_file.path,\
                        sep=self.datatable_obj.delimiter,\
                        )
        except pd.parser.CParserError as e:
            err_msg = ('Could not process the file. '
                        'At least one row had too many values. '
                        '(error: %s)' % e.message)
            self.add_error(err_msg)
            return False

        # (1b) Is the join column in the data frame?
        # --------------------------------------------
        if not self.table_attribute_for_join in df.columns:
            self.add_error('Failed to find column "%s" for formatting.'\
                % self.table_attribute_for_join)
            return False

        # (1c) Do we need a formatted column?
        #   The data may already be formatted
        # --------------------------------------------
        if not self.is_formatted_column_needed(df, zero_pad_length):
            # The existing column may be used for the join
            return True

        # Looks like we need a formatted column

        # ----------------------------------
        # (2) Add formatted column
        # ----------------------------------
        # new column name = existing name + "_formatted"
        new_column_name = '{0}_formatted'.format(\
                            self.table_attribute_for_join)

        # zero pad formatting
        zero_pad_fmt = '{0:0>%s}' % zero_pad_length

        # make the column
        # ----------------------------------
{'fraction_non_alphanumeric': 0.14868196472596246, 'fraction_numerical': 0.0015803780264239207, 'mean_word_length': 1.6704929101958137, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3518692', 'n_tokens_mistral': 4247, 'n_tokens_neox': 4092, 'n_words': 1174}