def add_first_request_timestamps_from_clean_data(self, data=None):
    clean_data = self._get_clean_data_collection()
    project_dict = self._get_clean_data_projection_dict()
    filter_dict = {'correctorStatus': 'done'}
    if (data is not None):
        for col in self._config.service_call_fields:
            data.loc[((data[col] == '-'), col)] = None
        filter_dict['$or'] = data.to_dict(orient='records')
    group_dict = {col: ('$%s' % col) for col in self._config.service_call_fields}
    res = clean_data.aggregate([{'$project': project_dict}, {'$match': filter_dict}, {'$group': {'_id': group_dict, self._config.timestamp_field: {'$min': ('$%s' % self._config.timestamp_field)}}}], allowDiskUse=True, maxTimeMS=14400000)
    results = []
    print((datetime.datetime.now().strftime('%H:%M:%s') + ' add_first_request_timestamps_from_clean_data_start '))
    for item_tmp in res:
        results.append(item_tmp)
    print((datetime.datetime.now().strftime('%H:%M:%s') + ' add_first_request_timestamps_from_clean_data_end '))
    res = deepcopy(results)
    if (len(res) == 0):
        return
    res = self._generate_dataframe(res)
    res = res.sort_values(self._config.timestamp_field, ascending=True).drop_duplicates(self._config.service_call_fields)
    existing_first_timestamps = self.get_first_timestamps_for_service_calls()
    if (len(existing_first_timestamps) > 0):
        res = res.merge(existing_first_timestamps[(self._config.service_call_fields + ['first_request_timestamp'])], on=self._config.service_call_fields, how='left')
        res = res[pd.isnull(res.first_request_timestamp)].drop('first_request_timestamp', axis=1)
    res = res.rename(columns={self._config.timestamp_field: 'first_request_timestamp'})
    res.first_request_timestamp = pd.to_datetime(res.first_request_timestamp, unit='ms')
    res = res.assign(first_incident_timestamp=None)
    res = res.assign(first_model_retrain_timestamp=None)
    res = res.assign(first_model_train_timestamp=None)
    scft = self._get_service_call_first_timestamps_collection()
    if (len(res) > 0):
        scft.insert_many(res.to_dict('records'))
def add_first_request_timestamps_from_clean_data(self, data=None):
    clean_data = self._get_clean_data_collection()
    project_dict = self._get_clean_data_projection_dict()
    filter_dict = {'correctorStatus': 'done'}
    if (data is not None):
        for col in self._config.service_call_fields:
            data.loc[((data[col] == '-'), col)] = None
        filter_dict['$or'] = data.to_dict(orient='records')
    group_dict = {col: ('$%s' % col) for col in self._config.service_call_fields}
    res = clean_data.aggregate([{'$project': project_dict}, {'$match': filter_dict}, {'$group': {'_id': group_dict, self._config.timestamp_field: {'$min': ('$%s' % self._config.timestamp_field)}}}], allowDiskUse=True, maxTimeMS=14400000)
    results = []
    print((datetime.datetime.now().strftime('%H:%M:%s') + ' add_first_request_timestamps_from_clean_data_start '))
    for item_tmp in res:
        results.append(item_tmp)
    print((datetime.datetime.now().strftime('%H:%M:%s') + ' add_first_request_timestamps_from_clean_data_end '))
    res = deepcopy(results)
    if (len(res) == 0):
        return
    res = self._generate_dataframe(res)
    res = res.sort_values(self._config.timestamp_field, ascending=True).drop_duplicates(self._config.service_call_fields)
    existing_first_timestamps = self.get_first_timestamps_for_service_calls()
    if (len(existing_first_timestamps) > 0):
        res = res.merge(existing_first_timestamps[(self._config.service_call_fields + ['first_request_timestamp'])], on=self._config.service_call_fields, how='left')
        res = res[pd.isnull(res.first_request_timestamp)].drop('first_request_timestamp', axis=1)
    res = res.rename(columns={self._config.timestamp_field: 'first_request_timestamp'})
    res.first_request_timestamp = pd.to_datetime(res.first_request_timestamp, unit='ms')
    res = res.assign(first_incident_timestamp=None)
    res = res.assign(first_model_retrain_timestamp=None)
    res = res.assign(first_model_train_timestamp=None)
    scft = self._get_service_call_first_timestamps_collection()
    if (len(res) > 0):
        scft.insert_many(res.to_dict('records'))