DatasetValidationError on import_data_task in Azure
`DatasetValidationError` on `import_data_task` in Azure

https://app-staging.auger.ai/admin/cluster_tasks/81400 - this happened when I tried to load CSV file through UI, after I retried task it worked

https://app-staging.auger.ai/admin/cluster_tasks/83283 - this happened on review process, after retry it also worked

It happens time to time, Probably it's race conditions

cc @skatedplrn
diff --git a/a2ml/api/azure/dataset.py b/a2ml/api/azure/dataset.py
index 0d0c453..e84908c 100644
--- a/a2ml/api/azure/dataset.py
+++ b/a2ml/api/azure/dataset.py
@@ -7,6 +7,8 @@ from .exceptions import AzureException
 from a2ml.api.utils.decorators import error_handler, authenticated
 from .credentials import Credentials
 from  a2ml.api.utils.s3_fsclient import S3FSClient
+from a2ml.api.utils.dataframe import DataFrame
+
 
 class AzureDataset(object):
 
@@ -59,6 +61,14 @@ class AzureDataset(object):
                 if self.ctx.config.path and not local_path.startswith("/"):
                     local_path = os.path.join(self.ctx.config.path, local_path)
 
+                # Do it on Auger Cloud only
+                if fsclient.is_s3_path(source) and \\
+                    not (source.endswith(".csv") or source.endswith(".parquet")):
+                    self.ctx.log("Convert file %s to parquet format."%source)
+                    df = DataFrame.create_dataframe(local_path)
+                    df.saveToFile(local_path+".parquet")
+                    local_path = local_path+".parquet"
+
                 ds.upload_files(files=[local_path], relative_root=None,
                     target_path=None, overwrite=True, show_progress=True)
                 dataset_name = os.path.basename(local_path)
['a2ml/api/azure/dataset.py']