<commit_before>import time
import pandas as pd
import binascii as ba

from tqdm import tqdm
from warnings import warn

from .columns import Column, normalize_names, normalize_name
from .geojson import load_geojson

from carto.exceptions import CartoException, CartoRateLimitException

# avoid _lock issue: https://github.com/tqdm/tqdm/issues/457
tqdm(disable=True, total=0)  # initialise internal lock


class Dataset(object):
    FAIL = 'fail'
    REPLACE = 'replace'
    APPEND = 'append'

    PRIVATE = 'private'
    PUBLIC = 'public'
    LINK = 'link'

    STATE_LOCAL = 'local'
    STATE_REMOTE = 'remote'

    GEOM_TYPE_POINT = 'point'
    GEOM_TYPE_LINE = 'line'
    GEOM_TYPE_POLYGON = 'polygon'

    DEFAULT_RETRY_TIMES = 3

    def __init__(self, table_name=None, schema=None, query=None, df=None, gdf=None, state=None, context=None):
        from .auth import _default_context
        self.cc = context or _default_context

        self.table_name = normalize_name(table_name)
        self.schema = schema or self._get_schema()
        self.query = query
        self.df = df
        self.gdf = gdf

        if not self._validate_init():
            raise ValueError('Wrong Dataset creation. You should use one of the class methods: '
                             'from_table, from_query, from_dataframe, from_geodataframe, from_geojson')

        self.state = state
        self.normalized_column_names = None

        if self.table_name != table_name:
            warn('Table will be named `{}`'.format(table_name))

    @classmethod
    def from_table(cls, table_name, context=None, schema=None):
        return cls(table_name=table_name, schema=schema, context=context, state=cls.STATE_REMOTE)

    @classmethod
    def from_query(cls, query, context=None):
        return cls(query=query, context=context, state=cls.STATE_REMOTE)

    @classmethod
    def from_dataframe(cls, df):
        dataset = cls(df=df, state=cls.STATE_LOCAL)
        _save_index_as_column(dataset.df)
        return dataset

    @classmethod
    def from_geodataframe(cls, gdf):
        dataset = cls(gdf=gdf, state=cls.STATE_LOCAL)
        _save_index_as_column(dataset.gdf)
        return dataset

    @classmethod
    def from_geojson(cls, geojson):
        return cls(gdf=load_geojson(geojson), state=cls.STATE_LOCAL)

    def upload(self, with_lnglat=None, if_exists=FAIL, table_name=None, schema=None, context=None):
        if table_name:
            self.table_name = normalize_name(table_name)
        if schema:
            self.schema = schema
        if context:
            self.cc = context

        if self.table_name is None or self.cc is None:
            raise ValueError('You should provide a table_name and context to upload data.')

        if self.gdf is None and self.df is None and self.query is None:
            raise ValueError('Nothing to upload.'
                             'We need data in a DataFrame or GeoDataFrame or a query to upload data to CARTO.')

        already_exists_error = CartoException('Table with name {t} and schema {s} already exists in CARTO.'
                                              'Please choose a different `table_name` or use'
                                              'if_exists="replace" to overwrite it'.format(
                                                  t=self.table_name, s=self.schema))

        # priority order: gdf, df, query
        if self.gdf is not None:
            warn('GeoDataFrame option is still under development. We will try the upload with DataFrame')
            # TODO: uncomment when we support GeoDataFrame
            # self.normalized_column_names = _normalize_column_names(self.gdf)
<commit_msg>set context implies changing schema<commit_after>import time
import pandas as pd
import binascii as ba

from tqdm import tqdm
from warnings import warn

from .columns import Column, normalize_names, normalize_name
from .geojson import load_geojson

from carto.exceptions import CartoException, CartoRateLimitException

# avoid _lock issue: https://github.com/tqdm/tqdm/issues/457
tqdm(disable=True, total=0)  # initialise internal lock


class Dataset(object):
    FAIL = 'fail'
    REPLACE = 'replace'
    APPEND = 'append'

    PRIVATE = 'private'
    PUBLIC = 'public'
    LINK = 'link'

    STATE_LOCAL = 'local'
    STATE_REMOTE = 'remote'

    GEOM_TYPE_POINT = 'point'
    GEOM_TYPE_LINE = 'line'
    GEOM_TYPE_POLYGON = 'polygon'

    DEFAULT_RETRY_TIMES = 3

    def __init__(self, table_name=None, schema=None, query=None, df=None, gdf=None, state=None, context=None):
        from .auth import _default_context
        self.cc = context or _default_context

        self.table_name = normalize_name(table_name)
        self.schema = schema or self._get_schema()
        self.query = query
        self.df = df
        self.gdf = gdf

        if not self._validate_init():
            raise ValueError('Wrong Dataset creation. You should use one of the class methods: '
                             'from_table, from_query, from_dataframe, from_geodataframe, from_geojson')

        self.state = state
        self.normalized_column_names = None

        if self.table_name != table_name:
            warn('Table will be named `{}`'.format(table_name))

    @classmethod
    def from_table(cls, table_name, context=None, schema=None):
        return cls(table_name=table_name, schema=schema, context=context, state=cls.STATE_REMOTE)

    @classmethod
    def from_query(cls, query, context=None):
        return cls(query=query, context=context, state=cls.STATE_REMOTE)

    @classmethod
    def from_dataframe(cls, df):
        dataset = cls(df=df, state=cls.STATE_LOCAL)
        _save_index_as_column(dataset.df)
        return dataset

    @classmethod
    def from_geodataframe(cls, gdf):
        dataset = cls(gdf=gdf, state=cls.STATE_LOCAL)
        _save_index_as_column(dataset.gdf)
        return dataset

    @classmethod
    def from_geojson(cls, geojson):
        return cls(gdf=load_geojson(geojson), state=cls.STATE_LOCAL)

    def upload(self, with_lnglat=None, if_exists=FAIL, table_name=None, schema=None, context=None):
        if table_name:
            self.table_name = normalize_name(table_name)
        if context:
            self.cc = context
            self.schema = context.get_default_schema()
        if schema:
            self.schema = schema

        if self.table_name is None or self.cc is None:
            raise ValueError('You should provide a table_name and context to upload data.')

        if self.gdf is None and self.df is None and self.query is None:
            raise ValueError('Nothing to upload.'
                             'We need data in a DataFrame or GeoDataFrame or a query to upload data to CARTO.')

        already_exists_error = CartoException('Table with name {t} and schema {s} already exists in CARTO.'
                                              'Please choose a different `table_name` or use'
                                              'if_exists="replace" to overwrite it'.format(
                                                  t=self.table_name, s=self.schema))

        # priority order: gdf, df, query
        if self.gdf is not None:
            warn('GeoDataFrame option is still under development. We will try the upload with DataFrame')
            # TODO: uncomment when we support GeoDataFrame
            # self.normalized_column_names = _normalize_column_names(self.gdf)

{'fraction_non_alphanumeric': 0.08088531187122737, 'fraction_numerical': 0.0013413816230717639, 'mean_word_length': 2.5070555032925683, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '882863', 'n_tokens_mistral': 2237, 'n_tokens_neox': 2154, 'n_words': 610}