<commit_before>from __future__ import absolute_import, division, print_function

import pytest
from dynd import nd
import numpy as np
import tables as tb
from pandas import DataFrame

from blaze.api.into import into, discover
from blaze.api.into import degrade_numpy_dtype_to_python
from blaze.utils import tmpfile
from datashape import dshape
import blaze
from blaze import Table, TableExpr, TableSymbol, compute, PyTables
import bcolz
from blaze.data import CSV
from blaze.sql import SQL
from datetime import datetime
from toolz import pluck
import os
from blaze.compatibility import xfail

dirname = os.path.dirname(__file__)


csv = CSV(os.path.join(dirname, 'accounts.csv'))

L = [[100, 1, 'Alice', datetime(2000, 12, 25, 0, 0, 1)],
     [200, 2, 'Bob', datetime(2001, 12, 25, 0, 0, 1)],
     [300, 3, 'Charlie', datetime(2002, 12, 25, 0, 0, 1)]]

df = DataFrame(L, columns=['amount', 'id', 'name', 'timestamp'])

x = np.array(list(map(tuple, L)),
             dtype=[('amount', 'i8'), ('id', 'i8'),
                    ('name', 'U7'), ('timestamp', 'M8[us]')])

schema = '{amount: int64, id: int64, name: string, timestamp: datetime}'
sql_schema = '{amount: int64, id: int64, name: string, timestamp: datetime[tz="UTC"]}'

arr = nd.array(L, dtype=schema)

bc = bcolz.ctable([np.array([100, 200, 300], dtype=np.int64),
                   np.array([1, 2, 3], dtype=np.int64),
                   np.array(['Alice', 'Bob', 'Charlie'], dtype='U7'),
                   np.array([datetime(2000, 12, 25, 0, 0, 1),
                             datetime(2001, 12, 25, 0, 0, 1),
                             datetime(2002, 12, 25, 0, 0, 1)], dtype='M8[us]')],
                  names=['amount', 'id', 'name', 'timestamp'])

sql = SQL('sqlite:///:memory:', 'accounts', schema=schema)
sql.extend(L)

data = {list: L,
        Table: Table(L, '{amount: int64, id: int64, name: string[7], timestamp: datetime}'),
        DataFrame: df,
        np.ndarray: x,
        nd.array: arr,
        bcolz.ctable: bc,
        CSV: csv,
        SQL: sql}

schema_no_date = '{amount: int64, id: int64, name: string[7]}'
sql_no_date = SQL('sqlite:///:memory:', 'accounts_no_date', schema=schema_no_date)

L_no_date = list(pluck([0, 1, 2], L))
sql_no_date.extend(L_no_date)

no_date = {list: list(pluck([0, 1, 2], L)),
           Table: Table(list(pluck([0, 1, 2], L)),
                        '{amount: int64, id: int64, name: string[7]}'),
           DataFrame: df[['amount', 'id', 'name']],
           np.ndarray: x[['amount', 'id', 'name']],
           nd.array: nd.fields(arr, 'amount', 'id', 'name'),
           bcolz.ctable: bc[['amount', 'id', 'name']],
           SQL: sql_no_date}


try:
    import pymongo
except ImportError:
    pymongo = None
    Collection = None
if pymongo:
    from pymongo.collection import Collection
    try:
        db = pymongo.MongoClient().db

        db.test.drop()
        data[Collection] = into(db.test, df)

        db.no_date.drop()
        no_date[Collection] = into(db.no_date, no_date[DataFrame])
    except pymongo.errors.ConnectionFailure:
        pymongo = None
        Collection = None

try:<commit_msg>Make into exhaustive use items rather than a dict<commit_after>from __future__ import absolute_import, division, print_function

import pytest
from dynd import nd
import numpy as np
import tables as tb
from pandas import DataFrame

from blaze.api.into import into
from blaze.api.into import degrade_numpy_dtype_to_python
from blaze.utils import tmpfile
from blaze import Table, TableExpr, TableSymbol, compute, PyTables
import bcolz
from blaze.data import CSV
from blaze.sql import SQL
from datetime import datetime
from toolz import pluck
import os

dirname = os.path.dirname(__file__)


csv = CSV(os.path.join(dirname, 'accounts.csv'))

L = [[100, 1, 'Alice', datetime(2000, 12, 25, 0, 0, 1)],
     [200, 2, 'Bob', datetime(2001, 12, 25, 0, 0, 1)],
     [300, 3, 'Charlie', datetime(2002, 12, 25, 0, 0, 1)]]

df = DataFrame(L, columns=['amount', 'id', 'name', 'timestamp'])

x = np.array(list(map(tuple, L)),
             dtype=[('amount', 'i8'), ('id', 'i8'),
                    ('name', 'U7'), ('timestamp', 'M8[us]')])

schema = '{amount: int64, id: int64, name: string, timestamp: datetime}'
sql_schema = '{amount: int64, id: int64, name: string, timestamp: datetime[tz="UTC"]}'

arr = nd.array(L, dtype=schema)

bc = bcolz.ctable([np.array([100, 200, 300], dtype=np.int64),
                   np.array([1, 2, 3], dtype=np.int64),
                   np.array(['Alice', 'Bob', 'Charlie'], dtype='U7'),
                   np.array([datetime(2000, 12, 25, 0, 0, 1),
                             datetime(2001, 12, 25, 0, 0, 1),
                             datetime(2002, 12, 25, 0, 0, 1)], dtype='M8[us]')],
                  names=['amount', 'id', 'name', 'timestamp'])

sql = SQL('sqlite:///:memory:', 'accounts', schema=schema)
sql.extend(L)

data = {list: L,
        Table: Table(L, '{amount: int64, id: int64, name: string[7], timestamp: datetime}'),
        DataFrame: df,
        np.ndarray: x,
        nd.array: arr,
        bcolz.ctable: bc,
        CSV: csv,
        SQL: sql}

schema_no_date = '{amount: int64, id: int64, name: string[7]}'
sql_no_date = SQL('sqlite:///:memory:', 'accounts_no_date', schema=schema_no_date)

L_no_date = list(pluck([0, 1, 2], L))
sql_no_date.extend(L_no_date)

no_date = {list: list(pluck([0, 1, 2], L)),
           Table: Table(list(pluck([0, 1, 2], L)),
                        '{amount: int64, id: int64, name: string[7]}'),
           DataFrame: df[['amount', 'id', 'name']],
           np.ndarray: x[['amount', 'id', 'name']],
           nd.array: nd.fields(arr, 'amount', 'id', 'name'),
           bcolz.ctable: bc[['amount', 'id', 'name']],
           SQL: sql_no_date}


try:
    import pymongo
except ImportError:
    pymongo = None
    Collection = None
if pymongo:
    from pymongo.collection import Collection
    try:
        db = pymongo.MongoClient().db

        db.test.drop()
        data.append((Collection, into(db.test, df)))

        db.no_date.drop()
        no_date.append((Collection, into(db.no_date, dict(no_date)[DataFrame])))
    except pymongo.errors.ConnectionFailure:
        pymongo = None
        Collection = None

try:
{'fraction_non_alphanumeric': 0.16554379210779596, 'fraction_numerical': 0.04234841193455245, 'mean_word_length': 2.953709575142676, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7135830', 'n_tokens_mistral': 2363, 'n_tokens_neox': 2131, 'n_words': 592}