<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
import os
import vaex.dataset as dataset
import numpy as np
import unittest
import vaex as vx
import tempfile
import vaex.webserver
import astropy.io.fits
import astropy.units
import pandas as pd
import vaex.execution
import contextlib
a = vaex.execution.buffer_size_default # will crash if we decide to rename it

basedir = os.path.dirname(__file__)
# this will make the test execute more code and may show up bugs
#vaex.execution.buffer_size_default = 3


@contextlib.contextmanager
def small_buffer(ds, size=3):
	previous = ds.executor.buffer_size
	ds.executor.buffer_size = size
	try:
		yield
	finally:
		ds.executor.buffer_size = previous


# these need to be global for pickling
def function_upper(x):
	return np.array(x.decode('ascii').upper())
import vaex.serialize
@vaex.serialize.register
class Multiply(object):
	def __init__(self, scale=0): self.scale = scale
	def state_set(self, state): self.scale = state
	def state_get(self): return self.scale
	def __call__(self, x): return x * self.scale


vx.set_log_level_exception()
#vx.set_log_level_off()
#vx.set_log_level_debug()<commit_msg>test: do not change buffer size for remote datasets (not supported)<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
import os
import vaex.dataset as dataset
import numpy as np
import unittest
import vaex as vx
import tempfile
import vaex.webserver
import astropy.io.fits
import astropy.units
import pandas as pd
import vaex.execution
import contextlib
a = vaex.execution.buffer_size_default # will crash if we decide to rename it

basedir = os.path.dirname(__file__)
# this will make the test execute more code and may show up bugs
#vaex.execution.buffer_size_default = 3


@contextlib.contextmanager
def small_buffer(ds, size=3):
	if ds.is_local():
		previous = ds.executor.buffer_size
		ds.executor.buffer_size = size
		try:
			yield
		finally:
			ds.executor.buffer_size = previous
	else:
		yield # for remote datasets we don't support this ... or should we?


# these need to be global for pickling
def function_upper(x):
	return np.array(x.decode('ascii').upper())
import vaex.serialize
@vaex.serialize.register
class Multiply(object):
	def __init__(self, scale=0): self.scale = scale
	def state_set(self, state): self.scale = state
	def state_get(self): return self.scale
	def __call__(self, x): return x * self.scale


vx.set_log_level_exception()
#vx.set_log_level_off()
#vx.set_log_level_debug()
{'fraction_non_alphanumeric': 0.09018355945730247, 'fraction_numerical': 0.0031923383878691143, 'mean_word_length': 5.011990407673861, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1353929', 'n_tokens_mistral': 843, 'n_tokens_neox': 796, 'n_words': 259}