<commit_before>import os

from magichour.api.local.sample.steps.evalapply import evalapply_step
from magichour.api.local.sample.steps.event import event_step
from magichour.api.local.sample.steps.genapply import genapply_step
from magichour.api.local.sample.steps.genwindow import genwindow_step
from magichour.api.local.sample.steps.preprocess import preprocess_step
from magichour.api.local.sample.steps.template import template_step

from magichour.api.local.util.log import get_logger, log_time
from magichour.api.local.util.pickl import read_pickle_file, write_pickle_file

logger = get_logger(__name__)
sample_dir = os.path.dirname(os.path.realpath(__file__))
data_dir = os.path.join(sample_dir, "data")

# Result caching (to help w/ debugging)

pickle_cache_dir = os.path.join(data_dir, "pickle")
loglines_file = os.path.join(pickle_cache_dir, "loglines.pickle")
gen_templates_file = os.path.join(pickle_cache_dir, "gen_templates.pickle")
eval_loglines_file = os.path.join(pickle_cache_dir, "eval_loglines.pickle")
gen_windows_file = os.path.join(pickle_cache_dir, "gen_windows.pickle")
gen_events_file = os.path.join(pickle_cache_dir, "gen_events.pickle")
timed_events_file = os.path.join(pickle_cache_dir, "timed_events.pickle")

###

# Inputs

input_dir = os.path.join(data_dir, "input")
log_file = os.path.join(input_dir, "tbird.log.500k")
transforms_file = os.path.join(data_dir, "sample.transforms")

###

# Parameters for pipeline
write_to_pickle_file = False
read_lines_args = [{}, 0, 10]
read_lines_kwargs = {"skip_num_chars": 22}

logcluster_kwargs = {"support": "50"}

genapply_kwargs = {'mp': True}
gen_windows_kwargs = {"window_size": 60, "tfidf_threshold": 0}
<commit_msg>Changing default back to writing pickle file<commit_after>import os

from magichour.api.local.sample.steps.evalapply import evalapply_step
from magichour.api.local.sample.steps.event import event_step
from magichour.api.local.sample.steps.genapply import genapply_step
from magichour.api.local.sample.steps.genwindow import genwindow_step
from magichour.api.local.sample.steps.preprocess import preprocess_step
from magichour.api.local.sample.steps.template import template_step

from magichour.api.local.util.log import get_logger, log_time
from magichour.api.local.util.pickl import read_pickle_file, write_pickle_file

logger = get_logger(__name__)
sample_dir = os.path.dirname(os.path.realpath(__file__))
data_dir = os.path.join(sample_dir, "data")

# Result caching (to help w/ debugging)

pickle_cache_dir = os.path.join(data_dir, "pickle")
loglines_file = os.path.join(pickle_cache_dir, "loglines.pickle")
gen_templates_file = os.path.join(pickle_cache_dir, "gen_templates.pickle")
eval_loglines_file = os.path.join(pickle_cache_dir, "eval_loglines.pickle")
gen_windows_file = os.path.join(pickle_cache_dir, "gen_windows.pickle")
gen_events_file = os.path.join(pickle_cache_dir, "gen_events.pickle")
timed_events_file = os.path.join(pickle_cache_dir, "timed_events.pickle")

###

# Inputs

input_dir = os.path.join(data_dir, "input")
log_file = os.path.join(input_dir, "tbird.log.500k")
transforms_file = os.path.join(data_dir, "sample.transforms")

###

# Parameters for pipeline
write_to_pickle_file = True
read_lines_args = [{}, 0, 10]
read_lines_kwargs = {"skip_num_chars": 22}

logcluster_kwargs = {"support": "50"}

genapply_kwargs = {'mp': True}
gen_windows_kwargs = {"window_size": 60, "tfidf_threshold": 0}

{'fraction_non_alphanumeric': 0.11826697892271663, 'fraction_numerical': 0.007611241217798595, 'mean_word_length': 6.891454965357967, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7642458', 'n_tokens_mistral': 1251, 'n_tokens_neox': 1224, 'n_words': 210}