<commit_before>"""
%load_ext autoreload
%autoreload 2
"""
import logging
from pathlib import Path
from jinja2 import Template

from dstools.pipeline.products import File
from dstools.pipeline.tasks import (BashCommand, BashScript, PythonCallable)
from dstools.pipeline import postgres as pg
from dstools.pipeline.dag import DAG
from dstools import testing
from dstools import Env
from dstools import mkfilename

from train import train_and_save_report
import util
from download_dataset import download_dataset
from sample import sample


logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)


env = Env()
home = env.path.home
path_to_sample = env.path.input / 'sample'

path_to_sample.mkdir(exist_ok=True)

pg.CONN = util.open_db_conn()
db = util.load_db_credentials()

dag = DAG()


get_data = BashCommand((home / 'get_data.sh').read_text(),
                       (File(env.path.input / 'raw' / 'red.csv'),
                        File(env.path.input / 'raw' / 'white.csv'),
                        File(env.path.input / 'raw' / 'names')),
                       dag, 'get_data',
                       split_source_code=False)

sample = PythonCallable(sample,
                        (File(env.path.input / 'sample' / 'red.csv'),
                         File(env.path.input / 'sample' / 'white.csv')),
                        dag, 'sample')
get_data >> sample

red_path = path_to_sample / 'red.csv'
red_task = BashCommand(Template('csvsql --db {{db}} --tables {{product.name}} --insert {{path}} '
                                '--overwrite'),
                       pg.PostgresRelation(('public', 'red', 'table')),
                       dag, 'red',<commit_msg>fixes example<commit_after>"""
%load_ext autoreload
%autoreload 2
"""
import logging
from pathlib import Path
from jinja2 import Template

from dstools.pipeline.products import File
from dstools.pipeline.tasks import (BashCommand, BashScript, PythonCallable)
from dstools.pipeline import postgres as pg
from dstools.pipeline.dag import DAG
from dstools import testing
from dstools import Env
from dstools import mkfilename

from train import train_and_save_report
import util
from download_dataset import download_dataset
from sample import sample


logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)


env = Env()
home = env.path.home
path_to_sample = env.path.input / 'sample'

(env.path.input / 'raw').mkdir(exist_ok=True, parents=True)
path_to_sample.mkdir(exist_ok=True)

pg.CONN = util.open_db_conn()
db = util.load_db_credentials()

dag = DAG()


get_data = BashCommand((home / 'get_data.sh').read_text(),
                       (File(env.path.input / 'raw' / 'red.csv'),
                        File(env.path.input / 'raw' / 'white.csv'),
                        File(env.path.input / 'raw' / 'names')),
                       dag, 'get_data',
                       split_source_code=False)

sample = PythonCallable(sample,
                        (File(env.path.input / 'sample' / 'red.csv'),
                         File(env.path.input / 'sample' / 'white.csv')),
                        dag, 'sample')
get_data >> sample

red_path = path_to_sample / 'red.csv'
red_task = BashCommand(Template('csvsql --db {{db}} --tables {{product.name}} --insert {{path}} '
                                '--overwrite'),
                       pg.PostgresRelation(('public', 'red', 'table')),
                       dag, 'red',
{'fraction_non_alphanumeric': 0.1210801393728223, 'fraction_numerical': 0.0011614401858304297, 'mean_word_length': 2.5010162601626016, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1437771', 'n_tokens_mistral': 1039, 'n_tokens_neox': 1009, 'n_words': 247}