<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""Azkaban example project configuration script.

Let us assume we have a flow with pig scripts to run, which share many
options. This example shows a way to concisely build the project.

"""

from azkaban import Job, PigJob, Project
from getpass import getuser


project = Project('for_loop_example', root=__file__)

defaults = {
  'user.to.proxy': getuser(),
  'mapred': {
    'max.split.size': 2684354560,
    'min.split.size': 2684354560,
  },
}

# dictionary of pig script options, keyed on the pig script path
pig_options = {
  'first.pig': {},
  'second.pig': {'dependencies': 'first.pig'},
  'third.pig': {'pig.param': {'foo': 48, 'bar': 'abc'}},
  'fourth.pig': {'dependencies': 'second.pig,third.pig'},
}

for path, options in pig_options.items():
  project.add_job(path, PigJob(path, defaults, options))<commit_msg>for loop example fix<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""Azkaban example project configuration script.

Let us assume we have a flow with pig scripts to run, which share many
options. This example shows a way to concisely build the project.

"""

from azkaban import PigJob, Project
from getpass import getuser


project = Project('for_loop_example', root=__file__)

default_options = {
  'user.to.proxy': getuser(),
  'mapred': {
    'max.split.size': 2684354560,
    'min.split.size': 2684354560,
  },
}

# dictionary of pig script options, keyed on the pig script path
pig_options = {
  'first.pig': {},
  'second.pig': {'dependencies': 'first.pig'},
  'third.pig': {'param': {'foo': 48, 'bar': 'abc'}},
  'fourth.pig': {'dependencies': 'second.pig,third.pig'},
}

for path, options in pig_options.items():
  project.add_job(path, PigJob(path, default_options, options))
{'fraction_non_alphanumeric': 0.14977477477477477, 'fraction_numerical': 0.0259009009009009, 'mean_word_length': 4.588050314465409, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '470273', 'n_tokens_mistral': 639, 'n_tokens_neox': 600, 'n_words': 181}