<commit_before>import storage.models
import job.models
from job.configuration.interface.job_interface import JobInterface
import ingest.models
from recipe.configuration.definition.recipe_definition import RecipeDefinition
import recipe.models
import storage.models
import trigger.models
import trigger.handler as trigger_handler

# Workspaces
if not storage.models.Workspace.objects.filter(name="raw").exists():
    storage.models.Workspace.objects.create(name="raw", title="raw", description="Raw ingested data",
        base_url="http://master:8080/raw",
        json_config={
        "version": "1.0", "broker": {"mount": "10.4.4.10:/raw", "type": "nfs"}}).save()

if not storage.models.Workspace.objects.filter(name="products").exists():
    storage.models.Workspace.objects.create(name="products", title="products", description="Product storage",
        base_url="http://master:8080/products",
        json_config={
        "version": "1.0", "broker": {"mount": "10.4.4.10:/products", "type": "nfs"}}).save()

# Triggers
if not trigger.models.TriggerRule.objects.filter(name="landsat-parse").exists():
    rule_handler = trigger_handler.get_trigger_rule_handler("INGEST")
    trigger_rule = rule_handler.create_trigger_rule({
        "version": "1.0",
        "condition": {
            "media_type": "application/x-tar",
            "data_types": ["landsat"]
        },
        "data": {
            "input_data_name": "infile",
            "workspace_name": "products"
        }
    }, "landsat-parse", True)

# Job types
if not job.models.JobType.objects.filter(name="landsat-parse").exists():
    jt = job.models.JobType.objects.create_job_type("landsat-parse", "1.0.0",
            JobInterface({"output_data": [
                {"media_type": "image/tiff", "required": True, "type": "file", "name": "multispectral"},
                {"media_type": "image/tiff", "required": True, "type": "file", "name": "panchromatic"},
                {"media_type": "image/tiff", "required": True, "type": "file", "name": "thermal"}
            ],
            "shared_resources": [],
            "command_arguments": "${infile} ${job_output_dir}",
            "input_data": [
                {"media_types": ["application/octet-stream"], "required": True, "type": "file", "name": "infile"}],
            "version": "1.0", "command": "./parse_landsat.sh"}),
        description="Parse landsat multi-tif files in tar.gz archives",
        docker_image="10.4.4.10:5000/landsat-parse_1.0:dev",
        priority=200, timeout=300, max_tries=3, cpus_required=0.25, mem_required=512.)
    jt.title = "Landsat Parse"
    jt.save()
if not job.models.JobType.objects.filter(name="landsat-ndwi").exists():<commit_msg>Fix error in example data<commit_after>import storage.models
import job.models
from job.configuration.interface.job_interface import JobInterface
import ingest.models
from recipe.configuration.definition.recipe_definition import RecipeDefinition
import recipe.models
import storage.models
import trigger.models
import trigger.handler as trigger_handler

# Workspaces
if not storage.models.Workspace.objects.filter(name="raw").exists():
    storage.models.Workspace.objects.create(name="raw", title="raw", description="Raw ingested data",
        base_url="http://master:8080/raw",
        json_config={
        "version": "1.0", "broker": {"mount": "10.4.4.10:/raw", "type": "nfs"}}).save()

if not storage.models.Workspace.objects.filter(name="products").exists():
    storage.models.Workspace.objects.create(name="products", title="products", description="Product storage",
        base_url="http://master:8080/products",
        json_config={
        "version": "1.0", "broker": {"mount": "10.4.4.10:/products", "type": "nfs"}}).save()

# Triggers
if not trigger.models.TriggerRule.objects.filter(name="landsat-parse").exists():
    rule_handler = trigger_handler.get_trigger_rule_handler("INGEST")
    trigger_rule = rule_handler.create_trigger_rule({
        "version": "1.0",
        "condition": {
            "media_type": "application/x-tar",
            "data_types": ["landsat"]
        },
        "data": {
            "input_data_name": "infile",
            "workspace_name": "raw"
        }
    }, "landsat-parse", True)

# Job types
if not job.models.JobType.objects.filter(name="landsat-parse").exists():
    jt = job.models.JobType.objects.create_job_type("landsat-parse", "1.0.0",
            JobInterface({"output_data": [
                {"media_type": "image/tiff", "required": True, "type": "file", "name": "multispectral"},
                {"media_type": "image/tiff", "required": True, "type": "file", "name": "panchromatic"},
                {"media_type": "image/tiff", "required": True, "type": "file", "name": "thermal"}
            ],
            "shared_resources": [],
            "command_arguments": "${infile} ${job_output_dir}",
            "input_data": [
                {"media_types": ["application/octet-stream"], "required": True, "type": "file", "name": "infile"}],
            "version": "1.0", "command": "./parse_landsat.sh"}),
        description="Parse landsat multi-tif files in tar.gz archives",
        docker_image="10.4.4.10:5000/landsat-parse_1.0:dev",
        priority=200, timeout=300, max_tries=3, cpus_required=0.25, mem_required=512.)
    jt.title = "Landsat Parse"
    jt.save()
if not job.models.JobType.objects.filter(name="landsat-ndwi").exists():
{'fraction_non_alphanumeric': 0.17476445593940512, 'fraction_numerical': 0.020690929244411602, 'mean_word_length': 3.507910074937552, 'pattern_counts': {'":': 74, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5318598', 'n_tokens_mistral': 1719, 'n_tokens_neox': 1654, 'n_words': 303}