<commit_before>import logging
import sys

from pathlib import Path
from datetime import date

from Pegasus.api import *

logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)

PEGASUS_LOCATION = "file:///usr/bin/pegasus-keg"

# --- Work Dir Setup -----------------------------------------------------------
RUN_ID = "black-diamond-5.0api-" + date.today().strftime("%s")
TOP_DIR = Path(Path.cwd())
WORK_DIR = TOP_DIR / "work"

try:
    Path.mkdir(WORK_DIR)
except FileExistsError:
    pass

# --- Configuration ------------------------------------------------------------

print("Generating pegasus.conf at: {}".format(TOP_DIR / "pegasus.properties"))

conf = Properties()

conf["pegasus.catalog.site"] = "YAML"
conf["pegasus.catalog.site.file"] = "sites.yml"
conf["pegasus.catalog.transformation"] = "YAML"
conf["pegasus.catalog.transformation.file"] = "transformations.yml"
conf["pegasus.catalog.replica"] = "YAML"
conf["pegasus.catalog.replica.file"] = "replicas.yml"
conf["pegasus.data.configuration"] = "condorio"

conf.write()

# --- Sites --------------------------------------------------------------------
LOCAL = "local"
CONDOR_POOL = "⿔condor-pool⼤"

shared_scratch_dir = str(WORK_DIR / RUN_ID)
local_storage_dir = str(WORK_DIR / "outputs" / RUN_ID)

print("Generating site catalog at: {}".format(TOP_DIR / "sites.yml"))

SiteCatalog().add_sites(
    Site(
        LOCAL, arch=Arch.X86_64, os_type=OS.LINUX, os_release="rhel", os_version="7"
    ).add_directories(
        Directory(Directory.SHAREDSCRATCH, shared_scratch_dir).add_file_servers(
            FileServer("file://" + shared_scratch_dir, Operation.ALL)
        ),
        Directory(Directory.LOCALSTORAGE, local_storage_dir).add_file_servers(
            FileServer("file://" + local_storage_dir, Operation.ALL)
        ),
    ),
    Site(CONDOR_POOL, arch=Arch.X86_64, os_type=OS.LINUX)
    .add_pegasus_profile(style="condor")
    .add_pegasus_profile(auxillary_local="true")
    .add_condor_profile(universe="vanilla"),
).write()

# --- Replicas -----------------------------------------------------------------

print("Generating replica catalog at: {}".format(TOP_DIR / "replicas.yml"))

# create initial input file
with open("f.å", "w") as f:
    f.write("This is sample input to KEG\n")

fa = File("f.å").add_metadata({"㐦": "㒦"})
ReplicaCatalog().add_replica(LOCAL, fa, "file://" + str(TOP_DIR / fa.lfn)).write()

# --- Transformations ----------------------------------------------------------

print("Generating transformation catalog at: {}".format(TOP_DIR / "transformations.yml"))

preprocess = Transformation("pЯёprocess", namespace="pέgasuζ", version="4.0").add_sites(
    TransformationSite(
        CONDOR_POOL,
        PEGASUS_LOCATION,
        is_stageable=False,
        arch=Arch.X86_64,
        os_type=OS.LINUX,
    )
)

findrage = Transformation("findrange", namespace="pέgasuζ", version="4.0").add_sites(
    TransformationSite(
        CONDOR_POOL,
        PEGASUS_LOCATION,
        is_stageable=False,
        arch=Arch.X86_64,
        os_type=OS.LINUX,
    )
)

analyze = Transformation("analyze", namespace="pέgasuζ", version="4.0").add_sites(
    TransformationSite(
        CONDOR_POOL,
        PEGASUS_LOCATION,
        is_stageable=False,
        arch=Arch.X86_64,
        os_type=OS.LINUX,
    )
)

TransformationCatalog().add_transformations(preprocess, findrage, analyze).write()

# --- Workflow -----------------------------------------------------------------
print("Generating workflow")

fb1 = File("f.ƀ1")<commit_msg>001-black-diamond-vanilla-condor-5.0api bug fixes and successful run on workflow<commit_after>import logging
import sys

from pathlib import Path
from datetime import datetime

from Pegasus.api import *

logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)

PEGASUS_LOCATION = "/usr/bin/pegasus-keg"

# --- Work Dir Setup -----------------------------------------------------------
RUN_ID = "black-diamond-5.0api-" + datetime.now().strftime("%s")
TOP_DIR = Path(Path.cwd())
WORK_DIR = TOP_DIR / "work"

try:
    Path.mkdir(WORK_DIR)
except FileExistsError:
    pass

# --- Configuration ------------------------------------------------------------

print("Generating pegasus.conf at: {}".format(TOP_DIR / "pegasus.properties"))

conf = Properties()

conf["pegasus.catalog.site"] = "YAML"
conf["pegasus.catalog.site.file"] = "sites.yml"
conf["pegasus.catalog.transformation"] = "YAML"
conf["pegasus.catalog.transformation.file"] = "transformations.yml"
conf["pegasus.catalog.replica"] = "YAML"
conf["pegasus.catalog.replica.file"] = "replicas.yml"
conf["pegasus.data.configuration"] = "condorio"
conf["pegasus.integrity.checking"] = "none"
conf.write()

# --- Sites --------------------------------------------------------------------
LOCAL = "local"
CONDOR_POOL = "⿔condor-pool⼤"

shared_scratch_dir = str(WORK_DIR / RUN_ID)
local_storage_dir = str(WORK_DIR / "outputs" / RUN_ID)

print("Generating site catalog at: {}".format(TOP_DIR / "sites.yml"))

SiteCatalog().add_sites(
    Site(
        LOCAL, arch=Arch.X86_64, os_type=OS.LINUX, os_release="rhel", os_version="7"
    ).add_directories(
        Directory(Directory.SHAREDSCRATCH, shared_scratch_dir).add_file_servers(
            FileServer("file://" + shared_scratch_dir, Operation.ALL)
        ),
        Directory(Directory.LOCALSTORAGE, local_storage_dir).add_file_servers(
            FileServer("file://" + local_storage_dir, Operation.ALL)
        ),
    ),
    Site(CONDOR_POOL, arch=Arch.X86_64, os_type=OS.LINUX)
    .add_pegasus_profile(style="condor")
    .add_pegasus_profile(auxillary_local="true")
    .add_condor_profile(universe="vanilla"),
).write()

# --- Replicas -----------------------------------------------------------------

print("Generating replica catalog at: {}".format(TOP_DIR / "replicas.yml"))

# create initial input file
with open("f.å", "w") as f:
    f.write("This is sample input to KEG\n")

fa = File("f.å").add_metadata({"㐦": "㒦"})
ReplicaCatalog().add_replica(LOCAL, fa, "file://" + str(TOP_DIR / fa.lfn)).write()

# --- Transformations ----------------------------------------------------------

print("Generating transformation catalog at: {}".format(TOP_DIR / "transformations.yml"))

preprocess = Transformation("pЯёprocess", namespace="pέgasuζ", version="4.0").add_sites(
    TransformationSite(
        CONDOR_POOL,
        PEGASUS_LOCATION,
        is_stageable=True,
        arch=Arch.X86_64,
        os_type=OS.LINUX,
    )
)

findrage = Transformation("findrange", namespace="pέgasuζ", version="4.0").add_sites(
    TransformationSite(
        CONDOR_POOL,
        PEGASUS_LOCATION,
        is_stageable=True,
        arch=Arch.X86_64,
        os_type=OS.LINUX,
    )
)

analyze = Transformation("analyze", namespace="pέgasuζ", version="4.0").add_sites(
    TransformationSite(
        CONDOR_POOL,
        PEGASUS_LOCATION,
        is_stageable=True,
        arch=Arch.X86_64,
        os_type=OS.LINUX,
    )
)

TransformationCatalog().add_transformations(preprocess, findrage, analyze).write()

# --- Workflow -----------------------------------------------------------------
print("Generating workflow")

fb1 = File("f.ƀ1")
{'fraction_non_alphanumeric': 0.24017832265254946, 'fraction_numerical': 0.009334076344385623, 'mean_word_length': 2.5294985250737465, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1090631', 'n_tokens_mistral': 2407, 'n_tokens_neox': 2305, 'n_words': 360}