<commit_before>/*
 *
 *       Copyright 2015 <NAME>
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 *
 */

package com.metawiring.load.activities;

import com.codahale.metrics.Counter;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Timer;
import com.datastax.driver.core.*;
import com.metawiring.load.activity.Activity;
import com.metawiring.load.activity.DefaultActivitySourceResolver;
import com.metawiring.load.config.YamlActivityDef;
import com.metawiring.load.core.ExecutionContext;
import com.metawiring.load.core.ReadyStatement;
import com.metawiring.load.core.ReadyStatements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;

import java.io.FileInputStream;
import java.io.InputStream;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;

import static com.codahale.metrics.MetricRegistry.name;

public class YamlConfigurableActivity implements Activity {

    private static Logger logger = LoggerFactory.getLogger(YamlConfigurableActivity.class);

    private String name;
    private ExecutionContext context;
    protected boolean diagnoseExceptions = true;
    private YamlActivityDef yamlActivityDef;

    // Since this class services multiple concurrent activities, synchronize each init independently
    private static Map<String,ReadyStatements> configuredActivities = new ConcurrentHashMap<>();

    private long endCycle, submittedCycle;
    private int pendingRq = 0;

    private long maxAsync = 0l;

    private ReadyStatements readyStatements;

    private LinkedList<TimedResultSetFuture> timedResultSetFutures = new LinkedList<>();
    private Timer timerOps;
    private Timer timerWaits;
    private Counter activityAsyncPendingCounter;
    private Histogram triesHistogram;

    private static Session session;

    private String[] statements;

    private PreparedStatement[] preparedStatements;
    private static boolean staticInitComplete = false;


    @Override
    public void init(String name, ExecutionContext context) {
        this.name = name;
        this.context = context;
        this.diagnoseExceptions = context.getConfig().diagnoseExceptions;

        Yaml yaml = new Yaml();
        for (String streamname : new String[]{"activities/" + name, "activities/" + name + ".yaml"}) {
            try {
                logger.debug("Looking for " + streamname + " on filesystem.");
                InputStream stream = null;
                stream =new FileInputStream(streamname);
                if ( stream == null ) {
                    logger.debug("Not found on filesystem, looking for " + streamname + " on in classpath.");
                    stream = DefaultActivitySourceResolver.class.getClassLoader().getResourceAsStream(streamname);
                }
                if (stream != null) {
                    try {
                        yamlActivityDef = yaml.loadAs(stream, YamlActivityDef.class);
                    } catch (Exception ye) {
                        logger.error("error loading yaml", ye);
                        throw ye;
                    }
                }
            } catch (Exception e) {
            }
            if (yamlActivityDef != null) {
                break;
            }
        }
        if (yamlActivityDef == null) {
            throw new RuntimeException("can not configure " + YamlConfigurableActivity.class.getSimpleName() + " without a yaml stream.");
        }<commit_msg>Allow yaml activities to find .yaml outside classpath<commit_after>/*
 *
 *       Copyright 2015 <NAME>
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 *
 */

package com.metawiring.load.activities;

import com.codahale.metrics.Counter;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Timer;
import com.datastax.driver.core.*;
import com.metawiring.load.activity.Activity;
import com.metawiring.load.activity.DefaultActivitySourceResolver;
import com.metawiring.load.config.YamlActivityDef;
import com.metawiring.load.core.ExecutionContext;
import com.metawiring.load.core.ReadyStatement;
import com.metawiring.load.core.ReadyStatements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;

import java.io.FileInputStream;
import java.io.InputStream;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;

import static com.codahale.metrics.MetricRegistry.name;

public class YamlConfigurableActivity implements Activity {

    private static Logger logger = LoggerFactory.getLogger(YamlConfigurableActivity.class);

    private String name;
    private ExecutionContext context;
    protected boolean diagnoseExceptions = true;
    private YamlActivityDef yamlActivityDef;

    // Since this class services multiple concurrent activities, synchronize each init independently
    private static Map<String,ReadyStatements> configuredActivities = new ConcurrentHashMap<>();

    private long endCycle, submittedCycle;
    private int pendingRq = 0;

    private long maxAsync = 0l;

    private ReadyStatements readyStatements;

    private LinkedList<TimedResultSetFuture> timedResultSetFutures = new LinkedList<>();
    private Timer timerOps;
    private Timer timerWaits;
    private Counter activityAsyncPendingCounter;
    private Histogram triesHistogram;

    private static Session session;

    private String[] statements;

    private PreparedStatement[] preparedStatements;
    private static boolean staticInitComplete = false;


    @Override
    public void init(String name, ExecutionContext context) {
        this.name = name;
        this.context = context;
        this.diagnoseExceptions = context.getConfig().diagnoseExceptions;

        Yaml yaml = new Yaml();
        for (String streamname : new String[]{"activities/" + name, "activities/" + name + ".yaml"}) {
            try {
                logger.debug("Looking for " + streamname + " on filesystem.");
                InputStream stream = null;
                try {
                    stream =new FileInputStream(streamname);
                } catch (Exception ignored) {
                }
                if ( stream == null ) {
                    logger.debug("Not found on filesystem, looking for " + streamname + " on in classpath.");
                    stream = DefaultActivitySourceResolver.class.getClassLoader().getResourceAsStream(streamname);
                }
                if (stream != null) {
                    try {
                        yamlActivityDef = yaml.loadAs(stream, YamlActivityDef.class);
                    } catch (Exception ye) {
                        logger.error("error loading yaml", ye);
                        throw ye;
                    }
                }
            } catch (Exception e) {
            }
            if (yamlActivityDef != null) {
                break;
            }
        }
        if (yamlActivityDef == null) {
            throw new RuntimeException("can not configure " + YamlConfigurableActivity.class.getSimpleName() + " without a yaml stream.");
        }
{'fraction_non_alphanumeric': 0.08217993079584775, 'fraction_numerical': 0.002965892239248641, 'mean_word_length': 2.731212540341171, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1892831', 'n_tokens_mistral': 2102, 'n_tokens_neox': 2000, 'n_words': 605}