<commit_before># Copyright 2011-2013 <NAME>
# Copyright 2011-2013 <NAME>
# Copyright 2012-2013 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''
Three control flow types for running the simulation forward.
  - Fuzzer: injects input events at random intervals, periodically checking
    for invariant violations
  - Interactive: presents an interactive prompt for injecting events and
    checking for invariants at the users' discretion
'''

from sts.control_flow.interactive import Interactive
from sts.topology import BufferedPatchPanel
from sts.traffic_generator import TrafficGenerator
from sts.replay_event import *
from pox.lib.util import TimeoutError
from pox.lib.packet.lldp import *
from config.invariant_checks import name_to_invariant_check
from sts.util.convenience import base64_encode
from sts.entities import FuzzSoftwareSwitch
from sts.openflow_buffer import OpenFlowBuffer

from sts.control_flow.base import ControlFlow, RecordingSyncCallback

import os
import re
import shutil
import signal
import sys
import time
import random
import logging

log = logging.getLogger("control_flow")

class Fuzzer(ControlFlow):
  '''
  Injects input events at random intervals, periodically checking
  for invariant violations. (Not the proper use of the term `Fuzzer`)
  '''
  def __init__(self, simulation_cfg, fuzzer_params="config.fuzzer_params",
               check_interval=None, traffic_inject_interval=10, random_seed=None,
               delay=0.1, steps=None, input_logger=None,
               invariant_check_name="InvariantChecker.check_correspondence",
               halt_on_violation=False, log_invariant_checks=True,
               delay_startup=True, print_buffers=True,
               record_deterministic_values=False,
               mock_link_discovery=False,
               never_drop_whitelisted_packets=True,
               initialization_rounds=0):
    '''
    Options:
      - fuzzer_params: path to event probabilities
      - check_interval: the period for checking invariants, in terms of
        logical rounds
      - traffic_inject_interval: how often to inject dataplane trace packets
      - random_seed: optionally set the seed of the random number generator
      - delay: how long to sleep between each logical round
      - input_logger: None, or a InputLogger instance
      - invariant_check_name: the name of the invariant check, from
        config/invariant_checks.py
      - halt_on_violation: whether to stop after a bug has been detected
      - log_invariant_checks: whether to log InvariantCheck events
      - delay_startup: whether to until the first OpenFlow message is received
        before proceeding with fuzzing
      - print_buffers: whether to print the remaining contents of the
        dataplane/controlplane buffers at the end of the execution
      - record_deterministic_values: whether to record gettimeofday requests
        for replay
      - mock_link_discovery: optional module for POX to experiment with
        better determinism -- tell POX exactly when links should be discovered
      - initialization_rounds: if non-zero, will wait the specified rounds to
        let the controller discover the topology before injecting inputs
    '''
    ControlFlow.__init__(self, simulation_cfg)
    self.sync_callback = RecordingSyncCallback(input_logger,
                           record_deterministic_values=record_deterministic_values)

    self.check_interval = check_interval
    if self.check_interval is None:
      log.warn("Check interval is not specified... not checking invariants")
    if invariant_check_name not in name_to_invariant_check:
      raise ValueError('''Unknown invariant check %s.\n'''
                       '''Invariant check name must be defined in config.invariant_checks''',
                       invariant_check_name)
    self.invariant_check_name = invariant_check_name
    self.invariant_check = name_to_invariant_check[invariant_check_name]
    self.log_invariant_checks = log_invariant_checks
    self.traffic_inject_interval = traffic_inject_interval
    # Make execution deterministic to allow the user to easily replay
    if random_seed is None:
      random_seed = random.randint(0, sys.maxint)

    self.random_seed = random_seed
    self.random = random.Random(random_seed)
    self.traffic_generator = TrafficGenerator(self.random)

    self.delay = delay
    self.steps = steps
    self.params = object()
    self._load_fuzzer_params(fuzzer_params)
    self._input_logger = input_logger
    self.halt_on_violation = halt_on_violation
    self.delay_startup = delay_startup
    self.print_buffers = print_buffers
    self.mock_link_discovery = mock_link_discovery
    # How many rounds to let the controller initialize:
    # send one round of packets directed at the source host itself (to facilitate
    # learning), then send all-to-all packets until all pairs have been
    # pinged. Tell MCSFinder not to prune initial inputs during this period.
    self.initialization_rounds = initialization_rounds
    # If initialization_rounds isn't 0, also make sure to send all-to-all
    # pings before starting any events
    self._pending_all_to_all = initialization_rounds != 0
    # Our current place in the all-to-all cycle. Stop when == len(hosts)
    self._all_to_all_iterations = 0
    # How often (in terms of logical rounds) to inject all-to-all packets
    self._all_to_all_interval = 5
    self.blocked_controller_pairs = []
    self.unblocked_controller_pairs = []

    # Logical time (round #) for the simulation execution
    self.logical_time = 0
    self.never_drop_whitelisted_packets = never_drop_whitelisted_packets

    # Determine whether to use delayed and randomized flow mod processing
    # (Set by fuzzer_params, not by an optional __init__ argument)
    self.delay_flow_mods = False

  def _log_input_event(self, event, **kws):
    if self._input_logger is not None:
      if self._initializing():
        # Tell MCSFinder never to prune this event
        event.prunable = False

      event.round = self.logical_time
      self._input_logger.log_input_event(event, **kws)

  def _load_fuzzer_params(self, fuzzer_params_path):
    if fuzzer_params_path.endswith('.py'):
      fuzzer_params_path = fuzzer_params_path[:-3].replace("/", ".")

    try:
      self.params = __import__(fuzzer_params_path, globals(), locals(), ["*"])
      # TODO(cs): temporary hack until we get determinism figured out
      self.params.link_discovery_rate = 0.1
    except:
      raise IOError("Could not find fuzzer params config file: %s" %
                    fuzzer_params_path)

  def _compute_unblocked_controller_pairs(self):
    sorted_controllers = sorted(self.simulation.controller_manager.controllers, key=lambda c: c.cid)
    unblocked_pairs = []
    for i in xrange(0, len(sorted_controllers)-1):
      for j in xrange(i+1, len(sorted_controllers)):
        unblocked_pairs.append((sorted_controllers[i].cid, sorted_controllers[j].cid))
    return unblocked_pairs

  def init_results(self, results_dir):
    if self._input_logger:
      self._input_logger.open(results_dir)
    params_file = re.sub(r'\.pyc$', '.py', self.params.__file__)
    # Move over our fuzzer params
    # TODO(cs): need to modify copied config file to point to the new fuzzer
    # params
    if os.path.exists(params_file):
      new_params_file = os.path.join(results_dir, os.path.basename(params_file))<commit_msg>Unblock all peers on startup<commit_after># Copyright 2011-2013 <NAME>
# Copyright 2011-2013 <NAME>
# Copyright 2012-2013 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''
Three control flow types for running the simulation forward.
  - Fuzzer: injects input events at random intervals, periodically checking
    for invariant violations
  - Interactive: presents an interactive prompt for injecting events and
    checking for invariants at the users' discretion
'''

from sts.control_flow.interactive import Interactive
from sts.topology import BufferedPatchPanel
from sts.traffic_generator import TrafficGenerator
from sts.replay_event import *
from pox.lib.util import TimeoutError
from pox.lib.packet.lldp import *
from config.invariant_checks import name_to_invariant_check
from sts.util.convenience import base64_encode
from sts.entities import FuzzSoftwareSwitch
from sts.openflow_buffer import OpenFlowBuffer

from sts.control_flow.base import ControlFlow, RecordingSyncCallback

import os
import re
import shutil
import signal
import sys
import time
import random
import logging

log = logging.getLogger("control_flow")

class Fuzzer(ControlFlow):
  '''
  Injects input events at random intervals, periodically checking
  for invariant violations. (Not the proper use of the term `Fuzzer`)
  '''
  def __init__(self, simulation_cfg, fuzzer_params="config.fuzzer_params",
               check_interval=None, traffic_inject_interval=10, random_seed=None,
               delay=0.1, steps=None, input_logger=None,
               invariant_check_name="InvariantChecker.check_correspondence",
               halt_on_violation=False, log_invariant_checks=True,
               delay_startup=True, print_buffers=True,
               record_deterministic_values=False,
               mock_link_discovery=False,
               never_drop_whitelisted_packets=True,
               initialization_rounds=0):
    '''
    Options:
      - fuzzer_params: path to event probabilities
      - check_interval: the period for checking invariants, in terms of
        logical rounds
      - traffic_inject_interval: how often to inject dataplane trace packets
      - random_seed: optionally set the seed of the random number generator
      - delay: how long to sleep between each logical round
      - input_logger: None, or a InputLogger instance
      - invariant_check_name: the name of the invariant check, from
        config/invariant_checks.py
      - halt_on_violation: whether to stop after a bug has been detected
      - log_invariant_checks: whether to log InvariantCheck events
      - delay_startup: whether to until the first OpenFlow message is received
        before proceeding with fuzzing
      - print_buffers: whether to print the remaining contents of the
        dataplane/controlplane buffers at the end of the execution
      - record_deterministic_values: whether to record gettimeofday requests
        for replay
      - mock_link_discovery: optional module for POX to experiment with
        better determinism -- tell POX exactly when links should be discovered
      - initialization_rounds: if non-zero, will wait the specified rounds to
        let the controller discover the topology before injecting inputs
    '''
    ControlFlow.__init__(self, simulation_cfg)
    self.sync_callback = RecordingSyncCallback(input_logger,
                           record_deterministic_values=record_deterministic_values)

    self.check_interval = check_interval
    if self.check_interval is None:
      log.warn("Check interval is not specified... not checking invariants")
    if invariant_check_name not in name_to_invariant_check:
      raise ValueError('''Unknown invariant check %s.\n'''
                       '''Invariant check name must be defined in config.invariant_checks''',
                       invariant_check_name)
    self.invariant_check_name = invariant_check_name
    self.invariant_check = name_to_invariant_check[invariant_check_name]
    self.log_invariant_checks = log_invariant_checks
    self.traffic_inject_interval = traffic_inject_interval
    # Make execution deterministic to allow the user to easily replay
    if random_seed is None:
      random_seed = random.randint(0, sys.maxint)

    self.random_seed = random_seed
    self.random = random.Random(random_seed)
    self.traffic_generator = TrafficGenerator(self.random)

    self.delay = delay
    self.steps = steps
    self.params = object()
    self._load_fuzzer_params(fuzzer_params)
    self._input_logger = input_logger
    self.halt_on_violation = halt_on_violation
    self.delay_startup = delay_startup
    self.print_buffers = print_buffers
    self.mock_link_discovery = mock_link_discovery
    # How many rounds to let the controller initialize:
    # send one round of packets directed at the source host itself (to facilitate
    # learning), then send all-to-all packets until all pairs have been
    # pinged. Tell MCSFinder not to prune initial inputs during this period.
    self.initialization_rounds = initialization_rounds
    # If initialization_rounds isn't 0, also make sure to send all-to-all
    # pings before starting any events
    self._pending_all_to_all = initialization_rounds != 0
    # Our current place in the all-to-all cycle. Stop when == len(hosts)
    self._all_to_all_iterations = 0
    # How often (in terms of logical rounds) to inject all-to-all packets
    self._all_to_all_interval = 5
    self.blocked_controller_pairs = []
    self.unblocked_controller_pairs = []

    # Logical time (round #) for the simulation execution
    self.logical_time = 0
    self.never_drop_whitelisted_packets = never_drop_whitelisted_packets

    # Determine whether to use delayed and randomized flow mod processing
    # (Set by fuzzer_params, not by an optional __init__ argument)
    self.delay_flow_mods = False

  def _log_input_event(self, event, **kws):
    if self._input_logger is not None:
      if self._initializing():
        # Tell MCSFinder never to prune this event
        event.prunable = False

      event.round = self.logical_time
      self._input_logger.log_input_event(event, **kws)

  def _load_fuzzer_params(self, fuzzer_params_path):
    if fuzzer_params_path.endswith('.py'):
      fuzzer_params_path = fuzzer_params_path[:-3].replace("/", ".")

    try:
      self.params = __import__(fuzzer_params_path, globals(), locals(), ["*"])
      # TODO(cs): temporary hack until we get determinism figured out
      self.params.link_discovery_rate = 0.1
    except:
      raise IOError("Could not find fuzzer params config file: %s" %
                    fuzzer_params_path)

  def _compute_unblocked_controller_pairs(self):
    sorted_controllers = sorted(self.simulation.controller_manager.controllers, key=lambda c: c.cid)
    unblocked_pairs = []
    for i in xrange(0, len(sorted_controllers)):
      for j in xrange(i+1, len(sorted_controllers)):
        c1 = sorted_controllers[i]
        c2 = sorted_controllers[j]
        # Make sure all controller pairs are unblocked on startup
        c1.unblock_peer(c2)
        c2.unblock_peer(c1)
        unblocked_pairs.append((c1.cid, c2.cid))
    return unblocked_pairs

  def init_results(self, results_dir):
    if self._input_logger:
      self._input_logger.open(results_dir)
    params_file = re.sub(r'\.pyc$', '.py', self.params.__file__)
    # Move over our fuzzer params
    # TODO(cs): need to modify copied config file to point to the new fuzzer
    # params
    if os.path.exists(params_file):
      new_params_file = os.path.join(results_dir, os.path.basename(params_file))
{'fraction_non_alphanumeric': 0.06599713055954089, 'fraction_numerical': 0.0063002931819599525, 'mean_word_length': 3.585812356979405, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1523212', 'n_tokens_mistral': 4633, 'n_tokens_neox': 4426, 'n_words': 1412}