<commit_before>import collections
import itertools
from typing import Iterable, List, NamedTuple, Optional, Tuple

from data import max_heap
from data.alphabets import morse
from puzzle.problems import problem

_IGNORE_DELIMITER = 'ignored'
_CHARACTER_DELIMITER = 'character delimiter'
_WORD_DELIMITER = 'word delimiter'
_DOT = '.'
_DASH = '-'

class _Interpretation(NamedTuple):
  dot: str
  dash: str
  character_delimiter: Optional[str]
  word_delimiter: Optional[str]
  ignored: set


# "Most common" interpretation.
_ALPHABET = _Interpretation(
    '.',
    '-',
    ' ',
    '/',
    set(),
)


class MorseProblem(problem.Problem):
  def __init__(self, name: str, lines: List[str], **kwargs) -> None:
    super(MorseProblem, self).__init__(name, lines, **kwargs)
    self._normalized = '\n'.join(lines)
    self._frequencies = collections.Counter(self._normalized)
    for c in list(self._frequencies.keys()):
      if c.isspace():  # Ignore whitespace in input.
        del self._frequencies[c]

  @staticmethod
  def score(lines: List[str]) -> float:
    return _score(lines)

  def _solve_iter(self) -> problem.Solutions:
    fringe: max_heap.MaxHeap[problem.Solutions] = max_heap.MaxHeap()
    solution_buffer: max_heap.MaxHeap[problem.Solutions] = max_heap.MaxHeap()
    # Initialize options with weights.
    for weight, interpretation in _generate_interpretations(self._normalized):
      fringe.push(weight, self._iter_interpretation(interpretation))
    while fringe:
      group_generator, group_weight = fringe.pop_with_weight()
      if fringe:
        next_best_weight = fringe.best_weight()
      else:
        next_best_weight = float('-inf')
      yield from solution_buffer.pop_with_weight_until(next_best_weight)
      for result in group_generator:
        (solution, weight), notes = result
        self._notes[solution] = notes  # Save (and throw away) notes.
        solution_weight = group_weight * weight
        if solution_weight > next_best_weight:
          yield solution, solution_weight
        else:
          solution_buffer.push(solution_weight, solution)
          # Abandon this generator for now.
          fringe.push(group_weight, group_generator)
          break
    yield from solution_buffer.pop_with_weight_until(float('-inf'))

  def _iter_interpretation(
      self, interpretation: _Interpretation) -> problem.Solutions:
    acc = []<commit_msg>Use types from new location.<commit_after>import collections
import itertools
from typing import Iterable, List, NamedTuple, Optional, Tuple

from data import max_heap
from data.alphabets import morse
from puzzle.problems import problem
from puzzle.steps import generate_solutions

_IGNORE_DELIMITER = 'ignored'
_CHARACTER_DELIMITER = 'character delimiter'
_WORD_DELIMITER = 'word delimiter'
_DOT = '.'
_DASH = '-'

class _Interpretation(NamedTuple):
  dot: str
  dash: str
  character_delimiter: Optional[str]
  word_delimiter: Optional[str]
  ignored: set


# "Most common" interpretation.
_ALPHABET = _Interpretation(
    '.',
    '-',
    ' ',
    '/',
    set(),
)


class MorseProblem(problem.Problem):
  def __init__(self, name: str, lines: List[str], **kwargs) -> None:
    super(MorseProblem, self).__init__(name, lines, **kwargs)
    self._normalized = '\n'.join(lines)
    self._frequencies = collections.Counter(self._normalized)
    for c in list(self._frequencies.keys()):
      if c.isspace():  # Ignore whitespace in input.
        del self._frequencies[c]

  @staticmethod
  def score(lines: List[str]) -> float:
    return _score(lines)

  def _solve_iter(self) -> generate_solutions.Solutions:
    fringe: max_heap.MaxHeap[generate_solutions.Solutions] = max_heap.MaxHeap()
    solution_buffer: max_heap.MaxHeap[generate_solutions.Solutions] = (
      max_heap.MaxHeap())
    # Initialize options with weights.
    for weight, interpretation in _generate_interpretations(self._normalized):
      fringe.push(weight, self._iter_interpretation(interpretation))
    while fringe:
      group_generator, group_weight = fringe.pop_with_weight()
      if fringe:
        next_best_weight = fringe.best_weight()
      else:
        next_best_weight = float('-inf')
      yield from solution_buffer.pop_with_weight_until(next_best_weight)
      for result in group_generator:
        (solution, weight), notes = result
        self._notes[solution] = notes  # Save (and throw away) notes.
        solution_weight = group_weight * weight
        if solution_weight > next_best_weight:
          yield solution, solution_weight
        else:
          solution_buffer.push(solution_weight, solution)
          # Abandon this generator for now.
          fringe.push(group_weight, group_generator)
          break
    yield from solution_buffer.pop_with_weight_until(float('-inf'))

  def _iter_interpretation(
      self, interpretation: _Interpretation) -> generate_solutions.Solutions:
    acc = []
{'fraction_non_alphanumeric': 0.09865470852017937, 'fraction_numerical': 0.0, 'mean_word_length': 3.6822519083969465, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3814861', 'n_tokens_mistral': 1514, 'n_tokens_neox': 1487, 'n_words': 346}