<commit_before># -*- coding: utf-8 -*-

#    Copyright 2013 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.


require 'erb'

module Astute
  module LogParser
    LOG_PORTION = 10000
    # Default values. Can be overrided by pattern_spec.
    # E.g. pattern_spec = {'separator' => 'new_separator', ...}
    PATH_PREFIX = '/var/log/remote/'
    SEPARATOR = "SEPARATOR\n"

    class NoParsing
      def initialize(*args)
      end

      def method_missing(*args)
        # We just eat the call if we don't want to deal with logs
      end

      def progress_calculate(*args)
        []
      end
    end

    class DirSizeCalculation
      attr_reader :nodes

      def initialize(nodes)
        @nodes = nodes.map{|n| n.dup}
        @nodes.each{|node| node[:path_items] = weight_reassignment(node[:path_items])}
      end

      def deploy_type=(*args)
        # Because we mimic the DeploymentParser, we should define all auxiliary method
        # even they do nothing.
      end

      def progress_calculate(uids_to_calc, nodes)
        uids_to_calc.map do |uid|
          node = @nodes.find{|n| n[:uid] == uid}
          node[:path_items] ||= []
          progress = 0
          node[:path_items].each do |item|
            size = recursive_size(item[:path])
            sub_progress = 100 * size / item[:max_size]
            sub_progress = 0 if sub_progress < 0
            sub_progress = 100 if sub_progress > 100
            progress += sub_progress * item[:weight]
          end
          {'uid' => uid, 'progress' => progress.to_i}
        end
      end

      private
      def recursive_size(path, opts={})
        return File.size?(path).to_i if not File.directory?(path)<commit_msg>Add stub prepare method for DirSizeCalculation<commit_after># -*- coding: utf-8 -*-

#    Copyright 2013 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.


require 'erb'

module Astute
  module LogParser
    LOG_PORTION = 10000
    # Default values. Can be overrided by pattern_spec.
    # E.g. pattern_spec = {'separator' => 'new_separator', ...}
    PATH_PREFIX = '/var/log/remote/'
    SEPARATOR = "SEPARATOR\n"

    class NoParsing
      def initialize(*args)
      end

      def method_missing(*args)
        # We just eat the call if we don't want to deal with logs
      end

      def progress_calculate(*args)
        []
      end
    end

    class DirSizeCalculation
      attr_reader :nodes

      def initialize(nodes)
        @nodes = nodes.map{|n| n.dup}
        @nodes.each{|node| node[:path_items] = weight_reassignment(node[:path_items])}
      end

      def deploy_type=(*args)
        # Because we mimic the DeploymentParser, we should define all auxiliary method
        # even they do nothing.
      end

      def prepare(nodes)
        # Because we mimic the DeploymentParser, we should define all auxiliary method
        # even they do nothing.
      end

      def progress_calculate(uids_to_calc, nodes)
        uids_to_calc.map do |uid|
          node = @nodes.find{|n| n[:uid] == uid}
          node[:path_items] ||= []
          progress = 0
          node[:path_items].each do |item|
            size = recursive_size(item[:path])
            sub_progress = 100 * size / item[:max_size]
            sub_progress = 0 if sub_progress < 0
            sub_progress = 100 if sub_progress > 100
            progress += sub_progress * item[:weight]
          end
          {'uid' => uid, 'progress' => progress.to_i}
        end
      end

      private
      def recursive_size(path, opts={})
        return File.size?(path).to_i if not File.directory?(path)
{'fraction_non_alphanumeric': 0.09262435677530018, 'fraction_numerical': 0.011149228130360206, 'mean_word_length': 2.365800865800866, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2937003', 'n_tokens_mistral': 1400, 'n_tokens_neox': 1314, 'n_words': 452}