<commit_before>module CDR

require 'csv'
require 'rubygems'
require 'net/ssh'
require 'net/sftp'
  
class Fetcher
  attr :hostname, false
  attr :username, false
  attr :password, false
  attr :base_dir, false
  attr :last_dir, true
  attr :last_file, true
  
  # Fetch CDR's from a remote server.
  #
  # :base_dir defaults to "/var/log/asterisk/cdr-csv".
  # :last_dir defaults to the first sub-directory
  # in :base_dir.
  # :last_file defaults to the first file in :last_dir.
  # 
  # Usage:
  #
  #  require 'cdr-fetcher'
  #  require 'pp'
  #  
  #  cdrs = CDR::Fetcher.new(
  #    'hostname', 
  #	   'username',
  #	   :password => 'password',
  #	   :base_dir => '/var/log/asterisk/cdr-csv',
  #	   :last_dir => 'old-cdrs',
  #	   :last_file=> 'old-cdr.csv')  
  #
  #  cdrs.each do |cdr|
  #    pp cdr
  #  end
  #
  def initialize(hostname, username, opts={})
    opts = { :password => nil,
             :base_dir => nil,
             :last_dir => nil,
             :last_file=> nil
           }.merge!(opts)
    @hostname   = hostname
    @username   = username
    @password   = opts[:password]
    @base_dir   = opts[:base_dir]
    @base_dir ||= '/var/log/asterisk/cdr-csv'

    # Setup SSH and SFTP Connection
    @ssh      = Net::SSH.start(@hostname, @username, :password => @password)
    @sftp     = Net::SFTP::Session.new(@ssh).connect!

    # Grab the list of directories, and the current directory attributes
    @directories  = directories
    @cur_dir      = directory(opts[:last_dir]) 
    # If we weren't passed an argument, use the first file in the list
    @cur_dir    ||= @directories[0]
        
    # Grab the list of files from the current directory
    @files      = files
    @cur_file   = file(opts[:last_file])
    # If we weren't passed an argument, use the first file in the list
    @cur_file ||= @files[0]
  end
  
  # Iterator for Files.  Returns one file path at a time, automatically
  # moving to the next file or directory until all CDRs have been
  # returned.
  def each_file
    while true
      file = @cur_file
      if file
        yield [@base_dir,@cur_dir.name,@cur_file.name].join("/")
        next_file!
      else
        break
      end
    end        
  end
  
  # Iterator for CDRs.  Returns one CDR at a time, automatically
  # moving to the next file or directory until all CDRs have been
  # returned.
  #
  # CDR files are read into memory one at a time.
  #
  # Line numbers are NOT zero base (i.e. line 1 = line 1 vs line 0)
  def each
    each_file do |file|
      @sftp.download!(file).split(/\n/).each_with_index do |line,i|
        cdr = {
          :hostname => @hostname, 
          :file => file, 
          :line => i+1,
          :cdr  => CSV.parse_line(line)
          }
        yield cdr
      end
    end
  end
  
  protected

    # Set @cur_file to the next file.
    # Changes to next directory if all files in the current directory<commit_msg>Fixes!<commit_after>module CDR

require 'csv'
require 'rubygems'
require 'net/ssh'
require 'net/sftp'

# Fetch CDR's from a remote server.
#
# :base_dir defaults to "/var/log/asterisk/cdr-csv".
# :last_dir defaults to the first sub-directory
# in :base_dir.
# :last_file defaults to the first file in :last_dir.
# 
# Usage:
#
#  require 'cdr-fetcher'
#  require 'pp'
#  
#  cdrs = CDR::Fetcher.new(
#    'hostname', 
#	   'username',
#	   :password => 'password',
#	   :base_dir => '/var/log/asterisk/cdr-csv',
#	   :last_dir => 'old-cdrs',
#	   :last_file=> 'old-cdr.csv')  
#
#  cdrs.each do |cdr|
#    pp cdr
#  end
#  
class Fetcher
  attr :hostname, false
  attr :username, false
  attr :password, false
  attr :base_dir, false
  attr :last_dir, true
  attr :last_file, true
  
  # Create a new instance of Fetcher
  #
  #  :base_dir defaults to "/var/log/asterisk/cdr-csv".
  #  :last_dir defaults to the first sub-directory
  #   in :base_dir.
  #  :last_file defaults to the first file in :last_dir.
  def initialize(hostname, username, opts={})
    # Process options and arguments
    opts = { :password => <PASSWORD>,
             :base_dir => nil,
             :last_dir => nil,
             :last_file=> nil
           }.merge!(opts)
    @hostname   = hostname
    @username   = username
    @password   = opts[:password]
    @base_dir   = opts[:base_dir]
    @base_dir ||= '/var/log/asterisk/cdr-csv'

    # Setup SSH and SFTP Connection
    @ssh      = Net::SSH.start(@hostname, @username, :password => @password)
    @sftp     = Net::SFTP::Session.new(@ssh).connect!

    # Grab the list of directories, and the current directory attributes
    @directories  = directories
    @cur_dir      = directory(opts[:last_dir]) 
    # If we weren't passed an argument, use the first file in the list
    @cur_dir    ||= @directories[0]
        
    # Grab the list of files from the current directory
    @files      = files
    @cur_file   = file(opts[:last_file])
    # If we weren't passed an argument, use the first file in the list
    @cur_file ||= @files[0]
  end
  
  # Iterator for Files.  Returns one file path at a time, automatically
  # moving to the next file or directory until all CDRs have been
  # returned.
  def each_file
    while true
      file = @cur_file
      if file
        yield [@base_dir,@cur_dir.name,@cur_file.name].join("/")
        next_file!
      else
        break
      end
    end        
  end
  
  # Iterator for CDRs.  Returns one CDR at a time, automatically
  # moving to the next file or directory until all CDRs have been
  # returned.
  #
  # CDR files are read into memory one at a time.
  #
  # Line numbers are NOT zero indexed (i.e. line 1 == line 1 NOT line 0)
  def each
    each_file do |file|
      @sftp.download!(file).split(/\n/).each_with_index do |line,i|
        cdr = {
          :hostname => @hostname, 
          :file => file, 
          :line => i+1,
          :cdr  => CSV.parse_line(line)
          }
        yield cdr
      end
    end
  end
  
  protected

    # Set @cur_file to the next file.
    # Changes to next directory if all files in the current directory
{'fraction_non_alphanumeric': 0.11193537751401253, 'fraction_numerical': 0.0019782393669634025, 'mean_word_length': 2.3335164835164837, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6784246', 'n_tokens_mistral': 2094, 'n_tokens_neox': 2050, 'n_words': 629}