<commit_before>#!/usr/bin/env ruby

# Add the path to your feedcache directory here
FEEDCACHE_DIR = '/path/to/your/wordpress/wp-content/plugins/feedcache'
# How many characters from each feed item do you want to display
CHAR_COUNT = 75
# Set to 'true' if you want to receive error emails from the CRON job
CRON_EMAILS = false
# Run as threaded
THREADED = false


#################################################################
#                                                               #
#  DO NOT EDIT BELOW THIS LINE                                  #
#                                                               #
#################################################################
$LOAD_PATH << File.expand_path(File.dirname(__FILE__))

require 'net/http'
require 'lib/feedparser'
require 'uri'
require 'yaml'

# Read master config settings
MASTER_CONFIG = "#{FEEDCACHE_DIR}/master-config.txt"
cfg = File.open(MASTER_CONFIG, 'r') do |f|
  @params = f.gets.split('~');
end
# parse the parameters from the config file
@groups_num   = @params[0].strip.to_i
@display_num  = @params[1].strip
@title_pre    = @params[2].strip
@title_post   = @params[3].strip
@format_text  = @params[4].strip == 'true' ? true : false
@link_target  = @params[5].strip == 'true' ? '_blank' : '_self'

# Load config and cache file variables
CONFIG_FILE = "#{FEEDCACHE_DIR}/files/feedcache-config.yml"
CACHE_FILES = []
1.upto(@groups_num) do |i|
  CACHE_FILES <<  "#{FEEDCACHE_DIR}/files/feedcache-cache#{i}.txt"
end

# RSS formatting function
def shorten_text(txt)
  if txt.size > CHAR_COUNT
    @text = "#{txt} ".slice(0,CHAR_COUNT)
    # need to break on the last space
    if @text.include?(' ') and @text.slice(@text.size-1, 1) != ' '
      @text.slice!(0, @text.size - (@text.reverse.index(' ') + 1))
      @text << '...'
    end
    return @text
  else
    return txt
  end
end

if THREADED
  # fork a thread for each config file here
  send_cron_emails = CRON_EMAILS ? '-e' : ''
  CONFIG_FILES.each do |config|
    pid = fork {
      # exec scripts here
      system("/usr/bin/env ruby feedcache-lite.rb -p #{config} -n #{@display_num.to_i} -f #{@format_text} -l #{@link_target} -c #{CHAR_COUNT.to_i} #{send_cron_emails}")
    }
    Process.detach(pid)
  end

else

  begin # read the config file settings
    @all_feeds = {}
    yaml_config = YAML.load_file(CONFIG_FILE)
    1.upto(@groups_num).each do |num|
      yaml_config["group#{num}"].each {|x| @feeds << x.strip unless x.strip.blank? }
      @all_feeds[num] = @feeds
    end
  rescue => e
    if CRON_EMAILS
      puts "Error reading YAML configuration file"
      puts YAML.dump(e)
    end
  end  

  @all_feeds.each do |k,v|
    @tmp = ''
    @processed = 0<commit_msg>need to set initial feeds array<commit_after>#!/usr/bin/env ruby

# Add the path to your feedcache directory here
FEEDCACHE_DIR = '/path/to/your/wordpress/wp-content/plugins/feedcache'
# How many characters from each feed item do you want to display
CHAR_COUNT = 75
# Set to 'true' if you want to receive error emails from the CRON job
CRON_EMAILS = false
# Run as threaded
THREADED = false


#################################################################
#                                                               #
#  DO NOT EDIT BELOW THIS LINE                                  #
#                                                               #
#################################################################
$LOAD_PATH << File.expand_path(File.dirname(__FILE__))

require 'net/http'
require 'lib/feedparser'
require 'uri'
require 'yaml'

# Read master config settings
MASTER_CONFIG = "#{FEEDCACHE_DIR}/master-config.txt"
cfg = File.open(MASTER_CONFIG, 'r') do |f|
  @params = f.gets.split('~');
end
# parse the parameters from the config file
@groups_num   = @params[0].strip.to_i
@display_num  = @params[1].strip
@title_pre    = @params[2].strip
@title_post   = @params[3].strip
@format_text  = @params[4].strip == 'true' ? true : false
@link_target  = @params[5].strip == 'true' ? '_blank' : '_self'

# Load config and cache file variables
CONFIG_FILE = "#{FEEDCACHE_DIR}/files/feedcache-config.yml"
CACHE_FILES = []
1.upto(@groups_num) do |i|
  CACHE_FILES <<  "#{FEEDCACHE_DIR}/files/feedcache-cache#{i}.txt"
end

# RSS formatting function
def shorten_text(txt)
  if txt.size > CHAR_COUNT
    @text = "#{txt} ".slice(0,CHAR_COUNT)
    # need to break on the last space
    if @text.include?(' ') and @text.slice(@text.size-1, 1) != ' '
      @text.slice!(0, @text.size - (@text.reverse.index(' ') + 1))
      @text << '...'
    end
    return @text
  else
    return txt
  end
end

if THREADED
  # fork a thread for each config file here
  send_cron_emails = CRON_EMAILS ? '-e' : ''
  CONFIG_FILES.each do |config|
    pid = fork {
      # exec scripts here
      system("/usr/bin/env ruby feedcache-lite.rb -p #{config} -n #{@display_num.to_i} -f #{@format_text} -l #{@link_target} -c #{CHAR_COUNT.to_i} #{send_cron_emails}")
    }
    Process.detach(pid)
  end

else

  begin # read the config file settings
    @all_feeds = {}
    yaml_config = YAML.load_file(CONFIG_FILE)
    1.upto(@groups_num).each do |num|
      feeds = []
      yaml_config["group#{num}"].each {|x| feeds << x.strip unless x.strip.blank? }
      @all_feeds[num] = feeds
    end
  rescue => e
    if CRON_EMAILS
      puts "Error reading YAML configuration file"
      puts YAML.dump(e)
    end
  end  

  @all_feeds.each do |k,v|
    @tmp = ''
    @processed = 0
{'fraction_non_alphanumeric': 0.17466691002007664, 'fraction_numerical': 0.005840481839751779, 'mean_word_length': 2.861874559548978, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5477480', 'n_tokens_mistral': 1853, 'n_tokens_neox': 1787, 'n_words': 469}