<commit_before>require 'nokogiri'

class Importer
  attr_accessor :wp_xml
  attr_accessor :posts

  def initialize( logger )
    @logger = logger
  end

  def parse_wordpress xml
    @posts = []
    @attachments = []
    @logger.notify "Loading posts from #{xml}\n"
    @wp_xml = Nokogiri::XML( File.open(xml) )
    @wp_xml.remove_namespaces!
    @wp_xml.css('item').each do |item|
      process_item item
    end
  end

  def process_item item
    if item.at_css('attachment_url')
      @attachments << AttachmentBuilder.from_item(item)
    else
      @posts << PostBuilder.from_item(item)
    end
  end

  def show_posts
    @posts.each do |p|
      @logger.warn "***********************"
      @logger.warn p.title
      @logger.warn "***********************"
      @logger.log p.content[0..60]
      @logger.error( "No content...") if p.content.length == 0
      @logger.warn "     \n"
    end
  end

  def show_attachments
    @attachments.each do |a|
      @logger.warn "***********************"<commit_msg>Better indicator of post type<commit_after>require 'nokogiri'

class Importer
  attr_accessor :wp_xml
  attr_accessor :posts

  def initialize( logger )
    @logger = logger
  end

  def parse_wordpress xml
    @posts = []
    @attachments = []
    @logger.notify "Loading posts from #{xml}\n"
    @wp_xml = Nokogiri::XML( File.open(xml) )
    @wp_xml.remove_namespaces!
    @wp_xml.css('item').each do |item|
      process_item item
    end
  end

  def process_item item
    if item.at_css('post_type').text == 'attachment'
      @attachments << AttachmentBuilder.from_item(item)
    else
      @posts << PostBuilder.from_item(item)
    end
  end

  def show_posts
    @posts.each do |p|
      @logger.warn "***********************"
      @logger.warn p.title
      @logger.warn "***********************"
      @logger.log p.content[0..60]
      @logger.error( "No content...") if p.content.length == 0
      @logger.warn "     \n"
    end
  end

  def show_attachments
    @attachments.each do |a|
      @logger.warn "***********************"
{'fraction_non_alphanumeric': 0.1793002915451895, 'fraction_numerical': 0.003887269193391642, 'mean_word_length': 2.841417910447761, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3909951', 'n_tokens_mistral': 693, 'n_tokens_neox': 681, 'n_words': 153}