<commit_before># encoding: utf-8

class Book < ActiveRecord::Base

  include Syncable

  has_and_belongs_to_many :notes

  default_scope { order('tag') }

  scope :citable, -> { where('title IS NOT ? AND tag IS NOT ? AND published_date IS NOT ?', nil, nil, nil) }
  scope :editable, -> { order('updated_at DESC') }
  scope :metadata_missing, -> { where('author IS ? OR title IS ? OR published_date IS ?', nil, nil, nil).order('updated_at DESC') }
  scope :cited, -> { where('title IS NOT ? AND tag IS NOT ?', nil, nil)
    .joins('left outer join books_notes on books.id = books_notes.book_id')
    .where('books_notes.book_id IS NOT ?', nil)
    .uniq } # OPTIMIZE: Notes must be active and not hidden (publishable) see http://stackoverflow.com/questions/3875564

  validates :isbn_10, :isbn_13, uniqueness: true, allow_blank: true
  validates :isbn_13, presence: true, if: 'isbn_10.blank?'
  validates :isbn_10, isbn_format: { with: :isbn10 }, allow_blank: true
  validates :isbn_13, isbn_format: { with: :isbn13 }, allow_blank: true
  validates :full_text_url, url: true, allow_blank: true

  before_validation :update_tag,
                    if: (:author_changed? || :editor_changed? || :published_date_changed?) && '!published_date.blank?',
                     unless: :tag_changed? # || '!tag.blank?'
  before_validation :scan_notes_for_references, if: :tag_changed?

  paginates_per Setting['advanced.books_index_per_page'].to_i

  extend FriendlyId
  friendly_id :tag, use: :slugged

  def to_param
    slug
  end

  def self.grab_isbns(text)
    text.scan(/([0-9\-]{9,17}X?)/).each { |isbn_candidate| add_task(isbn_candidate.first) }
  end

  def self.add_task(isbn_candidate)
    isbn_candidate.gsub!(/[^\dX]/, '')
    book = where(isbn_10: isbn_candidate).first_or_create if isbn_candidate.length == 10
    book = where(isbn_13: isbn_candidate).first_or_create if isbn_candidate.length == 13
    # We can't use dirtify here because this is a class method
    if book
      book.dirty = true
      book.attempts = 0
      book.save
      book
    end
  end

  def self.sync_all
    need_syncdown.each { |book| book.populate! }
  end

  def isbn
    (isbn_10.blank? ? isbn_13 : isbn_10)
  end

  def short_title
    title.gsub(/\:.*$/, '')
  end
<commit_msg>Only request books with missing metadata<commit_after># encoding: utf-8

class Book < ActiveRecord::Base

  include Syncable

  has_and_belongs_to_many :notes

  default_scope { order('tag') }

  scope :citable, -> { where('title IS NOT ? AND tag IS NOT ? AND published_date IS NOT ?', nil, nil, nil) }
  scope :editable, -> { order('updated_at DESC') }
  scope :metadata_missing, -> { where('author IS ? OR title IS ? OR published_date IS ?', nil, nil, nil).order('updated_at DESC') }
  scope :cited, -> { where('title IS NOT ? AND tag IS NOT ?', nil, nil)
    .joins('left outer join books_notes on books.id = books_notes.book_id')
    .where('books_notes.book_id IS NOT ?', nil)
    .uniq } # OPTIMIZE: Notes must be active and not hidden (publishable) see http://stackoverflow.com/questions/3875564

  validates :isbn_10, :isbn_13, uniqueness: true, allow_blank: true
  validates :isbn_13, presence: true, if: 'isbn_10.blank?'
  validates :isbn_10, isbn_format: { with: :isbn10 }, allow_blank: true
  validates :isbn_13, isbn_format: { with: :isbn13 }, allow_blank: true
  validates :full_text_url, url: true, allow_blank: true

  before_validation :update_tag,
                    if: (:author_changed? || :editor_changed? || :published_date_changed?) && '!published_date.blank?',
                     unless: :tag_changed? # || '!tag.blank?'
  before_validation :scan_notes_for_references, if: :tag_changed?

  paginates_per Setting['advanced.books_index_per_page'].to_i

  extend FriendlyId
  friendly_id :tag, use: :slugged

  def to_param
    slug
  end

  def self.grab_isbns(text)
    text.scan(/([0-9\-]{9,17}X?)/).each { |isbn_candidate| add_task(isbn_candidate.first) }
  end

  def self.add_task(isbn_candidate)
    isbn_candidate.gsub!(/[^\dX]/, '')
    book = where(isbn_10: isbn_candidate).first_or_create if isbn_candidate.length == 10
    book = where(isbn_13: isbn_candidate).first_or_create if isbn_candidate.length == 13
    # We can't use dirtify here because this is a class method
    if book
      book.dirty = true
      book.attempts = 0
      book.save
      book
    end
  end

  def self.sync_all
    need_syncdown.metadata_missing.each { |book| book.populate! }
  end

  def isbn
    (isbn_10.blank? ? isbn_13 : isbn_10)
  end

  def short_title
    title.gsub(/\:.*$/, '')
  end

{'fraction_non_alphanumeric': 0.1246180707114797, 'fraction_numerical': 0.01920558707987778, 'mean_word_length': 3.6153071500503526, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6914894', 'n_tokens_mistral': 1698, 'n_tokens_neox': 1611, 'n_words': 399}