<commit_before># Copyright 2013 Square Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

require 'fileutils'

# A state in a {Project}'s source history. A commit, in the context of this
# program, is a point in the history of a Project that is either approved or not
# approved for release, from a localization standpoint.
#
# Each new Commit, when created, is scanned for importable blobs. These blobs
# are scraped for {Key Keys} and {Translation Translations}, which are added to
# the corpus, and new {Blob} records are created. If the commit shares blobs
# with an already-imported commit, the blob is skipped.
#
# Existing keys are updated with new base translations, if the copy has changed,
# and translations are marked as pending review. Translations are created for
# any new Keys and marked as pending translation and review.
#
# Once all Translations of a Commit are translated and reviewed,
# that Commit is considered localized for that locale. Once all required
# locales have been reviewed, the Commit is ready for release. `after_save`
# hooks on {Translation} automatically manage the `ready` field, and the
# various workers method manage the `loading` field.
#
# Keys are child to a Project, not a Commit. A Commit has a many-to-many
# association that tracks which keys can be found under that commit.
#
# Associations
# ============
#
# |                |                                                      |
# |:---------------|:-----------------------------------------------------|
# | `project`      | The {Project} this is a Commit under.                |
# | `keys`         | All the {Key Keys} found in this Commit.             |
# | `translations` | The {Translation Translations} found in this Commit. |
#
# Properties
# ==========
#
# |                |                                                                                                                            |
# |:---------------|:---------------------------------------------------------------------------------------------------------------------------|
# | `committed_at` | The time this commit was made.                                                                                             |
# | `message`      | The commit message.                                                                                                        |
# | `ready`        | If `true`, all Keys under this Commit are marked as ready.                                                                 |
# | `revision`     | The SHA1 for this commit.                                                                                                  |
# | `loading`      | If `true`, there is at least one {BlobImporter} processing this Commit. |

class Commit < ActiveRecord::Base
  # @return [true, false] If `true`, does not perform an import after creating
  #   the Commit. Use this to avoid the overhead of making an HTTP request and
  #   spawning a worker for situations where Commits are being added in bulk.
  attr_accessor :skip_import

  belongs_to :project, inverse_of: :commits
  has_and_belongs_to_many :keys, uniq: true
  has_many :translations, through: :keys

  validates :project,
            presence: true
  validates :revision_raw,
            presence:   true,
            uniqueness: {scope: :project_id}
  validates :message,
            presence: true,
            length:   {maximum: 256}
  validates :committed_at,
            presence:   true,
            timeliness: {type: :time}

  extend GitObjectField
  git_object_field :revision,
                   git_type:        :commit,
                   repo:            ->(c) { c.project.try(:repo) },
                   repo_must_exist: true,
                   scope:           :for_revision

  before_validation :load_message, on: :create
  before_validation(on: :create) do |obj|
    obj.message = obj.message.truncate(256) if obj.message
  end

  after_commit(on: :create) do |commit|
    CommitImporter.perform_once(commit.id) unless commit.skip_import
  end
  after_commit :compile_and_cache_or_clear, on: :update

  attr_accessible :revision, :message, :committed_at, :skip_import,
                  :skip_sha_check, as: :system
  attr_accessible :revision, as: :admin
  attr_readonly :revision, :message

  # @private
  def to_param() revision end

  # Calculates the value of the `ready` field and saves the record.

  def recalculate_ready!
    ready = !keys.where(ready: false).exists?
    update_column :ready, ready
    compile_and_cache_or_clear(ready)
  end

  # Returns `true` if all Translations applying to this commit have been
  # translated to this locale and reviewed.
  #
  # @param [Locale] locale The locale.
  # @return [true, false] Whether localization is complete for that locale.

  def localized?(locale)
    translations.where(rfc5646_locale: locale.rfc5646).where('approved IS NOT TRUE').count == 0
  end

  # Recursively locates blobs in this commit, creates Blobs for each of them if
  # necessary, and calls {Blob#import_strings} on them.
  #
  # @param [Hash] options Import options.
  # @option options [Locale] locale The locale to assume the base copy is
  #   written in (by default it's the Project's base locale).
  # @option options [true, false] inline (false) If `true`, does not spawn
  #   Sidekiq workers to perform the import in parallel.
  # @option options [true, false] force (false) If `true`, blobs will be
  #   re-scanned for keys even if they have already been scanned.
  # @raise [CommitNotFoundError] If the commit could not be found in the Git
  #   repository.

  def import_strings(options={})
    raise CommitNotFoundError, "Commit no longer exists: #{revision}" unless commit!
    keys.clear
    import_tree commit!.gtree, '', options
    update_stats_at_end_of_loading if options[:inline]
  end

  # Returns a commit object used to interact with Git.
  #
  # @return [Git::Object::Commit, nil] The commit object.

  def commit
    project.repo.object(revision)
  end

  # Same as {#commit}, but fetches the upstream repository changes if the commit
  # is unrecognized.
  #<commit_msg>Fix a bug where a locale import would incorrectly delete all keys<commit_after># Copyright 2013 Square Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

require 'fileutils'

# A state in a {Project}'s source history. A commit, in the context of this
# program, is a point in the history of a Project that is either approved or not
# approved for release, from a localization standpoint.
#
# Each new Commit, when created, is scanned for importable blobs. These blobs
# are scraped for {Key Keys} and {Translation Translations}, which are added to
# the corpus, and new {Blob} records are created. If the commit shares blobs
# with an already-imported commit, the blob is skipped.
#
# Existing keys are updated with new base translations, if the copy has changed,
# and translations are marked as pending review. Translations are created for
# any new Keys and marked as pending translation and review.
#
# Once all Translations of a Commit are translated and reviewed,
# that Commit is considered localized for that locale. Once all required
# locales have been reviewed, the Commit is ready for release. `after_save`
# hooks on {Translation} automatically manage the `ready` field, and the
# various workers method manage the `loading` field.
#
# Keys are child to a Project, not a Commit. A Commit has a many-to-many
# association that tracks which keys can be found under that commit.
#
# Associations
# ============
#
# |                |                                                      |
# |:---------------|:-----------------------------------------------------|
# | `project`      | The {Project} this is a Commit under.                |
# | `keys`         | All the {Key Keys} found in this Commit.             |
# | `translations` | The {Translation Translations} found in this Commit. |
#
# Properties
# ==========
#
# |                |                                                                                                                            |
# |:---------------|:---------------------------------------------------------------------------------------------------------------------------|
# | `committed_at` | The time this commit was made.                                                                                             |
# | `message`      | The commit message.                                                                                                        |
# | `ready`        | If `true`, all Keys under this Commit are marked as ready.                                                                 |
# | `revision`     | The SHA1 for this commit.                                                                                                  |
# | `loading`      | If `true`, there is at least one {BlobImporter} processing this Commit. |

class Commit < ActiveRecord::Base
  # @return [true, false] If `true`, does not perform an import after creating
  #   the Commit. Use this to avoid the overhead of making an HTTP request and
  #   spawning a worker for situations where Commits are being added in bulk.
  attr_accessor :skip_import

  belongs_to :project, inverse_of: :commits
  has_and_belongs_to_many :keys, uniq: true
  has_many :translations, through: :keys

  validates :project,
            presence: true
  validates :revision_raw,
            presence:   true,
            uniqueness: {scope: :project_id}
  validates :message,
            presence: true,
            length:   {maximum: 256}
  validates :committed_at,
            presence:   true,
            timeliness: {type: :time}

  extend GitObjectField
  git_object_field :revision,
                   git_type:        :commit,
                   repo:            ->(c) { c.project.try(:repo) },
                   repo_must_exist: true,
                   scope:           :for_revision

  before_validation :load_message, on: :create
  before_validation(on: :create) do |obj|
    obj.message = obj.message.truncate(256) if obj.message
  end

  after_commit(on: :create) do |commit|
    CommitImporter.perform_once(commit.id) unless commit.skip_import
  end
  after_commit :compile_and_cache_or_clear, on: :update

  attr_accessible :revision, :message, :committed_at, :skip_import,
                  :skip_sha_check, as: :system
  attr_accessible :revision, as: :admin
  attr_readonly :revision, :message

  # @private
  def to_param() revision end

  # Calculates the value of the `ready` field and saves the record.

  def recalculate_ready!
    ready = !keys.where(ready: false).exists?
    update_column :ready, ready
    compile_and_cache_or_clear(ready)
  end

  # Returns `true` if all Translations applying to this commit have been
  # translated to this locale and reviewed.
  #
  # @param [Locale] locale The locale.
  # @return [true, false] Whether localization is complete for that locale.

  def localized?(locale)
    translations.where(rfc5646_locale: locale.rfc5646).where('approved IS NOT TRUE').count == 0
  end

  # Recursively locates blobs in this commit, creates Blobs for each of them if
  # necessary, and calls {Blob#import_strings} on them.
  #
  # @param [Hash] options Import options.
  # @option options [Locale] locale The locale to assume the base copy is
  #   written in (by default it's the Project's base locale).
  # @option options [true, false] inline (false) If `true`, does not spawn
  #   Sidekiq workers to perform the import in parallel.
  # @option options [true, false] force (false) If `true`, blobs will be
  #   re-scanned for keys even if they have already been scanned.
  # @raise [CommitNotFoundError] If the commit could not be found in the Git
  #   repository.

  def import_strings(options={})
    raise CommitNotFoundError, "Commit no longer exists: #{revision}" unless commit!

    # clear out existing keys so that we can import all new keys
    keys.clear unless options[:locale]
    # perform the recursive import
    import_tree commit!.gtree, '', options
    # normally this is performed once the last worker is removed from the list,
    # but if we're not using workers, we need to do it here
    update_stats_at_end_of_loading if options[:inline]
  end

  # Returns a commit object used to interact with Git.
  #
  # @return [Git::Object::Commit, nil] The commit object.

  def commit
    project.repo.object(revision)
  end

  # Same as {#commit}, but fetches the upstream repository changes if the commit
  # is unrecognized.
  #
{'fraction_non_alphanumeric': 0.10677829184108101, 'fraction_numerical': 0.003525005507821106, 'mean_word_length': 2.119102153000458, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1263268', 'n_tokens_mistral': 3509, 'n_tokens_neox': 3380, 'n_words': 1440}