<commit_before>require 'rollbar'
require_relative '../../models/carto/visualization'
require_relative '../../models/carto/external_source'
require_relative '../../models/common_data/singleton'

module CartoDB

  module Visualization

    class CommonDataService

      def initialize(datasets = nil)
        @datasets = datasets
      end

      def self.load_common_data(user, controller)
        if self.configured?
          common_data_url = CartoDB::Visualization::CommonDataService.build_url(controller)
          user.load_common_data(common_data_url)
        end
      end

      def self.configured?
        Cartodb.get_config(:common_data, 'base_url').present?
      end

      def self.build_url(controller)
        common_data_config = Cartodb.config[:common_data]
        return nil unless common_data_config

        common_data_base_url = common_data_config['base_url']
        common_data_username = common_data_config['username']
        common_data_user = Carto::User.where(username: common_data_username).first
        if !common_data_base_url.nil?
          # We set user_domain to nil to avoid duplication in the url for subdomainfull urls. Ie. user.carto.com/u/cartodb/...
          common_data_base_url + CartoDB.path(controller, 'api_v1_visualizations_index', {type: 'table', privacy: 'public', user_domain: nil})
        elsif !common_data_user.nil?
          CartoDB.url(controller, 'api_v1_visualizations_index', {type: 'table', privacy: 'public'}, common_data_user)
        else
          CartoDB.notify_error(
            'cant create common-data url. User doesn\'t exist and base_url is nil',
            username: common_data_username
          )
        end
      end

      def load_common_data_for_user(user, visualizations_api_url)
        update_user_date_flag(user)

        added = 0
        updated = 0
        not_modified = 0
        deleted = 0
        failed = 0

        carto_user = Carto::User.find(user.id)
        common_data_remotes_by_name = Carto::Visualization.remotes
                                                          .where(user_id: user.id)
                                                          .joins(:external_source)
                                                          .readonly(false) # joins causes readonly
                                                          .where(external_sources: { username: 'common-data' })
                                                          .map { |v| [v.name, v] }.to_h
        ActiveRecord::Base.transaction do
          get_datasets(visualizations_api_url).each do |dataset|
            begin
              visualization = common_data_remotes_by_name.delete(dataset['name'])
              if visualization
                visualization.attributes = dataset_visualization_attributes(dataset)
                if visualization.changed?
                  visualization.save!
                  updated += 1
                else
                  not_modified += 1<commit_msg>Fix hardcoded Common Data username<commit_after>require 'rollbar'
require_relative '../../models/carto/visualization'
require_relative '../../models/carto/external_source'
require_relative '../../models/common_data/singleton'

module CartoDB

  module Visualization

    class CommonDataService

      def initialize(datasets = nil)
        @datasets = datasets
      end

      def self.load_common_data(user, controller)
        if self.configured?
          common_data_url = CartoDB::Visualization::CommonDataService.build_url(controller)
          user.load_common_data(common_data_url)
        end
      end

      def self.configured?
        Cartodb.get_config(:common_data, 'base_url').present?
      end

      def self.build_url(controller)
        common_data_config = Cartodb.config[:common_data]
        return nil unless common_data_config

        common_data_base_url = common_data_config['base_url']
        common_data_username = common_data_config['username']
        common_data_user = Carto::User.where(username: common_data_username).first
        if !common_data_base_url.nil?
          # We set user_domain to nil to avoid duplication in the url for subdomainfull urls. Ie. user.carto.com/u/cartodb/...
          common_data_base_url + CartoDB.path(controller, 'api_v1_visualizations_index', {type: 'table', privacy: 'public', user_domain: nil})
        elsif !common_data_user.nil?
          CartoDB.url(controller, 'api_v1_visualizations_index', {type: 'table', privacy: 'public'}, common_data_user)
        else
          CartoDB.notify_error(
            'cant create common-data url. User doesn\'t exist and base_url is nil',
            username: common_data_username
          )
        end
      end

      def load_common_data_for_user(user, visualizations_api_url)
        update_user_date_flag(user)

        added = 0
        updated = 0
        not_modified = 0
        deleted = 0
        failed = 0

        carto_user = Carto::User.find(user.id)

        common_data_config = Cartodb.config[:common_data]
        common_data_username = common_data_config['username']
        common_data_remotes_by_name = Carto::Visualization.remotes
                                                          .where(user_id: user.id)
                                                          .joins(:external_source)
                                                          .readonly(false) # joins causes readonly
                                                          .where(external_sources: { username: common_data_username })
                                                          .map { |v| [v.name, v] }.to_h
        ActiveRecord::Base.transaction do
          get_datasets(visualizations_api_url).each do |dataset|
            begin
              visualization = common_data_remotes_by_name.delete(dataset['name'])
              if visualization
                visualization.attributes = dataset_visualization_attributes(dataset)
                if visualization.changed?
                  visualization.save!
                  updated += 1
                else
                  not_modified += 1
{'fraction_non_alphanumeric': 0.07951020408163266, 'fraction_numerical': 0.0029387755102040815, 'mean_word_length': 1.922709923664122, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3307339', 'n_tokens_mistral': 1640, 'n_tokens_neox': 1572, 'n_words': 319}