<commit_before>package google

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform-plugin-sdk/helper/customdiff"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"github.com/hashicorp/terraform-plugin-sdk/helper/validation"

	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	dataflow "google.golang.org/api/dataflow/v1b3"
	"google.golang.org/api/googleapi"
)

const resourceDataflowJobGoogleProvidedLabelPrefix = "labels.goog-dataflow-provided"

var dataflowTerminalStatesMap = map[string]struct{}{
	"JOB_STATE_DONE":      {},
	"JOB_STATE_FAILED":    {},
	"JOB_STATE_CANCELLED": {},
	"JOB_STATE_UPDATED":   {},
	"JOB_STATE_DRAINED":   {},
}

func resourceDataflowJobLabelDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
	// Example Diff: "labels.goog-dataflow-provided-template-version": "word_count" => ""
	if strings.HasPrefix(k, resourceDataflowJobGoogleProvidedLabelPrefix) && new == "" {
		// Suppress diff if field is a Google Dataflow-provided label key and has no explicitly set value in Config.
		return true
	}

	// Let diff be determined by labels (above)
	if strings.HasPrefix(k, "labels.%") {
		return true
	}

	// For other keys, don't suppress diff.
	return false
}

func resourceDataflowJob() *schema.Resource {
	return &schema.Resource{
		Create: resourceDataflowJobCreate,
		Read:   resourceDataflowJobRead,
		Update: resourceDataflowJobUpdateByReplacement,
		Delete: resourceDataflowJobDelete,
		Timeouts: &schema.ResourceTimeout{
			Update: schema.DefaultTimeout(10 * time.Minute),
		},
		CustomizeDiff: customdiff.All(
			resourceDataflowJobTypeCustomizeDiff,
		),
		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				// ForceNew applies to both stream and batch jobs
				ForceNew: true,
			},

			"template_gcs_path": {
				Type:     schema.TypeString,
				Required: true,
			},

			"temp_gcs_location": {
				Type:     schema.TypeString,
				Required: true,
			},

			"zone": {
				Type:     schema.TypeString,
				Optional: true,
				// ForceNew applies to both stream and batch jobs
				ForceNew: true,
			},

			"region": {
				Type:     schema.TypeString,
				Optional: true,
				// ForceNew applies to both stream and batch jobs
				ForceNew: true,
			},

			"max_workers": {
				Type:     schema.TypeInt,
				Optional: true,
				// ForceNew applies to both stream and batch jobs
				ForceNew: true,
			},

			"parameters": {
				Type:     schema.TypeMap,
				Optional: true,
			},

			"labels": {
				Type:             schema.TypeMap,
				Optional:         true,
				DiffSuppressFunc: resourceDataflowJobLabelDiffSuppress,
			},

			"on_delete": {
				Type:         schema.TypeString,
				ValidateFunc: validation.StringInSlice([]string{"cancel", "drain"}, false),
				Optional:     true,
				Default:      "drain",
			},

			"project": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				// ForceNew applies to both stream and batch jobs
				ForceNew: true,
			},

			"state": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"type": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"service_account_email": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"network": {
				Type:             schema.TypeString,
				Optional:         true,
				DiffSuppressFunc: compareSelfLinkOrResourceName,
			},

			"subnetwork": {
				Type:             schema.TypeString,
				Optional:         true,
				DiffSuppressFunc: compareSelfLinkOrResourceName,
			},

			"machine_type": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"ip_configuration": {
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validation.StringInSlice([]string{"WORKER_IP_PUBLIC", "WORKER_IP_PRIVATE", ""}, false),
			},

			"additional_experiments": {
				Type:     schema.TypeSet,
				Optional: true,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"job_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceDataflowJobTypeCustomizeDiff(d *schema.ResourceDiff, meta interface{}) error {
	// All non-virtual fields are ForceNew for batch jobs
	if d.Get("type") == "JOB_TYPE_BATCH" {
		resourceSchema := resourceDataflowJob().Schema
		for field := range resourceSchema {
			if field == "on_delete" {
				continue
			}
			// Labels map will likely have suppressed changes, so we check each key instead of the parent field
			if field == "labels" {
				resourceDataflowJobIterateMapForceNew(field, d)
			} else if d.HasChange(field) {
				d.ForceNew(field)
			}
		}<commit_msg>add desc to schema for google_dataflow_job (#3644)<commit_after>package google

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform-plugin-sdk/helper/customdiff"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"github.com/hashicorp/terraform-plugin-sdk/helper/validation"

	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	dataflow "google.golang.org/api/dataflow/v1b3"
	"google.golang.org/api/googleapi"
)

const resourceDataflowJobGoogleProvidedLabelPrefix = "labels.goog-dataflow-provided"

var dataflowTerminalStatesMap = map[string]struct{}{
	"JOB_STATE_DONE":      {},
	"JOB_STATE_FAILED":    {},
	"JOB_STATE_CANCELLED": {},
	"JOB_STATE_UPDATED":   {},
	"JOB_STATE_DRAINED":   {},
}

func resourceDataflowJobLabelDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
	// Example Diff: "labels.goog-dataflow-provided-template-version": "word_count" => ""
	if strings.HasPrefix(k, resourceDataflowJobGoogleProvidedLabelPrefix) && new == "" {
		// Suppress diff if field is a Google Dataflow-provided label key and has no explicitly set value in Config.
		return true
	}

	// Let diff be determined by labels (above)
	if strings.HasPrefix(k, "labels.%") {
		return true
	}

	// For other keys, don't suppress diff.
	return false
}

func resourceDataflowJob() *schema.Resource {
	return &schema.Resource{
		Create: resourceDataflowJobCreate,
		Read:   resourceDataflowJobRead,
		Update: resourceDataflowJobUpdateByReplacement,
		Delete: resourceDataflowJobDelete,
		Timeouts: &schema.ResourceTimeout{
			Update: schema.DefaultTimeout(10 * time.Minute),
		},
		CustomizeDiff: customdiff.All(
			resourceDataflowJobTypeCustomizeDiff,
		),
		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				// ForceNew applies to both stream and batch jobs
				ForceNew:    true,
				Description: `A unique name for the resource, required by Dataflow.`,
			},

			"template_gcs_path": {
				Type:        schema.TypeString,
				Required:    true,
				Description: `The GCS path to the Dataflow job template.`,
			},

			"temp_gcs_location": {
				Type:        schema.TypeString,
				Required:    true,
				Description: `A writeable location on GCS for the Dataflow job to dump its temporary data.`,
			},

			"zone": {
				Type:     schema.TypeString,
				Optional: true,
				// ForceNew applies to both stream and batch jobs
				ForceNew:    true,
				Description: `The zone in which the created job should run. If it is not provided, the provider zone is used.`,
			},

			"region": {
				Type:     schema.TypeString,
				Optional: true,
				// ForceNew applies to both stream and batch jobs
				ForceNew:    true,
				Description: `The region in which the created job should run.`,
			},

			"max_workers": {
				Type:     schema.TypeInt,
				Optional: true,
				// ForceNew applies to both stream and batch jobs
				ForceNew:    true,
				Description: `The number of workers permitted to work on the job. More workers may improve processing speed at additional cost.`,
			},

			"parameters": {
				Type:        schema.TypeMap,
				Optional:    true,
				Description: `Key/Value pairs to be passed to the Dataflow job (as used in the template).`,
			},

			"labels": {
				Type:             schema.TypeMap,
				Optional:         true,
				DiffSuppressFunc: resourceDataflowJobLabelDiffSuppress,
				Description:      `User labels to be specified for the job. Keys and values should follow the restrictions specified in the labeling restrictions page. NOTE: Google-provided Dataflow templates often provide default labels that begin with goog-dataflow-provided. Unless explicitly set in config, these labels will be ignored to prevent diffs on re-apply.`,
			},

			"on_delete": {
				Type:         schema.TypeString,
				ValidateFunc: validation.StringInSlice([]string{"cancel", "drain"}, false),
				Optional:     true,
				Default:      "drain",
				Description:  `One of "drain" or "cancel". Specifies behavior of deletion during terraform destroy.`,
			},

			"project": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				// ForceNew applies to both stream and batch jobs
				ForceNew:    true,
				Description: `The project in which the resource belongs.`,
			},

			"state": {
				Type:        schema.TypeString,
				Computed:    true,
				Description: `The current state of the resource, selected from the JobState enum.`,
			},
			"type": {
				Type:        schema.TypeString,
				Computed:    true,
				Description: `The type of this job, selected from the JobType enum.`,
			},
			"service_account_email": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: `The Service Account email used to create the job.`,
			},

			"network": {
				Type:             schema.TypeString,
				Optional:         true,
				DiffSuppressFunc: compareSelfLinkOrResourceName,
				Description:      `The network to which VMs will be assigned. If it is not provided, "default" will be used.`,
			},

			"subnetwork": {
				Type:             schema.TypeString,
				Optional:         true,
				DiffSuppressFunc: compareSelfLinkOrResourceName,
				Description:      `The subnetwork to which VMs will be assigned. Should be of the form "regions/REGION/subnetworks/SUBNETWORK".`,
			},

			"machine_type": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: `The machine type to use for the job.`,
			},

			"ip_configuration": {
				Type:         schema.TypeString,
				Optional:     true,
				Description:  `The configuration for VM IPs. Options are "WORKER_IP_PUBLIC" or "WORKER_IP_PRIVATE".`,
				ValidateFunc: validation.StringInSlice([]string{"WORKER_IP_PUBLIC", "WORKER_IP_PRIVATE", ""}, false),
			},

			"additional_experiments": {
				Type:        schema.TypeSet,
				Optional:    true,
				Description: `List of experiments that should be used by the job. An example value is ["enable_stackdriver_agent_metrics"].`,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},

			"job_id": {
				Type:        schema.TypeString,
				Computed:    true,
				Description: `The unique ID of this job.`,
			},
		},
	}
}

func resourceDataflowJobTypeCustomizeDiff(d *schema.ResourceDiff, meta interface{}) error {
	// All non-virtual fields are ForceNew for batch jobs
	if d.Get("type") == "JOB_TYPE_BATCH" {
		resourceSchema := resourceDataflowJob().Schema
		for field := range resourceSchema {
			if field == "on_delete" {
				continue
			}
			// Labels map will likely have suppressed changes, so we check each key instead of the parent field
			if field == "labels" {
				resourceDataflowJobIterateMapForceNew(field, d)
			} else if d.HasChange(field) {
				d.ForceNew(field)
			}
		}
{'fraction_non_alphanumeric': 0.10863095238095238, 'fraction_numerical': 0.0010504201680672268, 'mean_word_length': 3.0920487106017194, 'pattern_counts': {'":': 50, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5828416', 'n_tokens_mistral': 4196, 'n_tokens_neox': 3495, 'n_words': 932}