<commit_before>/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

use crate::config::ArtifactForPersister;
use crate::config::Config;
use crate::config::ProjectConfig;
use crate::errors::BuildProjectError;
use crate::Artifact;
use crate::ArtifactContent;
use crate::OperationPersister;
use common::sync::ParallelIterator;
use common::PerfLogEvent;
use lazy_static::lazy_static;
use log::debug;
use md5::Digest;
use md5::Md5;
use rayon::iter::IntoParallelRefMutIterator;
use regex::Regex;
use relay_codegen::QueryID;
use relay_transforms::Programs;
use std::fs;
use std::path::PathBuf;

lazy_static! {
    static ref RELAY_HASH_REGEX: Regex = Regex::new(r#"@relayHash (\w{32})\n"#).unwrap();
    static ref REQUEST_ID_REGEX: Regex = Regex::new(r#"@relayRequestID (.+)\n"#).unwrap();
}

pub async fn persist_operations(
    artifacts: &mut [Artifact],
    root_dir: &PathBuf,
    config: &Config,
    project_config: &ProjectConfig,
    operation_persister: &'_ (dyn OperationPersister + Send + Sync),
    log_event: &impl PerfLogEvent,
    programs: &Programs,
) -> Result<(), BuildProjectError> {
    let handles = artifacts
        .par_iter_mut()
        .flat_map(|artifact| {
            if let ArtifactContent::Operation {
                ref text,
                ref mut id_and_text_hash,
                ref reader_operation,
                ..
            } = artifact.content
            {
                if let Some(Some(virtual_id_file_name)) = config
                    .generate_virtual_id_file_name
                    .as_ref()
                    .map(|gen_name| gen_name(project_config, reader_operation, &programs.reader))
                {
                    *id_and_text_hash = Some(QueryID::External(virtual_id_file_name));
                    None
                } else if let Some(text) = text {
                    let text_hash = md5(text);
                    let artifact_path = root_dir.join(&artifact.path);
                    let relative_path = artifact.path.to_owned();<commit_msg>Do not require virtual perist IDs for operations without text (client-only)<commit_after>/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

use crate::config::ArtifactForPersister;
use crate::config::Config;
use crate::config::ProjectConfig;
use crate::errors::BuildProjectError;
use crate::Artifact;
use crate::ArtifactContent;
use crate::OperationPersister;
use common::sync::ParallelIterator;
use common::PerfLogEvent;
use lazy_static::lazy_static;
use log::debug;
use md5::Digest;
use md5::Md5;
use rayon::iter::IntoParallelRefMutIterator;
use regex::Regex;
use relay_codegen::QueryID;
use relay_transforms::Programs;
use std::fs;
use std::path::PathBuf;

lazy_static! {
    static ref RELAY_HASH_REGEX: Regex = Regex::new(r#"@relayHash (\w{32})\n"#).unwrap();
    static ref REQUEST_ID_REGEX: Regex = Regex::new(r#"@relayRequestID (.+)\n"#).unwrap();
}

pub async fn persist_operations(
    artifacts: &mut [Artifact],
    root_dir: &PathBuf,
    config: &Config,
    project_config: &ProjectConfig,
    operation_persister: &'_ (dyn OperationPersister + Send + Sync),
    log_event: &impl PerfLogEvent,
    programs: &Programs,
) -> Result<(), BuildProjectError> {
    let handles = artifacts
        .par_iter_mut()
        .flat_map(|artifact| {
            if let ArtifactContent::Operation {
                ref text,
                ref mut id_and_text_hash,
                ref reader_operation,
                ..
            } = artifact.content
            {
                if let Some(Some(virtual_id_file_name)) = config
                    .generate_virtual_id_file_name
                    .as_ref()
                    .map(|gen_name| gen_name(project_config, reader_operation, &programs.reader))
                {
                    if text.is_some() {
                        *id_and_text_hash = Some(QueryID::External(virtual_id_file_name));
                    }
                    None
                } else if let Some(text) = text {
                    let text_hash = md5(text);
                    let artifact_path = root_dir.join(&artifact.path);
                    let relative_path = artifact.path.to_owned();
{'fraction_non_alphanumeric': 0.1111111111111111, 'fraction_numerical': 0.002699055330634278, 'mean_word_length': 2.6661170651277826, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6817456', 'n_tokens_mistral': 1350, 'n_tokens_neox': 1305, 'n_words': 309}