<commit_before>{
  "type": "sequential",
  "config": {
    "item": {
      "output": {
        "path": "linear-test"
      }
    }
  },
  "steps": [
    {
      "type": "command",
      "value": "rm 10K_items.csv"
    },
    {
      "type": "command",
      "value": "rm $ITEM_INPUT_FILE"
    },
    {
      "type": "command",
      "value": "export ITEM_INPUT_FILE=delete.csv"
    },
    {
      "type": "precondition",
      "config": {
        "item": {
          "output": {
            "file": "$ITEM_INPUT_FILE"
          }
        },
        "test": {
          "step": {
            "limit": {
              "count": 10000
            }
          }
        }
      }
    },
    {
      "type": "for",
      "value": "itemSize",
      "in": [
        "10KB",
        "1MB",
        "100MB"
      ],
      "config": {
        "item": {
          "output": {
            "file": "10K_items.csv"
          }
        }
      },
      "steps": [
        {
          "type": "command",
          "value": "echo ${itemSize}"
        },
        {
          "type": "for",
          "value": "concurrencyLimit",
          "in": [
            1,
            10,
            100
          ],
          "steps": [
            {
              "type": "command",
              "value": "echo ${concurrencyLimit}"
            },
            {
              "type": "load",
              "config": {
                "item": {
                  "data": {
                    "size": "${itemSize}"
                  }
                },
                "load": {
                  "limit": {
                    "concurrency": "${concurrencyLimit}"
                  }
                },
                "test": {
                  "step": {
                    "limit": {
                      "time": "1m"
                    }
                  }
                }
              }
            }
          ]
        }
      ]
    }
  ]
}<commit_msg>add parallel-read in tmp-test-scenario & finished this task<commit_after>{
  "type": "sequential",
  "steps": [
    {
      "type": "command",
      "value": "rm $ITEM_INPUT_FILE"
    },
    {
      "type": "command",
      "value": "export ITEM_INPUT_FILE=10K_items.csv"
    },
    {
      "type": "precondition",
      "config": {
        "item": {
          "output": {
            "file": "{$ITEM_INPUT_FILE}"
          }
        },
        "test": {
          "step": {
            "limit": {
              "count": 10000
            }
          }
        }
      }
    },
    {
      "type": "parallel",
      "steps": [
        {
          "type": "load",
          "config": {
            "item": {
              "input": {
                "file": "{$ITEM_INPUT_FILE}"
              }
            },
            "load": {
              "generator": {
                "recycle": {
                  "enabled": true
                }
              },
              "type": "read"
            }
          }
        },
        {
          "type": "for",
          "value": "itemSize",
          "in": [
            "10KB",
            "1MB",
            "100MB"
          ],
          "steps": [
            {
              "type": "command",
              "value": "echo ${itemSize}"
            },
            {
              "type": "for",
              "value": "concurrencyLimit",
              "in": [
                1,
                10,
                100
              ],
              "steps": [
                {
                  "type": "command",
                  "value": "echo ${concurrencyLimit}"
                },
                {
                  "type": "load",
                  "config": {
                    "item": {
                      "data": {
                        "size": "${itemSize}"
                      }
                    },
                    "load": {
                      "limit": {
                        "concurrency": "${concurrencyLimit}"
                      }
                    },
                    "test": {
                      "step": {
                        "limit": {
                          "time": "1m"
                        }
                      }
                    }
                  }
                }
              ]
            }
          ]
        }
      ]
    }
  ]
}
{'fraction_non_alphanumeric': 0.15257779842080818, 'fraction_numerical': 0.009753831862517418, 'mean_word_length': 0.6745723172628305, 'pattern_counts': {'":': 100, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7367713', 'n_tokens_mistral': 1226, 'n_tokens_neox': 1148, 'n_words': 178}