<commit_before>{
  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "location": {
      "type": "string",
      "defaultValue": "[resourceGroup().location]",
      "metadata": {
        "description": "Location for the resources."
      }
    },
    "vmName": {
      "type": "string",
      "defaultValue": "linux-vm",
      "metadata": {
        "description": "Name for the Virtual Machine."
      }
    },
    "adminUsername": {
      "type": "string",
      "metadata": {
        "description": "User name for the Virtual Machine."
      }
    },
    "authenticationType": {
      "type": "string",
      "defaultValue": "password",
      "allowedValues": [
        "password",
        "sshPublicKey"
      ],
      "metadata": {
        "description": "Type of authentication to use on the Virtual Machine."
      }
    },
    "adminPasswordOrKey": {
      "type": "securestring",
      "metadata": {
        "description": "Password or ssh key for the Virtual Machine."
      }
    },
    "vmSize": {
      "type": "string",
      "defaultValue": "Standard_A2_v2",
      "metadata": {
        "description": "Size for the Virtual Machine."
      }
    },
    "storageNewOrExisting": {
      "type": "string",
      "defaultValue": "new",
      "metadata": {
        "description": "Determines whether or not a new storage account should be provisioned."
      }
    },
    "storageAccountName": {
      "type": "string",
      "defaultValue": "[concat('storage', uniqueString(resourceGroup().id))]",
      "metadata": {
        "description": "Name of the storage account"
      }
    },
    "storageAccountType": {
      "type": "string",
      "defaultValue": "Standard_LRS",
      "metadata": {
        "description": "Storage account type"
      }
    },
    "storageAccountResourceGroupName": {
      "type": "string",
      "defaultValue": "[resourceGroup().name]",
      "metadata": {
        "description": "Name of the resource group for the existing storage account"
      }
    },
    "virtualNetworkNewOrExisting": {
      "type": "string",
      "defaultValue": "new",
      "metadata": {
        "description": "Determines whether or not a new virtual network should be provisioned."
      }
    },
    "virtualNetworkName": {
      "type": "string",
      "defaultValue": "VirtualNetwork",
      "metadata": {
        "description": "Name of the virtual network"
      }
    },
    "addressPrefixes": {
      "type": "array",
      "defaultValue": [
        "10.0.0.0/16"
      ],
      "metadata": {
        "description": "Address prefix of the virtual network"
      }
    },
    "subnetName": {
      "type": "string",
      "defaultValue": "default",
      "metadata": {
        "description": "Name of the subnet"
      }
    },
    "subnetPrefix": {
      "type": "string",
      "defaultValue": "10.0.0.0/24",
      "metadata": {
        "description": "Subnet prefix of the virtual network"
      }
    },
    "virtualNetworkResourceGroupName": {
      "type": "string",
      "defaultValue": "[resourceGroup().name]",
      "metadata": {
        "description": "Name of the resource group for the existing virtual network"
      }
    },
    "publicIpNewOrExisting": {
      "type": "string",
      "defaultValue": "new",
      "metadata": {
        "description": "Determines whether or not a new public ip should be provisioned."
      }
    },
    "publicIpName": {
      "type": "string",
      "defaultValue": "PublicIp",
      "metadata": {
        "description": "Name of the public ip address"
      }
    },
    "publicIpDns": {
      "type": "string",
      "defaultValue": "[concat('linux-vm-', uniqueString(resourceGroup().id))]",
      "metadata": {
        "description": "DNS of the public ip address for the VM"
      }
    },
    "publicIpResourceGroupName": {
      "type": "string",
      "defaultValue": "[resourceGroup().name]",
      "metadata": {
        "description": "Name of the resource group for the public ip address"
      }
    },
    "publicIpAllocationMethod": {
      "type": "string",
      "defaultValue": "Dynamic",
      "allowedValues": [
        "Dynamic",
        "Static"
      ],
      "metadata": {
        "description": "Allocation method for the public ip address"
      }
    },
    "publicIpSku": {
      "type": "string",
      "defaultValue": "Basic",
      "allowedValues": [
        "Basic",
        "Standard"
      ],
      "metadata": {
        "description": "Name of the resource group for the public ip address"
      }
    },
    "_artifactsLocation": {
      "type": "string",
      "metadata": {
        "description": "The base URI where artifacts required by this template are located. When the template is deployed using the accompanying scripts, a private location in the subscription will be used and this value will be automatically generated."
      },
      //"defaultValue": "[deployment().properties.templateLink.uri]" use this for the Azure Marketplace
      "defaultValue": "https://raw.githubusercontent.com/Azure/azure-quickstart-templates/master/100-marketplace-sample/"
    },
    "_artifactsLocationSasToken": {
      "type": "securestring",
      "metadata": {
        "description": "The sasToken required to access _artifactsLocation.  When the template is deployed using the accompanying scripts, a sasToken will be automatically generated."
      },
      "defaultValue": ""
    }
  },
  "variables": {
    "nicName": "[concat(parameters('vmName'), '-nic')]",
    "linuxConfiguration": {
      "disablePasswordAuthentication": true,
      "ssh": {
        "publicKeys": [
          {
            "path": "[concat('/home/', parameters('adminUsername'), '/.ssh/authorized_keys')]",
            "keyData": "[parameters('adminPasswordOrKey')]"
          }
        ]
      }
    },
    "publicIpAddressId": {
      "id": "[resourceId(parameters('publicIpResourceGroupName'), 'Microsoft.Network/publicIPAddresses', parameters('publicIpName'))]"
    },
    "networkSecurityGroupName": "nsg-ssh",
    "fileToBeCopied": "FileToBeCopied.txt",
    "scriptFolder": "scripts",
    "scriptFileName": "copyfilefromazure.sh",
    "scriptArgs": "[concat('-a ', uri(parameters('_artifactsLocation'), '.'), ' -t \"', parameters('_artifactsLocationSasToken'), '\" -p ', variables('scriptFolder'), ' -f ', variables('fileToBeCopied'))]"
  },
  "resources": [
    {
      "apiVersion": "2018-02-01",
      "name": "pid-00000000-0000-0000-0000-000000000000",
      "type": "Microsoft.Resources/deployments",
      "properties": {
        "mode": "Incremental",
        "template": {
          "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
          "contentVersion": "1.0.0.0",
          "resources": []
        }
      }
    },
    {
      "condition": "[equals(parameters('storageNewOrExisting'), 'new')]",
      "type": "Microsoft.Storage/storageAccounts",
      "apiVersion": "2018-02-01",
      "name": "[parameters('storageAccountName')]",
      "location": "[parameters('location')]",
      "kind": "Storage",
      "sku": {
        "name": "[parameters('storageAccountType')]"
      }
    },
    {
      "condition": "[equals(parameters('publicIpNewOrExisting'), 'new')]",
      "type": "Microsoft.Network/publicIPAddresses",
      "apiVersion": "2018-04-01",
      "name": "[parameters('publicIpName')]",
      "location": "[parameters('location')]",
      "sku": {
        "name": "[parameters('publicIpSku')]"
      },    
      "properties": {
        "publicIPAllocationMethod": "[parameters('publicIpAllocationMethod')]",
        "dnsSettings": {
          "domainNameLabel": "[parameters('publicIpDns')]"
        }
      }
    },
    {
      "condition": "[equals(parameters('virtualNetworkNewOrExisting'), 'new')]",
      "type": "Microsoft.Network/virtualNetworks",
      "apiVersion": "2018-04-01",
      "name": "[parameters('virtualNetworkName')]",
      "location": "[parameters('location')]",
      "properties": {
        "addressSpace": {
          "addressPrefixes": "[parameters('addressPrefixes')]"
        },
        "subnets": [
          {
            "name": "[parameters('subnetName')]",
            "properties": {
              "addressPrefix": "[parameters('subnetPrefix')]"
            }
          }
        ]
      }
    },
    {
      "name": "[variables('networkSecurityGroupName')]",
      "type": "Microsoft.Network/networkSecurityGroups",
      "apiVersion": "2018-04-01",
      "location": "[parameters('location')]",
      "properties": {
        "securityRules": [
          {
            "name": "default-allow-ssh",
            "properties": {
              "priority": 1000,
              "sourceAddressPrefix": "*",
              "protocol": "Tcp",
              "destinationPortRange": "22",
              "access": "Allow",
              "direction": "Inbound",
              "sourcePortRange": "*",
              "destinationAddressPrefix": "*"
            }
          }
        ]
      }
    },
    {
      "apiVersion": "2018-04-01",
      "type": "Microsoft.Network/networkInterfaces",
      "name": "[variables('nicName')]",
      "location": "[parameters('location')]",
      "dependsOn": [
        "[parameters('publicIpName')]",
        "[parameters('virtualNetworkName')]",
        "[variables('networkSecurityGroupName')]"
      ],
      "properties": {
        "ipConfigurations": [
          {
            "name": "ipconfig1",
            "properties": {
              "privateIPAllocationMethod": "Dynamic",
              "subnet": {
                "id": "[resourceId(parameters('virtualNetworkResourceGroupName'), 'Microsoft.Network/virtualNetworks/subnets/', parameters('virtualNetworkName'), parameters('subnetName'))]"
              },
              "publicIPAddress": "[if(not(equals(parameters('publicIpNewOrExisting'), 'none')), variables('publicIpAddressId') , json('null'))]"
            }
          }
        ],
        "networkSecurityGroup": {
          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('networkSecurityGroupName'))]"
        }
      }
    },
    {
      "apiVersion": "2018-04-01",
      "type": "Microsoft.Compute/virtualMachines",
      "name": "[parameters('vmName')]",
      "location": "[parameters('location')]",
      "dependsOn": [
        "[parameters('storageAccountName')]",
        "[variables('nicName')]"
      ],
      "properties": {
        "hardwareProfile": {
          "vmSize": "[parameters('vmSize')]"
        },
        "osProfile": {
          "computerName": "[parameters('vmName')]",
          "adminUsername": "[parameters('adminUsername')]",
          "adminPassword": "[parameters('adminPasswordOrKey')]",
          "linuxConfiguration": "[if(equals(parameters('authenticationType'), 'password'), json('null'), variables('linuxConfiguration'))]"
        },
        "storageProfile": {
          "imageReference": {
            "publisher": "Canonical",
            "offer": "UbuntuServer",
            "sku": "16.04-LTS",
            "version": "latest"
          },
          "osDisk": {
            "caching": "ReadWrite",
            "createOption": "FromImage"
          }
        },
        "networkProfile": {
          "networkInterfaces": [
            {
              "id": "[resourceId('Microsoft.Network/networkInterfaces', variables('nicName'))]"
            }
          ]
        },
        "diagnosticsProfile": {
          "bootDiagnostics": {
            "enabled": true,
            "storageUri": "[reference(resourceId(parameters('storageAccountResourceGroupName'), 'Microsoft.Storage/storageAccounts/', parameters('storageAccountName')), '2018-02-01').primaryEndpoints.blob]"
          }
        }
      },
      "resources": [
        {
          "type": "extensions",
          "name": "configScript",
          "apiVersion": "2018-04-01",
          "location": "[parameters('location')]",
          "dependsOn": [
            "[parameters('vmName')]"
          ],
          "properties": {
            "publisher": "Microsoft.Azure.Extensions",
            "type": "CustomScript",
            "typeHandlerVersion": "2.0",
            "autoUpgradeMinorVersion": true,
            "settings": {
              "fileUris": [
                "[uri(parameters('_artifactsLocation'), concat('scripts/copyfilefromazure.sh', parameters('_artifactsLocationSasToken')))]"
              ]
            },
            "protectedSettings": {
              "commandToExecute": "[concat('bash ', variables('scriptFileName'), ' ', variables('scriptArgs'))]"
            }
          }
        }
      ]
    }
  ],
  "outputs": {
    "ssh-command": {
      "type": "string",
      "value": "[if(equals(parameters('publicIpNewOrExisting'), 'none'), 'no public ip', concat('ssh ', parameters('adminUsername'), '@', reference(resourceId(parameters('publicIpResourceGroupName'), 'Microsoft.Network/publicIPAddresses', parameters('publicIpName')), '2018-04-01').dnsSettings.fqdn))]"
    }
  }
}
<commit_msg>Update default authentication type to sshkey<commit_after>{
  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "location": {
      "type": "string",
      "defaultValue": "[resourceGroup().location]",
      "metadata": {
        "description": "Location for the resources."
      }
    },
    "vmName": {
      "type": "string",
      "defaultValue": "linux-vm",
      "metadata": {
        "description": "Name for the Virtual Machine."
      }
    },
    "adminUsername": {
      "type": "string",
      "metadata": {
        "description": "User name for the Virtual Machine."
      }
    },
    "authenticationType": {
      "type": "string",
      "defaultValue": "sshPublicKey",
      "allowedValues": [
        "password",
        "sshPublicKey"
      ],
      "metadata": {
        "description": "Type of authentication to use on the Virtual Machine."
      }
    },
    "adminPasswordOrKey": {
      "type": "securestring",
      "metadata": {
        "description": "Password or ssh key for the Virtual Machine."
      }
    },
    "vmSize": {
      "type": "string",
      "defaultValue": "Standard_A2_v2",
      "metadata": {
        "description": "Size for the Virtual Machine."
      }
    },
    "storageNewOrExisting": {
      "type": "string",
      "defaultValue": "new",
      "metadata": {
        "description": "Determines whether or not a new storage account should be provisioned."
      }
    },
    "storageAccountName": {
      "type": "string",
      "defaultValue": "[concat('storage', uniqueString(resourceGroup().id))]",
      "metadata": {
        "description": "Name of the storage account"
      }
    },
    "storageAccountType": {
      "type": "string",
      "defaultValue": "Standard_LRS",
      "metadata": {
        "description": "Storage account type"
      }
    },
    "storageAccountResourceGroupName": {
      "type": "string",
      "defaultValue": "[resourceGroup().name]",
      "metadata": {
        "description": "Name of the resource group for the existing storage account"
      }
    },
    "virtualNetworkNewOrExisting": {
      "type": "string",
      "defaultValue": "new",
      "metadata": {
        "description": "Determines whether or not a new virtual network should be provisioned."
      }
    },
    "virtualNetworkName": {
      "type": "string",
      "defaultValue": "VirtualNetwork",
      "metadata": {
        "description": "Name of the virtual network"
      }
    },
    "addressPrefixes": {
      "type": "array",
      "defaultValue": [
        "10.0.0.0/16"
      ],
      "metadata": {
        "description": "Address prefix of the virtual network"
      }
    },
    "subnetName": {
      "type": "string",
      "defaultValue": "default",
      "metadata": {
        "description": "Name of the subnet"
      }
    },
    "subnetPrefix": {
      "type": "string",
      "defaultValue": "10.0.0.0/24",
      "metadata": {
        "description": "Subnet prefix of the virtual network"
      }
    },
    "virtualNetworkResourceGroupName": {
      "type": "string",
      "defaultValue": "[resourceGroup().name]",
      "metadata": {
        "description": "Name of the resource group for the existing virtual network"
      }
    },
    "publicIpNewOrExisting": {
      "type": "string",
      "defaultValue": "new",
      "metadata": {
        "description": "Determines whether or not a new public ip should be provisioned."
      }
    },
    "publicIpName": {
      "type": "string",
      "defaultValue": "PublicIp",
      "metadata": {
        "description": "Name of the public ip address"
      }
    },
    "publicIpDns": {
      "type": "string",
      "defaultValue": "[concat('linux-vm-', uniqueString(resourceGroup().id))]",
      "metadata": {
        "description": "DNS of the public ip address for the VM"
      }
    },
    "publicIpResourceGroupName": {
      "type": "string",
      "defaultValue": "[resourceGroup().name]",
      "metadata": {
        "description": "Name of the resource group for the public ip address"
      }
    },
    "publicIpAllocationMethod": {
      "type": "string",
      "defaultValue": "Dynamic",
      "allowedValues": [
        "Dynamic",
        "Static"
      ],
      "metadata": {
        "description": "Allocation method for the public ip address"
      }
    },
    "publicIpSku": {
      "type": "string",
      "defaultValue": "Basic",
      "allowedValues": [
        "Basic",
        "Standard"
      ],
      "metadata": {
        "description": "Name of the resource group for the public ip address"
      }
    },
    "_artifactsLocation": {
      "type": "string",
      "metadata": {
        "description": "The base URI where artifacts required by this template are located. When the template is deployed using the accompanying scripts, a private location in the subscription will be used and this value will be automatically generated."
      },
      //"defaultValue": "[deployment().properties.templateLink.uri]" use this for the Azure Marketplace
      "defaultValue": "https://raw.githubusercontent.com/Azure/azure-quickstart-templates/master/100-marketplace-sample/"
    },
    "_artifactsLocationSasToken": {
      "type": "securestring",
      "metadata": {
        "description": "The sasToken required to access _artifactsLocation.  When the template is deployed using the accompanying scripts, a sasToken will be automatically generated."
      },
      "defaultValue": ""
    }
  },
  "variables": {
    "nicName": "[concat(parameters('vmName'), '-nic')]",
    "linuxConfiguration": {
      "disablePasswordAuthentication": true,
      "ssh": {
        "publicKeys": [
          {
            "path": "[concat('/home/', parameters('adminUsername'), '/.ssh/authorized_keys')]",
            "keyData": "[parameters('adminPasswordOrKey')]"
          }
        ]
      }
    },
    "publicIpAddressId": {
      "id": "[resourceId(parameters('publicIpResourceGroupName'), 'Microsoft.Network/publicIPAddresses', parameters('publicIpName'))]"
    },
    "networkSecurityGroupName": "nsg-ssh",
    "fileToBeCopied": "FileToBeCopied.txt",
    "scriptFolder": "scripts",
    "scriptFileName": "copyfilefromazure.sh",
    "scriptArgs": "[concat('-a ', uri(parameters('_artifactsLocation'), '.'), ' -t \"', parameters('_artifactsLocationSasToken'), '\" -p ', variables('scriptFolder'), ' -f ', variables('fileToBeCopied'))]"
  },
  "resources": [
    {
      "apiVersion": "2018-02-01",
      "name": "pid-00000000-0000-0000-0000-000000000000",
      "type": "Microsoft.Resources/deployments",
      "properties": {
        "mode": "Incremental",
        "template": {
          "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
          "contentVersion": "1.0.0.0",
          "resources": []
        }
      }
    },
    {
      "condition": "[equals(parameters('storageNewOrExisting'), 'new')]",
      "type": "Microsoft.Storage/storageAccounts",
      "apiVersion": "2018-02-01",
      "name": "[parameters('storageAccountName')]",
      "location": "[parameters('location')]",
      "kind": "Storage",
      "sku": {
        "name": "[parameters('storageAccountType')]"
      }
    },
    {
      "condition": "[equals(parameters('publicIpNewOrExisting'), 'new')]",
      "type": "Microsoft.Network/publicIPAddresses",
      "apiVersion": "2018-04-01",
      "name": "[parameters('publicIpName')]",
      "location": "[parameters('location')]",
      "sku": {
        "name": "[parameters('publicIpSku')]"
      },    
      "properties": {
        "publicIPAllocationMethod": "[parameters('publicIpAllocationMethod')]",
        "dnsSettings": {
          "domainNameLabel": "[parameters('publicIpDns')]"
        }
      }
    },
    {
      "condition": "[equals(parameters('virtualNetworkNewOrExisting'), 'new')]",
      "type": "Microsoft.Network/virtualNetworks",
      "apiVersion": "2018-04-01",
      "name": "[parameters('virtualNetworkName')]",
      "location": "[parameters('location')]",
      "properties": {
        "addressSpace": {
          "addressPrefixes": "[parameters('addressPrefixes')]"
        },
        "subnets": [
          {
            "name": "[parameters('subnetName')]",
            "properties": {
              "addressPrefix": "[parameters('subnetPrefix')]"
            }
          }
        ]
      }
    },
    {
      "name": "[variables('networkSecurityGroupName')]",
      "type": "Microsoft.Network/networkSecurityGroups",
      "apiVersion": "2018-04-01",
      "location": "[parameters('location')]",
      "properties": {
        "securityRules": [
          {
            "name": "default-allow-ssh",
            "properties": {
              "priority": 1000,
              "sourceAddressPrefix": "*",
              "protocol": "Tcp",
              "destinationPortRange": "22",
              "access": "Allow",
              "direction": "Inbound",
              "sourcePortRange": "*",
              "destinationAddressPrefix": "*"
            }
          }
        ]
      }
    },
    {
      "apiVersion": "2018-04-01",
      "type": "Microsoft.Network/networkInterfaces",
      "name": "[variables('nicName')]",
      "location": "[parameters('location')]",
      "dependsOn": [
        "[parameters('publicIpName')]",
        "[parameters('virtualNetworkName')]",
        "[variables('networkSecurityGroupName')]"
      ],
      "properties": {
        "ipConfigurations": [
          {
            "name": "ipconfig1",
            "properties": {
              "privateIPAllocationMethod": "Dynamic",
              "subnet": {
                "id": "[resourceId(parameters('virtualNetworkResourceGroupName'), 'Microsoft.Network/virtualNetworks/subnets/', parameters('virtualNetworkName'), parameters('subnetName'))]"
              },
              "publicIPAddress": "[if(not(equals(parameters('publicIpNewOrExisting'), 'none')), variables('publicIpAddressId') , json('null'))]"
            }
          }
        ],
        "networkSecurityGroup": {
          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('networkSecurityGroupName'))]"
        }
      }
    },
    {
      "apiVersion": "2018-04-01",
      "type": "Microsoft.Compute/virtualMachines",
      "name": "[parameters('vmName')]",
      "location": "[parameters('location')]",
      "dependsOn": [
        "[parameters('storageAccountName')]",
        "[variables('nicName')]"
      ],
      "properties": {
        "hardwareProfile": {
          "vmSize": "[parameters('vmSize')]"
        },
        "osProfile": {
          "computerName": "[parameters('vmName')]",
          "adminUsername": "[parameters('adminUsername')]",
          "adminPassword": "[parameters('adminPasswordOrKey')]",
          "linuxConfiguration": "[if(equals(parameters('authenticationType'), 'password'), json('null'), variables('linuxConfiguration'))]"
        },
        "storageProfile": {
          "imageReference": {
            "publisher": "Canonical",
            "offer": "UbuntuServer",
            "sku": "16.04-LTS",
            "version": "latest"
          },
          "osDisk": {
            "caching": "ReadWrite",
            "createOption": "FromImage"
          }
        },
        "networkProfile": {
          "networkInterfaces": [
            {
              "id": "[resourceId('Microsoft.Network/networkInterfaces', variables('nicName'))]"
            }
          ]
        },
        "diagnosticsProfile": {
          "bootDiagnostics": {
            "enabled": true,
            "storageUri": "[reference(resourceId(parameters('storageAccountResourceGroupName'), 'Microsoft.Storage/storageAccounts/', parameters('storageAccountName')), '2018-02-01').primaryEndpoints.blob]"
          }
        }
      },
      "resources": [
        {
          "type": "extensions",
          "name": "configScript",
          "apiVersion": "2018-04-01",
          "location": "[parameters('location')]",
          "dependsOn": [
            "[parameters('vmName')]"
          ],
          "properties": {
            "publisher": "Microsoft.Azure.Extensions",
            "type": "CustomScript",
            "typeHandlerVersion": "2.0",
            "autoUpgradeMinorVersion": true,
            "settings": {
              "fileUris": [
                "[uri(parameters('_artifactsLocation'), concat('scripts/copyfilefromazure.sh', parameters('_artifactsLocationSasToken')))]"
              ]
            },
            "protectedSettings": {
              "commandToExecute": "[concat('bash ', variables('scriptFileName'), ' ', variables('scriptArgs'))]"
            }
          }
        }
      ]
    }
  ],
  "outputs": {
    "ssh-command": {
      "type": "string",
      "value": "[if(equals(parameters('publicIpNewOrExisting'), 'none'), 'no public ip', concat('ssh ', parameters('adminUsername'), '@', reference(resourceId(parameters('publicIpResourceGroupName'), 'Microsoft.Network/publicIPAddresses', parameters('publicIpName')), '2018-04-01').dnsSettings.fqdn))]"
    }
  }
}

{'fraction_non_alphanumeric': 0.1709708182338356, 'fraction_numerical': 0.01281708945260347, 'mean_word_length': 2.367934224049332, 'pattern_counts': {'":': 520, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3091042', 'n_tokens_mistral': 7487, 'n_tokens_neox': 7198, 'n_words': 1361}