<commit_before>﻿using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Xml.Linq;
using System.Xml.XPath;
using Microsoft.Win32;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;

namespace AzureBlobFileSystem
{
    public class AzureBlobStorageProvider  : IStorageProvider
    {
        private readonly CloudStorageAccount _storageAccount;
        public CloudBlobClient BlobClient { get; private set; }
        public IList<CloudBlobContainer> Containers { get; private set; }
        public Func<string, CloudBlobContainer> ContainerFactory;

        public AzureBlobStorageProvider(CloudStorageAccount storageAccount)
        {
            _storageAccount = storageAccount;
            BlobClient = _storageAccount.CreateCloudBlobClient();
            Containers = new List<CloudBlobContainer>();
            ContainerFactory = CreateContainer;
        }

        private CloudBlobContainer EnsurePathIsRelativeAndEnsureContainer(ref string path)
        {
            var containerName = path.Split('/').First();

            var container = Containers.SingleOrDefault(c => c.Name == containerName);

            if (container == null)
            {
                container = BlobClient.GetContainerReference(containerName);
                
                if (!container.Exists())
                {
                    container = ContainerFactory(containerName);
                }

                Containers.Add(container);
            }

            if (path.StartsWith("/") || path.StartsWith("http://") || path.StartsWith("https://"))
                throw new ArgumentException("Path must be relative");

            path = string.Join("/", path.Split('/').Skip(1));

            return container;
        }

        public string Combine(string path1, string path2)
        {
            if (path1 == null)
            {
                throw new ArgumentNullException("path1");
            }

            if (path2 == null)
            {
                throw new ArgumentNullException("path2");
            }
<commit_msg>fix #6 add lock to Ensure...<commit_after>﻿using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Xml.Linq;
using System.Xml.XPath;
using Microsoft.Win32;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;

namespace AzureBlobFileSystem
{
    public class AzureBlobStorageProvider  : IStorageProvider
    {
        private readonly CloudStorageAccount _storageAccount;
        private Object _lock = new Object();
        public CloudBlobClient BlobClient { get; private set; }
        public IList<CloudBlobContainer> Containers { get; private set; }
        public Func<string, CloudBlobContainer> ContainerFactory;

        public AzureBlobStorageProvider(CloudStorageAccount storageAccount)
        {
            _storageAccount = storageAccount;
            BlobClient = _storageAccount.CreateCloudBlobClient();
            Containers = new List<CloudBlobContainer>();
            ContainerFactory = CreateContainer;
        }

        private CloudBlobContainer EnsurePathIsRelativeAndEnsureContainer(ref string path)
        {
            var containerName = path.Split('/').First();

            CloudBlobContainer container;
            lock (_lock)
            {
                container = Containers.SingleOrDefault(c => c.Name == containerName);

                if (container == null)
                {
                    container = BlobClient.GetContainerReference(containerName);

                    if (!container.Exists())
                    {
                        container = ContainerFactory(containerName);
                    }

                    Containers.Add(container);
                }
            }

            if (path.StartsWith("/") || path.StartsWith("http://") || path.StartsWith("https://"))
                throw new ArgumentException("Path must be relative");

            path = string.Join("/", path.Split('/').Skip(1));

            return container;
        }

        public string Combine(string path1, string path2)
        {
            if (path1 == null)
            {
                throw new ArgumentNullException("path1");
            }

            if (path2 == null)
            {
                throw new ArgumentNullException("path2");
            }

{'fraction_non_alphanumeric': 0.09090909090909091, 'fraction_numerical': 0.004264870931537598, 'mean_word_length': 2.035422343324251, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1077138', 'n_tokens_mistral': 1059, 'n_tokens_neox': 1024, 'n_words': 241}