<commit_before>﻿/*
    Copyright 2014 <NAME>

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;
using System.Web;
using Newtonsoft.Json.Linq;
using TinCan.Documents;
using TinCan.LRSResponses;

namespace TinCan
{
    public class RemoteLRS : ILRS
    {
        public Uri endpoint { get; set; }
        public TCAPIVersion version { get; set; }
        public String auth { get; set; }
        public Dictionary<String, String> extended { get; set; } = new Dictionary<String, String>();

        public void SetAuth(String username, String password)
        {
            auth = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes(username + ":" + password));
        }

        public RemoteLRS() { }
        public RemoteLRS(Uri endpoint, TCAPIVersion version, String username, String password)
        {
            this.endpoint = endpoint;
            this.version = version;
            this.SetAuth(username, password);
        }
        public RemoteLRS(String endpoint, TCAPIVersion version, String username, String password) : this(new Uri(endpoint), version, username, password) { }
        public RemoteLRS(String endpoint, String username, String password) : this(endpoint, TCAPIVersion.latest(), username, password) { }

        private class MyHTTPRequest
        {
            public String method { get; set; }
            public String resource { get; set; }
            public Dictionary<String, String> queryParams { get; set; } = new Dictionary<String, String>();
            public Dictionary<String, String> headers { get; set; } = new Dictionary<String, String>();
            public String contentType { get; set; }
            public byte[] content { get; set; }
        }

        private class MyHTTPResponse
        {
            public HttpStatusCode status { get; set; }
            public String contentType { get; set; }
            public byte[] content { get; set; }
            public DateTime lastModified { get; set; }
            public String etag { get; set; }
            public Exception ex { get; set; }

            public MyHTTPResponse() { }
            public MyHTTPResponse(HttpWebResponse webResp)
            {
                status = webResp.StatusCode;
                contentType = webResp.ContentType;
                etag = webResp.Headers.Get("Etag");
                lastModified = webResp.LastModified;

                using (var stream = webResp.GetResponseStream())
                {
                    content = ReadFully(stream, (int)webResp.ContentLength);
                }
            }
        }

        private string AppendParamsToExistingQueryString(string currentQueryString, Dictionary<string, string> parameters)
        {
            foreach (KeyValuePair<String, String> entry in parameters)
            {
                if (currentQueryString != "")
                {
                    currentQueryString += "&";
                }
                currentQueryString += HttpUtility.UrlEncode(entry.Key) + "=" + HttpUtility.UrlEncode(entry.Value);
            }

            return currentQueryString;
        }

        private MyHTTPResponse MakeSyncRequest(MyHTTPRequest req)
        {
            String url;
            if (req.resource.StartsWith("http", StringComparison.InvariantCultureIgnoreCase))
            {
                url = req.resource;
            }
            else
            {
                url = endpoint.ToString();
                if (! url.EndsWith("/") && ! req.resource.StartsWith("/")) {
                    url += "/";
                }
                url += req.resource;
            }

            String qs = "";
            qs = AppendParamsToExistingQueryString(qs, req.queryParams);
            qs = AppendParamsToExistingQueryString(qs, extended);

            if (qs != "")
            {
                url += "?" + qs;
            }

            // TODO: handle special properties we recognize, such as content type, modified since, etc.
            var webReq = (HttpWebRequest)WebRequest.Create(url);
            webReq.Method = req.method;

            webReq.Headers.Add("X-Experience-API-Version", version.ToString());
            if (auth != null)
            {
                webReq.Headers.Add("Authorization", auth);
            }
            foreach (KeyValuePair<String, String> entry in req.headers)
            {
                webReq.Headers.Add(entry.Key, entry.Value);
            }
            
<commit_msg>Don't overwrite params in queryParams<commit_after>﻿/*
    Copyright 2014 Rustici Software

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;
using System.Web;
using Newtonsoft.Json.Linq;
using TinCan.Documents;
using TinCan.LRSResponses;
using System.Linq;

namespace TinCan
{
    public class RemoteLRS : ILRS
    {
        public Uri endpoint { get; set; }
        public TCAPIVersion version { get; set; }
        public String auth { get; set; }
        public Dictionary<String, String> extended { get; set; } = new Dictionary<String, String>();

        public void SetAuth(String username, String password)
        {
            auth = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes(username + ":" + password));
        }

        public RemoteLRS() { }
        public RemoteLRS(Uri endpoint, TCAPIVersion version, String username, String password)
        {
            this.endpoint = endpoint;
            this.version = version;
            this.SetAuth(username, password);
        }
        public RemoteLRS(String endpoint, TCAPIVersion version, String username, String password) : this(new Uri(endpoint), version, username, password) { }
        public RemoteLRS(String endpoint, String username, String password) : this(endpoint, TCAPIVersion.latest(), username, password) { }

        private class MyHTTPRequest
        {
            public String method { get; set; }
            public String resource { get; set; }
            public Dictionary<String, String> queryParams { get; set; } = new Dictionary<String, String>();
            public Dictionary<String, String> headers { get; set; } = new Dictionary<String, String>();
            public String contentType { get; set; }
            public byte[] content { get; set; }
        }

        private class MyHTTPResponse
        {
            public HttpStatusCode status { get; set; }
            public String contentType { get; set; }
            public byte[] content { get; set; }
            public DateTime lastModified { get; set; }
            public String etag { get; set; }
            public Exception ex { get; set; }

            public MyHTTPResponse() { }
            public MyHTTPResponse(HttpWebResponse webResp)
            {
                status = webResp.StatusCode;
                contentType = webResp.ContentType;
                etag = webResp.Headers.Get("Etag");
                lastModified = webResp.LastModified;

                using (var stream = webResp.GetResponseStream())
                {
                    content = ReadFully(stream, (int)webResp.ContentLength);
                }
            }
        }

        private string AppendParamsToExistingQueryString(string currentQueryString,  IEnumerable<KeyValuePair<string,  string>> parameters)
        {
            foreach (KeyValuePair<String, String> entry in parameters)
            {
                if (currentQueryString != "")
                {
                    currentQueryString += "&";
                }
                currentQueryString += HttpUtility.UrlEncode(entry.Key) + "=" + HttpUtility.UrlEncode(entry.Value);
            }

            return currentQueryString;
        }

        private MyHTTPResponse MakeSyncRequest(MyHTTPRequest req)
        {
            String url;
            if (req.resource.StartsWith("http", StringComparison.InvariantCultureIgnoreCase))
            {
                url = req.resource;
            }
            else
            {
                url = endpoint.ToString();
                if (! url.EndsWith("/") && ! req.resource.StartsWith("/")) {
                    url += "/";
                }
                url += req.resource;
            }

            String qs = "";
            qs = AppendParamsToExistingQueryString(qs, req.queryParams);
            qs = AppendParamsToExistingQueryString(qs, extended.Where(w => !req.queryParams.ContainsKey(w.Key)));

            if (qs != "")
            {
                url += "?" + qs;
            }

            // TODO: handle special properties we recognize, such as content type, modified since, etc.
            var webReq = (HttpWebRequest)WebRequest.Create(url);
            webReq.Method = req.method;

            webReq.Headers.Add("X-Experience-API-Version", version.ToString());
            if (auth != null)
            {
                webReq.Headers.Add("Authorization", auth);
            }
            foreach (KeyValuePair<String, String> entry in req.headers)
            {
                webReq.Headers.Add(entry.Key, entry.Value);
            }
            

{'fraction_non_alphanumeric': 0.09262870514820593, 'fraction_numerical': 0.0021450858034321374, 'mean_word_length': 1.9048428207306711, 'pattern_counts': {'":': 2, '<': 23, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6018394', 'n_tokens_mistral': 2582, 'n_tokens_neox': 2491, 'n_words': 732}