<commit_before>﻿using System;
using System.Collections.Generic;
using System.Text;
using Newtonsoft.Json;

namespace Akavache
{
    /// <summary>
    /// Since we use BSON at places, we want to just store ticks to avoid loosing precision.
    /// By default BSON will use JSON ticks.
    /// </summary>
    internal class JsonDateTimeTickConverter : JsonConverter
    {
        public static JsonDateTimeTickConverter Default { get; } = new JsonDateTimeTickConverter();

        public override bool CanConvert(Type objectType)
        {
            return objectType == typeof(DateTime) || objectType == typeof(DateTime?);
        }

        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
        {
            if (reader.TokenType != JsonToken.Integer && reader.TokenType != JsonToken.Date)
            {
                return null;
            }

            if (reader.TokenType == JsonToken.Date)
            {
                return (DateTime)reader.Value;
            }

            if (objectType == typeof(DateTime) || objectType == typeof(DateTime?))
            {
                var ticks = (long)reader.Value;
                var dateTime = new DateTime(ticks, DateTimeKind.Utc);
                return dateTime;
            }

            return null;
        }

        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
        {
            if (value is DateTime dateTime)
            {
                serializer.Serialize(writer, dateTime.Ticks);
            }
        }
    }
}
<commit_msg>Fix: datetime tick storage (#459)<commit_after>﻿using System;
using System.Collections.Generic;
using System.Text;
using Newtonsoft.Json;

namespace Akavache
{
    /// <summary>
    /// Since we use BSON at places, we want to just store ticks to avoid loosing precision.
    /// By default BSON will use JSON ticks.
    /// </summary>
    internal class JsonDateTimeTickConverter : JsonConverter
    {
        public static JsonDateTimeTickConverter Default { get; } = new JsonDateTimeTickConverter();

        public override bool CanConvert(Type objectType)
        {
            return objectType == typeof(DateTime) || objectType == typeof(DateTime?);
        }

        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
        {
            if (reader.TokenType != JsonToken.Integer && reader.TokenType != JsonToken.Date)
            {
                return null;
            }

            if (reader.TokenType == JsonToken.Date)
            {
                return (DateTime)reader.Value;
            }

            if (objectType == typeof(DateTime) || objectType == typeof(DateTime?))
            {
                var ticks = (long)reader.Value;
                var dateTime = new DateTime(ticks, DateTimeKind.Utc);
                return dateTime;
            }

            return null;
        }

        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
        {
            if (value is DateTime dateTime)
            {
                serializer.Serialize(writer, dateTime.ToUniversalTime().Ticks);
            }
        }
    }
}

{'fraction_non_alphanumeric': 0.08628048780487804, 'fraction_numerical': 0.0009146341463414635, 'mean_word_length': 1.9190391459074734, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1076848', 'n_tokens_mistral': 816, 'n_tokens_neox': 782, 'n_words': 228}