<commit_before>#include "property.h"

#include <iostream>
#include <xmmintrin.h>

#define MAX_STRING_LENGTH 0x200

uint32_t read_int(Bitstream &stream, const SendProp *prop) {
  if (prop->flags & SP_EncodedAgainstTickcount) {
    if (prop->flags & SP_Unsigned) {
      return stream.read_var_35();
    } else {
      uint32_t value = stream.read_var_35();
      return (-(value & 1)) ^ (value >> 1);
    }
  } else {
    uint32_t value = stream.get_bits(prop->num_bits);
    uint32_t signer = (0x80000000 >> (32 - prop->num_bits)) & ((prop->flags & SP_Unsigned) - 1);

    value = value ^ signer;
    return value - signer;
  }
}

float read_float_coord(Bitstream &stream) {
  uint32_t first = stream.get_bits(1);
  uint32_t second = stream.get_bits(1);

  __m128 a = (__m128)_mm_setzero_si128();
  __m128 b = (__m128)_mm_setzero_si128();

  if (first || second) {
    uint32_t third = stream.get_bits(1);

    if (first) {
      first = stream.get_bits(0x0E) + 1;
    }

    if (second) {
      second = stream.get_bits(5);
    }

    b = (__m128)_mm_cvtsi32_si128(first);
    a = _mm_cvtsi32_ss(a, second);

    __m128 special = (__m128)_mm_setr_epi32(0x3D000000, 0, 0, 0);
    a = _mm_mul_ss(a, special);
    a = (__m128)_mm_cvtps_pd(a);

    b = (__m128)_mm_cvtepi32_pd((__m128i)b);
    
    a = (__m128)_mm_add_sd((__m128d)a, (__m128d)b);
    a = (__m128)_mm_cvtpd_ps((__m128d)a);

    if (third) {
      __m128 mask = (__m128)_mm_set1_epi32(0x80000000);
      a = _mm_xor_ps(a, mask);
    }
  }

  float f;
  _mm_store_ss(&f, a);
  return f;
}

enum FloatType {
  FT_None,
  FT_LowPrecision,
  FT_Integral,
};

float read_float_coord_mp(Bitstream &stream, FloatType type) {
  uint32_t value;

  if (type == FT_LowPrecision || type == FT_None) {
    uint32_t a = stream.get_bits(1);
    uint32_t b = stream.get_bits(1);
    uint32_t c = stream.get_bits(1);

    value = 4 * c + 2 * b + a;

    XERROR("please no");

  } else if (type == FT_Integral) {
    uint32_t a = stream.get_bits(1);
    uint32_t b = stream.get_bits(1);
    a = a + 2 * b;

    if (!b) {
      return 0;
    } else {
      if (a) {
        value = stream.get_bits(12);
      } else {
        value = stream.get_bits(15);
      }

      if (value & 1) {
        value = -((value >> 1) + 1);
      } else {
        value = (value >> 1) + 1;
      }
    }
  } else {
    XERROR("Unknown coord type.");
  }

  __m128 a = (__m128)_mm_setzero_si128();
  a = _mm_cvtsi32_ss(a, value);
  float f;
  _mm_store_ss(&f, a);

  return f;
}

float read_float_no_scale(Bitstream &stream) {
  uint32_t value = stream.get_bits(32);

  __m128 a = (__m128)_mm_setzero_si128();
  a = _mm_cvtsi32_ss(a, value);
  float f;
  _mm_store_ss(&f, a);

  return f;
}

float read_float_normal(Bitstream &stream) {
  uint32_t first = stream.get_bits(1);
  uint32_t second = stream.get_bits(11);

  float f = second;

  if (second >> 31) {
    f += 4.2949673e9;
  }

  f *= 4.885197850512946e-4;

  __m128 a = _mm_load_ss(&f);
  __m128 mask = (__m128)_mm_set_epi32(0x80000000, 0x80000000, 0x80000000, 0x80000000);
  a = _mm_xor_ps(a, mask);

  _mm_store_ss(&f, a);

  return f;
}

float read_float_cell_coord(Bitstream &stream, FloatType type, uint32_t bits) {
  uint32_t value = stream.get_bits(bits);

  if (type == FT_None || type == FT_LowPrecision) {
    bool lp = type  == FT_LowPrecision;

    uint32_t second;
    if (!lp) {
      second = stream.get_bits(5);
    } else {
      second = stream.get_bits(3);
    }

    __m128 a;
    if (!lp) {
      a = (__m128)_mm_setr_epi32(0x3FA00000, 0, 0, 0);
    } else {
      a = (__m128)_mm_setr_epi32(0x3FC00000, 0, 0, 0);
    }

    __m128 b = (__m128)_mm_setzero_si128();
    b = _mm_cvtsi32_ss(b, second);
    b = (__m128)_mm_cvtps_pd(b);
    b = (__m128)_mm_mul_sd((__m128d)b, (__m128d)a);
    
    a = (__m128)_mm_setzero_si128();
    a = _mm_cvtsi32_ss(a, value);
    
    b = (__m128)_mm_add_sd((__m128d)b, (__m128d)a);
    a = (__m128)_mm_cvtpd_ps((__m128d)b);

    float f;
    _mm_store_ss(&f, a);

    return f;
  } else if (type == FT_Integral) {
    __m128 a = (__m128)_mm_setzero_si128();
    a = _mm_cvtsi32_ss(a, value);
    float f;
    _mm_store_ss(&f, a);

    if (value >> 31) {
      f += 4.2949673e9;
    } 

    return f;
  } else {
    XERROR("Unknown float type");
    return 0;
  }
}

float read_float(Bitstream &stream, const SendProp *prop) {
  if (prop->flags & SP_Coord) {
    return read_float_coord(stream);
  } else if (prop->flags & SP_CoordMp) {
    return read_float_coord_mp(stream, FT_None);
  } else if (prop->flags & SP_CoordMpLowPrecision) {
    return read_float_coord_mp(stream, FT_LowPrecision);
  } else if (prop->flags & SP_CoordMpIntegral) {
    return read_float_coord_mp(stream, FT_Integral);
  } else if (prop->flags & SP_NoScale) {
    return read_float_no_scale(stream);
  } else if (prop->flags & SP_Normal) {
    return read_float_normal(stream);
  } else if (prop->flags & SP_CellCoord) {
    return read_float_cell_coord(stream, FT_None, prop->num_bits);
  } else if (prop->flags & SP_CellCoordLowPrecision) {
    return read_float_cell_coord(stream, FT_LowPrecision, prop->num_bits);
  } else if (prop->flags & SP_CellCoordIntegral) {
    return read_float_cell_coord(stream, FT_Integral, prop->num_bits);
  } else {
    unsigned int value = stream.get_bits(prop->num_bits);
    __m128 a = (__m128)_mm_setzero_si128();
    a = _mm_cvtsi32_ss(a, value);

    __m128 b = (__m128)_mm_setzero_si128();
    b = _mm_cvtsi32_ss(b, (1 << prop->num_bits) - 1);

    a = _mm_div_ss(a, b);

    float range = prop->high_value - prop->low_value;
    b = _mm_load_ss(&range);
    a = _mm_mul_ss(a, b);

    range = prop->low_value;
    b = _mm_load_ss(&range);
    a = _mm_add_ss(a, b);

    float f;
    _mm_store_ss(&f, a);

    return f;
  }
}

void read_vector(float vector[3], Bitstream &stream, const SendProp *prop) {
  vector[0] = read_float(stream, prop);
  vector[1] = read_float(stream, prop);

  if (prop->flags & SP_Normal) {
    uint32_t first = stream.get_bits(1);

    __m128 a = _mm_load_ss(&vector[0]);
    __m128 b = _mm_load_ss(&vector[1]);

    a = _mm_mul_ss(a, a);
    b = _mm_mul_ss(b, b);

    a = _mm_add_ss(a, b);

    b = (__m128)_mm_setr_epi32(0x3D000000, 0, 0, 0);

    if (_mm_comile_ss(b, a)) {
      a = (__m128)_mm_setzero_si128();
    } else {
      b = _mm_sub_ss(b, a);
      a = _mm_sqrt_ss(b);
    }

    if (first) {
      __m128 special = (__m128)_mm_setr_epi32(0x3D000000, 0, 0, 0);
      a = _mm_mul_ss(a, special);
    }

    _mm_store_ss(&vector[2], a);
  } else {
    vector[2] = read_float(stream, prop);
  }
}

void read_vector_xy(float vector[2], Bitstream &stream, const SendProp *prop) {
  vector[0] = read_float(stream, prop);
  vector[1] = read_float(stream, prop);
}

size_t read_string(char *buf, size_t max_length, Bitstream &stream, const SendProp *prop) {
  uint32_t length = stream.get_bits(9);
  XASSERT(length <= max_length, "String too long %d > %d", length, max_length);

  stream.read_bits(buf, 8 * length);

  return length;
}
<commit_msg>Removed sse stuff and fixed several bugs<commit_after>#include "property.h"

#include <cmath>

#define MAX_STRING_LENGTH 0x200

uint32_t read_int(Bitstream &stream, const SendProp *prop) {
  if (prop->flags & SP_EncodedAgainstTickcount) {
    if (prop->flags & SP_Unsigned) {
      return stream.read_var_35();
    } else {
      uint32_t value = stream.read_var_35();
      return (-(value & 1)) ^ (value >> 1);
    }
  } else {
    uint32_t value = stream.get_bits(prop->num_bits);
    uint32_t signer = (0x80000000 >> (32 - prop->num_bits)) & ((prop->flags & SP_Unsigned) - 1);

    value = value ^ signer;
    return value - signer;
  }
}

float read_float_coord(Bitstream &stream) {
  uint32_t integer = stream.get_bits(1);
  uint32_t fraction = stream.get_bits(1);

  if (integer || fraction) {
    uint32_t sign = stream.get_bits(1);

    if (integer) {
      integer = stream.get_bits(0x0E) + 1;
    }

    if (fraction) {
      fraction = stream.get_bits(5);
    }

    double d = 0.03125 * fraction;
    d += integer;

    if (sign) {
      d *= -1;
    }

    return (float) d;
  } else {
    return 0;
  }
}

enum FloatType {
  FT_None,
  FT_LowPrecision,
  FT_Integral,
};

float read_float_coord_mp(Bitstream &stream, FloatType type) {
  uint32_t value;

  if (type == FT_LowPrecision || type == FT_None) {
    uint32_t a = stream.get_bits(1);
    uint32_t b = stream.get_bits(1);
    uint32_t c = stream.get_bits(1);

    value = 4 * c + 2 * b + a;

    XERROR("please no");
  } else if (type == FT_Integral) {
    uint32_t a = stream.get_bits(1);
    uint32_t b = stream.get_bits(1);
    a = a + 2 * b;

    if (!b) {
      return 0;
    } else {
      if (a) {
        value = stream.get_bits(12);
      } else {
        value = stream.get_bits(15);
      }

      if (value & 1) {
        value = -((value >> 1) + 1);
      } else {
        value = (value >> 1) + 1;
      }

      return value;
    }
  } else {
    XERROR("Unknown coord type.");
  }
}

float read_float_no_scale(Bitstream &stream) {
  return stream.get_bits(32);
}

float read_float_normal(Bitstream &stream) {
  uint32_t sign = stream.get_bits(1);
  uint32_t value = stream.get_bits(11);

  float f = value;

  if (value >> 31) {
    f += 4.2949673e9;
  }

  f *= 4.885197850512946e-4;

  if (sign) {
    f = -1 * f;
  }

  return f;
}

float read_float_cell_coord(Bitstream &stream, FloatType type, uint32_t bits) {
  uint32_t value = stream.get_bits(bits);

  if (type == FT_None || type == FT_LowPrecision) {
    bool lp = type == FT_LowPrecision;

    uint32_t fraction;
    if (!lp) {
      fraction = stream.get_bits(5);
    } else {
      fraction = stream.get_bits(3);
    }

    double d = value + (lp ? 1.5 : 1.25) * fraction;
    return (float) d;
  } else if (type == FT_Integral) {
    double d = value;

    if (value >> 31) {
      d += 4.2949673e9;
    } 

    return (float) d;
  } else {
    XERROR("Unknown float type");
    return 0;
  }
}

float read_float(Bitstream &stream, const SendProp *prop) {
  if (prop->flags & SP_Coord) {
    return read_float_coord(stream);
  } else if (prop->flags & SP_CoordMp) {
    return read_float_coord_mp(stream, FT_None);
  } else if (prop->flags & SP_CoordMpLowPrecision) {
    return read_float_coord_mp(stream, FT_LowPrecision);
  } else if (prop->flags & SP_CoordMpIntegral) {
    return read_float_coord_mp(stream, FT_Integral);
  } else if (prop->flags & SP_NoScale) {
    return read_float_no_scale(stream);
  } else if (prop->flags & SP_Normal) {
    return read_float_normal(stream);
  } else if (prop->flags & SP_CellCoord) {
    return read_float_cell_coord(stream, FT_None, prop->num_bits);
  } else if (prop->flags & SP_CellCoordLowPrecision) {
    return read_float_cell_coord(stream, FT_LowPrecision, prop->num_bits);
  } else if (prop->flags & SP_CellCoordIntegral) {
    return read_float_cell_coord(stream, FT_Integral, prop->num_bits);
  } else {
    uint32_t dividend = stream.get_bits(prop->num_bits);
    uint32_t divisor = (1 << prop->num_bits) - 1;

    float f = ((float) dividend) / divisor;
    float range = prop->high_value - prop->low_value;

    return f * range + prop->low_value;
  }
}

void read_vector(float vector[3], Bitstream &stream, const SendProp *prop) {
  vector[0] = read_float(stream, prop);
  vector[1] = read_float(stream, prop);

  if (prop->flags & SP_Normal) {
    uint32_t sign = stream.get_bits(1);

    float f = vector[0] * vector[0] + vector[1] * vector[1];

    if (1 >= f) {
      vector[2] = 0;
    } else {
      vector[2] = sqrt(1 - f);
    }

    if (sign) {
      vector[2] = -1 * vector[2];
    }
  } else {
    vector[2] = read_float(stream, prop);
  }
}

void read_vector_xy(float vector[2], Bitstream &stream, const SendProp *prop) {
  vector[0] = read_float(stream, prop);
  vector[1] = read_float(stream, prop);
}

size_t read_string(char *buf, size_t max_length, Bitstream &stream, const SendProp *prop) {
  uint32_t length = stream.get_bits(9);
  XASSERT(length <= max_length, "String too long %d > %d", length, max_length);

  stream.read_bits(buf, 8 * length);

  return length;
}

{'fraction_non_alphanumeric': 0.14269246603540553, 'fraction_numerical': 0.05319061342116097, 'mean_word_length': 3.0649263721552877, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 7, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1408085', 'n_tokens_mistral': 5297, 'n_tokens_neox': 4798, 'n_words': 1013}