<commit_before>// Copyright (c) 2006- Facebook
// Distributed under the Thrift Software License
//
// See accompanying file LICENSE or visit the Thrift site at:
// http://developers.facebook.com/thrift/
//
// NOTE:  This code was contributed by an external developer.
//        The internal Thrift team has reviewed and tested it,
//        but we cannot guarantee that it is production-ready.
//        Please feel free to report bugs and/or success stories
//        to the public mailing list.

#include <Python.h>
#include "cStringIO.h"
#include <stdbool.h>
#include <stdint.h>
#include <netinet/in.h>

// TODO(dreiss): defval appears to be unused.  Look into removing it.
// TODO(dreiss): Make parse_spec_args recursive, and cache the output
//               permanently in the object.  (Malloc and orphan.)
// TODO(dreiss): Why do we need cStringIO for reading, why not just char*?
//               Can cStringIO let us work with a BufferedTransport?
// TODO(dreiss): Don't ignore the rv from cwrite (maybe).

/* ====== BEGIN UTILITIES ====== */

#define INIT_OUTBUF_SIZE 128

// Stolen out of TProtocol.h.
// It would be a huge pain to have both get this from one place.
typedef enum TType {
  T_STOP       = 0,
  T_VOID       = 1,
  T_BOOL       = 2,
  T_BYTE       = 3,
  T_I08        = 3,
  T_I16        = 6,
  T_I32        = 8,
  T_U64        = 9,
  T_I64        = 10,
  T_DOUBLE     = 4,
  T_STRING     = 11,
  T_UTF7       = 11,
  T_STRUCT     = 12,
  T_MAP        = 13,
  T_SET        = 14,
  T_LIST       = 15,
  T_UTF8       = 16,
  T_UTF16      = 17
} TType;
<commit_msg>THRIFT-340. python: Detect endianness properly on FreeBSD<commit_after>// Copyright (c) 2006- Facebook
// Distributed under the Thrift Software License
//
// See accompanying file LICENSE or visit the Thrift site at:
// http://developers.facebook.com/thrift/
//
// NOTE:  This code was contributed by an external developer.
//        The internal Thrift team has reviewed and tested it,
//        but we cannot guarantee that it is production-ready.
//        Please feel free to report bugs and/or success stories
//        to the public mailing list.

#include <Python.h>
#include "cStringIO.h"
#include <stdbool.h>
#include <stdint.h>
#include <netinet/in.h>

// TODO(dreiss): defval appears to be unused.  Look into removing it.
// TODO(dreiss): Make parse_spec_args recursive, and cache the output
//               permanently in the object.  (Malloc and orphan.)
// TODO(dreiss): Why do we need cStringIO for reading, why not just char*?
//               Can cStringIO let us work with a BufferedTransport?
// TODO(dreiss): Don't ignore the rv from cwrite (maybe).

/* ====== BEGIN UTILITIES ====== */

#define INIT_OUTBUF_SIZE 128

// Stolen out of TProtocol.h.
// It would be a huge pain to have both get this from one place.
typedef enum TType {
  T_STOP       = 0,
  T_VOID       = 1,
  T_BOOL       = 2,
  T_BYTE       = 3,
  T_I08        = 3,
  T_I16        = 6,
  T_I32        = 8,
  T_U64        = 9,
  T_I64        = 10,
  T_DOUBLE     = 4,
  T_STRING     = 11,
  T_UTF7       = 11,
  T_STRUCT     = 12,
  T_MAP        = 13,
  T_SET        = 14,
  T_LIST       = 15,
  T_UTF8       = 16,
  T_UTF16      = 17
} TType;

#ifndef __BYTE_ORDER
# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN)
#  define __BYTE_ORDER BYTE_ORDER
#  define __LITTLE_ENDIAN LITTLE_ENDIAN
#  define __BIG_ENDIAN BIG_ENDIAN
# else
#  error "Cannot determine endianness"
# endif
#endif

{'fraction_non_alphanumeric': 0.10157819225251076, 'fraction_numerical': 0.028407460545193688, 'mean_word_length': 2.3105413105413106, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4531893', 'n_tokens_mistral': 1277, 'n_tokens_neox': 1170, 'n_words': 407}