<commit_before>cdef extern from "hdfs.h":
  ctypedef struct HdfsFileSystemInternalWrapper:
    pass
  ctypedef HdfsFileSystemInternalWrapper hdfsFS


  ctypedef struct HdfsFileInternalWrapper:
    pass
  ctypedef HdfsFileInternalWrapper hdfsFile

  ctypedef struct hdfsFileInfo:
      # tObjectKind mKind
      char *mName
      # tTime mLastMod
      # tOffset mSize
      short mReplication
      # tOffset mBlockSize
      char *mOwner
      char *mGroup
      short mPermissions
      # tTime mLastAccess

  struct hdfsBuilder:
    pass
  hdfsBuilder *hdfsNewBuilder()
  void hdfsFreeBuilder(hdfsBuilder *bld)

  char* hdfsGetLastError()

  void hdfsBuilderSetNameNode(hdfsBuilder *bld, char *nn)
  void hdfsBuilderSetNameNodePort(hdfsBuilder *bld, unsigned int port)

  hdfsFS hdfsBuilderConnect(hdfsBuilder *bld)

  hdfsFileInfo *hdfsListDirectory(hdfsFS fs, char *path, int *numEntries)

  hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, int bufferSize, short replication, int blocksize)
  int hdfsCloseFile(hdfsFS fs, hdfsFile file)

  int hdfsFileIsOpenForRead(hdfsFile file)
  int hdfsFileIsOpenForWrite(hdfsFile file)

  int hdfsRead(hdfsFS fs, hdfsFile file, void *buffer, int length)
  int hdfsWrite(hdfsFS fs, hdfsFile file, void *buffer, int length)
  int hdfsFlush(hdfsFS fs, hdfsFile file)
  int hdfsHFlush(hdfsFS fs, hdfsFile file)
  int hdfsSync(hdfsFS fs, hdfsFile file)<commit_msg>Add BlockLocation and FreeMemory functions. Add reused ctypedef<commit_after>cdef extern from "hdfs.h":
  ctypedef int tSize
  ctypedef int tTime
  ctypedef int tOffset
  ctypedef unsigned int tPort

  ctypedef struct HdfsFileSystemInternalWrapper:
    pass
  ctypedef HdfsFileSystemInternalWrapper hdfsFS

  ctypedef struct HdfsFileInternalWrapper:
    pass
  ctypedef HdfsFileInternalWrapper hdfsFile

  ctypedef struct BlockLocation:
    int numOfNodes
    char **hosts
    char **names
    char **topologyPaths
    tOffset length
    tOffset offset
    int corrupt

  ctypedef struct hdfsFileInfo:
      tObjectKind mKind
      char *mName
      tOffset mSize
      short mReplication
      tOffset mBlockSize
      char *mOwner
      char *mGroup
      short mPermissions
      tTime mLastMod
      tTime mLastAccess

  cdef enum tObjectKind:
    kObjectKindFile = 0
    kObjectKindDirectory = 1

  struct hdfsBuilder:
    pass
  hdfsBuilder *hdfsNewBuilder()
  void hdfsFreeBuilder(hdfsBuilder *bld)

  char* hdfsGetLastError()

  void hdfsBuilderSetNameNode(hdfsBuilder *bld, char *nn)
  void hdfsBuilderSetNameNodePort(hdfsBuilder *bld, tPort port)

  hdfsFS hdfsBuilderConnect(hdfsBuilder *bld)
  int hdfsDisconnect(hdfsFS fs)

  hdfsFileInfo *hdfsListDirectory(hdfsFS fs, char *path, int *numEntries)
  void hdfsFreeFileInfo(hdfsFileInfo *infos, int numEntries)

  hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, int bufferSize,
                        short replication, tOffset blocksize)
  int hdfsCloseFile(hdfsFS fs, hdfsFile file)

  int hdfsFileIsOpenForRead(hdfsFile file)
  int hdfsFileIsOpenForWrite(hdfsFile file)

  int hdfsRead(hdfsFS fs, hdfsFile file, void *buffer, tSize length)
  int hdfsWrite(hdfsFS fs, hdfsFile file, void *buffer, tSize length)
  int hdfsFlush(hdfsFS fs, hdfsFile file)
  int hdfsHFlush(hdfsFS fs, hdfsFile file)
  int hdfsSync(hdfsFS fs, hdfsFile file)

  BlockLocation *hdfsGetFileBlockLocations(hdfsFS fs, char *path,
                                           int start, int length, int *numOfBlock);
  void hdfsFreeFileBlockLocations(BlockLocation *locations, int numOfBlock);
{'fraction_non_alphanumeric': 0.05424395727937043, 'fraction_numerical': 0.0005621135469364812, 'mean_word_length': 3.9706703910614527, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2323819', 'n_tokens_mistral': 1269, 'n_tokens_neox': 1212, 'n_words': 326}