<commit_before>/* bzflag
 * Copyright (c) 1993 - 2004 <NAME>
 *
 * This package is free software;  you can redistribute it and/or
 * modify it under the terms of the license found in the file
 * named LICENSE that should have accompanied this file.
 *
 * THIS PACKAGE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 */

// interface header
#include "BZWReader.h"

// implementation-specific system headers
#include <fstream>
#include <sstream>

// implementation-specific bzflag headers
#include "URLManager.h"

// implementation-specific bzfs-specific headers
#include "WorldFileObject.h"
#include "CustomBox.h"
#include "CustomPyramid.h"
#include "CustomGate.h"
#include "CustomLink.h"
#include "CustomBase.h"
#include "CustomWeapon.h"
#include "CustomWorld.h"
#include "CustomZone.h"
#include "CustomTetra.h"
#include "CustomDynamicColor.h"
#include "CustomTextureMatrix.h"
#include "CustomMesh.h"
#include "CustomArc.h"
#include "CustomCone.h"
#include "CustomSphere.h"
#include "CustomWaterLevel.h"

static const std::string urlProtocol("http://");

BZWReader::BZWReader(std::string filename) : location(filename), input(NULL)
{
  errorHandler = new BZWError(location);
  if (filename.substr(0, urlProtocol.size()) != urlProtocol) {
    input = new std::ifstream(filename.c_str(), std::ios::in);
  } else {
    URLManager::instance().getURL(location, httpData);
    input = new std::istringstream(httpData);
  }

  if (input->peek() == EOF) {
    errorHandler->fatalError(std::string("could not find bzflag world file"), 0);
  }
}

BZWReader::~BZWReader()
{
  // clean up
  delete errorHandler;
  delete input;
}

void BZWReader::readToken(char *buffer, int n)
{
  int c = -1;

  // skip whitespace
  while (input->good() && (c = input->get()) != -1 && isspace(c) && c != '\n')
    ;

  // read up to whitespace or n - 1 characters into buffer
  int i = 0;
  if (c != -1 && c != '\n') {
    buffer[i++] = c;<commit_msg>Adding ftp: and file: urls<commit_after>/* bzflag
 * Copyright (c) 1993 - 2004 <NAME>
 *
 * This package is free software;  you can redistribute it and/or
 * modify it under the terms of the license found in the file
 * named LICENSE that should have accompanied this file.
 *
 * THIS PACKAGE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 */

// interface header
#include "BZWReader.h"

// implementation-specific system headers
#include <fstream>
#include <sstream>

// implementation-specific bzflag headers
#include "URLManager.h"

// implementation-specific bzfs-specific headers
#include "WorldFileObject.h"
#include "CustomBox.h"
#include "CustomPyramid.h"
#include "CustomGate.h"
#include "CustomLink.h"
#include "CustomBase.h"
#include "CustomWeapon.h"
#include "CustomWorld.h"
#include "CustomZone.h"
#include "CustomTetra.h"
#include "CustomDynamicColor.h"
#include "CustomTextureMatrix.h"
#include "CustomMesh.h"
#include "CustomArc.h"
#include "CustomCone.h"
#include "CustomSphere.h"
#include "CustomWaterLevel.h"

BZWReader::BZWReader(std::string filename) : location(filename), input(NULL)
{
  static const std::string httpProtocol("http://");
  static const std::string ftpProtocol("ftp://");
  static const std::string fileProtocol("file:/");

  errorHandler = new BZWError(location);

  if ((filename.substr(0, httpProtocol.size()) == httpProtocol)
      || (filename.substr(0, ftpProtocol.size()) == ftpProtocol)
      || (filename.substr(0, fileProtocol.size()) == fileProtocol)) {
    URLManager::instance().getURL(location, httpData);
    input = new std::istringstream(httpData);
  } else {
    input = new std::ifstream(filename.c_str(), std::ios::in);
  }

  if (input->peek() == EOF) {
    errorHandler->fatalError(std::string("could not find bzflag world file"), 0);
  }
}

BZWReader::~BZWReader()
{
  // clean up
  delete errorHandler;
  delete input;
}

void BZWReader::readToken(char *buffer, int n)
{
  int c = -1;

  // skip whitespace
  while (input->good() && (c = input->get()) != -1 && isspace(c) && c != '\n')
    ;

  // read up to whitespace or n - 1 characters into buffer
  int i = 0;
  if (c != -1 && c != '\n') {
    buffer[i++] = c;
{'fraction_non_alphanumeric': 0.14680365296803652, 'fraction_numerical': 0.0073059360730593605, 'mean_word_length': 4.401972872996301, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6777937', 'n_tokens_mistral': 1455, 'n_tokens_neox': 1414, 'n_words': 411}