<commit_before>/**
 * SUBLEQ Assembly: main.cpp
 *
 *  Created on: Apr 9, 2014
 *      Author:       Pimenta
 *      Collaborator: <NAME>
 *
 * This program reads a SUBLEQ Assembly file and outputs
 * a well-formed object file.
 * It's supposed to be fed to the Linker.
 */

#include <cstdint>
#include <cstdio>
#include <fstream>
#include <map>
#include <set>
#include <string>
#include <sstream>
#include <iostream>

using namespace std;

typedef uint32_t uword_t;

#ifndef MEM_WORDS
#define MEM_WORDS 0x2000
#endif

/// Buffer that contains the entire contents
/// of the input file, with normalized line endings.
static stringstream f;

/// Global buffer that always contains the current
/// token being read by `readToken`
static string buf;

/// Set of all exported symbols.
/// They're used together with other object files
/// on the linker.
static set<string> exported;

/// All current symbols declared on this file.
/// Maps from a label to it's address in memory.
static map<string, uword_t> symbols;

/// Contains all symbols not found on this file.
///
/// It has labels supposed to be found on other
/// Assembly files.
/// They'll get resolved with the external symbols.
static map<string, set<uword_t>> references;

/// Relative addresses found on this file.
///
/// When you "call" a symbol on this file, it's address
/// is relative to the start of this file.
///
/// When the Linker joins several object files, it will
/// use the values on this set to relocate them all.
///
static set<uword_t> relatives;

/// Current size of the memory.
/// While parsing: points the current memory address
///                being read.
/// At the end:    has the entire size of the memory
///                read.
static uword_t mem_size = 0;

/// Raw memory.
/// It contains the assembled memory that will get
/// written at the end of the object file.
static uword_t* mem = new uword_t[MEM_WORDS];

static int currentLine = 1, lastTokenLine = 1, currentTokenLine = 1;

/// Reads a string from `is` to the end-of-line, saving it on `t`.
///
/// @note It doesn't save the end-of-line character on `t`.
///
/// This is a "safe" implementation of `std::getline`.
/// It is "safe" in the sense that it does work with all possible
/// line endings ("\r", "\n" and "\r\n").
///
/// Taken straight from StackOverflow. Source:
/// http://stackoverflow.com/a/6089413
///
std::istream& safeGetline(std::istream& is, std::string& t)
{
  t.clear();

  // The characters in the stream are read one-by-one using a
  // std::streambuf. That is faster than reading them one-by-one
  // using the std::istream.
  //
  // Code that uses streambuf this way must be guarded by a sentry
  // object.
  // The sentry object performs various tasks, such as thread
  // synchronization and updating the stream state.

  std::istream::sentry se(is, true);

  std::streambuf* sb = is.rdbuf();

  while (true) {
    int c = sb->sbumpc();

    switch (c) {

    case '\n':
      return is;

    case '\r':
      if (sb->sgetc() == '\n')
        sb->sbumpc();

      return is;

    case EOF:
      // Also handle the case when the last line has no line ending
      if (t.empty())
        is.setstate(std::ios::eofbit);

      return is;

    default:
      t += (char)c;
    }
  }
}

/// Reads a token (separated by whitespaces) from the file,
/// char by char, putting on `buf`
inline static void readToken() {
  buf = "";
  lastTokenLine = currentTokenLine;

  // Will read all chars until:
  //
  // - Finding a whitespace (or tab)
  // - Finding a comment
  // - Finding an end-of-line
  // - File somehow ends
  //
  while (true)
  {
    char c = f.get();

    if (!f.good())
      break;

    // comment found
    if (c == '/') {

      // token was read
      if (buf.size()) {
        currentTokenLine = currentLine;
        return;
      }

      // ignoring entire comment
      do {
        c = f.get();
      } while (c != '\r' && c != '\n' && f.good());

      // checking for CR or CRLF line endings
      if (c == '\r') {
        c = f.get();

        if (f.good() && c != '\n')
          f.unget();
      }

      currentLine++;
      continue;
    }

    // line break found
    if (c == '\r' || c == '\n') {

      // checking for CR or CRLF line endings
      if (c == '\r') {
        c = f.get();

        if (f.good() && c != '\n')
          f.unget();
      }

      // token was read
      if (buf.size()) {
        currentTokenLine = currentLine++;
        return;
      }

      currentLine++;
      continue;
    }

    // white space found<commit_msg>No need to worry about different line endings anymore<commit_after>/**
 * SUBLEQ Assembly: main.cpp
 *
 *  Created on: Apr 9, 2014
 *      Author:       Pimenta
 *      Collaborator: <NAME>
 *
 * This program reads a SUBLEQ Assembly file and outputs
 * a well-formed object file.
 * It's supposed to be fed to the Linker.
 */

#include <cstdint>
#include <cstdio>
#include <fstream>
#include <map>
#include <set>
#include <string>
#include <sstream>
#include <iostream>

using namespace std;

typedef uint32_t uword_t;

#ifndef MEM_WORDS
#define MEM_WORDS 0x2000
#endif

/// Buffer that contains the entire contents
/// of the input file, with normalized line endings.
static stringstream f;

/// Global buffer that always contains the current
/// token being read by `readToken`
static string buf;

/// Set of all exported symbols.
/// They're used together with other object files
/// on the linker.
static set<string> exported;

/// All current symbols declared on this file.
/// Maps from a label to it's address in memory.
static map<string, uword_t> symbols;

/// Contains all symbols not found on this file.
///
/// It has labels supposed to be found on other
/// Assembly files.
/// They'll get resolved with the external symbols.
static map<string, set<uword_t>> references;

/// Relative addresses found on this file.
///
/// When you "call" a symbol on this file, it's address
/// is relative to the start of this file.
///
/// When the Linker joins several object files, it will
/// use the values on this set to relocate them all.
///
static set<uword_t> relatives;

/// Current size of the memory.
/// While parsing: points the current memory address
///                being read.
/// At the end:    has the entire size of the memory
///                read.
static uword_t mem_size = 0;

/// Raw memory.
/// It contains the assembled memory that will get
/// written at the end of the object file.
static uword_t* mem = new uword_t[MEM_WORDS];

static int currentLine = 1, lastTokenLine = 1, currentTokenLine = 1;

/// Reads a string from `is` to the end-of-line, saving it on `t`.
///
/// @note It doesn't save the end-of-line character on `t`.
///
/// This is a "safe" implementation of `std::getline`.
/// It is "safe" in the sense that it does work with all possible
/// line endings ("\r", "\n" and "\r\n").
///
/// Taken straight from StackOverflow. Source:
/// http://stackoverflow.com/a/6089413
///
std::istream& safeGetline(std::istream& is, std::string& t)
{
  t.clear();

  // The characters in the stream are read one-by-one using a
  // std::streambuf. That is faster than reading them one-by-one
  // using the std::istream.
  //
  // Code that uses streambuf this way must be guarded by a sentry
  // object.
  // The sentry object performs various tasks, such as thread
  // synchronization and updating the stream state.

  std::istream::sentry se(is, true);

  std::streambuf* sb = is.rdbuf();

  while (true) {
    int c = sb->sbumpc();

    switch (c) {

    case '\n':
      return is;

    case '\r':
      if (sb->sgetc() == '\n')
        sb->sbumpc();

      return is;

    case EOF:
      // Also handle the case when the last line has no line ending
      if (t.empty())
        is.setstate(std::ios::eofbit);

      return is;

    default:
      t += (char)c;
    }
  }
}

/// Reads a token (separated by whitespaces) from the file,
/// char by char, putting on `buf`
inline static void readToken() {
  buf = "";
  lastTokenLine = currentTokenLine;

  // Will read all chars until:
  //
  // - Finding a whitespace (or tab)
  // - Finding a comment
  // - Finding an end-of-line
  // - File somehow ends
  //
  while (true)
  {
    char c = f.get();

    if (!f.good())
      break;

    // comment found
    if (c == '/') {

      // token was read
      if (buf.size()) {
        currentTokenLine = currentLine;
        return;
      }

      // ignoring entire comment
      do {
        c = f.get();
      } while (c != '\n' && f.good());

      currentLine++;
      continue;
    }

    // line break found
    if (c == '\n') {

      // token was read
      if (buf.size()) {
        currentTokenLine = currentLine++;
        return;
      }

      currentLine++;
      continue;
    }

    // white space found
{'fraction_non_alphanumeric': 0.13398322375878485, 'fraction_numerical': 0.0052142371344366355, 'mean_word_length': 2.256921373200443, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2778874', 'n_tokens_mistral': 2824, 'n_tokens_neox': 2684, 'n_words': 1075}