<commit_before>/*
 * This file is part of Mapyrus, software for plotting maps.
 * Copyright (C) 2003 <NAME>.
 *
 * Mapyrus is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * Mapyrus is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Mapyrus; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */

/*
 * @(#) $Id$
 */
package org.mapyrus;

import java.io.*;
import java.net.MalformedURLException;
import java.util.LinkedList;
import java.lang.String;

/**
 * Wrapper around a Reader to read from a file or URL, whilst expanding
 * any included files and URLs.
 * Also allows for read characters to be pushed back onto input stream.
 */
class Preprocessor
{
	private static final String INCLUDE_KEYWORD = "include";

	/*
	 * Files we are reading from and their names.
	 * Built up as a stack as we include nested files.
	 */
	private LinkedList mFileStack;
	
	/*
	 * First file we are reading from.
	 */
	private FileOrURL mInitialFile;
	
	/*
	 * Line we are currently reading from.
	 */
	private StringBuffer mCurrentLine = null;
	private int mCurrentLineIndex = 0;

	/**
	 * Create stack of files being read.
	 * @param f is the first file to push onto the stack.
	 */
	private void initFileStack(FileOrURL f)
	{
		mFileStack = new LinkedList();
		mFileStack.add(f);
		mInitialFile = f;
	}

	/**
	 * Create new user input producer from an already open Reader.
	 * @param f is a file or URL to read from.
	 */
	public Preprocessor(FileOrURL f)
	{
		initFileStack(f);
	}

	/**
	 * Create new user input producer from a file.
	 * @param filename is a file to open and read from.
	 */
	public Preprocessor(String filename) throws IOException, MapyrusException
	{
		FileOrURL f = new FileOrURL(filename);
		initFileStack(f);
	}

	/*
	 * Open new file to read from and push it on stack of files being read.
	 */
	private void openIncludedFile(String filename) throws MalformedURLException, IOException, MapyrusException
	{
		FileOrURL f;

		FileOrURL includingFile = (FileOrURL)mFileStack.getLast();

		if (includingFile.isURL())
			f = new FileOrURL(filename, includingFile);
		else
			f = new FileOrURL(filename);

		if (f.isURL())
		{
			String contentType = f.getURLContentType();
			if (!contentType.startsWith("text/"))
			{
				throw new MapyrusException(MapyrusMessages.get(MapyrusMessages.NOT_TEXT_FILE) +
					": " + f.toString());
			}
		}
		mFileStack.add(f);
	}


	/**
	 * Reads next character that is not a space.
	 * @return next non-space character.
	 */
	public int readNonSpace() throws IOException, MapyrusException
	{
		int c;

		do
		{
			c = read();
		}
		while (c != -1 && Character.isWhitespace((char)c) && c != '\n');
		return(c);
	}

	/**
	 * Reads next character.
	 * Blocks if another character is not available.
	 * @return next character from wherever user input is coming from, or -1
	 * if at EOF.
	 */
	public int read() throws IOException, MapyrusException
	{
		int c;
		LineNumberReader reader;

		/*
		 * Return next character from current line.
		 */
		if (mCurrentLine != null && mCurrentLineIndex < mCurrentLine.length())
		{
			c = mCurrentLine.charAt(mCurrentLineIndex++);
			return(c);
		}

		/*
		 * Need to read a new line.
		 */
		mCurrentLineIndex = 0;
		FileOrURL f = (FileOrURL)mFileStack.getLast();
		reader = f.getReader();
		String s = reader.readLine();
		if (s == null)
		{
			/*
			 * Got end-of-file.  Close file and continue reading any file that included
			 * this one.
			 */
			reader.close();
			mFileStack.removeLast();
			if (mFileStack.size() > 0)
			{
				mCurrentLine = null;
				return(read());
			}
			else
			{
				return(-1);
			}
		}

		/*
		 * Check if this line includes another file.
		 */
		mCurrentLine = new StringBuffer(s);
		mCurrentLine.append('\n');

		String trimmed = s.trim();
		if (trimmed.toLowerCase().startsWith(INCLUDE_KEYWORD))
		{
			if (trimmed.length() == INCLUDE_KEYWORD.length())
			{
				throw new MapyrusException(getCurrentFilenameAndLineNumber() +
					": " + MapyrusMessages.get(MapyrusMessages.MISSING_FILENAME));
			}

			if (Character.isWhitespace(s.charAt(INCLUDE_KEYWORD.length())))
			{
				String filename = trimmed.substring(INCLUDE_KEYWORD.length() + 1).trim();

				/*
				 * Open included file and start
				 * reading from it.
				 */
				try
				{
					openIncludedFile(filename);
				}
				catch (Exception e)
				{
					throw new MapyrusException(getCurrentFilenameAndLineNumber() +
						": " + e.getMessage());
				}

				mCurrentLine = null;
				return(read());<commit_msg>Join lines ending with a '\' with the next line and return them both as a single line.<commit_after>/*
 * This file is part of Mapyrus, software for plotting maps.
 * Copyright (C) 2003 <NAME>y.
 *
 * Mapyrus is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * Mapyrus is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Mapyrus; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */

/*
 * @(#) $Id$
 */
package org.mapyrus;

import java.io.*;
import java.net.MalformedURLException;
import java.util.LinkedList;
import java.lang.String;

/**
 * Wrapper around a Reader to read from a file or URL, whilst expanding
 * any included files and URLs.
 * Also allows for read characters to be pushed back onto input stream.
 */
class Preprocessor
{
	private static final String INCLUDE_KEYWORD = "include";

	/*
	 * Files we are reading from and their names.
	 * Built up as a stack as we include nested files.
	 */
	private LinkedList mFileStack;
	
	/*
	 * First file we are reading from.
	 */
	private FileOrURL mInitialFile;
	
	/*
	 * Line we are currently reading from.
	 */
	private StringBuffer mCurrentLine = null;
	private int mCurrentLineIndex = 0;

	/**
	 * Create stack of files being read.
	 * @param f is the first file to push onto the stack.
	 */
	private void initFileStack(FileOrURL f)
	{
		mFileStack = new LinkedList();
		mFileStack.add(f);
		mInitialFile = f;
	}

	/**
	 * Create new user input producer from an already open Reader.
	 * @param f is a file or URL to read from.
	 */
	public Preprocessor(FileOrURL f)
	{
		initFileStack(f);
	}

	/**
	 * Create new user input producer from a file.
	 * @param filename is a file to open and read from.
	 */
	public Preprocessor(String filename) throws IOException, MapyrusException
	{
		FileOrURL f = new FileOrURL(filename);
		initFileStack(f);
	}

	/*
	 * Open new file to read from and push it on stack of files being read.
	 */
	private void openIncludedFile(String filename) throws MalformedURLException, IOException, MapyrusException
	{
		FileOrURL f;

		FileOrURL includingFile = (FileOrURL)mFileStack.getLast();

		if (includingFile.isURL())
			f = new FileOrURL(filename, includingFile);
		else
			f = new FileOrURL(filename);

		if (f.isURL())
		{
			String contentType = f.getURLContentType();
			if (!contentType.startsWith("text/"))
			{
				throw new MapyrusException(MapyrusMessages.get(MapyrusMessages.NOT_TEXT_FILE) +
					": " + f.toString());
			}
		}
		mFileStack.add(f);
	}


	/**
	 * Reads next character that is not a space.
	 * @return next non-space character.
	 */
	public int readNonSpace() throws IOException, MapyrusException
	{
		int c;

		do
		{
			c = read();
		}
		while (c != -1 && Character.isWhitespace((char)c) && c != '\n');
		return(c);
	}

	/**
	 * Reads next character.
	 * Blocks if another character is not available.
	 * @return next character from wherever user input is coming from, or -1
	 * if at EOF.
	 */
	public int read() throws IOException, MapyrusException
	{
		int c;
		LineNumberReader reader;

		/*
		 * Return next character from current line.
		 */
		if (mCurrentLine != null && mCurrentLineIndex < mCurrentLine.length())
		{
			c = mCurrentLine.charAt(mCurrentLineIndex++);
			return(c);
		}

		/*
		 * Need to read a new line.
		 */
		mCurrentLineIndex = 0;
		FileOrURL f = (FileOrURL)mFileStack.getLast();
		reader = f.getReader();
		String s = reader.readLine();
		if (s == null)
		{
			/*
			 * Got end-of-file.  Close file and continue reading any file that included
			 * this one.
			 */
			reader.close();
			mFileStack.removeLast();
			if (mFileStack.size() > 0)
			{
				mCurrentLine = null;
				return(read());
			}
			else
			{
				return(-1);
			}
		}

		/*
		 * Join line with next line if it ends with '\'.
		 */
		mCurrentLine = new StringBuffer(s);
		while (s != null && s.endsWith("\\"))
		{
			/*
			 * Remove '\' at end of buffer, read and append next line
			 * to buffer.
			 */
			int len = mCurrentLine.length();
			mCurrentLine.deleteCharAt(len - 1);

			s = reader.readLine();
			if (s != null)
				mCurrentLine.append(s);
		}
		mCurrentLine.append('\n');

		/*
		 * Check if this line includes another file.
		 */
		String trimmed = mCurrentLine.toString().trim();
		if (trimmed.toLowerCase().startsWith(INCLUDE_KEYWORD))
		{
			if (trimmed.length() == INCLUDE_KEYWORD.length())
			{
				throw new MapyrusException(getCurrentFilenameAndLineNumber() +
					": " + MapyrusMessages.get(MapyrusMessages.MISSING_FILENAME));
			}

			if (Character.isWhitespace(trimmed.charAt(INCLUDE_KEYWORD.length())))
			{
				String filename = trimmed.substring(INCLUDE_KEYWORD.length() + 1).trim();

				/*
				 * Open included file and start
				 * reading from it.
				 */
				try
				{
					openIncludedFile(filename);
				}
				catch (Exception e)
				{
					throw new MapyrusException(getCurrentFilenameAndLineNumber() +
						": " + e.getMessage());
				}

				mCurrentLine = null;
				return(read());
{'fraction_non_alphanumeric': 0.1093465990624701, 'fraction_numerical': 0.005070314742179278, 'mean_word_length': 3.1060487038491753, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1059853', 'n_tokens_mistral': 3746, 'n_tokens_neox': 3156, 'n_words': 1087}