<commit_before>using System;
using System.IO;
using System.Text;

namespace ProtocolBuffers
{
	public static class ProtoParser
	{
		public static Proto Parse (string path)
		{
			Proto p = new Proto ();
			
			string t = "";
			
			using (TextReader reader = new StreamReader(path, Encoding.UTF8)) {
				while (true) {
					string line = reader.ReadLine ();
					if (line == null)
						break;
					
					//Remove comment
					int comment = line.IndexOf ("//");
					if (comment >= 0)
						line = line.Substring (0, comment);
					t += line + "\n";
					
				}
			}
			
			TokenReader tr = new TokenReader (t);
			Exception e = null;
			try {
				ParseMessages (tr, p);
				return p;
			} catch (InvalidDataException ide) {
				e = ide;
			} catch (FormatException fe) {
				e = fe;
			}
			Console.Write (tr.Parsed);
			Console.WriteLine (" <---");
			Console.WriteLine (e.Message);
			Console.WriteLine ("Got: " + tr.Next);
			return null;
		}

		static void ParseMessages (TokenReader tr, Proto p)
		{
			while (true) {
				string token;
				try {
					token = tr.ReadNext ();
				} catch (EndOfStreamException) {
					return;
				}
				switch (token) {
				case "message":
					p.Messages.Add (ParseMessage (tr, p));
					break;
				case "option":
					//Save options
					ParseOption (tr, p);
					break;
				default:
					throw new InvalidDataException ("Expected: message or option");
				}
			}
		}

		static Message ParseMessage (TokenReader tr, Message parent)
		{
			Message msg = new Message (parent);
			msg.ProtoName = tr.ReadNext ();
			
			//Expect "{"
			if (tr.ReadNext () != "{")
				throw new InvalidDataException ("Ecpected: {");
			
			while (ParseField (tr, msg))
				continue;
			
			return msg;			
		}

		static bool ParseField (TokenReader tr, Message m)
		{
			string rule = tr.ReadNext ();
			if (rule == "}")
				return false;
			
			if (rule == "enum") {
				MessageEnum me = ParseEnum (tr, m);
				m.Enums.Add (me);
				return true;
			}

			Field f = new Field ();
			
			//Rule
			switch (rule) {
			case "required":
				f.Rule = Rules.Required;
				break;
			case "optional":
				f.Rule = Rules.Optional;
				break;
			case "repeated":
				f.Rule = Rules.Repeated;
				break;
			case "option":
				//Save options
				ParseOption (tr, m);
				return true;
			case "message":
				m.Messages.Add (ParseMessage (tr, m));
				return true;
			default:
				throw new InvalidDataException ("unknown rule: " + rule);
			}
			m.Fields.Add (f);

			//Type
			f.ProtoTypeName = tr.ReadNext ();
			
			//Name
			f.Name = tr.ReadNext ();
			
			//ID
			if (tr.ReadNext () != "=")
				throw new InvalidDataException ("Expected: =");
			f.ID = uint.Parse (tr.ReadNext ());
			if (19000 <= f.ID && f.ID <= 19999)
				throw new InvalidDataException ("Can't use reserved field ID 19000-19999");
			if (f.ID > 536870911)
				throw new InvalidDataException ("Maximum field id is 2^29 - 1");
			
			//Determine if extra options
			string extra = tr.ReadNext ();
			if (extra == ";")
				return true;
			
			//Field options
			if (extra != "[")
				throw new InvalidDataException ("Expected: [");<commit_msg>Check for duplicate keys in proto messages<commit_after>using System;
using System.IO;
using System.Text;

namespace ProtocolBuffers
{
	public static class ProtoParser
	{
		public static Proto Parse (string path)
		{
			Proto p = new Proto ();
			
			string t = "";
			
			using (TextReader reader = new StreamReader(path, Encoding.UTF8)) {
				while (true) {
					string line = reader.ReadLine ();
					if (line == null)
						break;
					
					//Remove comment
					int comment = line.IndexOf ("//");
					if (comment >= 0)
						line = line.Substring (0, comment);
					t += line + "\n";
					
				}
			}
			
			TokenReader tr = new TokenReader (t);
			Exception e = null;
			try {
				ParseMessages (tr, p);
				return p;
			} catch (InvalidDataException ide) {
				e = ide;
			} catch (FormatException fe) {
				e = fe;
			}
			Console.Write (tr.Parsed);
			Console.WriteLine (" <---");
			Console.WriteLine (e.Message);
			Console.WriteLine ("Got: " + tr.Next);
			return null;
		}

		static void ParseMessages (TokenReader tr, Proto p)
		{
			while (true) {
				string token;
				try {
					token = tr.ReadNext ();
				} catch (EndOfStreamException) {
					return;
				}
				switch (token) {
				case "message":
					p.Messages.Add (ParseMessage (tr, p));
					break;
				case "option":
					//Save options
					ParseOption (tr, p);
					break;
				default:
					throw new InvalidDataException ("Expected: message or option");
				}
			}
		}

		static Message ParseMessage (TokenReader tr, Message parent)
		{
			Message msg = new Message (parent);
			msg.ProtoName = tr.ReadNext ();
			
			//Expect "{"
			if (tr.ReadNext () != "{")
				throw new InvalidDataException ("Ecpected: {");
			
			while (ParseField (tr, msg))
				continue;
			
			return msg;			
		}

		static bool ParseField (TokenReader tr, Message m)
		{
			string rule = tr.ReadNext ();
			if (rule == "}")
				return false;
			
			if (rule == "enum") {
				MessageEnum me = ParseEnum (tr, m);
				m.Enums.Add (me);
				return true;
			}

			Field f = new Field ();
			
			//Rule
			switch (rule) {
			case "required":
				f.Rule = Rules.Required;
				break;
			case "optional":
				f.Rule = Rules.Optional;
				break;
			case "repeated":
				f.Rule = Rules.Repeated;
				break;
			case "option":
				//Save options
				ParseOption (tr, m);
				return true;
			case "message":
				m.Messages.Add (ParseMessage (tr, m));
				return true;
			default:
				throw new InvalidDataException ("unknown rule: " + rule);
			}
			m.Fields.Add (f);

			//Type
			f.ProtoTypeName = tr.ReadNext ();
			
			//Name
			f.Name = tr.ReadNext ();
			
			//ID
			if (tr.ReadNext () != "=")
				throw new InvalidDataException ("Expected: =");
			f.ID = uint.Parse (tr.ReadNext ());
			if (19000 <= f.ID && f.ID <= 19999)
				throw new InvalidDataException ("Can't use reserved field ID 19000-19999");
			if (f.ID > 536870911)
				throw new InvalidDataException ("Maximum field id is 2^29 - 1");
			foreach (Field fd in m.Fields)
				if (fd.ID == f.ID && fd != f)
					throw new InvalidDataException ("Duplicate key id: " + f.ID);

			
			//Determine if extra options
			string extra = tr.ReadNext ();
			if (extra == ";")
				return true;
			
			//Field options
			if (extra != "[")
				throw new InvalidDataException ("Expected: [");
{'fraction_non_alphanumeric': 0.13911007025761124, 'fraction_numerical': 0.011241217798594848, 'mean_word_length': 2.3088842975206614, 'pattern_counts': {'":': 14, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '490399', 'n_tokens_mistral': 2813, 'n_tokens_neox': 2050, 'n_words': 609}