<commit_before>package main

import (
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Command Line Lexer", func() {

	var lexer *CommandLineLexer

	BeforeEach(func(){
		lexer = NewCommandLineLexer()
	})

	It("Lex a line twice", func(){
			method := "GET"
			applicationJson := "Content-type:application/json"
			applicationSoapXml := "Content-type:application/soap+xml"
			line := fmt.Sprintf(`http://127.0.0.1:8000/A -X %s -H "%s" -H "%s"`, method, applicationJson, applicationSoapXml)

		result := lexer.Lex(line)
		Expect(result).To(Equal([]string{"http://127.0.0.1:8000/A", "-X", method, "-H", applicationJson, "-H", applicationSoapXml}))

		result = lexer.Lex(line)
		Expect(result).To(Equal([]string{"http://127.0.0.1:8000/A", "-X", method, "-H", applicationJson, "-H", applicationSoapXml}))

	})

	It("Lex one single dashed flag", func(){
		result := lexer.Lex("-X POST")
		Expect(result).To(Equal([]string{"-X","POST"}))
	})

	It("Lex a flag value which contains a space", func(){
		result := lexer.Lex(`-H "Content-type: application/json" -H "Accept: text/xml"`)
		Expect(result).To(Equal([]string{"-H", "Content-type: application/json","-H","Accept: text/xml"}))
	})

	It("Lex a flag value which is inside single quotes", func(){
		result := lexer.Lex(`-H 'Content-type: application/json' -H 'Accept: text/xml'`)
		Expect(result).To(Equal([]string{"-H", "Content-type: application/json","-H","Accept: text/xml"}))
	})

	It("Lex a flag value which contains a non-flag at beginning", func(){
		result := lexer.Lex(` A -H "Content-type: application/json"`)
		Expect(result).To(Equal([]string{"A", "-H", "Content-type: application/json"}))
	})

	It("Lex a flag value with double quotes inside single quotes",func(){
		result := lexer.Lex(`-d '{"name":"bob"}'`)
		Expect(result).To(Equal([]string{"-d", "{\"name\":\"bob\"}"}))
	})

	It("Lex a flag value with single quotes inside double quotes",func(){
		result := lexer.Lex(`-d "{'name':'bob'}"`)
		Expect(result).To(Equal([]string{"-d", "{'name':'bob'}"}))
	})

	It("Lex a double dashed flag", func(){
		result := lexer.Lex("--cert ca.pem")
		Expect(result).To(Equal([]string{"--cert","ca.pem"}))
	})

	It("Lex a double dashed flag without value", func(){
		result := lexer.Lex("--dummy --cert ca.pem")<commit_msg>Got rid of the back ticks and just escaped the double quotes<commit_after>package main

import (
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Command Line Lexer", func() {

	var lexer *CommandLineLexer

	BeforeEach(func(){
		lexer = NewCommandLineLexer()
	})

	It("Lex a line twice", func(){
			method := "GET"
			applicationJson := "Content-type:application/json"
			applicationSoapXml := "Content-type:application/soap+xml"
			line := fmt.Sprintf("http://127.0.0.1:8000/A -X %s -H \"%s\" -H \"%s\"", method, applicationJson, applicationSoapXml)

		result := lexer.Lex(line)
		Expect(result).To(Equal([]string{"http://127.0.0.1:8000/A", "-X", method, "-H", applicationJson, "-H", applicationSoapXml}))

		result = lexer.Lex(line)
		Expect(result).To(Equal([]string{"http://127.0.0.1:8000/A", "-X", method, "-H", applicationJson, "-H", applicationSoapXml}))

	})

	It("Lex one single dashed flag", func(){
		result := lexer.Lex("-X POST")
		Expect(result).To(Equal([]string{"-X","POST"}))
	})

	It("Lex a flag value which contains a space", func(){
		result := lexer.Lex("-H \"Content-type: application/json\" -H \"Accept: text/xml\"")
		Expect(result).To(Equal([]string{"-H", "Content-type: application/json","-H","Accept: text/xml"}))
	})

	It("Lex a flag value which is inside single quotes", func(){
		result := lexer.Lex("-H 'Content-type: application/json' -H 'Accept: text/xml'")
		Expect(result).To(Equal([]string{"-H", "Content-type: application/json","-H","Accept: text/xml"}))
	})

	It("Lex a flag value which contains a non-flag at beginning", func(){
		result := lexer.Lex(" A -H \"Content-type: application/json\"")
		Expect(result).To(Equal([]string{"A", "-H", "Content-type: application/json"}))
	})

	It("Lex a flag value with double quotes inside single quotes",func(){
		result := lexer.Lex("-d '{\"name\":\"bob\"}'")
		Expect(result).To(Equal([]string{"-d", "{\"name\":\"bob\"}"}))
	})

	It("Lex a flag value with single quotes inside double quotes",func(){
		result := lexer.Lex("-d \"{'name':'bob'}\"")
		Expect(result).To(Equal([]string{"-d", "{'name':'bob'}"}))
	})

	It("Lex a double dashed flag", func(){
		result := lexer.Lex("--cert ca.pem")
		Expect(result).To(Equal([]string{"--cert","ca.pem"}))
	})

	It("Lex a double dashed flag without value", func(){
		result := lexer.Lex("--dummy --cert ca.pem")
{'fraction_non_alphanumeric': 0.22877913192217234, 'fraction_numerical': 0.012828736369467608, 'mean_word_length': 4.529550827423168, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5075320', 'n_tokens_mistral': 1803, 'n_tokens_neox': 1618, 'n_words': 372}