<commit_before>REBOL [
	title: {source-tool}
	purpose: {Automatic source code modifications.}
	author: {<NAME>}
	license: {Apache 2.0}
]

; --------------------------------------------------------------
; Change the CONFIG section below as necessary.
; Debugging mode writes old and new files to subfolders of current.
;
; INITIAL CONVERSION of old format to new
;
;	Run it once for the conversion to the new format.
;
;
; NORMAL USAGE:
;
;	source-tool/update/all
;
; AD HOC USAGE:
;
;	source-tool/init ; Need this to load and index data.
;
;	help source-tool/list
;
;
; OBJECTIVES:
;
;	* Write function specification into c source file comments.
;	* Index native function arguments.
;	* Not write files unless necessary.
;	* Not overwrite any existing notes in the comments.
;	* Be robust in the face of introduced parse rule bugs.
;
; NOTE:
;
;	Some C identifiers are different to the words they define.
;	- See C-ID-TO-WORD for the mapping.
;
;	Supports tools (e.g. coverity that use comments
;	to annotate declarations by maintaing a comment intact
;	that follows the intro comment but just prior to the
;	declaration.
;
;	Runs on Rebol 2 and Rebol 3.
;
; --------------------------------------------------------------

script-needs [
	%apropos.reb
	%parse-kit.reb
	%trees.reb
	%read-below.reb
	%text-lines.reb
	%../rebol-source-conventions.reb
]

source-tool: context [

	; --- Config 

	rebol.source.folder: clean-path %../../ren-c/src/
	rebol.output.folder: rebol.source.folder

	boot.natives.file: rebol.source.folder/boot/natives.r

	notes-edit-list: %notes-edit-list.reb

	max-line-length: 79 ; Not counting newline.

	logfile: clean-path %source-tool.log.txt
	log: func [message] [write/append logfile join newline mold new-line/all compose/only message false]

	debug: none
	; Set to NONE or :LOG ...

	timing: function [code /local result] [
		started: now/precise
		log [started (code)]
		set/any 'result do code
		finished: now/precise
		log [finished (code) (difference finished started)]
		get/any 'result
	]

	; --- End Config

	stats: context [
		parsed: none
		not-parsed: none
		decl-updated: none
		files-written: none
		width-exceeded: none
	]


	init: func [{Load and index the data.}] [

		reset ; Start fresh.

		timing [r-source/process]
		timing [c-source/process]

		log [words-missing-specs (new-line/all list/missing/specs false)]
		log [words-missing-rebnatives (new-line/all list/missing/rebnatives false)]
	]

	list: context [

		missing: context [

			rebnatives: func [] [
				exclude r-source/native/names c-source/rebnatives
			]

			specs: func [] [
				exclude c-source/rebnatives r-source/native/names
			]

		]
	]

	reset: func [{Clear caches.}] [

		r-source/reset
		c-source/reset

		set stats none
	]

	update: context [

		all: func [] [

			timing [init]
			timing [code]
			timing [files]

			log [stats (body-of stats)]

			reset ; Allow caches to be garbage collected.
		]

		code: func [{Update source (in-memory).}] [

			debug [update-code]

			apropos c-source/decl [
				foreach def list [sync-to-code def]
			]
		]

		files: func [{Write changes to source files.}] [

			debug [update-files]

			apropos c-source [
				foreach name list/changed [file/update name]
			]

			exit
		]
	]

	c-source: context [

		rebnative-index: none

		comment: context [

			format: context [

				slashed: func [text] [

					encode-lines copy text {//} { }
				]

				starred: function [text] [

					width: max-line-length - 2

					rejoin [
						{/*} line* width newline
						encode-lines copy text {**} {  }
						line* width {*/} newline
					]
				]
			]

			line*: func [{Return a line of *.} count] [

				head insert/dup copy {} #"*" count
			]

			load: func [string /local lines prefix] [

				if none? string [return none]

				parse/all string [

					; Slashed.
					copy lines some [{//} [newline | #" " thru newline]]
					(prefix: {//})

					| [ ; Starred
						{/*} 20 200 #"*" newline
						copy lines some [{**} [newline | #" " thru newline]]
						20 200 #"*" #"/" newline
						to end
						(prefix: {**})
					]
				]

				if lines [
					lines: decode-lines lines prefix {  }
					decode-function-meta lines
				]
			]
		]

		decl: context [

			list: none

			format: function [def][

				intro: comment/format/slashed def/intro-notes

				if lines-exceeding max-line-length intro [
					log [line-width-exceeded intro (mold def/file) (def/name) (def/param)]
					stats/width-exceeded: 1 + any [stats/width-exceeded 0]
				]

				parts: collect [

					if def/pre-comment [
						keep def/pre-comment
						keep newline
					]

					foreach word def/keywords [
						keep word
						if "*" <> word [keep #" "]
					]

					keep def/name
					keep rejoin [#"(" def/param #")" newline]

					keep #"^{"
				]

				rest: rejoin parts

				if lines-exceeding max-line-length rest [
					log [line-width-exceeded non-intro (mold def/file) (def/name) (def/param)]
				]

				join intro rest
			]

			normalise: function [def][

				debug [normalising (def)]

				name: def/name
				r.id: none

				either def/style = 'new-style-decl [

					set [meta notes] comment/load def/intro-notes
					if none? meta [
						log [warning meta-not-decoded (def)]
					]
				][

					either native: rebnative? def [

						r.id: c-id-to-word name: def/param
						r-info: attempt [copy r-source/native/cache/:r.id]

						meta: compose/only [
							(to set-word! r.id) native (r-info)
						]
					][

						meta: compose/only [
							(to set-word! form name) C
						]
					]

					new-line meta true

					if def/post-notes [def/post-notes: rejoin [newline def/post-notes newline]]
					notes: def/post-notes
					def/post-notes: none

					if def/intro-notes [
						def/pre-comment: rejoin [{/*} def/intro-notes {*/}]
						def/intro-notes: none
					]

					if all [
						notes
						r.id
						edit: attempt [r-source/notes-to-edit/:r.id]
						edit/checksum = checksum/secure to binary! notes
					][
						notes: edit/new
						if empty? notes [notes: none]
					]
				]
<commit_msg>Discard empty notes with just blank lines.<commit_after>REBOL [
	title: {source-tool}
	purpose: {Automatic source code modifications.}
	author: {<NAME>}
	license: {Apache 2.0}
]

; --------------------------------------------------------------
; Change the CONFIG section below as necessary.
; Debugging mode writes old and new files to subfolders of current.
;
; INITIAL CONVERSION of old format to new
;
;	Run it once for the conversion to the new format.
;
;
; NORMAL USAGE:
;
;	source-tool/update/all
;
; AD HOC USAGE:
;
;	source-tool/init ; Need this to load and index data.
;
;	help source-tool/list
;
;
; OBJECTIVES:
;
;	* Write function specification into c source file comments.
;	* Index native function arguments.
;	* Not write files unless necessary.
;	* Not overwrite any existing notes in the comments.
;	* Be robust in the face of introduced parse rule bugs.
;
; NOTE:
;
;	Some C identifiers are different to the words they define.
;	- See C-ID-TO-WORD for the mapping.
;
;	Supports tools (e.g. coverity that use comments
;	to annotate declarations by maintaing a comment intact
;	that follows the intro comment but just prior to the
;	declaration.
;
;	Runs on Rebol 2 and Rebol 3.
;
; --------------------------------------------------------------

script-needs [
	%apropos.reb
	%parse-kit.reb
	%trees.reb
	%read-below.reb
	%text-lines.reb
	%../rebol-source-conventions.reb
]

source-tool: context [

	; --- Config 

	rebol.source.folder: clean-path %../../ren-c/src/
	rebol.output.folder: rebol.source.folder

	boot.natives.file: rebol.source.folder/boot/natives.r

	notes-edit-list: %notes-edit-list.reb

	max-line-length: 79 ; Not counting newline.

	logfile: clean-path %source-tool.log.txt
	log: func [message] [write/append logfile join newline mold new-line/all compose/only message false]

	debug: none
	; Set to NONE or :LOG ...

	timing: function [code /local result] [
		started: now/precise
		log [started (code)]
		set/any 'result do code
		finished: now/precise
		log [finished (code) (difference finished started)]
		get/any 'result
	]

	; --- End Config

	stats: context [
		parsed: none
		not-parsed: none
		decl-updated: none
		files-written: none
		width-exceeded: none
	]


	init: func [{Load and index the data.}] [

		reset ; Start fresh.

		timing [r-source/process]
		timing [c-source/process]

		log [words-missing-specs (new-line/all list/missing/specs false)]
		log [words-missing-rebnatives (new-line/all list/missing/rebnatives false)]
	]

	list: context [

		missing: context [

			rebnatives: func [] [
				exclude r-source/native/names c-source/rebnatives
			]

			specs: func [] [
				exclude c-source/rebnatives r-source/native/names
			]

		]
	]

	reset: func [{Clear caches.}] [

		r-source/reset
		c-source/reset

		set stats none
	]

	update: context [

		all: func [] [

			timing [init]
			timing [code]
			timing [files]

			log [stats (body-of stats)]

			reset ; Allow caches to be garbage collected.
		]

		code: func [{Update source (in-memory).}] [

			debug [update-code]

			apropos c-source/decl [
				foreach def list [sync-to-code def]
			]
		]

		files: func [{Write changes to source files.}] [

			debug [update-files]

			apropos c-source [
				foreach name list/changed [file/update name]
			]

			exit
		]
	]

	c-source: context [

		rebnative-index: none

		comment: context [

			format: context [

				slashed: func [text] [

					encode-lines copy text {//} { }
				]

				starred: function [text] [

					width: max-line-length - 2

					rejoin [
						{/*} line* width newline
						encode-lines copy text {**} {  }
						line* width {*/} newline
					]
				]
			]

			line*: func [{Return a line of *.} count] [

				head insert/dup copy {} #"*" count
			]

			load: func [string /local lines prefix] [

				if none? string [return none]

				parse/all string [

					; Slashed.
					copy lines some [{//} [newline | #" " thru newline]]
					(prefix: {//})

					| [ ; Starred
						{/*} 20 200 #"*" newline
						copy lines some [{**} [newline | #" " thru newline]]
						20 200 #"*" #"/" newline
						to end
						(prefix: {**})
					]
				]

				if lines [
					lines: decode-lines lines prefix {  }
					decode-function-meta lines
				]
			]
		]

		decl: context [

			list: none

			format: function [def][

				intro: comment/format/slashed def/intro-notes

				if lines-exceeding max-line-length intro [
					log [line-width-exceeded intro (mold def/file) (def/name) (def/param)]
					stats/width-exceeded: 1 + any [stats/width-exceeded 0]
				]

				parts: collect [

					if def/pre-comment [
						keep def/pre-comment
						keep newline
					]

					foreach word def/keywords [
						keep word
						if "*" <> word [keep #" "]
					]

					keep def/name
					keep rejoin [#"(" def/param #")" newline]

					keep #"^{"
				]

				rest: rejoin parts

				if lines-exceeding max-line-length rest [
					log [line-width-exceeded non-intro (mold def/file) (def/name) (def/param)]
				]

				join intro rest
			]

			normalise: function [def][

				debug [normalising (def)]

				name: def/name
				r.id: none

				either def/style = 'new-style-decl [

					set [meta notes] comment/load def/intro-notes
					if none? meta [
						log [warning meta-not-decoded (def)]
					]
				][

					either native: rebnative? def [

						r.id: c-id-to-word name: def/param
						r-info: attempt [copy r-source/native/cache/:r.id]

						meta: compose/only [
							(to set-word! r.id) native (r-info)
						]
					][

						meta: compose/only [
							(to set-word! form name) C
						]
					]

					new-line meta true

					if def/post-notes [def/post-notes: rejoin [newline def/post-notes newline]]
					notes: def/post-notes
					def/post-notes: none

					if def/intro-notes [
						def/pre-comment: rejoin [{/*} def/intro-notes {*/}]
						def/intro-notes: none
					]

					if all [
						notes
						r.id
						edit: attempt [r-source/notes-to-edit/:r.id]
						edit/checksum = checksum/secure to binary! notes
					][
						notes: edit/new
						if empty? trim copy notes [notes: none]
					]
				]

{'fraction_non_alphanumeric': 0.14523711340206186, 'fraction_numerical': 0.003134020618556701, 'mean_word_length': 2.21474019088017, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6635637', 'n_tokens_mistral': 5224, 'n_tokens_neox': 4031, 'n_words': 1158}