<commit_before>/* ###
 * IP: GHIDRA
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package ghidra.app.util.bin.format.dwarf4.next;

import java.util.*;
import java.util.stream.Collectors;

import java.io.IOException;

import ghidra.app.cmd.comments.AppendCommentCmd;
import ghidra.app.cmd.label.SetLabelPrimaryCmd;
import ghidra.app.util.bin.format.dwarf4.*;
import ghidra.app.util.bin.format.dwarf4.encoding.DWARFAttribute;
import ghidra.app.util.bin.format.dwarf4.encoding.DWARFSourceLanguage;
import ghidra.app.util.bin.format.dwarf4.encoding.DWARFTag;
import ghidra.app.util.bin.format.dwarf4.expression.*;
import ghidra.program.database.function.OverlappingFunctionException;
import ghidra.program.model.address.Address;
import ghidra.program.model.address.AddressSet;
import ghidra.program.model.data.*;
import ghidra.program.model.data.DataUtilities.ClearDataMode;
import ghidra.program.model.data.Enum;
import ghidra.program.model.lang.*;
import ghidra.program.model.listing.*;
import ghidra.program.model.listing.Function.FunctionUpdateType;
import ghidra.program.model.pcode.Varnode;
import ghidra.program.model.symbol.*;
import ghidra.program.model.util.CodeUnitInsertionException;
import ghidra.util.Msg;
import ghidra.util.exception.*;
import ghidra.util.task.TaskMonitor;

/**
 * Iterates through all DIEAs in a {@link DWARFProgram} and creates Ghidra functions
 * and variables.
 */
public class DWARFFunctionImporter {

	/**
	 * Inline funcs shorter than this value receive comments at EOL instead of PRE
	 * (ie. inline funcs that reduce down to a single operand or operand value)
	 */
	private static final int INLINE_FUNC_SHORT_LEN = 8;

	private final DWARFProgram prog;
	private final Program currentProgram;
	private final DWARFDataTypeManager dwarfDTM;
	private final DWARFImportOptions importOptions;

	private ProgramModule rootModule;// Program tree module for DWARF
	private Set<Long> processedOffsets = new HashSet<>();
	private Map<Address, String> functionsProcessed = new HashMap<>();
	private Set<Address> variablesProcesesed = new HashSet<>();

	private TaskMonitor monitor;

	private DWARFImportSummary importSummary;

	public static boolean hasDWARFProgModule(Program prog, String progModuleName) {
		ProgramModule dwarfModule = prog.getListing().getRootModule(progModuleName);

		return dwarfModule != null;
	}

	public DWARFFunctionImporter(DWARFProgram prog, DWARFDataTypeManager dwarfDTM,
			DWARFImportOptions importOptions, DWARFImportSummary importSummary,
			TaskMonitor monitor) {
		this.prog = prog;
		this.monitor = monitor;
		this.currentProgram = prog.getGhidraProgram();
		this.dwarfDTM = dwarfDTM;
		this.importOptions = importOptions;
		this.importSummary = importSummary;
	}

	private boolean shouldProcess(DIEAggregate diea) {
		if (processedOffsets.contains(diea.getOffset())) {
			return false;
		}
		processedOffsets.add(diea.getOffset());
		return true;
	}

	public void importFunctions() throws CancelledException {
		rootModule = currentProgram.getListing().getRootModule(DWARFProgram.DWARF_ROOT_NAME);
		if (rootModule == null) {
			try {
				rootModule =
					currentProgram.getListing().createRootModule(DWARFProgram.DWARF_ROOT_NAME);
			}
			catch (DuplicateNameException e) {
				// should not happen
			}
		}

		for (DIEAggregate diea : DIEAMonitoredIterator.iterable(prog,
			"DWARF - Create Funcs & Symbols", monitor)) {
			monitor.checkCanceled();

			try {
				switch (diea.getTag()) {
					case DWARFTag.DW_TAG_subprogram:
						try {
							processSubprogram(diea);
						}
						catch (InvalidInputException e) {
							Msg.error(this, "Failed to process subprog " + diea.getHexOffset(), e);
						}
						break;
					case DWARFTag.DW_TAG_variable:
						// only process variable definitions that are static variables
						// (ie. they are children of the compunit root, ie. depth == 1)
						// local variables should be children of dw_tag_subprograms
						// and will be handled in processFuncChildren()
						if (diea.getDepth() == 1) {
							try {
								processVariable(diea, null, null, -1);
							}
							catch (InvalidInputException e) {
								Msg.error(this, "Failed to process var " + diea.getHexOffset(), e);
							}
						}
						break;
					case DWARFTag.DW_TAG_label:
						processLabel(diea);
						break;

					case DWARFTag.DW_TAG_gnu_call_site:
					case DWARFTag.DW_TAG_call_site:
						DIEAggregate partDIEA = DIEAggregate.createSkipHead(diea);
						if (partDIEA != null && !isBadSubprogramDef(partDIEA)) {
							processSubprogram(partDIEA);
						}
						break;
				}
			}
			catch (OutOfMemoryError oom) {
				throw oom;
			}
			catch (Throwable th) {
				Msg.error(this,
					"Error when processing DWARF information for DIE " + diea.getHexOffset(), th);
				Msg.info(this, "DIE info:\n" + diea.toString());
			}
		}
		logImportErrorSummary();
	}

	private void logImportErrorSummary() {
		if (!importSummary.unknownRegistersEncountered.isEmpty()) {
			Msg.error(this, "Found " + importSummary.unknownRegistersEncountered.size() +
				" unknown registers referenced in DWARF expression operands:");
			List<Integer> sortedUnknownRegs =
				new ArrayList<>(importSummary.unknownRegistersEncountered);
			Collections.sort(sortedUnknownRegs);
			Msg.error(this,
				"  unknown registers: " +
					sortedUnknownRegs.stream().map(i -> Integer.toString(i)).collect(
						Collectors.joining(", ")));
		}
	}

	private boolean isBadSubprogramDef(DIEAggregate diea) {
		if (diea.isDanglingDeclaration() || !diea.hasAttribute(DWARFAttribute.DW_AT_low_pc)) {
			return true;
		}

		if (diea.getLowPC(-1) == 0) {
			return true;
		}

		return false;
	}

	private void markAllChildrenAsProcessed(DebugInfoEntry die) {
		for (DebugInfoEntry child : die.getChildren()) {
			processedOffsets.add(child.getOffset());
			markAllChildrenAsProcessed(child);
		}
	}

	private void processSubprogram(DIEAggregate diea)
			throws IOException, InvalidInputException, DWARFExpressionException {

		if (!shouldProcess(diea)) {
			return;
		}

		if (isBadSubprogramDef(diea)) {
			markAllChildrenAsProcessed(diea.getHeadFragment());
			return;
		}

		DWARFFunction dfunc = new DWARFFunction(prog.getName(diea));
		dfunc.namespace = dfunc.dni.getParentNamespace(currentProgram);

		Number lowPC = diea.getLowPC(0);
		dfunc.address = toAddr(lowPC);
		dfunc.highAddress =
			diea.hasAttribute(DWARFAttribute.DW_AT_high_pc) ? toAddr(diea.getHighPC()) : null;

		String previousFunctionProcessed = functionsProcessed.get(dfunc.address);
		if (previousFunctionProcessed != null) {
//			Msg.info(this, "Duplicate function defintion found for " + dni.getCategoryPath() +
//				" at " + function.address + " in DIE " + diea.getHexOffset() + ", skipping");
			markAllChildrenAsProcessed(diea.getHeadFragment());
			return;
		}
		functionsProcessed.put(dfunc.address,
			dfunc.dni.getNamespacePath() + " DIE: " + diea.getHexOffset());

		// Check if the function is an external function
		dfunc.isExternal = diea.getBool(DWARFAttribute.DW_AT_external, false);

		// Retrieve the frame base if it exists
		DWARFLocation frameLoc = null;
		if (diea.hasAttribute(DWARFAttribute.DW_AT_frame_base)) {
			List<DWARFLocation> frameBase = diea.getAsLocation(DWARFAttribute.DW_AT_frame_base);
			// get the framebase register, find where the frame is finally set
			// up.
			frameLoc = getTopLocation(frameBase, dfunc.address.getOffset());
			if (frameLoc != null) {
				dfunc.frameBase = (int) diea.evaluateLocation(frameLoc);
			}
		}

		// Get it's return type
		// TODO: Sometimes the return type may actually be a pointer parameter
		// passed into
		// the given function - figure out how to determine this. For example,
		// C++ can
		// return object types defined in the function but may be implemented as
		// the caller
		// function passing a pointer to the callee function where the object is
		// then operated on.
		DIEAggregate typeRef = diea.getTypeRef();
		DataType formalReturnType = (typeRef != null)
				? dwarfDTM.getDataType(typeRef, DataType.DEFAULT)
				: dwarfDTM.getVoidType();
		dfunc.retval = new DWARFVariable();
		dfunc.retval.type = formalReturnType;

		boolean formalParamsOnly = false;
		boolean skipFuncSignature = false;
		List<Parameter> formalParams = new ArrayList<>();

		for (DebugInfoEntry childEntry : diea.getHeadFragment().getChildren(
			DWARFTag.DW_TAG_formal_parameter)) {
			DIEAggregate childDIEA = prog.getAggregate(childEntry);

			Parameter formalParam = createFormalParameter(childDIEA);
			if (formalParam == null) {
				skipFuncSignature = true;
				break;
			}
			formalParams.add(formalParam);

			if (!formalParamsOnly) {
				DWARFVariable var = processVariable(childDIEA, dfunc, null, -1);
				if (var == null) {
					// we had an error, can't rely on detailed param data, fallback to
					// formal params
					formalParamsOnly = true;
					dfunc.params.clear();
				}
				else {
					dfunc.params.add(var);
				}
			}
		}
		dfunc.varArg =
			!diea.getHeadFragment().getChildren(DWARFTag.DW_TAG_unspecified_parameters).isEmpty();

		processFuncChildren(diea, dfunc);

		Function gfunc = createFunction(dfunc, diea);

		boolean updateSignature = true;

		DWARFCompilationUnit firstCompilationUnit = prog.getCompilationUnits().get(0);
		if (firstCompilationUnit != null) {
			Number dwarfLanguage = firstCompilationUnit.getCompileUnit().getLanguage();

			// Rust does not have associated function signature
			if (dwarfLanguage == (Number) DWARFSourceLanguage.DW_LANG_Rust) {
				updateSignature = false;
			}
		}

		if (gfunc != null && updateSignature) {

			if (formalParams.isEmpty() && dfunc.localVarErrors) {
				// if there were no defined parameters and we had problems decoding local variables,
				// don't force the method to have an empty param signature because there are other
				// issues afoot.
				skipFuncSignature = true;
			}

			if (skipFuncSignature) {
				Msg.error(this,
					"Failed to get function signature information, leaving undefined: " +
						gfunc.getName() + "@" + gfunc.getEntryPoint());
				Msg.debug(this, "DIE info: " + diea.toString());
				return;
			}

			if (formalParamsOnly) {
				updateFunctionSignatureWithFormalParams(gfunc, formalParams,
					formalReturnType, dfunc.varArg, diea);
			}
			else {
				updateFunctionSignatureWithDetailParams(gfunc, dfunc, diea);
			}
		}
<commit_msg>allow debug info used for Rust lang when present<commit_after>/* ###
 * IP: GHIDRA
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package ghidra.app.util.bin.format.dwarf4.next;

import java.util.*;
import java.util.stream.Collectors;

import java.io.IOException;

import ghidra.app.cmd.comments.AppendCommentCmd;
import ghidra.app.cmd.label.SetLabelPrimaryCmd;
import ghidra.app.util.bin.format.dwarf4.*;
import ghidra.app.util.bin.format.dwarf4.encoding.DWARFAttribute;
import ghidra.app.util.bin.format.dwarf4.encoding.DWARFSourceLanguage;
import ghidra.app.util.bin.format.dwarf4.encoding.DWARFTag;
import ghidra.app.util.bin.format.dwarf4.expression.*;
import ghidra.program.database.function.OverlappingFunctionException;
import ghidra.program.model.address.Address;
import ghidra.program.model.address.AddressSet;
import ghidra.program.model.data.*;
import ghidra.program.model.data.DataUtilities.ClearDataMode;
import ghidra.program.model.data.Enum;
import ghidra.program.model.lang.*;
import ghidra.program.model.listing.*;
import ghidra.program.model.listing.Function.FunctionUpdateType;
import ghidra.program.model.pcode.Varnode;
import ghidra.program.model.symbol.*;
import ghidra.program.model.util.CodeUnitInsertionException;
import ghidra.util.Msg;
import ghidra.util.exception.*;
import ghidra.util.task.TaskMonitor;

/**
 * Iterates through all DIEAs in a {@link DWARFProgram} and creates Ghidra functions
 * and variables.
 */
public class DWARFFunctionImporter {

	/**
	 * Inline funcs shorter than this value receive comments at EOL instead of PRE
	 * (ie. inline funcs that reduce down to a single operand or operand value)
	 */
	private static final int INLINE_FUNC_SHORT_LEN = 8;

	private final DWARFProgram prog;
	private final Program currentProgram;
	private final DWARFDataTypeManager dwarfDTM;
	private final DWARFImportOptions importOptions;

	private ProgramModule rootModule;// Program tree module for DWARF
	private Set<Long> processedOffsets = new HashSet<>();
	private Map<Address, String> functionsProcessed = new HashMap<>();
	private Set<Address> variablesProcesesed = new HashSet<>();

	private TaskMonitor monitor;

	private DWARFImportSummary importSummary;

	public static boolean hasDWARFProgModule(Program prog, String progModuleName) {
		ProgramModule dwarfModule = prog.getListing().getRootModule(progModuleName);

		return dwarfModule != null;
	}

	public DWARFFunctionImporter(DWARFProgram prog, DWARFDataTypeManager dwarfDTM,
			DWARFImportOptions importOptions, DWARFImportSummary importSummary,
			TaskMonitor monitor) {
		this.prog = prog;
		this.monitor = monitor;
		this.currentProgram = prog.getGhidraProgram();
		this.dwarfDTM = dwarfDTM;
		this.importOptions = importOptions;
		this.importSummary = importSummary;
	}

	private boolean shouldProcess(DIEAggregate diea) {
		if (processedOffsets.contains(diea.getOffset())) {
			return false;
		}
		processedOffsets.add(diea.getOffset());
		return true;
	}

	public void importFunctions() throws CancelledException {
		rootModule = currentProgram.getListing().getRootModule(DWARFProgram.DWARF_ROOT_NAME);
		if (rootModule == null) {
			try {
				rootModule =
					currentProgram.getListing().createRootModule(DWARFProgram.DWARF_ROOT_NAME);
			}
			catch (DuplicateNameException e) {
				// should not happen
			}
		}

		for (DIEAggregate diea : DIEAMonitoredIterator.iterable(prog,
			"DWARF - Create Funcs & Symbols", monitor)) {
			monitor.checkCanceled();

			try {
				switch (diea.getTag()) {
					case DWARFTag.DW_TAG_subprogram:
						try {
							processSubprogram(diea);
						}
						catch (InvalidInputException e) {
							Msg.error(this, "Failed to process subprog " + diea.getHexOffset(), e);
						}
						break;
					case DWARFTag.DW_TAG_variable:
						// only process variable definitions that are static variables
						// (ie. they are children of the compunit root, ie. depth == 1)
						// local variables should be children of dw_tag_subprograms
						// and will be handled in processFuncChildren()
						if (diea.getDepth() == 1) {
							try {
								processVariable(diea, null, null, -1);
							}
							catch (InvalidInputException e) {
								Msg.error(this, "Failed to process var " + diea.getHexOffset(), e);
							}
						}
						break;
					case DWARFTag.DW_TAG_label:
						processLabel(diea);
						break;

					case DWARFTag.DW_TAG_gnu_call_site:
					case DWARFTag.DW_TAG_call_site:
						DIEAggregate partDIEA = DIEAggregate.createSkipHead(diea);
						if (partDIEA != null && !isBadSubprogramDef(partDIEA)) {
							processSubprogram(partDIEA);
						}
						break;
				}
			}
			catch (OutOfMemoryError oom) {
				throw oom;
			}
			catch (Throwable th) {
				Msg.error(this,
					"Error when processing DWARF information for DIE " + diea.getHexOffset(), th);
				Msg.info(this, "DIE info:\n" + diea.toString());
			}
		}
		logImportErrorSummary();
	}

	private void logImportErrorSummary() {
		if (!importSummary.unknownRegistersEncountered.isEmpty()) {
			Msg.error(this, "Found " + importSummary.unknownRegistersEncountered.size() +
				" unknown registers referenced in DWARF expression operands:");
			List<Integer> sortedUnknownRegs =
				new ArrayList<>(importSummary.unknownRegistersEncountered);
			Collections.sort(sortedUnknownRegs);
			Msg.error(this,
				"  unknown registers: " +
					sortedUnknownRegs.stream().map(i -> Integer.toString(i)).collect(
						Collectors.joining(", ")));
		}
	}

	private boolean isBadSubprogramDef(DIEAggregate diea) {
		if (diea.isDanglingDeclaration() || !diea.hasAttribute(DWARFAttribute.DW_AT_low_pc)) {
			return true;
		}

		if (diea.getLowPC(-1) == 0) {
			return true;
		}

		return false;
	}

	private void markAllChildrenAsProcessed(DebugInfoEntry die) {
		for (DebugInfoEntry child : die.getChildren()) {
			processedOffsets.add(child.getOffset());
			markAllChildrenAsProcessed(child);
		}
	}

	private void processSubprogram(DIEAggregate diea)
			throws IOException, InvalidInputException, DWARFExpressionException {

		if (!shouldProcess(diea)) {
			return;
		}

		if (isBadSubprogramDef(diea)) {
			markAllChildrenAsProcessed(diea.getHeadFragment());
			return;
		}

		DWARFFunction dfunc = new DWARFFunction(prog.getName(diea));
		dfunc.namespace = dfunc.dni.getParentNamespace(currentProgram);

		Number lowPC = diea.getLowPC(0);
		dfunc.address = toAddr(lowPC);
		dfunc.highAddress =
			diea.hasAttribute(DWARFAttribute.DW_AT_high_pc) ? toAddr(diea.getHighPC()) : null;

		String previousFunctionProcessed = functionsProcessed.get(dfunc.address);
		if (previousFunctionProcessed != null) {
//			Msg.info(this, "Duplicate function defintion found for " + dni.getCategoryPath() +
//				" at " + function.address + " in DIE " + diea.getHexOffset() + ", skipping");
			markAllChildrenAsProcessed(diea.getHeadFragment());
			return;
		}
		functionsProcessed.put(dfunc.address,
			dfunc.dni.getNamespacePath() + " DIE: " + diea.getHexOffset());

		// Check if the function is an external function
		dfunc.isExternal = diea.getBool(DWARFAttribute.DW_AT_external, false);

		// Retrieve the frame base if it exists
		DWARFLocation frameLoc = null;
		if (diea.hasAttribute(DWARFAttribute.DW_AT_frame_base)) {
			List<DWARFLocation> frameBase = diea.getAsLocation(DWARFAttribute.DW_AT_frame_base);
			// get the framebase register, find where the frame is finally set
			// up.
			frameLoc = getTopLocation(frameBase, dfunc.address.getOffset());
			if (frameLoc != null) {
				dfunc.frameBase = (int) diea.evaluateLocation(frameLoc);
			}
		}

		// Get it's return type
		// TODO: Sometimes the return type may actually be a pointer parameter
		// passed into
		// the given function - figure out how to determine this. For example,
		// C++ can
		// return object types defined in the function but may be implemented as
		// the caller
		// function passing a pointer to the callee function where the object is
		// then operated on.
		DIEAggregate typeRef = diea.getTypeRef();
		DataType formalReturnType = (typeRef != null)
				? dwarfDTM.getDataType(typeRef, DataType.DEFAULT)
				: dwarfDTM.getVoidType();
		dfunc.retval = new DWARFVariable();
		dfunc.retval.type = formalReturnType;

		boolean formalParamsOnly = false;
		boolean skipFuncSignature = false;
		List<Parameter> formalParams = new ArrayList<>();

		for (DebugInfoEntry childEntry : diea.getHeadFragment().getChildren(
			DWARFTag.DW_TAG_formal_parameter)) {
			DIEAggregate childDIEA = prog.getAggregate(childEntry);

			Parameter formalParam = createFormalParameter(childDIEA);
			if (formalParam == null) {
				skipFuncSignature = true;
				break;
			}
			formalParams.add(formalParam);

			if (!formalParamsOnly) {
				DWARFVariable var = processVariable(childDIEA, dfunc, null, -1);
				if (var == null) {
					// we had an error, can't rely on detailed param data, fallback to
					// formal params
					formalParamsOnly = true;
					dfunc.params.clear();
				}
				else {
					dfunc.params.add(var);
				}
			}
		}
		dfunc.varArg =
			!diea.getHeadFragment().getChildren(DWARFTag.DW_TAG_unspecified_parameters).isEmpty();

		processFuncChildren(diea, dfunc);

		Function gfunc = createFunction(dfunc, diea);

		Number dwarfLanguage = -1;


		DWARFCompilationUnit firstCompilationUnit = prog.getCompilationUnits().get(0);
		if (firstCompilationUnit != null) {
			dwarfLanguage = firstCompilationUnit.getCompileUnit().getLanguage();
		}

		if (gfunc != null) {

			if (formalParams.isEmpty() && dfunc.localVarErrors) {
				// if there were no defined parameters and we had problems decoding local variables,
				// don't force the method to have an empty param signature because there are other
				// issues afoot.
				skipFuncSignature = true;
			} else if (formalParams.isEmpty() && dwarfLanguage == (Number) DWARFSourceLanguage.DW_LANG_Rust) {
				// if there were no defined parameters and the language is Rust, don't force an
				// empty param signature. Rust language emit dwarf info without types (signatures)
				// when used without -g.
				skipFuncSignature = true;
			}

			if (skipFuncSignature) {
				Msg.error(this,
					"Failed to get function signature information, leaving undefined: " +
						gfunc.getName() + "@" + gfunc.getEntryPoint());
				Msg.debug(this, "DIE info: " + diea.toString());
				return;
			}

			if (formalParamsOnly) {
				updateFunctionSignatureWithFormalParams(gfunc, formalParams,
					formalReturnType, dfunc.varArg, diea);
			}
			else {
				updateFunctionSignatureWithDetailParams(gfunc, dfunc, diea);
			}
		}

{'fraction_non_alphanumeric': 0.10396897810218977, 'fraction_numerical': 0.0017791970802919707, 'mean_word_length': 3.968495013599275, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3639203', 'n_tokens_mistral': 7951, 'n_tokens_neox': 6788, 'n_words': 1632}