<commit_before>## @file
# process FV generation
#
#  Copyright (c) 2007 - 2017, Intel Corporation. All rights reserved.<BR>
#
#  This program and the accompanying materials
#  are licensed and made available under the terms and conditions of the BSD License
#  which accompanies this distribution.  The full text of the license may be found at
#  http://opensource.org/licenses/bsd-license.php
#
#  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
#  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
#

##
# Import Modules
#
import Common.LongFilePathOs as os
import subprocess
import StringIO
from struct import *

import Ffs
import AprioriSection
import FfsFileStatement
from GenFdsGlobalVariable import GenFdsGlobalVariable
from GenFds import GenFds
from CommonDataClass.FdfClass import FvClassObject
from Common.Misc import SaveFileOnChange
from Common.LongFilePathSupport import CopyLongFilePath
from Common.LongFilePathSupport import OpenLongFilePath as open

T_CHAR_LF = '\n'
FV_UI_EXT_ENTY_GUID = 'A67DF1FA-8DE8-4E98-AF09-4BDF2EFFBC7C'

## generate FV
#
#
class FV (FvClassObject):
    ## The constructor
    #
    #   @param  self        The object pointer
    #
    def __init__(self):
        FvClassObject.__init__(self)
        self.FvInfFile = None
        self.FvAddressFile = None
        self.BaseAddress = None
        self.InfFileName = None
        self.FvAddressFileName = None
        self.CapsuleName = None
        self.FvBaseAddress = None
        self.FvForceRebase = None
        self.FvRegionInFD = None
        self.UsedSizeEnable = False
        
    ## AddToBuffer()
    #
    #   Generate Fv and add it to the Buffer
    #
    #   @param  self        The object pointer
    #   @param  Buffer      The buffer generated FV data will be put
    #   @param  BaseAddress base address of FV
    #   @param  BlockSize   block size of FV
    #   @param  BlockNum    How many blocks in FV
    #   @param  ErasePolarity      Flash erase polarity
    #   @param  VtfDict     VTF objects
    #   @param  MacroDict   macro value pair
    #   @retval string      Generated FV file path
    #
    def AddToBuffer (self, Buffer, BaseAddress=None, BlockSize= None, BlockNum=None, ErasePloarity='1', VtfDict=None, MacroDict = {}, Flag=False) :

        if BaseAddress == None and self.UiFvName.upper() + 'fv' in GenFds.ImageBinDict.keys():
            return GenFds.ImageBinDict[self.UiFvName.upper() + 'fv']
        
        #
        # Check whether FV in Capsule is in FD flash region.
        # If yes, return error. Doesn't support FV in Capsule image is also in FD flash region.
        #
        if self.CapsuleName != None:
            for FdName in GenFdsGlobalVariable.FdfParser.Profile.FdDict.keys():
                FdObj = GenFdsGlobalVariable.FdfParser.Profile.FdDict[FdName]
                for RegionObj in FdObj.RegionList:
                    if RegionObj.RegionType == 'FV':
                        for RegionData in RegionObj.RegionDataList:
                            if RegionData.endswith(".fv"):
                                continue
                            elif RegionData.upper() + 'fv' in GenFds.ImageBinDict.keys():
                                continue
                            elif self.UiFvName.upper() == RegionData.upper():
                                GenFdsGlobalVariable.ErrorLogger("Capsule %s in FD region can't contain a FV %s in FD region." % (self.CapsuleName, self.UiFvName.upper()))
        if not Flag:
            GenFdsGlobalVariable.InfLogger( "\nGenerating %s FV" %self.UiFvName)
        GenFdsGlobalVariable.LargeFileInFvFlags.append(False)
        FFSGuid = None
        
        if self.FvBaseAddress != None:
            BaseAddress = self.FvBaseAddress
        if not Flag:
            self.__InitializeInf__(BaseAddress, BlockSize, BlockNum, ErasePloarity, VtfDict)
        #
        # First Process the Apriori section
        #
        MacroDict.update(self.DefineVarDict)

        GenFdsGlobalVariable.VerboseLogger('First generate Apriori file !')
        FfsFileList = []
        for AprSection in self.AprioriSectionList:
            FileName = AprSection.GenFfs (self.UiFvName, MacroDict, IsMakefile=Flag)
            FfsFileList.append(FileName)
            # Add Apriori file name to Inf file
            if not Flag:
                self.FvInfFile.writelines("EFI_FILE_NAME = " + \
                                            FileName          + \
                                            T_CHAR_LF)

        # Process Modules in FfsList
        for FfsFile in self.FfsList :
            if Flag:
                if isinstance(FfsFile, FfsFileStatement.FileStatement):
                    continue
            if GenFdsGlobalVariable.EnableGenfdsMultiThread and GenFdsGlobalVariable.ModuleFile and GenFdsGlobalVariable.ModuleFile.Path.find(os.path.normpath(FfsFile.InfFileName)) == -1:
                continue
            FileName = FfsFile.GenFfs(MacroDict, FvParentAddr=BaseAddress, IsMakefile=Flag, FvName=self.UiFvName)
            FfsFileList.append(FileName)
            if not Flag:
                self.FvInfFile.writelines("EFI_FILE_NAME = " + \
                                            FileName          + \
                                            T_CHAR_LF)
        if not Flag:
            SaveFileOnChange(self.InfFileName, self.FvInfFile.getvalue(), False)
            self.FvInfFile.close()
        #
        # Call GenFv tool
        #
        FvOutputFile = os.path.join(GenFdsGlobalVariable.FvDir, self.UiFvName)
        FvOutputFile = FvOutputFile + '.Fv'
        # BUGBUG: FvOutputFile could be specified from FDF file (FV section, CreateFile statement)
        if self.CreateFileName != None:
            FvOutputFile = self.CreateFileName

        if Flag:
            GenFds.ImageBinDict[self.UiFvName.upper() + 'fv'] = FvOutputFile
            return FvOutputFile

        FvInfoFileName = os.path.join(GenFdsGlobalVariable.FfsDir, self.UiFvName + '.inf')
        if not Flag:
            CopyLongFilePath(GenFdsGlobalVariable.FvAddressFileName, FvInfoFileName)
            OrigFvInfo = None
            if os.path.exists (FvInfoFileName):
                OrigFvInfo = open(FvInfoFileName, 'r').read()
            if GenFdsGlobalVariable.LargeFileInFvFlags[-1]:
                FFSGuid = GenFdsGlobalVariable.EFI_FIRMWARE_FILE_SYSTEM3_GUID
            GenFdsGlobalVariable.GenerateFirmwareVolume(
                                    FvOutputFile,
                                    [self.InfFileName],
                                    AddressFile=FvInfoFileName,
                                    FfsList=FfsFileList,
                                    ForceRebase=self.FvForceRebase,
                                    FileSystemGuid=FFSGuid
                                    )

            NewFvInfo = None
            if os.path.exists (FvInfoFileName):
                NewFvInfo = open(FvInfoFileName, 'r').read()
            if NewFvInfo != None and NewFvInfo != OrigFvInfo:
                FvChildAddr = []
                AddFileObj = open(FvInfoFileName, 'r')
                AddrStrings = AddFileObj.readlines()
                AddrKeyFound = False
                for AddrString in AddrStrings:
                    if AddrKeyFound:
                        #get base address for the inside FvImage
                        FvChildAddr.append (AddrString)
                    elif AddrString.find ("[FV_BASE_ADDRESS]") != -1:
                        AddrKeyFound = True
                AddFileObj.close()

                if FvChildAddr != []:
                    # Update Ffs again
                    for FfsFile in self.FfsList :
                        FileName = FfsFile.GenFfs(MacroDict, FvChildAddr, BaseAddress, IsMakefile=Flag)

                    if GenFdsGlobalVariable.LargeFileInFvFlags[-1]:
                        FFSGuid = GenFdsGlobalVariable.EFI_FIRMWARE_FILE_SYSTEM3_GUID;
                    #Update GenFv again
                    GenFdsGlobalVariable.GenerateFirmwareVolume(
                                                FvOutputFile,
                                                [self.InfFileName],
                                                AddressFile=FvInfoFileName,
                                                FfsList=FfsFileList,
                                                ForceRebase=self.FvForceRebase,
                                                FileSystemGuid=FFSGuid
                                                )

            #
            # Write the Fv contents to Buffer
            #
            if os.path.isfile(FvOutputFile):
                FvFileObj = open(FvOutputFile, 'rb')
                GenFdsGlobalVariable.VerboseLogger("\nGenerate %s FV Successfully" % self.UiFvName)
                GenFdsGlobalVariable.SharpCounter = 0

                Buffer.write(FvFileObj.read())
                FvFileObj.seek(0)
                # PI FvHeader is 0x48 byte
                FvHeaderBuffer = FvFileObj.read(0x48)
                # FV alignment position.
                FvAlignmentValue = 1 << (ord(FvHeaderBuffer[0x2E]) & 0x1F)
                if FvAlignmentValue >= 0x400:
                    if FvAlignmentValue >= 0x100000:
                        if FvAlignmentValue >= 0x1000000:<commit_msg>BaseTools: Fix the regression bug of a74398 for SubFv Image<commit_after>## @file
# process FV generation
#
#  Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
#
#  This program and the accompanying materials
#  are licensed and made available under the terms and conditions of the BSD License
#  which accompanies this distribution.  The full text of the license may be found at
#  http://opensource.org/licenses/bsd-license.php
#
#  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
#  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
#

##
# Import Modules
#
import Common.LongFilePathOs as os
import subprocess
import StringIO
from struct import *

import Ffs
import AprioriSection
import FfsFileStatement
from GenFdsGlobalVariable import GenFdsGlobalVariable
from GenFds import GenFds
from CommonDataClass.FdfClass import FvClassObject
from Common.Misc import SaveFileOnChange
from Common.LongFilePathSupport import CopyLongFilePath
from Common.LongFilePathSupport import OpenLongFilePath as open

T_CHAR_LF = '\n'
FV_UI_EXT_ENTY_GUID = 'A67DF1FA-8DE8-4E98-AF09-4BDF2EFFBC7C'

## generate FV
#
#
class FV (FvClassObject):
    ## The constructor
    #
    #   @param  self        The object pointer
    #
    def __init__(self):
        FvClassObject.__init__(self)
        self.FvInfFile = None
        self.FvAddressFile = None
        self.BaseAddress = None
        self.InfFileName = None
        self.FvAddressFileName = None
        self.CapsuleName = None
        self.FvBaseAddress = None
        self.FvForceRebase = None
        self.FvRegionInFD = None
        self.UsedSizeEnable = False
        
    ## AddToBuffer()
    #
    #   Generate Fv and add it to the Buffer
    #
    #   @param  self        The object pointer
    #   @param  Buffer      The buffer generated FV data will be put
    #   @param  BaseAddress base address of FV
    #   @param  BlockSize   block size of FV
    #   @param  BlockNum    How many blocks in FV
    #   @param  ErasePolarity      Flash erase polarity
    #   @param  VtfDict     VTF objects
    #   @param  MacroDict   macro value pair
    #   @retval string      Generated FV file path
    #
    def AddToBuffer (self, Buffer, BaseAddress=None, BlockSize= None, BlockNum=None, ErasePloarity='1', VtfDict=None, MacroDict = {}, Flag=False) :

        if BaseAddress == None and self.UiFvName.upper() + 'fv' in GenFds.ImageBinDict.keys():
            return GenFds.ImageBinDict[self.UiFvName.upper() + 'fv']
        
        #
        # Check whether FV in Capsule is in FD flash region.
        # If yes, return error. Doesn't support FV in Capsule image is also in FD flash region.
        #
        if self.CapsuleName != None:
            for FdName in GenFdsGlobalVariable.FdfParser.Profile.FdDict.keys():
                FdObj = GenFdsGlobalVariable.FdfParser.Profile.FdDict[FdName]
                for RegionObj in FdObj.RegionList:
                    if RegionObj.RegionType == 'FV':
                        for RegionData in RegionObj.RegionDataList:
                            if RegionData.endswith(".fv"):
                                continue
                            elif RegionData.upper() + 'fv' in GenFds.ImageBinDict.keys():
                                continue
                            elif self.UiFvName.upper() == RegionData.upper():
                                GenFdsGlobalVariable.ErrorLogger("Capsule %s in FD region can't contain a FV %s in FD region." % (self.CapsuleName, self.UiFvName.upper()))
        if not Flag:
            GenFdsGlobalVariable.InfLogger( "\nGenerating %s FV" %self.UiFvName)
        GenFdsGlobalVariable.LargeFileInFvFlags.append(False)
        FFSGuid = None
        
        if self.FvBaseAddress != None:
            BaseAddress = self.FvBaseAddress
        if not Flag:
            self.__InitializeInf__(BaseAddress, BlockSize, BlockNum, ErasePloarity, VtfDict)
        #
        # First Process the Apriori section
        #
        MacroDict.update(self.DefineVarDict)

        GenFdsGlobalVariable.VerboseLogger('First generate Apriori file !')
        FfsFileList = []
        for AprSection in self.AprioriSectionList:
            FileName = AprSection.GenFfs (self.UiFvName, MacroDict, IsMakefile=Flag)
            FfsFileList.append(FileName)
            # Add Apriori file name to Inf file
            if not Flag:
                self.FvInfFile.writelines("EFI_FILE_NAME = " + \
                                            FileName          + \
                                            T_CHAR_LF)

        # Process Modules in FfsList
        for FfsFile in self.FfsList :
            if Flag:
                if isinstance(FfsFile, FfsFileStatement.FileStatement):
                    continue
            if GenFdsGlobalVariable.EnableGenfdsMultiThread and GenFdsGlobalVariable.ModuleFile and GenFdsGlobalVariable.ModuleFile.Path.find(os.path.normpath(FfsFile.InfFileName)) == -1:
                continue
            FileName = FfsFile.GenFfs(MacroDict, FvParentAddr=BaseAddress, IsMakefile=Flag, FvName=self.UiFvName)
            FfsFileList.append(FileName)
            if not Flag:
                self.FvInfFile.writelines("EFI_FILE_NAME = " + \
                                            FileName          + \
                                            T_CHAR_LF)
        if not Flag:
            SaveFileOnChange(self.InfFileName, self.FvInfFile.getvalue(), False)
            self.FvInfFile.close()
        #
        # Call GenFv tool
        #
        FvOutputFile = os.path.join(GenFdsGlobalVariable.FvDir, self.UiFvName)
        FvOutputFile = FvOutputFile + '.Fv'
        # BUGBUG: FvOutputFile could be specified from FDF file (FV section, CreateFile statement)
        if self.CreateFileName != None:
            FvOutputFile = self.CreateFileName

        if Flag:
            GenFds.ImageBinDict[self.UiFvName.upper() + 'fv'] = FvOutputFile
            return FvOutputFile

        FvInfoFileName = os.path.join(GenFdsGlobalVariable.FfsDir, self.UiFvName + '.inf')
        if not Flag:
            CopyLongFilePath(GenFdsGlobalVariable.FvAddressFileName, FvInfoFileName)
            OrigFvInfo = None
            if os.path.exists (FvInfoFileName):
                OrigFvInfo = open(FvInfoFileName, 'r').read()
            if GenFdsGlobalVariable.LargeFileInFvFlags[-1]:
                FFSGuid = GenFdsGlobalVariable.EFI_FIRMWARE_FILE_SYSTEM3_GUID
            GenFdsGlobalVariable.GenerateFirmwareVolume(
                                    FvOutputFile,
                                    [self.InfFileName],
                                    AddressFile=FvInfoFileName,
                                    FfsList=FfsFileList,
                                    ForceRebase=self.FvForceRebase,
                                    FileSystemGuid=FFSGuid
                                    )

            NewFvInfo = None
            if os.path.exists (FvInfoFileName):
                NewFvInfo = open(FvInfoFileName, 'r').read()
            if NewFvInfo != None and NewFvInfo != OrigFvInfo:
                FvChildAddr = []
                AddFileObj = open(FvInfoFileName, 'r')
                AddrStrings = AddFileObj.readlines()
                AddrKeyFound = False
                for AddrString in AddrStrings:
                    if AddrKeyFound:
                        #get base address for the inside FvImage
                        FvChildAddr.append (AddrString)
                    elif AddrString.find ("[FV_BASE_ADDRESS]") != -1:
                        AddrKeyFound = True
                AddFileObj.close()

                if FvChildAddr != []:
                    # Update Ffs again
                    for FfsFile in self.FfsList :
                        FileName = FfsFile.GenFfs(MacroDict, FvChildAddr, BaseAddress, IsMakefile=Flag, FvName=self.UiFvName)

                    if GenFdsGlobalVariable.LargeFileInFvFlags[-1]:
                        FFSGuid = GenFdsGlobalVariable.EFI_FIRMWARE_FILE_SYSTEM3_GUID;
                    #Update GenFv again
                    GenFdsGlobalVariable.GenerateFirmwareVolume(
                                                FvOutputFile,
                                                [self.InfFileName],
                                                AddressFile=FvInfoFileName,
                                                FfsList=FfsFileList,
                                                ForceRebase=self.FvForceRebase,
                                                FileSystemGuid=FFSGuid
                                                )

            #
            # Write the Fv contents to Buffer
            #
            if os.path.isfile(FvOutputFile):
                FvFileObj = open(FvOutputFile, 'rb')
                GenFdsGlobalVariable.VerboseLogger("\nGenerate %s FV Successfully" % self.UiFvName)
                GenFdsGlobalVariable.SharpCounter = 0

                Buffer.write(FvFileObj.read())
                FvFileObj.seek(0)
                # PI FvHeader is 0x48 byte
                FvHeaderBuffer = FvFileObj.read(0x48)
                # FV alignment position.
                FvAlignmentValue = 1 << (ord(FvHeaderBuffer[0x2E]) & 0x1F)
                if FvAlignmentValue >= 0x400:
                    if FvAlignmentValue >= 0x100000:
                        if FvAlignmentValue >= 0x1000000:
{'fraction_non_alphanumeric': 0.06832694763729247, 'fraction_numerical': 0.006651766709237974, 'mean_word_length': 1.643922341024198, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 58, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2746707', 'n_tokens_mistral': 5287, 'n_tokens_neox': 5034, 'n_words': 1191}