<commit_before>/*
  Copyright (C) 2003-2006 by <NAME>
		2005-2006 by <NAME>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Library General Public
  License as published by the Free Software Foundation; either
  version 2 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  Library General Public License for more details.

  You should have received a copy of the GNU Library General Public
  License along with this library; if not, write to the Free
  Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/

#include "cssysdef.h"
#include <ctype.h>

#include "imap/services.h"
#include "iutil/hiercache.h"
#include "iutil/vfs.h"
#include "ivaria/reporter.h"
#include "ivideo/rendermesh.h"

#include "csplugincommon/shader/shadercachehelper.h"
#include "csutil/base64.h"
#include "csutil/csendian.h"
#include "csutil/cspmeter.h"
#include "csutil/documenthelper.h"
#include "csutil/parasiticdatabuffer.h"
#include "csutil/scfarray.h"
#include "csutil/xmltiny.h"
#include "cstool/vfsdirchange.h"

#include "shader.h"
#include "xmlshader.h"

CS_PLUGIN_NAMESPACE_BEGIN(XMLShader)
{

  CS_LEAKGUARD_IMPLEMENT (csXMLShader);

  //---------------------------------------------------------------------------

  csShaderConditionResolver::csShaderConditionResolver (
    csConditionEvaluator& evaluator, bool keepVariantToConditionsMap)
    : rootNode (0), nextVariant (0),
    keepVariantToConditionsMap (keepVariantToConditionsMap),
    evaluator (evaluator)
  {
    SetEvalParams (0, 0);
  }

  csShaderConditionResolver::~csShaderConditionResolver ()
  {
    delete rootNode;
  }

  const char* csShaderConditionResolver::SetLastError (const char* msg, ...)
  {
    va_list args;
    va_start (args, msg);
    lastError.FormatV (msg, args);
    va_end (args);
    return lastError;
  }

  const char* csShaderConditionResolver::ParseCondition (const char* str, 
    size_t len, 
    csConditionID& result)
  {
    CondOperation op;
    const char* err = ParseCondition (str, len, op);
    if (err) return err;
    result = evaluator.FindOptimizedCondition (op);
    return 0;
  }

  const char* csShaderConditionResolver::ParseCondition (const char* str, 
    size_t len, 
    CondOperation& result)
  {
    csExpressionTokenList tokens;
    const char* err = tokenizer.Tokenize (str, len, tokens);
    if (err)
      return SetLastError ("Tokenization: %s", err);
    csExpression* newExpression;
    err = parser.Parse (tokens, newExpression);
    if (err)
    {
      delete newExpression;
      return SetLastError ("Parsing: %s", err);
    }

    err = evaluator.ProcessExpression (newExpression, result);
    delete newExpression;
    if (err)
      return SetLastError ("Processing: %s", err);

    return 0;
  }

  bool csShaderConditionResolver::Evaluate (csConditionID condition)
  {
    const csRenderMeshModes* modes = csShaderConditionResolver::modes;
    const csShaderVariableStack* stack = csShaderConditionResolver::stack;

    return evaluator.Evaluate (condition, modes ? *modes : csRenderMeshModes(),
      stack);
  }

  csConditionNode* csShaderConditionResolver::NewNode (csConditionNode* parent)
  {
    csConditionNode* newNode = new csConditionNode (parent);
    return newNode;
  }

  csConditionNode* csShaderConditionResolver::GetRoot ()
  {
    if (rootNode == 0)
      rootNode = NewNode (0);
    return rootNode;
  }

  size_t csShaderConditionResolver::GetVariant (csConditionNode* node)
  {
    MyBitArrayTemp bits (evaluator.GetNumConditions ());
    node->FillConditionArray (bits);
    size_t* var = variantIDs.GetElementPointer (bits);
    if (var)
      return *var;
    else
    {
      variantIDs.Put (bits, nextVariant);
      return nextVariant++;
    }
  }

  void csShaderConditionResolver::AddNode (csConditionNode* parent,
    csConditionID condition, 
    csConditionNode*& trueNode,
    csConditionNode*& falseNode,
    const MyBitArrayTemp& conditionResultsTrue,
    const MyBitArrayTemp& conditionResultsFalse)
  {
    if (rootNode == 0)
    {
      // This is the first condition, new node gets root
      CS_ASSERT_MSG ("No root but parent? Weird.", parent == 0);
      parent = GetRoot ();

      parent->condition = condition;

      parent->trueNode = trueNode = NewNode (parent);
      trueNode->variant = GetVariant (trueNode);
      if (keepVariantToConditionsMap)
      {
        variantConditions.PutUnique (trueNode->variant, conditionResultsTrue);
      }

      parent->falseNode = falseNode = NewNode (parent);
      falseNode->variant = GetVariant (falseNode);
      if (keepVariantToConditionsMap)
      {
        variantConditions.PutUnique (falseNode->variant, conditionResultsFalse);
      }
    }
    else
    {
      if (parent == 0)
        parent = GetRoot ();

      parent->trueNode = trueNode = NewNode (parent);
      parent->falseNode = falseNode = NewNode (parent);

      CS_ASSERT(parent->variant != csArrayItemNotFound);
      MyBitArrayTemp bits (evaluator.GetNumConditions ());
      parent->condition = condition;

      trueNode->variant = GetVariant (trueNode);
      if (keepVariantToConditionsMap)
      {
        variantConditions.PutUnique (trueNode->variant, conditionResultsTrue);
      }

      falseNode->variant = parent->variant;
      falseNode->FillConditionArray (bits);
      variantIDs.PutUnique (bits, falseNode->variant);
      if (keepVariantToConditionsMap)
      {
        variantConditions.PutUnique (falseNode->variant, conditionResultsFalse);
      }

      parent->variant = csArrayItemNotFound;
    }
  }

  void csShaderConditionResolver::FinishAdding ()
  {
    variantIDs.Empty();
  }

  void csShaderConditionResolver::SetEvalParams (const csRenderMeshModes* modes,
    const csShaderVariableStack* stack)
  {
    csShaderConditionResolver::modes = modes;
    csShaderConditionResolver::stack = stack;
  }

  size_t csShaderConditionResolver::GetVariant ()
  {
    const csRenderMeshModes& modes = *csShaderConditionResolver::modes;
    const csShaderVariableStack* stack = csShaderConditionResolver::stack;

    if (rootNode == 0)
    {
      return 0;
    }
    else
    {
      csConditionNode* currentRoot = 0;
      csConditionNode* nextRoot = rootNode;

      while (nextRoot != 0)
      {
        currentRoot = nextRoot;
        if (evaluator.Evaluate (currentRoot->condition, modes, stack))
        {
          nextRoot = currentRoot->trueNode;
        }
        else
        {
          nextRoot = currentRoot->falseNode;
        }
      }
      CS_ASSERT (currentRoot != 0);
      return currentRoot->variant;
    }
  }

  void csShaderConditionResolver::SetVariant (size_t variant)
  {
    if (rootNode == 0) return;
    CS_ASSERT(keepVariantToConditionsMap);

    csBitArray conditionResults (evaluator.GetNumConditions());
    csBitArray conditionSet (evaluator.GetNumConditions());
    MyBitArrayTemp* conditionResultsPtr = variantConditions.GetElementPointer (
      variant);
    CS_ASSERT(conditionResultsPtr);

    size_t i = 0;
    for (; i < conditionResultsPtr->GetSize(); i++)
    {
      conditionResults.Set (i, (*conditionResultsPtr)[i]);
      conditionSet.SetBit (i);
    }
    evaluator.ForceConditionResults (conditionSet, conditionResults);
  }

  void csShaderConditionResolver::DumpConditionTree (csString& out)
  {
    if (rootNode == 0)
      return;

    out += "\n";
    DumpConditionNode (out, rootNode, 0);
  }

  static void Indent (csString& out, int n)
  {
    while (n-- > 0)
      out += "| ";
  }

  void csShaderConditionResolver::DumpConditionNode (csString& out,
    csConditionNode* node, 
    int level)
  {
    if (node == 0)
    {
      Indent (out, level);
      out.Append ("<none>\n");
    }
    else
    {
      if (node->variant != csArrayItemNotFound)
      {
        out.AppendFmt ("variant: %zu", node->variant);
      }
      else
      {
        out.Append ("\n");
        Indent (out, level);
        out.AppendFmt ("condition %zu = true: ", node->condition);
        DumpConditionNode (out, node->trueNode, level + 1);
        out.Append ("\n");
        Indent (out, level);
        out.AppendFmt ("condition %zu = false: ", node->condition);
        DumpConditionNode (out, node->falseNode, level + 1);
      }
    }
  }

  void csShaderConditionResolver::CollectUsedConditions (csConditionNode* node,
    ConditionsWriter& condWrite)
  {
    if (node == 0) return;
    if (node->condition == csCondAlwaysTrue) return;
    if (node->condition == csCondAlwaysFalse) return;

    csFIFO<csConditionID> condStack;
    condStack.Push (node->condition);
    while (condStack.GetSize() > 0)
    {
      csConditionID cond = condStack.PopTop();
      condWrite.GetDiskID (cond);
      const CondOperation& op = evaluator.GetCondition (cond);
      if (op.left.type == operandOperation) condStack.Push (op.left.operation);
      if (op.right.type == operandOperation) condStack.Push (op.right.operation);
    }
    CollectUsedConditions (node->falseNode, condWrite);
    CollectUsedConditions (node->trueNode, condWrite);
  }

  bool csShaderConditionResolver::ReadFromCache (iFile* cacheFile,
    ConditionsReader& condReader)
  {
    uint32 nextVarLE;
    if (cacheFile->Read ((char*)&nextVarLE, sizeof (nextVarLE))
      != sizeof (nextVarLE))
      return false;
    nextVariant = csLittleEndian::UInt32 (nextVarLE);

    if (!ReadNode (cacheFile, condReader, 0, rootNode))
    {
      delete rootNode; rootNode = 0;
      nextVariant = 0;
      return false;
    }

    return true;
  }

  bool csShaderConditionResolver::WriteToCache (iFile* cacheFile,
    ConditionsWriter& condWriter)
  {
    uint32 nextVarLE = nextVariant;
    nextVarLE = csLittleEndian::UInt32 (nextVarLE);
    if (cacheFile->Write ((char*)&nextVarLE, sizeof (nextVarLE))
      != sizeof (nextVarLE))
      return false;

    if (rootNode == 0)
    {
      // Special case
      uint32 varLE = 0xffffffff;
      if (cacheFile->Write ((char*)&varLE, sizeof (varLE)) != sizeof (varLE))
        return false;
    }
    else
    {
      if (!WriteNode (cacheFile, rootNode, condWriter)) return false;
    }

    return true;
  }

  bool csShaderConditionResolver::ReadNode (iFile* cacheFile, 
    const ConditionsReader& condRead, csConditionNode* parent,
    csConditionNode*& node)
  {
    uint32 condLE;
    if (cacheFile->Read ((char*)&condLE, sizeof (condLE)) != sizeof (condLE))
      return false;
    if (condLE == 0xffffffff)
    {
      // Special case 'no root'
      node = 0;
      return true;
    }
    node = NewNode (parent);
    condLE = csLittleEndian::UInt32 (condLE);
    if (condLE & 0x80000000)
    {
      // Leaf, ie variant
      node->variant = condLE & 0x7fffffff;
      return true;
    }
    else
    {
      // Node, ie condition
      node->condition = condRead.GetConditionID (condLE);
      if (node->condition == (csConditionID)~0) return false;
      return ReadNode (cacheFile, condRead, node, node->trueNode)
        && ReadNode (cacheFile, condRead, node, node->falseNode);
    }
  }

  bool csShaderConditionResolver::WriteNode (iFile* cacheFile, csConditionNode* node,
    const ConditionsWriter& condWrite)
  {
    if (node->variant == csArrayItemNotFound)
    {
      CS_ASSERT(node->condition != csCondAlwaysTrue);
      CS_ASSERT(node->condition != csCondAlwaysFalse);

      uint32 condLE = condWrite.GetDiskID (node->condition);
      condLE = csLittleEndian::UInt32 (condLE);
      if (cacheFile->Write ((char*)&condLE, sizeof (condLE)) != sizeof (condLE))
        return false;
      return WriteNode (cacheFile, node->trueNode, condWrite)
        && WriteNode (cacheFile, node->falseNode, condWrite);
    }
    else
    {
      uint32 varLE = node->variant;
      varLE |= 0x80000000;
      varLE = csLittleEndian::UInt32 (varLE);
      return cacheFile->Write ((char*)&varLE, sizeof (varLE)) == sizeof (varLE);
    }
  }

  //---------------------------------------------------------------------------

  csXMLShader::csXMLShader (csXMLShaderCompiler* compiler, 
    iLoaderContext* ldr_context,
    iDocumentNode* source,
    int forcepriority)
    : scfImplementationType (this), techsResolver (0),
    sharedEvaluator (compiler->sharedEvaluator),
    fallbackTried (false)
  {
    InitTokenTable (xmltokens);

    csXMLShader::ldr_context = ldr_context;

    activeTech = 0;
    filename = 0;
    csXMLShader::compiler = compiler;
    g3d = compiler->g3d;
    csXMLShader::forcepriority = forcepriority;
    useFallbackContext = false;

    allShaderMeta.numberOfLights = source->GetAttributeValueAsInt ("lights");

    shadermgr = csQueryRegistry<iShaderManager> (compiler->objectreg);
    CS_ASSERT (shadermgr); // Should be present - loads us, after all

    vfsStartDir = CS::StrDup (compiler->vfs->GetCwd ());
    shaderCache = shadermgr->GetShaderCache();
    Load (source, false);
  }

  csXMLShader::csXMLShader (csXMLShaderCompiler* compiler)
    : scfImplementationType (this), techsResolver (0),
    sharedEvaluator (compiler->sharedEvaluator),
    fallbackTried (false)
  {
    InitTokenTable (xmltokens);

    csXMLShader::ldr_context = ldr_context;

    activeTech = 0;
    filename = 0;
    csXMLShader::compiler = compiler;
    g3d = compiler->g3d;
    csXMLShader::forcepriority = forcepriority;
    useFallbackContext = false;

    shadermgr = csQueryRegistry<iShaderManager> (compiler->objectreg);
    CS_ASSERT (shadermgr); // Should be present - loads us, after all

    vfsStartDir = CS::StrDup (compiler->vfs->GetCwd ());
  }

  csXMLShader::~csXMLShader ()
  {
    for (size_t i = 0; i < techVariants.GetSize (); i++)
    {
      techVariants[i].Free();
    }

    cs_free (filename);
    delete techsResolver;
    cs_free (vfsStartDir);
    cs_free (const_cast<char*> (allShaderMeta.description));
  }

  /* Magic value for cache file.
  * The most significant byte serves as a "version", increase when the
  * cache file format changes. */
  static const uint32 cacheFileMagic = 0x06737863;

  void csXMLShader::Load (iDocumentNode* source, bool forPrecache)
  {
    techsResolver = new csShaderConditionResolver (*sharedEvaluator,
      forPrecache);

    CS::PluginCommon::ShaderCacheHelper::ShaderDocHasher hasher (
      compiler->objectreg, source);

    csString cacheType = source->GetAttributeValue ("name");
    cacheTag = source->GetAttributeValue ("_cachetag");
    csString cacheID_header;
    {
      csString cacheID_base (source->GetAttributeValue ("_cacheid"));
      if (cacheID_base.IsEmpty())
      {
        csMD5::Digest sourceDigest (csMD5::Encode (CS::DocSystem::FlattenNode (source)));
        cacheID_base = sourceDigest.HexString();
      }
      cacheID_header.Format ("%sXH", cacheID_base.GetData());
      cacheScope_tech.Format ("%sXT", cacheID_base.GetData());
    }
    if (shaderCache.IsValid ()) shaderCache = shaderCache->GetRootedCache (
      csString().Format ("/%s", cacheType.GetData()));
    bool cacheValid = (shaderCache != 0) && !cacheType.IsEmpty()
      && !cacheID_header.IsEmpty();
    if (!cacheValid) shaderCache.Invalidate();
    bool readFromCache = cacheValid && !forPrecache;

    csRef<iFile> cacheFile;
    if (cacheValid)
    {
      csRef<iDataBuffer> cacheData;
      if (readFromCache)
        cacheData = shaderCache->ReadCache (csString().Format ("/%s", cacheID_header.GetData()));
      if (cacheData.IsValid())
      {
        cacheFile.AttachNew (new csMemFile (cacheData, true));
      }
      else
        readFromCache = false;
      if (cacheFile.IsValid())
      {
        do
        {
          // Read magic header
          uint32 diskMagic;
          size_t read = cacheFile->Read ((char*)&diskMagic, sizeof (diskMagic));
          if (read != sizeof (diskMagic)) break;
          if (csLittleEndian::UInt32 (diskMagic) != cacheFileMagic) break;

          csString cacheFileTag =
            CS::PluginCommon::ShaderCacheHelper::ReadString (cacheFile);
          if (cacheFileTag != cacheTag) break;

          // Extract hash stream
          csRef<iDataBuffer> hashStream = 
            CS::PluginCommon::ShaderCacheHelper::ReadDataBuffer (cacheFile);
          if (!hashStream.IsValid()) break;

          readFromCache = hasher.ValidateHashStream (hashStream);
        }
        while (false);
      }
      if (!readFromCache)
      {
        // Getting from cache failed, so prep for writing to cache
        cacheFile.AttachNew (new csMemFile ());
        // Write magic header
        uint32 diskMagic = csLittleEndian::UInt32 (cacheFileMagic);
        cacheFile->Write ((char*)&diskMagic, sizeof (diskMagic));
        CS::PluginCommon::ShaderCacheHelper::WriteString (cacheFile, cacheTag);
        // Write hash stream
        csRef<iDataBuffer> hashStream = hasher.GetHashStream ();
        if (!CS::PluginCommon::ShaderCacheHelper::WriteDataBuffer (
          cacheFile, hashStream))
          cacheFile.Invalidate();
      }
    }

    if (cacheTag.IsEmpty())
    {
      csRef<iDataBuffer> hashStream = hasher.GetHashStream ();
      /* @@@ Actually, the cache tag wouldn't have to be that large.
      In theory, anything would work as long as (a) it changes when
      some file the shader uses changes (b) the tag is reasonably
      unique (also over multiple program runs).
      E.g. a UUID, recomputed when the shader is 'touched',
      could do as well. */
      cacheTag = CS::Utility::EncodeBase64 (hashStream);
    }

    ConditionsReader* condReader = 0;
    if (readFromCache)
    {
      do
      {
        readFromCache = false;<commit_msg>Fix crash when no 'hash stream' is available<commit_after>/*
  Copyright (C) 2003-2006 by <NAME>
		2005-2006 by <NAME>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Library General Public
  License as published by the Free Software Foundation; either
  version 2 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  Library General Public License for more details.

  You should have received a copy of the GNU Library General Public
  License along with this library; if not, write to the Free
  Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/

#include "cssysdef.h"
#include <ctype.h>

#include "imap/services.h"
#include "iutil/hiercache.h"
#include "iutil/vfs.h"
#include "ivaria/reporter.h"
#include "ivideo/rendermesh.h"

#include "csplugincommon/shader/shadercachehelper.h"
#include "csutil/base64.h"
#include "csutil/csendian.h"
#include "csutil/cspmeter.h"
#include "csutil/documenthelper.h"
#include "csutil/parasiticdatabuffer.h"
#include "csutil/scfarray.h"
#include "csutil/xmltiny.h"
#include "cstool/vfsdirchange.h"

#include "shader.h"
#include "xmlshader.h"

CS_PLUGIN_NAMESPACE_BEGIN(XMLShader)
{

  CS_LEAKGUARD_IMPLEMENT (csXMLShader);

  //---------------------------------------------------------------------------

  csShaderConditionResolver::csShaderConditionResolver (
    csConditionEvaluator& evaluator, bool keepVariantToConditionsMap)
    : rootNode (0), nextVariant (0),
    keepVariantToConditionsMap (keepVariantToConditionsMap),
    evaluator (evaluator)
  {
    SetEvalParams (0, 0);
  }

  csShaderConditionResolver::~csShaderConditionResolver ()
  {
    delete rootNode;
  }

  const char* csShaderConditionResolver::SetLastError (const char* msg, ...)
  {
    va_list args;
    va_start (args, msg);
    lastError.FormatV (msg, args);
    va_end (args);
    return lastError;
  }

  const char* csShaderConditionResolver::ParseCondition (const char* str, 
    size_t len, 
    csConditionID& result)
  {
    CondOperation op;
    const char* err = ParseCondition (str, len, op);
    if (err) return err;
    result = evaluator.FindOptimizedCondition (op);
    return 0;
  }

  const char* csShaderConditionResolver::ParseCondition (const char* str, 
    size_t len, 
    CondOperation& result)
  {
    csExpressionTokenList tokens;
    const char* err = tokenizer.Tokenize (str, len, tokens);
    if (err)
      return SetLastError ("Tokenization: %s", err);
    csExpression* newExpression;
    err = parser.Parse (tokens, newExpression);
    if (err)
    {
      delete newExpression;
      return SetLastError ("Parsing: %s", err);
    }

    err = evaluator.ProcessExpression (newExpression, result);
    delete newExpression;
    if (err)
      return SetLastError ("Processing: %s", err);

    return 0;
  }

  bool csShaderConditionResolver::Evaluate (csConditionID condition)
  {
    const csRenderMeshModes* modes = csShaderConditionResolver::modes;
    const csShaderVariableStack* stack = csShaderConditionResolver::stack;

    return evaluator.Evaluate (condition, modes ? *modes : csRenderMeshModes(),
      stack);
  }

  csConditionNode* csShaderConditionResolver::NewNode (csConditionNode* parent)
  {
    csConditionNode* newNode = new csConditionNode (parent);
    return newNode;
  }

  csConditionNode* csShaderConditionResolver::GetRoot ()
  {
    if (rootNode == 0)
      rootNode = NewNode (0);
    return rootNode;
  }

  size_t csShaderConditionResolver::GetVariant (csConditionNode* node)
  {
    MyBitArrayTemp bits (evaluator.GetNumConditions ());
    node->FillConditionArray (bits);
    size_t* var = variantIDs.GetElementPointer (bits);
    if (var)
      return *var;
    else
    {
      variantIDs.Put (bits, nextVariant);
      return nextVariant++;
    }
  }

  void csShaderConditionResolver::AddNode (csConditionNode* parent,
    csConditionID condition, 
    csConditionNode*& trueNode,
    csConditionNode*& falseNode,
    const MyBitArrayTemp& conditionResultsTrue,
    const MyBitArrayTemp& conditionResultsFalse)
  {
    if (rootNode == 0)
    {
      // This is the first condition, new node gets root
      CS_ASSERT_MSG ("No root but parent? Weird.", parent == 0);
      parent = GetRoot ();

      parent->condition = condition;

      parent->trueNode = trueNode = NewNode (parent);
      trueNode->variant = GetVariant (trueNode);
      if (keepVariantToConditionsMap)
      {
        variantConditions.PutUnique (trueNode->variant, conditionResultsTrue);
      }

      parent->falseNode = falseNode = NewNode (parent);
      falseNode->variant = GetVariant (falseNode);
      if (keepVariantToConditionsMap)
      {
        variantConditions.PutUnique (falseNode->variant, conditionResultsFalse);
      }
    }
    else
    {
      if (parent == 0)
        parent = GetRoot ();

      parent->trueNode = trueNode = NewNode (parent);
      parent->falseNode = falseNode = NewNode (parent);

      CS_ASSERT(parent->variant != csArrayItemNotFound);
      MyBitArrayTemp bits (evaluator.GetNumConditions ());
      parent->condition = condition;

      trueNode->variant = GetVariant (trueNode);
      if (keepVariantToConditionsMap)
      {
        variantConditions.PutUnique (trueNode->variant, conditionResultsTrue);
      }

      falseNode->variant = parent->variant;
      falseNode->FillConditionArray (bits);
      variantIDs.PutUnique (bits, falseNode->variant);
      if (keepVariantToConditionsMap)
      {
        variantConditions.PutUnique (falseNode->variant, conditionResultsFalse);
      }

      parent->variant = csArrayItemNotFound;
    }
  }

  void csShaderConditionResolver::FinishAdding ()
  {
    variantIDs.Empty();
  }

  void csShaderConditionResolver::SetEvalParams (const csRenderMeshModes* modes,
    const csShaderVariableStack* stack)
  {
    csShaderConditionResolver::modes = modes;
    csShaderConditionResolver::stack = stack;
  }

  size_t csShaderConditionResolver::GetVariant ()
  {
    const csRenderMeshModes& modes = *csShaderConditionResolver::modes;
    const csShaderVariableStack* stack = csShaderConditionResolver::stack;

    if (rootNode == 0)
    {
      return 0;
    }
    else
    {
      csConditionNode* currentRoot = 0;
      csConditionNode* nextRoot = rootNode;

      while (nextRoot != 0)
      {
        currentRoot = nextRoot;
        if (evaluator.Evaluate (currentRoot->condition, modes, stack))
        {
          nextRoot = currentRoot->trueNode;
        }
        else
        {
          nextRoot = currentRoot->falseNode;
        }
      }
      CS_ASSERT (currentRoot != 0);
      return currentRoot->variant;
    }
  }

  void csShaderConditionResolver::SetVariant (size_t variant)
  {
    if (rootNode == 0) return;
    CS_ASSERT(keepVariantToConditionsMap);

    csBitArray conditionResults (evaluator.GetNumConditions());
    csBitArray conditionSet (evaluator.GetNumConditions());
    MyBitArrayTemp* conditionResultsPtr = variantConditions.GetElementPointer (
      variant);
    CS_ASSERT(conditionResultsPtr);

    size_t i = 0;
    for (; i < conditionResultsPtr->GetSize(); i++)
    {
      conditionResults.Set (i, (*conditionResultsPtr)[i]);
      conditionSet.SetBit (i);
    }
    evaluator.ForceConditionResults (conditionSet, conditionResults);
  }

  void csShaderConditionResolver::DumpConditionTree (csString& out)
  {
    if (rootNode == 0)
      return;

    out += "\n";
    DumpConditionNode (out, rootNode, 0);
  }

  static void Indent (csString& out, int n)
  {
    while (n-- > 0)
      out += "| ";
  }

  void csShaderConditionResolver::DumpConditionNode (csString& out,
    csConditionNode* node, 
    int level)
  {
    if (node == 0)
    {
      Indent (out, level);
      out.Append ("<none>\n");
    }
    else
    {
      if (node->variant != csArrayItemNotFound)
      {
        out.AppendFmt ("variant: %zu", node->variant);
      }
      else
      {
        out.Append ("\n");
        Indent (out, level);
        out.AppendFmt ("condition %zu = true: ", node->condition);
        DumpConditionNode (out, node->trueNode, level + 1);
        out.Append ("\n");
        Indent (out, level);
        out.AppendFmt ("condition %zu = false: ", node->condition);
        DumpConditionNode (out, node->falseNode, level + 1);
      }
    }
  }

  void csShaderConditionResolver::CollectUsedConditions (csConditionNode* node,
    ConditionsWriter& condWrite)
  {
    if (node == 0) return;
    if (node->condition == csCondAlwaysTrue) return;
    if (node->condition == csCondAlwaysFalse) return;

    csFIFO<csConditionID> condStack;
    condStack.Push (node->condition);
    while (condStack.GetSize() > 0)
    {
      csConditionID cond = condStack.PopTop();
      condWrite.GetDiskID (cond);
      const CondOperation& op = evaluator.GetCondition (cond);
      if (op.left.type == operandOperation) condStack.Push (op.left.operation);
      if (op.right.type == operandOperation) condStack.Push (op.right.operation);
    }
    CollectUsedConditions (node->falseNode, condWrite);
    CollectUsedConditions (node->trueNode, condWrite);
  }

  bool csShaderConditionResolver::ReadFromCache (iFile* cacheFile,
    ConditionsReader& condReader)
  {
    uint32 nextVarLE;
    if (cacheFile->Read ((char*)&nextVarLE, sizeof (nextVarLE))
      != sizeof (nextVarLE))
      return false;
    nextVariant = csLittleEndian::UInt32 (nextVarLE);

    if (!ReadNode (cacheFile, condReader, 0, rootNode))
    {
      delete rootNode; rootNode = 0;
      nextVariant = 0;
      return false;
    }

    return true;
  }

  bool csShaderConditionResolver::WriteToCache (iFile* cacheFile,
    ConditionsWriter& condWriter)
  {
    uint32 nextVarLE = nextVariant;
    nextVarLE = csLittleEndian::UInt32 (nextVarLE);
    if (cacheFile->Write ((char*)&nextVarLE, sizeof (nextVarLE))
      != sizeof (nextVarLE))
      return false;

    if (rootNode == 0)
    {
      // Special case
      uint32 varLE = 0xffffffff;
      if (cacheFile->Write ((char*)&varLE, sizeof (varLE)) != sizeof (varLE))
        return false;
    }
    else
    {
      if (!WriteNode (cacheFile, rootNode, condWriter)) return false;
    }

    return true;
  }

  bool csShaderConditionResolver::ReadNode (iFile* cacheFile, 
    const ConditionsReader& condRead, csConditionNode* parent,
    csConditionNode*& node)
  {
    uint32 condLE;
    if (cacheFile->Read ((char*)&condLE, sizeof (condLE)) != sizeof (condLE))
      return false;
    if (condLE == 0xffffffff)
    {
      // Special case 'no root'
      node = 0;
      return true;
    }
    node = NewNode (parent);
    condLE = csLittleEndian::UInt32 (condLE);
    if (condLE & 0x80000000)
    {
      // Leaf, ie variant
      node->variant = condLE & 0x7fffffff;
      return true;
    }
    else
    {
      // Node, ie condition
      node->condition = condRead.GetConditionID (condLE);
      if (node->condition == (csConditionID)~0) return false;
      return ReadNode (cacheFile, condRead, node, node->trueNode)
        && ReadNode (cacheFile, condRead, node, node->falseNode);
    }
  }

  bool csShaderConditionResolver::WriteNode (iFile* cacheFile, csConditionNode* node,
    const ConditionsWriter& condWrite)
  {
    if (node->variant == csArrayItemNotFound)
    {
      CS_ASSERT(node->condition != csCondAlwaysTrue);
      CS_ASSERT(node->condition != csCondAlwaysFalse);

      uint32 condLE = condWrite.GetDiskID (node->condition);
      condLE = csLittleEndian::UInt32 (condLE);
      if (cacheFile->Write ((char*)&condLE, sizeof (condLE)) != sizeof (condLE))
        return false;
      return WriteNode (cacheFile, node->trueNode, condWrite)
        && WriteNode (cacheFile, node->falseNode, condWrite);
    }
    else
    {
      uint32 varLE = node->variant;
      varLE |= 0x80000000;
      varLE = csLittleEndian::UInt32 (varLE);
      return cacheFile->Write ((char*)&varLE, sizeof (varLE)) == sizeof (varLE);
    }
  }

  //---------------------------------------------------------------------------

  csXMLShader::csXMLShader (csXMLShaderCompiler* compiler, 
    iLoaderContext* ldr_context,
    iDocumentNode* source,
    int forcepriority)
    : scfImplementationType (this), techsResolver (0),
    sharedEvaluator (compiler->sharedEvaluator),
    fallbackTried (false)
  {
    InitTokenTable (xmltokens);

    csXMLShader::ldr_context = ldr_context;

    activeTech = 0;
    filename = 0;
    csXMLShader::compiler = compiler;
    g3d = compiler->g3d;
    csXMLShader::forcepriority = forcepriority;
    useFallbackContext = false;

    allShaderMeta.numberOfLights = source->GetAttributeValueAsInt ("lights");

    shadermgr = csQueryRegistry<iShaderManager> (compiler->objectreg);
    CS_ASSERT (shadermgr); // Should be present - loads us, after all

    vfsStartDir = CS::StrDup (compiler->vfs->GetCwd ());
    shaderCache = shadermgr->GetShaderCache();
    Load (source, false);
  }

  csXMLShader::csXMLShader (csXMLShaderCompiler* compiler)
    : scfImplementationType (this), techsResolver (0),
    sharedEvaluator (compiler->sharedEvaluator),
    fallbackTried (false)
  {
    InitTokenTable (xmltokens);

    csXMLShader::ldr_context = ldr_context;

    activeTech = 0;
    filename = 0;
    csXMLShader::compiler = compiler;
    g3d = compiler->g3d;
    csXMLShader::forcepriority = forcepriority;
    useFallbackContext = false;

    shadermgr = csQueryRegistry<iShaderManager> (compiler->objectreg);
    CS_ASSERT (shadermgr); // Should be present - loads us, after all

    vfsStartDir = CS::StrDup (compiler->vfs->GetCwd ());
  }

  csXMLShader::~csXMLShader ()
  {
    for (size_t i = 0; i < techVariants.GetSize (); i++)
    {
      techVariants[i].Free();
    }

    cs_free (filename);
    delete techsResolver;
    cs_free (vfsStartDir);
    cs_free (const_cast<char*> (allShaderMeta.description));
  }

  /* Magic value for cache file.
  * The most significant byte serves as a "version", increase when the
  * cache file format changes. */
  static const uint32 cacheFileMagic = 0x06737863;

  void csXMLShader::Load (iDocumentNode* source, bool forPrecache)
  {
    techsResolver = new csShaderConditionResolver (*sharedEvaluator,
      forPrecache);

    CS::PluginCommon::ShaderCacheHelper::ShaderDocHasher hasher (
      compiler->objectreg, source);

    csString cacheType = source->GetAttributeValue ("name");
    cacheTag = source->GetAttributeValue ("_cachetag");
    csString cacheID_header;
    {
      csString cacheID_base (source->GetAttributeValue ("_cacheid"));
      if (cacheID_base.IsEmpty())
      {
        csMD5::Digest sourceDigest (csMD5::Encode (CS::DocSystem::FlattenNode (source)));
        cacheID_base = sourceDigest.HexString();
      }
      cacheID_header.Format ("%sXH", cacheID_base.GetData());
      cacheScope_tech.Format ("%sXT", cacheID_base.GetData());
    }
    if (shaderCache.IsValid ()) shaderCache = shaderCache->GetRootedCache (
      csString().Format ("/%s", cacheType.GetData()));
    bool cacheValid = (shaderCache != 0) && !cacheType.IsEmpty()
      && !cacheID_header.IsEmpty();
    if (!cacheValid) shaderCache.Invalidate();
    bool readFromCache = cacheValid && !forPrecache;

    csRef<iFile> cacheFile;
    if (cacheValid)
    {
      csRef<iDataBuffer> cacheData;
      if (readFromCache)
        cacheData = shaderCache->ReadCache (csString().Format ("/%s", cacheID_header.GetData()));
      if (cacheData.IsValid())
      {
        cacheFile.AttachNew (new csMemFile (cacheData, true));
      }
      else
        readFromCache = false;
      if (cacheFile.IsValid())
      {
        do
        {
          // Read magic header
          uint32 diskMagic;
          size_t read = cacheFile->Read ((char*)&diskMagic, sizeof (diskMagic));
          if (read != sizeof (diskMagic)) break;
          if (csLittleEndian::UInt32 (diskMagic) != cacheFileMagic) break;

          csString cacheFileTag =
            CS::PluginCommon::ShaderCacheHelper::ReadString (cacheFile);
          if (cacheFileTag != cacheTag) break;

          // Extract hash stream
          csRef<iDataBuffer> hashStream = 
            CS::PluginCommon::ShaderCacheHelper::ReadDataBuffer (cacheFile);
          if (!hashStream.IsValid()) break;

          readFromCache = hasher.ValidateHashStream (hashStream);
        }
        while (false);
      }
      if (!readFromCache)
      {
        // Getting from cache failed, so prep for writing to cache
        cacheFile.AttachNew (new csMemFile ());
        // Write magic header
        uint32 diskMagic = csLittleEndian::UInt32 (cacheFileMagic);
        cacheFile->Write ((char*)&diskMagic, sizeof (diskMagic));
        CS::PluginCommon::ShaderCacheHelper::WriteString (cacheFile, cacheTag);
        // Write hash stream
        csRef<iDataBuffer> hashStream = hasher.GetHashStream ();
        if (!CS::PluginCommon::ShaderCacheHelper::WriteDataBuffer (
          cacheFile, hashStream))
          cacheFile.Invalidate();
      }
    }

    if (cacheTag.IsEmpty())
    {
      csRef<iDataBuffer> hashStream = hasher.GetHashStream ();
      if (hashStream.IsValid())
      {
	/* @@@ Actually, the cache tag wouldn't have to be that large.
	In theory, anything would work as long as (a) it changes when
	some file the shader uses changes (b) the tag is reasonably
	unique (also over multiple program runs).
	E.g. a UUID, recomputed when the shader is 'touched',
	could do as well. */
	cacheTag = CS::Utility::EncodeBase64 (hashStream);
      }
    }

    ConditionsReader* condReader = 0;
    if (readFromCache)
    {
      do
      {
        readFromCache = false;
{'fraction_non_alphanumeric': 0.11919504643962849, 'fraction_numerical': 0.007824373768646215, 'mean_word_length': 3.0293717396234974, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 181, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 10, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5325348', 'n_tokens_mistral': 10948, 'n_tokens_neox': 10510, 'n_words': 2552}