<commit_before>﻿using System;
using System.Collections.Generic;
using Globalization = System.Globalization;
using System.Linq;
using System.Text;
using Microsoft.Build.Framework;
using System.IO;
using Microsoft.Build.Utilities;
using System.Diagnostics;
using Cosmos.Debug.Common;

namespace Cosmos.Build.MSBuild {
  public class ExtractMapFromElfFile : BaseToolTask {
    [Required]
    public string InputFile {
      get;
      set;
    }

    [Required]
    public string DebugInfoFile {
      get;
      set;
    }

    [Required]
    public string WorkingDir {
      get;
      set;
    }

    [Required]
    public string CosmosBuildDir {
      get;
      set;
    }

    public override bool Execute() {
      // Important! A given address can have more than one label.
      // Do NOT filter by duplicate addresses as this causes serious lookup problems.

      string xSymbolString;
      //TODO: This reads the file (13MB currently) into RAM...
      // Thats not needed.. read it line by line instead. Wait till we move to direct
      // DB only use though as we can do this at one time.
      if (!RunObjDump(out xSymbolString)) {
        return false;
      }

      var xResult = new List<Label>();
      var xLines = xSymbolString.Split(new string[] { "\r\n", "\n", "\r" }, StringSplitOptions.RemoveEmptyEntries);
      
      // Parse file
      bool xListStarted = false;
      foreach (string xLine in xLines) {
        if (String.IsNullOrEmpty(xLine)) {
          continue;
        } else if (!xListStarted) {
          // Find start of the data
          if (xLine == "SYMBOL TABLE:") {
            xListStarted = true;
          }
          continue;
        }

        uint xAddress;
        try {
          xAddress = UInt32.Parse(xLine.Substring(0, 8), Globalization.NumberStyles.HexNumber);
        } catch (Exception) {
          Log.LogError("Error processing line '" + xLine + "'");
          throw;
        }
        
        string xSection = xLine.Substring(17, 5);
        if (xSection != ".text" && xSection != ".data") {
          continue;
        }
        string xLabel = xLine.Substring(32);
        if (xLabel == xSection) {
          // Non label, skip
          continue;
        }
        xResult.Add(new Label() {
          LABELNAME = xLabel,
          ADDRESS = xAddress
        });
      }

      using (var xDebugInfo = new DebugInfo(DebugInfoFile)) {
        xDebugInfo.WriteLabels(xResult);
      }
      return true;
    }

    private bool RunObjDump(out string result) {
      result = "";
      var xTempBatFile = Path.Combine(WorkingDir, "ExtractMapFromElfFileTemp.bat");
      if (File.Exists(xTempBatFile)) {
        File.Delete(xTempBatFile);
        if (File.Exists(xTempBatFile)) {
          Log.LogError("ExtractMapFromElfFileTemp.bat already exists!");
          return false;
        }
      }

      var xTempOutFile = Path.Combine(WorkingDir, "ExtractMapFromElfFileTemp.out");
      if (File.Exists(xTempOutFile)) {
        File.Delete(xTempOutFile);
        if (File.Exists(xTempOutFile)) {
          Log.LogError("ExtractMapFromElfFileTemp.out already exists!");
          return false;
        }
      }

      File.WriteAllText(xTempBatFile, "@ECHO OFF\r\n\"" + Path.Combine(CosmosBuildDir, @"tools\cygwin\objdump.exe") + "\" --wide --syms \"" + InputFile + "\" > ExtractMapFromElfFileTemp.out");
      if (!ExecuteTool(WorkingDir, xTempBatFile, "", "objdump")) {
        return false;
      }
      result = File.ReadAllText(xTempOutFile);

      return true;
    }
  }
}<commit_msg>Sql Bulk inserts are now batched.<commit_after>﻿using System;
using System.Collections.Generic;
using Globalization = System.Globalization;
using System.Linq;
using System.Text;
using Microsoft.Build.Framework;
using System.IO;
using Microsoft.Build.Utilities;
using System.Diagnostics;
using Cosmos.Debug.Common;

namespace Cosmos.Build.MSBuild {
  public class ExtractMapFromElfFile : BaseToolTask {
    [Required]
    public string InputFile {
      get;
      set;
    }

    [Required]
    public string DebugInfoFile {
      get;
      set;
    }

    [Required]
    public string WorkingDir {
      get;
      set;
    }

    [Required]
    public string CosmosBuildDir {
      get;
      set;
    }

    public override bool Execute() {
      // Important! A given address can have more than one label.
      // Do NOT filter by duplicate addresses as this causes serious lookup problems.

      string xSymbolString;
      //TODO: This reads the file (13MB currently) into RAM...
      // Thats not needed.. read it line by line instead. Wait till we move to direct
      // DB only use though as we can do this at one time.
      if (!RunObjDump(out xSymbolString)) {
        return false;
      }

      var xLabels = new List<Label>();
      var xLines = xSymbolString.Split(new string[] { "\r\n", "\n", "\r" }, StringSplitOptions.RemoveEmptyEntries);

      using (var xDebugInfo = new DebugInfo(DebugInfoFile)) {
        bool xListStarted = false;
        foreach (string xLine in xLines) {
          if (String.IsNullOrEmpty(xLine)) {
            continue;
          } else if (!xListStarted) {
            // Find start of the data
            if (xLine == "SYMBOL TABLE:") {
              xListStarted = true;
            }
            continue;
          }

          uint xAddress;
          try {
            xAddress = UInt32.Parse(xLine.Substring(0, 8), Globalization.NumberStyles.HexNumber);
          } catch (Exception) {
            Log.LogError("Error processing line '" + xLine + "'");
            throw;
          }

          string xSection = xLine.Substring(17, 5);
          if (xSection != ".text" && xSection != ".data") {
            continue;
          }
          string xLabel = xLine.Substring(32);
          if (xLabel == xSection) {
            // Non label, skip
            continue;
          }

          xLabels.Add(new Label() {
            LABELNAME = xLabel,
            ADDRESS = xAddress
          });

          // We dont want to issue individual inserts to SQL as this is very slow.
          // But accumulating too many records in RAM also is a problem. For example 
          // at time of writing the full structure would take up 11 MB of RAM just for this structure.
          // This is not a huge amount, but as we compile in more and more this figure will grow.
          // So as a compromise, we collect 2500 records then bulk insert.
          if (xLabels.Count > 2500) {
            xDebugInfo.WriteLabels(xLabels);
            xLabels.Clear();
          }
        }

        if (xLabels.Count > 0) {
          xDebugInfo.WriteLabels(xLabels);
        }
      }
      return true;
    }

    private bool RunObjDump(out string result) {
      result = "";
      var xTempBatFile = Path.Combine(WorkingDir, "ExtractMapFromElfFileTemp.bat");
      if (File.Exists(xTempBatFile)) {
        File.Delete(xTempBatFile);
        if (File.Exists(xTempBatFile)) {
          Log.LogError("ExtractMapFromElfFileTemp.bat already exists!");
          return false;
        }
      }

      var xTempOutFile = Path.Combine(WorkingDir, "ExtractMapFromElfFileTemp.out");
      if (File.Exists(xTempOutFile)) {
        File.Delete(xTempOutFile);
        if (File.Exists(xTempOutFile)) {
          Log.LogError("ExtractMapFromElfFileTemp.out already exists!");
          return false;
        }
      }

      File.WriteAllText(xTempBatFile, "@ECHO OFF\r\n\"" + Path.Combine(CosmosBuildDir, @"tools\cygwin\objdump.exe") + "\" --wide --syms \"" + InputFile + "\" > ExtractMapFromElfFileTemp.out");
      if (!ExecuteTool(WorkingDir, xTempBatFile, "", "objdump")) {
        return false;
      }
      result = File.ReadAllText(xTempOutFile);

      return true;
    }
  }
}
{'fraction_non_alphanumeric': 0.10281304456061738, 'fraction_numerical': 0.004107542942494399, 'mean_word_length': 1.8706680957484816, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3137583', 'n_tokens_mistral': 2364, 'n_tokens_neox': 2257, 'n_words': 607}