<commit_before>var http = require('http'),
    events = require('events'),
    path = require('path'),
    debugr = require('./debugger');

///////////////////////////////////////////////////////////
// exports

exports.create = function (conn, debuggerPort, config) {
  var debug = null,
      //map from sourceID:lineNumber to breakpoint
      breakpoints = {},
      //map from sourceID to filename
      sourceIDs = {},
      //milliseconds to wait for a lookup
      LOOKUP_TIMEOUT = 2500;
  
  function wrapperObject(type, description, hasChildren, frame, scope, ref) {
    return {
      type:type,
      description:description,
      hasChildren:hasChildren,
      objectId:frame + ':' + scope + ':' + ref
    };
  }
  
  function refToObject(ref) {
    var desc = '',
        name,
        kids = ref.properties ? ref.properties.length : false;
    switch (ref.type) {
      case 'object':
        name = /#<an?\s(\w+)>/.exec(ref.text);
        if (name && name.length > 1) {
          desc = name[1];
          if (desc === 'Array') {
            desc += '[' + (ref.properties.length - 1) + ']';
          }
          else if (desc === 'Buffer') {
            desc += '[' + (ref.properties.length - 4) + ']';
          }
        }
        else {
          desc = ref.className || 'Object';
        }
      break;
      case 'function':
        desc = ref.text || 'function()';
      break;
      default:
        desc = ref.text || '';
      break;
    }
    if (desc.length > 100) {
      desc = desc.substring(0, 100) + "\u2026";
    }
    return wrapperObject(ref.type, desc, kids, 0, 0, ref.handle);
  }
  
  function callFrames(bt) {
      if (bt.body.totalFrames > 0) {
        return bt.body.frames.map(function(frame) {
          var f = {
            type: 'function',
            functionName: frame.func.inferredName,
            sourceID: frame.func.scriptId,
            line: frame.line + 1,
            id: frame.index,
            worldId: 1,
            scopeChain: frame.scopes.map(
              function(scope) {
                var c = {};
                switch (scope.type) {
                  case 0:
                    break;
                  case 1:
                    c.isLocal = true;
                    c.thisObject =
                      wrapperObject(
                        'object',
                        frame.receiver.className,
                        true,
                        frame.index,
                        scope.index,
                        frame.receiver.ref);
                    break;
                  case 2:
                    c.isWithBlock = true;
                    break;
                  case 3:
                    c.isClosure = true;
                    break;
                  case 4:
                    c.isElement = true;
                    break;
                  default:
                    break;
                }
                c.objectId = frame.index + ':' + scope.index + ':backtrace';
                return c;
              })
            };
            return f;
          });
      }
      return [{type:'program', sourceID:'internal', line: 0, id: 0, worldId: 1}];
  }

  function evaluate(expr, frame, andThen) {
    var args = { expression: expr, disable_break: true, global: true };
    if (frame != null) {
      args.frame = frame;
      args.global = false;
    }
    debug.request(
      'evaluate',
      { arguments: args},
      andThen);
  }

  function sendBacktrace() {
    debug.request(
      'backtrace',
      {arguments: { inlineRefs: true }},
      function(msg) {
        sendEvent('pausedScript', { details: { callFrames: callFrames(msg) }});
      });
  }

  function breakEvent(obj) {
    var data = {},
        source = sourceIDs[obj.body.script.id],
        args;
    if(!source) {
      args = {
        arguments: {
          includeSource: true,
          types: 4,
          ids: [obj.body.script.id]
      }};
      debug.request('scripts', args, parsedScripts);
    }
    else if (source.hidden) {
      debug.request('continue', { arguments: {stepaction:'out'}});
    }
    else {
      sendBacktrace();
    }
  }
  
  function parsedScripts(msg) {
    var scripts = msg.body.map(function(s) {
      return {
        sourceID: s.id.toString(),
        url: s.name,
        data: s.source,
        firstLine: s.lineOffset,
        scriptWorldType: 0,
        path: s.name.split('/')
      };
    });
    scripts.sort(function(a, b) {
      return a.path.length - b.path.length;
    });
    var paths = [];
    function shorten(s) {
      var p;
      for(var i = s.length - 1; i > 0; i--) {
        p = s.slice(i).join('/');
        if (paths.indexOf(p) === -1) {
          paths.push(p);
          return p;
        }
      }
      return s.join('/');
    }
    scripts.forEach(function(s) {
      var hidden;
      s.url = shorten(s.path);
      hidden = config.hidden &&
          config.hidden.some(function(r) { return r.test(s.url); });
      sourceIDs[s.sourceID] = { url: s.url, hidden: hidden };
      delete s.path;
      if (!hidden) {
        sendEvent('parsedScriptSource', s);
      }
    });
  }
  
  function sendEvent(name, data) {
    data = data || {};
    conn.write(JSON.stringify({<commit_msg>test for hidden scripts before shortening<commit_after>var http = require('http'),
    events = require('events'),
    path = require('path'),
    debugr = require('./debugger');

///////////////////////////////////////////////////////////
// exports

exports.create = function (conn, debuggerPort, config) {
  var debug = null,
      //map from sourceID:lineNumber to breakpoint
      breakpoints = {},
      //map from sourceID to filename
      sourceIDs = {},
      //milliseconds to wait for a lookup
      LOOKUP_TIMEOUT = 2500;
  
  function wrapperObject(type, description, hasChildren, frame, scope, ref) {
    return {
      type:type,
      description:description,
      hasChildren:hasChildren,
      objectId:frame + ':' + scope + ':' + ref
    };
  }
  
  function refToObject(ref) {
    var desc = '',
        name,
        kids = ref.properties ? ref.properties.length : false;
    switch (ref.type) {
      case 'object':
        name = /#<an?\s(\w+)>/.exec(ref.text);
        if (name && name.length > 1) {
          desc = name[1];
          if (desc === 'Array') {
            desc += '[' + (ref.properties.length - 1) + ']';
          }
          else if (desc === 'Buffer') {
            desc += '[' + (ref.properties.length - 4) + ']';
          }
        }
        else {
          desc = ref.className || 'Object';
        }
      break;
      case 'function':
        desc = ref.text || 'function()';
      break;
      default:
        desc = ref.text || '';
      break;
    }
    if (desc.length > 100) {
      desc = desc.substring(0, 100) + "\u2026";
    }
    return wrapperObject(ref.type, desc, kids, 0, 0, ref.handle);
  }
  
  function callFrames(bt) {
      if (bt.body.totalFrames > 0) {
        return bt.body.frames.map(function(frame) {
          var f = {
            type: 'function',
            functionName: frame.func.inferredName,
            sourceID: frame.func.scriptId,
            line: frame.line + 1,
            id: frame.index,
            worldId: 1,
            scopeChain: frame.scopes.map(
              function(scope) {
                var c = {};
                switch (scope.type) {
                  case 0:
                    break;
                  case 1:
                    c.isLocal = true;
                    c.thisObject =
                      wrapperObject(
                        'object',
                        frame.receiver.className,
                        true,
                        frame.index,
                        scope.index,
                        frame.receiver.ref);
                    break;
                  case 2:
                    c.isWithBlock = true;
                    break;
                  case 3:
                    c.isClosure = true;
                    break;
                  case 4:
                    c.isElement = true;
                    break;
                  default:
                    break;
                }
                c.objectId = frame.index + ':' + scope.index + ':backtrace';
                return c;
              })
            };
            return f;
          });
      }
      return [{type:'program', sourceID:'internal', line: 0, id: 0, worldId: 1}];
  }

  function evaluate(expr, frame, andThen) {
    var args = { expression: expr, disable_break: true, global: true };
    if (frame != null) {
      args.frame = frame;
      args.global = false;
    }
    debug.request(
      'evaluate',
      { arguments: args},
      andThen);
  }

  function sendBacktrace() {
    debug.request(
      'backtrace',
      {arguments: { inlineRefs: true }},
      function(msg) {
        sendEvent('pausedScript', { details: { callFrames: callFrames(msg) }});
      });
  }

  function breakEvent(obj) {
    var data = {},
        source = sourceIDs[obj.body.script.id],
        args;
    if(!source) {
      args = {
        arguments: {
          includeSource: true,
          types: 4,
          ids: [obj.body.script.id]
      }};
      debug.request('scripts', args, parsedScripts);
    }
    else if (source.hidden) {
      debug.request('continue', { arguments: {stepaction:'out'}});
    }
    else {
      sendBacktrace();
    }
  }
  
  function parsedScripts(msg) {
    var scripts = msg.body.map(function(s) {
      return {
        sourceID: s.id.toString(),
        url: s.name,
        data: s.source,
        firstLine: s.lineOffset,
        scriptWorldType: 0,
        path: s.name.split('/')
      };
    });
    scripts.sort(function(a, b) {
      return a.path.length - b.path.length;
    });
    var paths = [];
    function shorten(s) {
      var p;
      for(var i = s.length - 1; i > 0; i--) {
        p = s.slice(i).join('/');
        if (paths.indexOf(p) === -1) {
          paths.push(p);
          return p;
        }
      }
      return s.join('/');
    }
    scripts.forEach(function(s) {
      var hidden;
      hidden = config.hidden &&
          config.hidden.some(function(r) { return r.test(s.url); });
      s.url = shorten(s.path);
      sourceIDs[s.sourceID] = { url: s.url, hidden: hidden };
      delete s.path;
      if (!hidden) {
        sendEvent('parsedScriptSource', s);
      }
    });
  }
  
  function sendEvent(name, data) {
    data = data || {};
    conn.write(JSON.stringify({
{'fraction_non_alphanumeric': 0.14711465451784358, 'fraction_numerical': 0.007023538344722855, 'mean_word_length': 1.368929856115108, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3774467', 'n_tokens_mistral': 3167, 'n_tokens_neox': 3025, 'n_words': 664}