<commit_before>'use strict';

const { normalize } = require('path');
const { platform, pointerSize } = Process;
const { toByteArray } = require('base64-js');

module.exports = {
  ls,
  cat,
  open,
  transformVirtualPath,
  exist
};

function debase(a) {
  if (a.startsWith('base64:')) {
    try {
      const data = toByteArray(a.slice(7));
      a = String.fromCharCode.apply(null, data);
    } catch(e) {
      // invalid base64
    }
  }
  return normalize(a);
}

let fs = null;

const direntSpecs = {
  'linux-32': {
    d_name: [11, 'Utf8String'],
    d_type: [10, 'U8']
  },
  'linux-64': {
    d_name: [19, 'Utf8String'],
    d_type: [18, 'U8']
  },
  'darwin-32': {
    d_name: [21, 'Utf8String'],
    d_type: [20, 'U8']
  },
  'darwin-64': {
    d_name: [
      [8, 'Utf8String'],
      [21, 'Utf8String']
    ],
    d_type: [
      [6, 'U8'],
      [20, 'U8']
    ]
  }
};

const statSpecs = {
  'linux-32': {
    size: [44, 'S32']
  },
  'linux-64': {
    size: [48, 'S64']
  },
  'darwin-32': {
    size: [60, 'S64']
  },
  'darwin-64': {
    size: [96, 'S64']
  }
};

const statxSpecs = {
  'linux-64': {
    size: [40, 'S64']
  }
};

const STATX_SIZE = 0x200;

let has64BitInode = null;
const direntSpec = direntSpecs[`${platform}-${pointerSize * 8}`];
const statSpec = statSpecs[`${platform}-${pointerSize * 8}`] || null;
const statxSpec = statxSpecs[`${platform}-${pointerSize * 8}`] || null;

function ls (path) {
  if (fs === null) {
    fs = new FridaFS();
  }
  return fs.ls(debase(path));
}

function cat (path, mode, offset, size) {
  if (fs === null) {
    fs = new FridaFS();
  }

  return fs.cat(debase(path), mode, offset, size);
}

function open (path) {
  if (fs === null) {
    fs = new FridaFS();
  }

  return fs.open(debase(path));
}

function transformVirtualPath (path) {
  if (fs === null) {
    fs = new FridaFS();
  }
  return fs.transformVirtualPath(normalize(path));
}

function exist (path) {
  if (fs === null) {
    fs = new FridaFS();
  }
  return fs.exist(debase(path));
}

class FridaFS {
  constructor () {
    this._api = null;
    this._entryTypes = null;
    this._excludeSet = new Set(['.', '..']);
    this._transform = null;
  }

  exist (path) {
    return this.api.getFileSize(path) >= 0;
  }

  ls (path) {
    const result = [];

    const actualPath = this.transform.toActual(path);
    if (actualPath !== null) {
      const entryBuf = Memory.alloc(Process.pageSize);
      const resultPtr = Memory.alloc(Process.pointerSize);
      const dir = this.api.opendir(actualPath);
      if (dir === null) {
        return '';
      }
      let entry;
      while ((entry = this.api.readdir(dir, entryBuf, resultPtr)) !== null) {
        if (!this._excludeSet.has(entry.name)) {
          // result.push(`${this._getEntryType(entry.type)} ${entry.name}`);
          result.push([this._getEntryType(entry.type), entry.name].join(' '));
        }
      }
      this.api.closedir(dir);
    } else {
      const virtualDir = this.transform.getVirtualDir(path);
      for (const entry of virtualDir) {
        result.push(`d ${entry.name}`);
      }
    }
    return result.join('\n');
  }

  cat (path, mode, offset, size) {
    const actualPath = this.transform.toActual(path);
    if (actualPath !== null) {
      const fileSize = this.api.getFileSize(actualPath);
      if (fileSize < 0) {
        console.log(`ERROR: cannot stat ${actualPath}`);
        return '';
      }

      size = parseInt(size);
      offset = parseInt(offset);
      size = (size === null) ? fileSize : size;
      if (size < 0) {
        console.log(`ERROR: invalid size ${size}`);
        return '';
      }
      let weak = false;
      if (size === 0) {
        console.log('weak');
        weak = true;
        size = 1024 * 32;
      }
      if (size > 1024 * 512) {
        console.log('ERROR: file chunk is too big. (' + size + ' bytes)');
        return '';
      }

      const buf = Memory.alloc(size);
      const f = this.api.fopen(actualPath, 'rb');
      if (offset > 0) {
        this.api.fseek(f, offset, 0);<commit_msg>Increasing file limit to 4MB<commit_after>'use strict';

const { normalize } = require('path');
const { platform, pointerSize } = Process;
const { toByteArray } = require('base64-js');

module.exports = {
  ls,
  cat,
  open,
  transformVirtualPath,
  exist
};

function debase(a) {
  if (a.startsWith('base64:')) {
    try {
      const data = toByteArray(a.slice(7));
      a = String.fromCharCode.apply(null, data);
    } catch(e) {
      // invalid base64
    }
  }
  return normalize(a);
}

let fs = null;

const direntSpecs = {
  'linux-32': {
    d_name: [11, 'Utf8String'],
    d_type: [10, 'U8']
  },
  'linux-64': {
    d_name: [19, 'Utf8String'],
    d_type: [18, 'U8']
  },
  'darwin-32': {
    d_name: [21, 'Utf8String'],
    d_type: [20, 'U8']
  },
  'darwin-64': {
    d_name: [
      [8, 'Utf8String'],
      [21, 'Utf8String']
    ],
    d_type: [
      [6, 'U8'],
      [20, 'U8']
    ]
  }
};

const statSpecs = {
  'linux-32': {
    size: [44, 'S32']
  },
  'linux-64': {
    size: [48, 'S64']
  },
  'darwin-32': {
    size: [60, 'S64']
  },
  'darwin-64': {
    size: [96, 'S64']
  }
};

const statxSpecs = {
  'linux-64': {
    size: [40, 'S64']
  }
};

const STATX_SIZE = 0x200;

let has64BitInode = null;
const direntSpec = direntSpecs[`${platform}-${pointerSize * 8}`];
const statSpec = statSpecs[`${platform}-${pointerSize * 8}`] || null;
const statxSpec = statxSpecs[`${platform}-${pointerSize * 8}`] || null;

function ls (path) {
  if (fs === null) {
    fs = new FridaFS();
  }
  return fs.ls(debase(path));
}

function cat (path, mode, offset, size) {
  if (fs === null) {
    fs = new FridaFS();
  }

  return fs.cat(debase(path), mode, offset, size);
}

function open (path) {
  if (fs === null) {
    fs = new FridaFS();
  }

  return fs.open(debase(path));
}

function transformVirtualPath (path) {
  if (fs === null) {
    fs = new FridaFS();
  }
  return fs.transformVirtualPath(normalize(path));
}

function exist (path) {
  if (fs === null) {
    fs = new FridaFS();
  }
  return fs.exist(debase(path));
}

class FridaFS {
  constructor () {
    this._api = null;
    this._entryTypes = null;
    this._excludeSet = new Set(['.', '..']);
    this._transform = null;
  }

  exist (path) {
    return this.api.getFileSize(path) >= 0;
  }

  ls (path) {
    const result = [];

    const actualPath = this.transform.toActual(path);
    if (actualPath !== null) {
      const entryBuf = Memory.alloc(Process.pageSize);
      const resultPtr = Memory.alloc(Process.pointerSize);
      const dir = this.api.opendir(actualPath);
      if (dir === null) {
        return '';
      }
      let entry;
      while ((entry = this.api.readdir(dir, entryBuf, resultPtr)) !== null) {
        if (!this._excludeSet.has(entry.name)) {
          // result.push(`${this._getEntryType(entry.type)} ${entry.name}`);
          result.push([this._getEntryType(entry.type), entry.name].join(' '));
        }
      }
      this.api.closedir(dir);
    } else {
      const virtualDir = this.transform.getVirtualDir(path);
      for (const entry of virtualDir) {
        result.push(`d ${entry.name}`);
      }
    }
    return result.join('\n');
  }

  cat (path, mode, offset, size) {
    const actualPath = this.transform.toActual(path);
    if (actualPath !== null) {
      const fileSize = this.api.getFileSize(actualPath);
      if (fileSize < 0) {
        console.log(`ERROR: cannot stat ${actualPath}`);
        return '';
      }

      size = parseInt(size);
      offset = parseInt(offset);
      size = (size === null) ? fileSize : size;
      if (size < 0) {
        console.log(`ERROR: invalid size ${size}`);
        return '';
      }
      let weak = false;
      if (size === 0) {
        console.log('weak');
        weak = true;
        size = 1024 * 32;
      }
      if (size > 1024 * 4096) {
        console.log('ERROR: file chunk is too big. (' + size + ' bytes)');
        return '';
      }

      const buf = Memory.alloc(size);
      const f = this.api.fopen(actualPath, 'rb');
      if (offset > 0) {
        this.api.fseek(f, offset, 0);
{'fraction_non_alphanumeric': 0.18168426221458384, 'fraction_numerical': 0.025042965872821017, 'mean_word_length': 2.4477359289039358, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1405524', 'n_tokens_mistral': 3088, 'n_tokens_neox': 2965, 'n_words': 647}