<commit_before>import argparse
import csv
import os.path

from collections import namedtuple

# POLICY pdir: Always include the entire (flat) parent directory
class SpecPoint:
  def __init__(self, path, mode):
    self.path = path
    self.mode = mode
    self.subfiles = []
  def isParentOf(self, path):
    return path[:len(self.path)+1] == self.path+"/"\
      or path == self.path or self.path==""
  def __str__(self):
    if self.mode==1:
      return "^" + self.path + "/*"
    elif self.mode==0:
      return "^" + self.path
  def __eq__(self, other):
    return self.path == other.path
  def __ne__(self, other):
    return self.path != other.path
  def __lt__(self, other):
    return self.path < other.path
  def __gt__(self, other):
    return self.path > other.path

def peek(stack, pos = 1):
  return stack[-pos] if len(stack)>0 else None

def get_parent(path):
  pos = path.rfind("/")
  return path[:pos] if pos>0 else ""

def parent_dir_parser(pathsToInclude):
  specsToInclude = []
  # Go through tracer points and build specs based on made calls
  for curPoint in pathsToInclude:
    specPoint = None
    if curPoint.action in parent_dir_parser.parentDirFlat:
      specPoint = SpecPoint(get_parent(curPoint.path), 1)
    elif curPoint.action in parent_dir_parser.dirFlat:
      specPoint = SpecPoint(curPoint.path, 1)
    else:
      specPoint = SpecPoint(curPoint.path, 0)
    specsToInclude.append(specPoint)
  # Sort generated specs alphabetically for reduction
  specsToInclude.sort()
  rootEl = SpecPoint("", 0)
  workStack = [rootEl]
  results = []
  for specPoint in specsToInclude:
    while not peek(workStack).isParentOf(specPoint.path):
      # Backtrack up to nearest parent
      el = workStack.pop()
      results.append(el)
    topEl = peek(workStack)
    if topEl.path == specPoint.path and topEl.mode < specPoint.mode:
      # If stack top element is same path: Update if necessary
      topEl.mode = specPoint.mode
    elif topEl.path != specPoint.path:
      # If stack top is some parent: Add if necessary
      if topEl.mode==0\
        or topEl.path != get_parent(specPoint.path)\
        or specPoint.mode!=0:
        workStack.append(specPoint)
  results+=workStack
  return results


parent_dir_parser.parentDirFlat = ["open()", "getattr()", "getxattr()", "listxattr()", "statfs()"]
parent_dir_parser.dirFlat = ["opendir()"]

ParsingPolicies = {
  "pdir": parent_dir_parser
}
<commit_msg>Only including parent directory for open (no longer for getattr, getxattr, listxattr or statfs)<commit_after>import argparse
import csv
import os.path

from collections import namedtuple

# POLICY pdir: Always include the entire (flat) parent directory
class SpecPoint:
  def __init__(self, path, mode):
    self.path = path
    self.mode = mode
    self.subfiles = []
  def isParentOf(self, path):
    return path[:len(self.path)+1] == self.path+"/"\
      or path == self.path or self.path==""
  def __str__(self):
    if self.mode==1:
      return "^" + self.path + "/*"
    elif self.mode==0:
      return "^" + self.path
  def __eq__(self, other):
    return self.path == other.path
  def __ne__(self, other):
    return self.path != other.path
  def __lt__(self, other):
    return self.path < other.path
  def __gt__(self, other):
    return self.path > other.path

def peek(stack, pos = 1):
  return stack[-pos] if len(stack)>0 else None

def get_parent(path):
  pos = path.rfind("/")
  return path[:pos] if pos>0 else ""

def parent_dir_parser(pathsToInclude):
  specsToInclude = []
  # Go through tracer points and build specs based on made calls
  for curPoint in pathsToInclude:
    specPoint = None
    if curPoint.action in parent_dir_parser.parentDirFlat:
      specPoint = SpecPoint(get_parent(curPoint.path), 1)
    elif curPoint.action in parent_dir_parser.dirFlat:
      specPoint = SpecPoint(curPoint.path, 1)
    else:
      specPoint = SpecPoint(curPoint.path, 0)
    specsToInclude.append(specPoint)
  # Sort generated specs alphabetically for reduction
  specsToInclude.sort()
  rootEl = SpecPoint("", 0)
  workStack = [rootEl]
  results = []
  for specPoint in specsToInclude:
    while not peek(workStack).isParentOf(specPoint.path):
      # Backtrack up to nearest parent
      el = workStack.pop()
      results.append(el)
    topEl = peek(workStack)
    if topEl.path == specPoint.path and topEl.mode < specPoint.mode:
      # If stack top element is same path: Update if necessary
      topEl.mode = specPoint.mode
    elif topEl.path != specPoint.path:
      # If stack top is some parent: Add if necessary
      if topEl.mode==0\
        or topEl.path != get_parent(specPoint.path)\
        or specPoint.mode!=0:
        workStack.append(specPoint)
  results+=workStack
  return results


parent_dir_parser.parentDirFlat = ["open()"]
parent_dir_parser.dirFlat = ["opendir()"]

ParsingPolicies = {
  "pdir": parent_dir_parser
}

{'fraction_non_alphanumeric': 0.1132924335378323, 'fraction_numerical': 0.0049079754601227, 'mean_word_length': 3.422242314647378, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4786463', 'n_tokens_mistral': 1606, 'n_tokens_neox': 1572, 'n_words': 439}