<commit_before>from docopt import docopt
from farmfs import getvol
from farmfs import reverse
from farmfs.util import empty2dot
from farmfs.volume import encode_snapshot
from func_prototypes import constructors
from os import getcwdu
from fs import Path
from json import loads, JSONEncoder
from functools import partial

def printNotNone(value):
  if value is not None:
    print value

def print_file((path, type_)):
  if type_ == "link":
    print type_, path, path.readlink()
  else:
    print type_, path

def walk(parents, exclude, match):
  for parent in parents:
    for (path, type_) in parent.entries(exclude):
      if type_ in match:
        yield (path, type_)

USAGE = \
"""
FarmDBG

Usage:
  farmdbg findvol
  farmdbg reverse <link>
  farmdbg key read <key>
  farmdbg key write <key> <value>
  farmdbg key delete <key>
  farmdbg key list [<key>]
  farmdbg walk (keys|userdata|root)
  farmdbg checksum <path>...
  farmdbg fix link <file> <target>
  farmdbg rewrite-links <target>
"""

def main():
  args = docopt(USAGE)
  cwd = Path(getcwdu())
  vol = getvol(cwd)
  if args['findvol']:
    #TODO Volume found at: <Path from root>
    print "Volume found at: %s" % vol.root
  elif args['reverse']:
    path = Path(args['<link>'], cwd)
    for p in reverse(vol, path):
      print p
  elif args['key']:
    db = vol.keydb
    key = args['<key>']
    if args['read']:
      printNotNone(db.readraw(key))
    elif args['delete']:
      db.delete(key)
    elif args['list']:
      for v in db.list(key):
        print v
    elif args['write']:
      value = args['<value>']
      db.write(key, value)
  elif args['walk']:
    if args['root']:
      print JSONEncoder(ensure_ascii=False).encode(encode_snapshot(vol.tree()))
    elif args['userdata']:
      #TODO file <pathfromroot>/.farmfs/userdata/<ref_path>
      map(print_file, walk([vol.udd], [str(vol.mdd)], ["file"]))
    elif args['keys']:
      print JSONEncoder(ensure_ascii=False).encode(vol.keydb.list())
  elif args['checksum']:
    #TODO <checksum> <full path>
    paths = map(lambda x: Path(x, cwd), empty2dot(args['<path>']))
    for p in paths:
      print p.checksum(), p
  elif args['link']:
    f = Path(args['<file>'], cwd)
    t = Path(args['<target>'], cwd)<commit_msg>Add snapshot walker to farmdbg.<commit_after>from docopt import docopt
from farmfs import getvol
from farmfs import reverse
from farmfs.util import empty2dot
from farmfs.volume import encode_snapshot
from func_prototypes import constructors
from os import getcwdu
from fs import Path
from json import loads, JSONEncoder
from functools import partial

def printNotNone(value):
  if value is not None:
    print value

def print_file((path, type_)):
  if type_ == "link":
    print type_, path, path.readlink()
  else:
    print type_, path

def walk(parents, exclude, match):
  for parent in parents:
    for (path, type_) in parent.entries(exclude):
      if type_ in match:
        yield (path, type_)

USAGE = \
"""
FarmDBG

Usage:
  farmdbg findvol
  farmdbg reverse <link>
  farmdbg key read <key>
  farmdbg key write <key> <value>
  farmdbg key delete <key>
  farmdbg key list [<key>]
  farmdbg walk (keys|userdata|root|snap <snapshot>)
  farmdbg checksum <path>...
  farmdbg fix link <file> <target>
  farmdbg rewrite-links <target>
"""

def main():
  args = docopt(USAGE)
  cwd = Path(getcwdu())
  vol = getvol(cwd)
  if args['findvol']:
    #TODO Volume found at: <Path from root>
    print "Volume found at: %s" % vol.root
  elif args['reverse']:
    path = Path(args['<link>'], cwd)
    for p in reverse(vol, path):
      print p
  elif args['key']:
    db = vol.keydb
    key = args['<key>']
    if args['read']:
      printNotNone(db.readraw(key))
    elif args['delete']:
      db.delete(key)
    elif args['list']:
      for v in db.list(key):
        print v
    elif args['write']:
      value = args['<value>']
      db.write(key, value)
  elif args['walk']:
    if args['root']:
      print JSONEncoder(ensure_ascii=False).encode(encode_snapshot(vol.tree()))
    elif args['snap']:
      print JSONEncoder(ensure_ascii=False).encode(encode_snapshot(vol.snapdb.read(args['<snapshot>'])))
    elif args['userdata']:
      #TODO file <pathfromroot>/.farmfs/userdata/<ref_path>
      map(print_file, walk([vol.udd], [str(vol.mdd)], ["file"]))
    elif args['keys']:
      print JSONEncoder(ensure_ascii=False).encode(vol.keydb.list())
  elif args['checksum']:
    #TODO <checksum> <full path>
    paths = map(lambda x: Path(x, cwd), empty2dot(args['<path>']))
    for p in paths:
      print p.checksum(), p
  elif args['link']:
    f = Path(args['<file>'], cwd)
    t = Path(args['<target>'], cwd)
{'fraction_non_alphanumeric': 0.1404976404976405, 'fraction_numerical': 0.000858000858000858, 'mean_word_length': 3.5537109375, 'pattern_counts': {'":': 2, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1153912', 'n_tokens_mistral': 1604, 'n_tokens_neox': 1612, 'n_words': 468}