<commit_before>#!/usr/bin/env python

import os
import re
import sys
import time
import logging
import progressbar
from errno import EPERM, ENOENT, ENODATA
from treelib import Node, Tree
from dateutil.parser import parse
from hanzo.warctools import WarcRecord
from hanzo.warctools.stream import open_record_stream
from stat import S_IFDIR, S_IFLNK, S_IFREG
from fuse import FUSE, FuseOSError, Operations, LoggingMixIn

LOGGING_FORMAT="[%(asctime)s] %(levelname)s: %(message)s"
logging.basicConfig( format=LOGGING_FORMAT, level=logging.DEBUG )
logger = logging.getLogger( "webarchivefuse" )

class WarcRecordNode( Node ):
	"""Node with a WarcRecord and offset."""
	def __init__( self, record, offset, tag=None, identifier=None, expanded=True ):
		Node.__init__( self, tag=tag, identifier=identifier, expanded=expanded )
		self.record = record
		self.offset = offset
		self.payload_offset = 0
		self.xattrs = {}
		for k, v in record.headers:
			self.xattrs[ k ] = v
		if record.type == WarcRecord.RESPONSE and record.url.startswith( "http" ):
			mime, data = record.content
			if data.startswith( "HTTP" ):
				match = re.search( "\r?\n(\r?\n)+", data, re.MULTILINE )
				self.xattrs[ "http.headers" ] = data[ 0:match.end() ]
				self.payload_offset = match.end()


class WarcFileSystem( LoggingMixIn, Operations ):
	"""Filesystem built on a WARC's URI paths."""
	def __init__( self, warc ):
		self.warc = warc
		logger.debug( "Mounting %s" % self.warc )
		self.fh = WarcRecord.open_archive( warc, gzip="auto", mode="rb" )
		self.tree = Tree()<commit_msg>Added offset to extended attributes.<commit_after>#!/usr/bin/env python

import os
import re
import sys
import time
import logging
import progressbar
from errno import EPERM, ENOENT, ENODATA
from treelib import Node, Tree
from dateutil.parser import parse
from hanzo.warctools import WarcRecord
from stat import S_IFDIR, S_IFLNK, S_IFREG
from fuse import FUSE, FuseOSError, Operations, LoggingMixIn

LOGGING_FORMAT="[%(asctime)s] %(levelname)s: %(message)s"
logging.basicConfig( format=LOGGING_FORMAT, level=logging.DEBUG )
logger = logging.getLogger( "webarchivefuse" )

class WarcRecordNode( Node ):
	"""Node with a WarcRecord and offset."""
	def __init__( self, record, offset, tag=None, identifier=None, expanded=True ):
		Node.__init__( self, tag=tag, identifier=identifier, expanded=expanded )
		self.record = record
		self.offset = offset
		self.payload_offset = 0
		self.xattrs = {}
		self.xattrs[ "offset" ] = str( offset )
		for k, v in record.headers:
			self.xattrs[ k ] = v
		if record.type == WarcRecord.RESPONSE and record.url.startswith( "http" ):
			mime, data = record.content
			if data.startswith( "HTTP" ):
				match = re.search( "\r?\n(\r?\n)+", data, re.MULTILINE )
				self.xattrs[ "http.headers" ] = data[ 0:match.end() ]
				self.payload_offset = match.end()


class WarcFileSystem( LoggingMixIn, Operations ):
	"""Filesystem built on a WARC's URI paths."""
	def __init__( self, warc ):
		self.warc = warc
		logger.debug( "Mounting %s" % self.warc )
		self.fh = WarcRecord.open_archive( warc, gzip="auto", mode="rb" )
		self.tree = Tree()
{'fraction_non_alphanumeric': 0.12672441450112287, 'fraction_numerical': 0.0012832852101379532, 'mean_word_length': 4.648550724637682, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2815410', 'n_tokens_mistral': 1150, 'n_tokens_neox': 1068, 'n_words': 297}