<commit_before>( function _Extract_s_() {

'use strict';

if( typeof module !== 'undefined' )
{

  var _ = _global_.wTools;
  if( !_.FileProvider )
  require( '../UseMid.s' );

}

var _global = _global_;
var _ = _global_.wTools;
var Abstract = _.FileProvider.Abstract;
var Partial = _.FileProvider.Partial;
var FileRecord = _.FileRecord;
var Find = _.FileProvider.Find;

_.assert( _.routineIs( _.FileRecord ) );
_.assert( _.routineIs( Abstract ) );
_.assert( _.routineIs( Partial ) );
_.assert( !!Find );
_.assert( !_.FileProvider.Extract );

//

var Parent = Partial;
var Self = function wFileProviderExtract( o )
{
  return _.instanceConstructor( Self, this, arguments );
}

Self.shortName = 'Extract';

// --
// inter
// --

function init( o )
{
  var self = this;
  Parent.prototype.init.call( self,o );

  if( self.filesTree === null )
  self.filesTree = Object.create( null );

}

// --
// path
// --

function pathCurrentAct()
{
  var self = this;

  _.assert( arguments.length === 0 || arguments.length === 1 );

  if( arguments.length === 1 && arguments[ 0 ] )
  {
    var path = arguments[ 0 ];
    _.assert( self.path.is( path ) );
    self._currentPath = path;
  }

  var result = self._currentPath;

  return result;
}

//

function pathResolveSoftLinkAct( o )
{
  var self = this;

  _.assert( arguments.length === 1, 'expects single argument' );
  _.assert( self.path.isAbsolute( o.filePath ) );

  /* using self.resolvingSoftLink causes recursion problem in pathResolveLink */
  if( !self.fileIsSoftLink( o.filePath ) )
  return o.filePath;

  var descriptor = self._descriptorRead( o.filePath );
  var resolved = self._descriptorResolveSoftLinkPath( descriptor );

  _.assert( _.strIs( resolved ) )

  return resolved;
}

var defaults = pathResolveSoftLinkAct.defaults = Object.create( Parent.prototype.pathResolveSoftLinkAct.defaults );
var paths = pathResolveSoftLinkAct.paths = Object.create( Parent.prototype.pathResolveSoftLinkAct.paths );
var having = pathResolveSoftLinkAct.having = Object.create( Parent.prototype.pathResolveSoftLinkAct.having );

//

function pathResolveHardLinkAct( o )
{
  var self = this;

  _.assert( arguments.length === 1, 'expects single argument' );
  _.assert( self.path.isAbsolute( o.filePath ) );

  if( /*!self.resolvingHardLink ||*/ !self.fileIsHardLink( o.filePath ) )
  return o.filePath;

  var descriptor = self._descriptorRead( o.filePath );
  var resolved = self._descriptorResolveHardLinkPath( descriptor );

  if( !self._descriptorRead( resolved ) )
  return o.filePath;

  _.assert( _.strIs( resolved ) )

  return resolved;
}

//

// function linkSoftReadAct( o )
// {
//   let self = this;

//   _.assert( arguments.length === 1, 'expects single argument' );
//   _.assert( self.path.isAbsolute( o.filePath ) );

//   if( !self.fileIsSoftLink( o.filePath ) )
//   return o.filePath;

//   let descriptor = self._descriptorRead( o.filePath );
//   let result = self._descriptorResolveSoftLinkPath( descriptor );

//   _.assert( _.strIs( result ) );

//   return result;
// }

// _.routineExtend( linkSoftReadAct, Parent.prototype.linkSoftReadAct );

// --
// read
// --

function fileReadAct( o )
{
  var self = this;
  var con = new _.Consequence();
  var result = null;

  _.assert( arguments.length === 1, 'expects single argument' );
  _.assertRoutineOptions( fileReadAct,o );
  _.assert( _.strIs( o.encoding ) );

  var encoder = fileReadAct.encoders[ o.encoding ];

  if( o.encoding )
  if( !encoder )
  return handleError( _.err( 'Encoding: ' + o.encoding + ' is not supported!' ) )

  /* exec */

  handleBegin();

  // if( _.strHas( o.filePath, 'icons.woff2' ) )
  // debugger;

  o.filePath = self.pathResolveLink
  ({
    filePath : o.filePath,
    resolvingSoftLink : o.resolvingSoftLink,
    resolvingTextLink : o.resolvingTextLink,
  });

  if( self.hub && _.uri.isGlobal( o.filePath ) )
  {
    _.assert( self.hub !== self );
    return self.hub.fileReadAct( o );
  }

  var result = self._descriptorRead( o.filePath );

  // if( self._descriptorIsLink( result ) )
  // {
  //   result = self._descriptorResolve({ descriptor : result });
  //   if( result === undefined )
  //   return handleError( _.err( 'Cant resolve :', result ) );
  // }

  if( self._descriptorIsHardLink( result ) )
  {
    var resolved = self._descriptorResolve({ descriptor : result });
    if( resolved === undefined )
    return handleError( _.err( 'Cant resolve :', result ) );
    result = resolved;
  }

  if( result === undefined || result === null )
  {
    debugger;
    result = self._descriptorRead( o.filePath );
    return handleError( _.err( 'File at :', o.filePath, 'doesn`t exist!' ) );
  }

  if( self._descriptorIsDir( result ) )
  return handleError( _.err( 'Can`t read from dir : ' + _.strQuote( o.filePath ) + ' method expects file' ) );
  else if( self._descriptorIsLink( result ) )
  return handleError( _.err( 'Can`t read from link : ' + _.strQuote( o.filePath ) + ', without link resolving enabled' ) );
  else if( !self._descriptorIsTerminal( result ) )
  return handleError( _.err( 'Can`t read file : ' + _.strQuote( o.filePath ), result ) );

  if( self.usingTime )
  self._fileTimeSetAct({ filePath : o.filePath, atime : _.timeNow() });

  return handleEnd( result );

  /* begin */

  function handleBegin()
  {

    if( encoder && encoder.onBegin )
    _.sure( encoder.onBegin.call( self, { operation : o, encoder : encoder }) === undefined );

  }

  /* end */

  function handleEnd( data )
  {

    let context = { data : data, operation : o, encoder : encoder };
    if( encoder && encoder.onEnd )
    _.sure( encoder.onEnd.call( self, context ) === undefined );
    data = context.data;

    if( o.sync )
    {
      return data;
    }
    else
    {
      return con.give( data );
    }

  }

  /* error */

  function handleError( err )
  {

    debugger;

    if( encoder && encoder.onError )
    try
    {
      err = _._err
      ({
        args : [ stack,'\nfileReadAct( ',o.filePath,' )\n',err ],
        usingSourceCode : 0,
        level : 0,
      });
      err = encoder.onError.call( self,{ error : err, operation : o, encoder : encoder })
    }
    catch( err2 )
    {
      console.error( err2 );
      console.error( err.toString() + '\n' + err.stack );
    }

    if( o.sync )
    {
      throw err;
    }
    else
    {
      return con.error( err );
    }

  }

}

_.routineExtend( fileReadAct, Parent.prototype.fileReadAct );

// var defaults = fileReadAct.defaults = Object.create( Parent.prototype.fileReadAct.defaults );
// var having = fileReadAct.having = Object.create( Parent.prototype.fileReadAct.having );

//

// var fileHashAct = ( function()
// {

//   var crypto;

//   return function fileHashAct( o )
//   {
//     var result=NaN;
//     var self = this;

//     if( _.strIs( o ) )
//     o = { filePath : o };

//     _.assertRoutineOptions( fileHashAct,o );
//     _.assert( _.strIs( o.filePath ) );
//     _.assert( arguments.length === 1, 'expects single argument' );

//     /* */

//     if( !crypto )
//     crypto = require( 'crypto' );
//     var md5sum = crypto.createHash( 'md5' );

//     /* */
//     function makeHash()
//     {
//       try
//       {
//         var read = self.fileReadAct( { filePath : o.filePath, sync : 1 } );
//         md5sum.update( read );
//         result = md5sum.digest( 'hex' );
//       }
//       catch( err )
//       {
//         if( o.throwing )
//         {
//           throw _.err( err );
//         }
//       }
//     }

//    if( o.sync )
//    {
//      makeHash( );
//      return result;
//    }
//    else
//    {
//      return _.timeOut( 0, function()
//      {
//        makeHash();
//        return result;
//      })
//    }
//   }
// })();

// fileHashAct.defaults = {};
// fileHashAct.defaults.__proto__ = Parent.prototype.fileHashAct.defaults;

//

function directoryReadAct( o )
{
  var self = this;

  _.assert( arguments.length === 1, 'expects single argument' );
  _.assertRoutineOptions( directoryReadAct,o );

  var result;

  if( o.sync )
  {
    readDir();
    return result;
  }
  else
  {
    return _.timeOut( 0, function()
    {
      readDir();
      return result;
    });
  }

  /* */

  function readDir()
  {
    o.filePath = self.pathResolveLink({ filePath : o.filePath, resolvingSoftLink : 1 });

    var file = self._descriptorRead( o.filePath );

    // if( self._descriptorIsLink( file ) )
    // file = self._descriptorResolve({ descriptor : result, resolvingSoftLink : 1 });

    if( file !== undefined )
    {
      if( _.objectIs( file ) )
      {
        result = Object.keys( file );
        _.assert( _.arrayIs( result ),'readdirSync returned not array' );
      }
      else
      {
        result = [ self.path.name({ path : o.filePath, withExtension : 1 }) ];
      }
    }
    else
    {
      if( o.throwing )
      throw _.err( 'Path : ', o.filePath, 'doesn`t exist!' );;
      result = null;
    }
  }

}

var defaults = directoryReadAct.defaults = Object.create( Parent.prototype.directoryReadAct.defaults );
var having = directoryReadAct.having = Object.create( Parent.prototype.directoryReadAct.having );

// --
// read stat
// -

function fileStatAct( o )
{
  var self = this;

  _.assert( arguments.length === 1, 'expects single argument' );
  _.assertRoutineOptions( fileStatAct,o );

  /* */

  if( o.sync )
  {
    return _fileStatAct( o.filePath );
  }
  else
  {
    return _.timeOut( 0, function()
    {
      return _fileStatAct( o.filePath );
    })
  }

  /* */

  function _fileStatAct( filePath )
  {
    var result = null;

    // if( filePath === '/out/icons' )
    // debugger;

    filePath = self.pathResolveLink({ filePath : filePath, resolvingSoftLink : o.resolvingSoftLink });

    var file = self._descriptorRead( filePath );

    if( !_.definedIs( file ) )
    {
      // _.assert( !file );
      if( o.throwing )
      throw _.err( 'Path :', filePath, 'doesn`t exist!' );
      return result;
    }

    result = new _.FileStat();

    if( self.timeStats && self.timeStats[ filePath ] )
    {
      var timeStats = self.timeStats[ filePath ];
      for( var k in timeStats )<commit_msg>Extract.fileStatAct : adjust behavior for cycle links<commit_after>( function _Extract_s_() {

'use strict';

if( typeof module !== 'undefined' )
{

  var _ = _global_.wTools;
  if( !_.FileProvider )
  require( '../UseMid.s' );

}

var _global = _global_;
var _ = _global_.wTools;
var Abstract = _.FileProvider.Abstract;
var Partial = _.FileProvider.Partial;
var FileRecord = _.FileRecord;
var Find = _.FileProvider.Find;

_.assert( _.routineIs( _.FileRecord ) );
_.assert( _.routineIs( Abstract ) );
_.assert( _.routineIs( Partial ) );
_.assert( !!Find );
_.assert( !_.FileProvider.Extract );

//

var Parent = Partial;
var Self = function wFileProviderExtract( o )
{
  return _.instanceConstructor( Self, this, arguments );
}

Self.shortName = 'Extract';

// --
// inter
// --

function init( o )
{
  var self = this;
  Parent.prototype.init.call( self,o );

  if( self.filesTree === null )
  self.filesTree = Object.create( null );

}

// --
// path
// --

function pathCurrentAct()
{
  var self = this;

  _.assert( arguments.length === 0 || arguments.length === 1 );

  if( arguments.length === 1 && arguments[ 0 ] )
  {
    var path = arguments[ 0 ];
    _.assert( self.path.is( path ) );
    self._currentPath = path;
  }

  var result = self._currentPath;

  return result;
}

//

function pathResolveSoftLinkAct( o )
{
  var self = this;

  _.assert( arguments.length === 1, 'expects single argument' );
  _.assert( self.path.isAbsolute( o.filePath ) );

  /* using self.resolvingSoftLink causes recursion problem in pathResolveLink */
  if( !self.fileIsSoftLink( o.filePath ) )
  return o.filePath;

  var descriptor = self._descriptorRead( o.filePath );
  var resolved = self._descriptorResolveSoftLinkPath( descriptor );

  _.assert( _.strIs( resolved ) )

  return resolved;
}

var defaults = pathResolveSoftLinkAct.defaults = Object.create( Parent.prototype.pathResolveSoftLinkAct.defaults );
var paths = pathResolveSoftLinkAct.paths = Object.create( Parent.prototype.pathResolveSoftLinkAct.paths );
var having = pathResolveSoftLinkAct.having = Object.create( Parent.prototype.pathResolveSoftLinkAct.having );

//

function pathResolveHardLinkAct( o )
{
  var self = this;

  _.assert( arguments.length === 1, 'expects single argument' );
  _.assert( self.path.isAbsolute( o.filePath ) );

  if( /*!self.resolvingHardLink ||*/ !self.fileIsHardLink( o.filePath ) )
  return o.filePath;

  var descriptor = self._descriptorRead( o.filePath );
  var resolved = self._descriptorResolveHardLinkPath( descriptor );

  if( !self._descriptorRead( resolved ) )
  return o.filePath;

  _.assert( _.strIs( resolved ) )

  return resolved;
}

//

// function linkSoftReadAct( o )
// {
//   let self = this;

//   _.assert( arguments.length === 1, 'expects single argument' );
//   _.assert( self.path.isAbsolute( o.filePath ) );

//   if( !self.fileIsSoftLink( o.filePath ) )
//   return o.filePath;

//   let descriptor = self._descriptorRead( o.filePath );
//   let result = self._descriptorResolveSoftLinkPath( descriptor );

//   _.assert( _.strIs( result ) );

//   return result;
// }

// _.routineExtend( linkSoftReadAct, Parent.prototype.linkSoftReadAct );

// --
// read
// --

function fileReadAct( o )
{
  var self = this;
  var con = new _.Consequence();
  var result = null;

  _.assert( arguments.length === 1, 'expects single argument' );
  _.assertRoutineOptions( fileReadAct,o );
  _.assert( _.strIs( o.encoding ) );

  var encoder = fileReadAct.encoders[ o.encoding ];

  if( o.encoding )
  if( !encoder )
  return handleError( _.err( 'Encoding: ' + o.encoding + ' is not supported!' ) )

  /* exec */

  handleBegin();

  // if( _.strHas( o.filePath, 'icons.woff2' ) )
  // debugger;

  o.filePath = self.pathResolveLink
  ({
    filePath : o.filePath,
    resolvingSoftLink : o.resolvingSoftLink,
    resolvingTextLink : o.resolvingTextLink,
  });

  if( self.hub && _.uri.isGlobal( o.filePath ) )
  {
    _.assert( self.hub !== self );
    return self.hub.fileReadAct( o );
  }

  var result = self._descriptorRead( o.filePath );

  // if( self._descriptorIsLink( result ) )
  // {
  //   result = self._descriptorResolve({ descriptor : result });
  //   if( result === undefined )
  //   return handleError( _.err( 'Cant resolve :', result ) );
  // }

  if( self._descriptorIsHardLink( result ) )
  {
    var resolved = self._descriptorResolve({ descriptor : result });
    if( resolved === undefined )
    return handleError( _.err( 'Cant resolve :', result ) );
    result = resolved;
  }

  if( result === undefined || result === null )
  {
    debugger;
    result = self._descriptorRead( o.filePath );
    return handleError( _.err( 'File at :', o.filePath, 'doesn`t exist!' ) );
  }

  if( self._descriptorIsDir( result ) )
  return handleError( _.err( 'Can`t read from dir : ' + _.strQuote( o.filePath ) + ' method expects file' ) );
  else if( self._descriptorIsLink( result ) )
  return handleError( _.err( 'Can`t read from link : ' + _.strQuote( o.filePath ) + ', without link resolving enabled' ) );
  else if( !self._descriptorIsTerminal( result ) )
  return handleError( _.err( 'Can`t read file : ' + _.strQuote( o.filePath ), result ) );

  if( self.usingTime )
  self._fileTimeSetAct({ filePath : o.filePath, atime : _.timeNow() });

  return handleEnd( result );

  /* begin */

  function handleBegin()
  {

    if( encoder && encoder.onBegin )
    _.sure( encoder.onBegin.call( self, { operation : o, encoder : encoder }) === undefined );

  }

  /* end */

  function handleEnd( data )
  {

    let context = { data : data, operation : o, encoder : encoder };
    if( encoder && encoder.onEnd )
    _.sure( encoder.onEnd.call( self, context ) === undefined );
    data = context.data;

    if( o.sync )
    {
      return data;
    }
    else
    {
      return con.give( data );
    }

  }

  /* error */

  function handleError( err )
  {

    debugger;

    if( encoder && encoder.onError )
    try
    {
      err = _._err
      ({
        args : [ stack,'\nfileReadAct( ',o.filePath,' )\n',err ],
        usingSourceCode : 0,
        level : 0,
      });
      err = encoder.onError.call( self,{ error : err, operation : o, encoder : encoder })
    }
    catch( err2 )
    {
      console.error( err2 );
      console.error( err.toString() + '\n' + err.stack );
    }

    if( o.sync )
    {
      throw err;
    }
    else
    {
      return con.error( err );
    }

  }

}

_.routineExtend( fileReadAct, Parent.prototype.fileReadAct );

// var defaults = fileReadAct.defaults = Object.create( Parent.prototype.fileReadAct.defaults );
// var having = fileReadAct.having = Object.create( Parent.prototype.fileReadAct.having );

//

// var fileHashAct = ( function()
// {

//   var crypto;

//   return function fileHashAct( o )
//   {
//     var result=NaN;
//     var self = this;

//     if( _.strIs( o ) )
//     o = { filePath : o };

//     _.assertRoutineOptions( fileHashAct,o );
//     _.assert( _.strIs( o.filePath ) );
//     _.assert( arguments.length === 1, 'expects single argument' );

//     /* */

//     if( !crypto )
//     crypto = require( 'crypto' );
//     var md5sum = crypto.createHash( 'md5' );

//     /* */
//     function makeHash()
//     {
//       try
//       {
//         var read = self.fileReadAct( { filePath : o.filePath, sync : 1 } );
//         md5sum.update( read );
//         result = md5sum.digest( 'hex' );
//       }
//       catch( err )
//       {
//         if( o.throwing )
//         {
//           throw _.err( err );
//         }
//       }
//     }

//    if( o.sync )
//    {
//      makeHash( );
//      return result;
//    }
//    else
//    {
//      return _.timeOut( 0, function()
//      {
//        makeHash();
//        return result;
//      })
//    }
//   }
// })();

// fileHashAct.defaults = {};
// fileHashAct.defaults.__proto__ = Parent.prototype.fileHashAct.defaults;

//

function directoryReadAct( o )
{
  var self = this;

  _.assert( arguments.length === 1, 'expects single argument' );
  _.assertRoutineOptions( directoryReadAct,o );

  var result;

  if( o.sync )
  {
    readDir();
    return result;
  }
  else
  {
    return _.timeOut( 0, function()
    {
      readDir();
      return result;
    });
  }

  /* */

  function readDir()
  {
    o.filePath = self.pathResolveLink({ filePath : o.filePath, resolvingSoftLink : 1 });

    var file = self._descriptorRead( o.filePath );

    // if( self._descriptorIsLink( file ) )
    // file = self._descriptorResolve({ descriptor : result, resolvingSoftLink : 1 });

    if( file !== undefined )
    {
      if( _.objectIs( file ) )
      {
        result = Object.keys( file );
        _.assert( _.arrayIs( result ),'readdirSync returned not array' );
      }
      else
      {
        result = [ self.path.name({ path : o.filePath, withExtension : 1 }) ];
      }
    }
    else
    {
      if( o.throwing )
      throw _.err( 'Path : ', o.filePath, 'doesn`t exist!' );;
      result = null;
    }
  }

}

var defaults = directoryReadAct.defaults = Object.create( Parent.prototype.directoryReadAct.defaults );
var having = directoryReadAct.having = Object.create( Parent.prototype.directoryReadAct.having );

// --
// read stat
// -

function fileStatAct( o )
{
  var self = this;

  _.assert( arguments.length === 1, 'expects single argument' );
  _.assertRoutineOptions( fileStatAct,o );

  /* */

  if( o.sync )
  {
    return _fileStatAct( o.filePath );
  }
  else
  {
    return _.timeOut( 0, function()
    {
      return _fileStatAct( o.filePath );
    })
  }

  /* */

  function _fileStatAct( filePath )
  {
    var result = null;

    // if( filePath === '/out/icons' )
    // debugger;

    let o2 = { filePath : filePath, resolvingSoftLink : o.resolvingSoftLink };
    try
    {
      filePath = self.pathResolveLink( o2 );
    }
    catch( err )
    {
      if( !o.throwing && o2.err && o2.err.cycleInLinks )
      return result;

      throw err;
    }

    var file = self._descriptorRead( filePath );

    if( !_.definedIs( file ) )
    {
      // _.assert( !file );
      if( o.throwing )
      throw _.err( 'Path :', filePath, 'doesn`t exist!' );
      return result;
    }

    result = new _.FileStat();

    if( self.timeStats && self.timeStats[ filePath ] )
    {
      var timeStats = self.timeStats[ filePath ];
      for( var k in timeStats )
{'fraction_non_alphanumeric': 0.15545685279187818, 'fraction_numerical': 0.0029285435376805937, 'mean_word_length': 2.43659845689366, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4377366', 'n_tokens_mistral': 6865, 'n_tokens_neox': 6658, 'n_words': 1549}