<commit_before>require 'virtualfs/backend'

require 'base64'
require 'stringio'
require 'octokit'

module VirtualFS
  class Github < Backend
    attr_reader :user, :repo, :branch

    def initialize(opts={})
      super opts

      @user = opts.fetch(:user)
      @repo = opts.fetch(:repo)
      @branch = opts.fetch(:branch, 'master')
      auth = opts[:authentication]

      # @gh = ::Github::GitData.new(opts)
      @gh = Octokit::Client.new(auth)
    end

    def entries(path='/')
      path = '/' << path unless path.start_with? '/'
      p = path[1..-1]

      contents = path == '/' ? toplevel_contents : tree_contents(p)
      dotfolders_for(path) + map_entries(contents.values, :path) { |item| item.type == 'tree' }
    end

    def glob(pattern, path='/')
      path = '/' << path unless path.start_with? '/'
      p = path[1..-1]

      contents = path == '/' ? internal_items : tree_contents(p, true)
      map_entries(contents.select { |path, _| ::File.fnmatch(pattern, path, ::File::FNM_PATHNAME) }.values, :path) { |item| item.type == 'tree' }<commit_msg>Fixed stupid empty authentication issue<commit_after>require 'virtualfs/backend'

require 'base64'
require 'stringio'
require 'octokit'

module VirtualFS
  class Github < Backend
    attr_reader :user, :repo, :branch

    def initialize(opts={})
      super opts

      @user = opts.fetch(:user)
      @repo = opts.fetch(:repo)
      @branch = opts.fetch(:branch, 'master')
      auth = opts.fetch(:authentication, {})

      # @gh = ::Github::GitData.new(opts)
      @gh = Octokit::Client.new(auth)
    end

    def entries(path='/')
      path = '/' << path unless path.start_with? '/'
      p = path[1..-1]

      contents = path == '/' ? toplevel_contents : tree_contents(p)
      dotfolders_for(path) + map_entries(contents.values, :path) { |item| item.type == 'tree' }
    end

    def glob(pattern, path='/')
      path = '/' << path unless path.start_with? '/'
      p = path[1..-1]

      contents = path == '/' ? internal_items : tree_contents(p, true)
      map_entries(contents.select { |path, _| ::File.fnmatch(pattern, path, ::File::FNM_PATHNAME) }.values, :path) { |item| item.type == 'tree' }
{'fraction_non_alphanumeric': 0.16788990825688074, 'fraction_numerical': 0.005504587155963303, 'mean_word_length': 3.0314232902033273, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4787899', 'n_tokens_mistral': 748, 'n_tokens_neox': 730, 'n_words': 160}