<commit_before>nfsMounter:
  enabled: true

nfsPVC:
  enabled: false
jupyterhub:
  scheduling:
    podPriority:
      enabled: true
      globalDefault: false
      defaultPriority: 0
      userPlaceholderPriority: -10
    userScheduler:
      enabled: true
      resources:
        requests:
          memory: 64Mi
        limits:
          memory: 512Mi
  proxy:
    chp:
      resources:
        requests:
          memory: 320Mi
        limits:
          memory: 320Mi
    nginx:
      resources:
        requests:
          memory: 512Mi
        limits:
          memory: 512Mi
    https:
      letsencrypt:
        contactEmail: <EMAIL>
    networkPolicy:
      enabled: true
  singleuser:
    defaultUrl: /tree
    networkPolicy:
      # In clusters with NetworkPolicy enabled, do not
      # allow outbound internet access that's not DNS, HTTP or HTTPS
      # We can override this on a case to case basesi where
      # required.
      enabled: true
      egress:
        - ports:
            - port: 53
              protocol: UDP
        - ports:
            - port: 80
              protocol: TCP
        - ports:
            - port: 443
              protocol: TCP
  auth:
    google:
      hostedDomain: ['berkeley.edu']
    # common settings when using Canvas Auth
    # Enabled by setting 'custom' as auth.type in individual hub config
    custom:
      className: 'canvasauthenticator.CanvasAuthenticator'
      config:
        canvas_url: https://bcourses.berkeley.edu/
        strip_email_domain: berkeley.edu
        login_service: bCourses
        scope: ['url:GET|/api/v1/users/:user_id/profile']
        username_key: primary_email
  hub:
    # Generated by chartpress
    image:
      name: gcr.io/ucb-datahub-2018/jupyterhub-hub
      tag: '0.1.0-5eb0c77'
    networkPolicy:
      enabled: true
    resources:
      requests:
        memory: 256Mi
      limits:
        memory: 1Gi
    extraEnv:
      # This is unfortunately still needed by canvas auth
      OAUTH2_AUTHORIZE_URL: https://bcourses.berkeley.edu/login/oauth2/auth
    extraConfig:
      01-custom-attr-spawner: |
        import os
        from kubespawner import KubeSpawner
        from tornado import gen
        import z2jh

        hosted_domain = 'berkeley.edu'
        course_profile_tmpl = '/srv/jupyterhub/profiles.d/{}-{}.txt'

        def memparse(val):
          '''Parse memory for relative comparisons.'''
          if type(val) != str or len(val) == 0: return val
          mem = int(val.upper()[0:-1])
          unit = val[-1]
          n = {'B':0, 'K':1, 'M':2, 'G':3}[unit]
          return mem * 1024**n

        def mem_cmp(a, b):
          '''Compare memory values.'''
          x = memparse(a) ; y = memparse(b)
          return (x > y) - (x < y)

        def course_profile_filename(course, constituent):
          return course_profile_tmpl.format(course, constituent)

        def course_members(course, constituent, hd=None):
          '''Extracts usernames from files containing email addresses.'''
          members = []
          filename = course_profile_filename(course, constituent)
          if not os.path.exists(filename): return members
          with open(filename) as f:
            line = f.readline()
            while line != '':
              email = line.strip()
              if hd and email.endswith('@' + hd):
                members.append(email.split('@')[0])
              elif not hd:
                members.append(email)
              line = f.readline()
          return members

        class CustomAttrSpawner(KubeSpawner):

          @gen.coroutine
          def start(self):
            # custom.memory
            custom_memory = z2jh.get_config('custom.memory', {})
            for attr, users in custom_memory.items():
              if self.user.name in users:
                self.mem_limit = attr
                self.mem_guarantee = attr
                break

            # custom.profiles
            custom_profiles = z2jh.get_config('custom.profiles', {})
            is_student = False ; is_instructor = False
            for course, profile_data in custom_profiles.items():
              students    = course_members(course, 'students',    hosted_domain)
              instructors = course_members(course, 'instructors', hosted_domain)
              is_student    |= (self.user.name in students)
              is_instructor |= (self.user.name in instructors)
              if is_student or is_instructor:
                self.log.warning(f'using profile {course}')
                self.volumes += profile_data.get('extraVolumes', [])
                self.volume_mounts += profile_data.get('extraVolumeMounts', [])
                # set new mem thresholds if specified are bigger than current
                if 'mem_limit' in profile_data and \
                mem_cmp(profile_data['mem_limit'], self.mem_limit) == 1:
                  self.mem_limit = profile_data['mem_limit']
                if 'mem_guarantee' in profile_data and \
                mem_cmp(profile_data['mem_guarantee'], self.mem_guarantee) == 1:
                  self.mem_guarantee = profile_data['mem_guarantee']

            # if the user is a student in any course specified by a profile,
            # they never get to be an admin
            if is_student:
              self.user.admin = False
            elif is_instructor:
              self.user.admin = True

            # custom.admin
            custom_admin = z2jh.get_config('custom.admin', {})
            if custom_admin and self.user.admin:
              self.init_containers += custom_admin.get('initContainers', [])
              self.volume_mounts += custom_admin.get('extraVolumeMounts', [])

            return (yield super().start())

        c.JupyterHub.spawner_class = CustomAttrSpawner

        c.JupyterHub.extra_log_file = '/srv/jupyterhub/jupyterhub.log'

      02-lab-availability: |
        c.Spawner.cmd = ['jupyter-labhub']

      03-working-dir: |
        # Make sure working directory is ${HOME}
        # hubploy has a bug where it unconditionally puts workingdir to be /srv/repo
        c.KubeSpawner.working_dir = '/home/jovyan'
      05-prometheus: |
        # Allow unauthenticated prometheus requests
        # Otherwise our prometheus server can't get to these
        c.JupyterHub.authenticate_prometheus = False
<commit_msg>Request canvas scope for getting course enrollment info<commit_after>nfsMounter:
  enabled: true

nfsPVC:
  enabled: false
jupyterhub:
  scheduling:
    podPriority:
      enabled: true
      globalDefault: false
      defaultPriority: 0
      userPlaceholderPriority: -10
    userScheduler:
      enabled: true
      resources:
        requests:
          memory: 64Mi
        limits:
          memory: 512Mi
  proxy:
    chp:
      resources:
        requests:
          memory: 320Mi
        limits:
          memory: 320Mi
    nginx:
      resources:
        requests:
          memory: 512Mi
        limits:
          memory: 512Mi
    https:
      letsencrypt:
        contactEmail: <EMAIL>
    networkPolicy:
      enabled: true
  singleuser:
    defaultUrl: /tree
    networkPolicy:
      # In clusters with NetworkPolicy enabled, do not
      # allow outbound internet access that's not DNS, HTTP or HTTPS
      # We can override this on a case to case basesi where
      # required.
      enabled: true
      egress:
        - ports:
            - port: 53
              protocol: UDP
        - ports:
            - port: 80
              protocol: TCP
        - ports:
            - port: 443
              protocol: TCP
  auth:
    google:
      hostedDomain: ['berkeley.edu']
    # common settings when using Canvas Auth
    # Enabled by setting 'custom' as auth.type in individual hub config
    custom:
      className: 'canvasauthenticator.CanvasAuthenticator'
      config:
        canvas_url: https://bcourses.berkeley.edu/
        strip_email_domain: berkeley.edu
        login_service: bCourses
        scope:
          - "url:GET|/api/v1/users/:user_id/profile"
          - "url:GET|/api/v1/courses"
        username_key: primary_email
  hub:
    # Generated by chartpress
    image:
      name: gcr.io/ucb-datahub-2018/jupyterhub-hub
      tag: '0.1.0-5eb0c77'
    networkPolicy:
      enabled: true
    resources:
      requests:
        memory: 256Mi
      limits:
        memory: 1Gi
    extraEnv:
      # This is unfortunately still needed by canvas auth
      OAUTH2_AUTHORIZE_URL: https://bcourses.berkeley.edu/login/oauth2/auth
    extraConfig:
      01-custom-attr-spawner: |
        import os
        from kubespawner import KubeSpawner
        from tornado import gen
        import z2jh

        hosted_domain = 'berkeley.edu'
        course_profile_tmpl = '/srv/jupyterhub/profiles.d/{}-{}.txt'

        def memparse(val):
          '''Parse memory for relative comparisons.'''
          if type(val) != str or len(val) == 0: return val
          mem = int(val.upper()[0:-1])
          unit = val[-1]
          n = {'B':0, 'K':1, 'M':2, 'G':3}[unit]
          return mem * 1024**n

        def mem_cmp(a, b):
          '''Compare memory values.'''
          x = memparse(a) ; y = memparse(b)
          return (x > y) - (x < y)

        def course_profile_filename(course, constituent):
          return course_profile_tmpl.format(course, constituent)

        def course_members(course, constituent, hd=None):
          '''Extracts usernames from files containing email addresses.'''
          members = []
          filename = course_profile_filename(course, constituent)
          if not os.path.exists(filename): return members
          with open(filename) as f:
            line = f.readline()
            while line != '':
              email = line.strip()
              if hd and email.endswith('@' + hd):
                members.append(email.split('@')[0])
              elif not hd:
                members.append(email)
              line = f.readline()
          return members

        class CustomAttrSpawner(KubeSpawner):

          @gen.coroutine
          def start(self):
            # custom.memory
            custom_memory = z2jh.get_config('custom.memory', {})
            for attr, users in custom_memory.items():
              if self.user.name in users:
                self.mem_limit = attr
                self.mem_guarantee = attr
                break

            # custom.profiles
            custom_profiles = z2jh.get_config('custom.profiles', {})
            is_student = False ; is_instructor = False
            for course, profile_data in custom_profiles.items():
              students    = course_members(course, 'students',    hosted_domain)
              instructors = course_members(course, 'instructors', hosted_domain)
              is_student    |= (self.user.name in students)
              is_instructor |= (self.user.name in instructors)
              if is_student or is_instructor:
                self.log.warning(f'using profile {course}')
                self.volumes += profile_data.get('extraVolumes', [])
                self.volume_mounts += profile_data.get('extraVolumeMounts', [])
                # set new mem thresholds if specified are bigger than current
                if 'mem_limit' in profile_data and \
                mem_cmp(profile_data['mem_limit'], self.mem_limit) == 1:
                  self.mem_limit = profile_data['mem_limit']
                if 'mem_guarantee' in profile_data and \
                mem_cmp(profile_data['mem_guarantee'], self.mem_guarantee) == 1:
                  self.mem_guarantee = profile_data['mem_guarantee']

            # if the user is a student in any course specified by a profile,
            # they never get to be an admin
            if is_student:
              self.user.admin = False
            elif is_instructor:
              self.user.admin = True

            # custom.admin
            custom_admin = z2jh.get_config('custom.admin', {})
            if custom_admin and self.user.admin:
              self.init_containers += custom_admin.get('initContainers', [])
              self.volume_mounts += custom_admin.get('extraVolumeMounts', [])

            return (yield super().start())

        c.JupyterHub.spawner_class = CustomAttrSpawner

        c.JupyterHub.extra_log_file = '/srv/jupyterhub/jupyterhub.log'

      02-lab-availability: |
        c.Spawner.cmd = ['jupyter-labhub']

      03-working-dir: |
        # Make sure working directory is ${HOME}
        # hubploy has a bug where it unconditionally puts workingdir to be /srv/repo
        c.KubeSpawner.working_dir = '/home/jovyan'
      05-prometheus: |
        # Allow unauthenticated prometheus requests
        # Otherwise our prometheus server can't get to these
        c.JupyterHub.authenticate_prometheus = False

{'fraction_non_alphanumeric': 0.09199624824136314, 'fraction_numerical': 0.011333437548851024, 'mean_word_length': 1.8560267857142858, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6367825', 'n_tokens_mistral': 3835, 'n_tokens_neox': 3641, 'n_words': 912}