Spaces:
Running
Running
File size: 6,677 Bytes
aaef8e0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 |
import os
import logging
from typing import Optional
import pandas as pd
import requests
class LeaderboardServer:
def __init__(self):
self._LOG = logging.getLogger('leaderboard_server')
self._server_address = os.environ['LEADERBOARD_SERVER_ADDRESS']
def get_leaderboard(self, submission_type: str) -> pd.DataFrame:
"""
Gets the leaderboard of the given submission type
Args:
submission_type: the type of the submission to get the leaderboard of:
'SC' / 'MC-specific' / 'MC-agnostic' / 'MC-agnostic-all'
"""
self._LOG.info(f'Getting leaderboard for submission type: {submission_type}')
endpoint = f'{self._server_address}/leaderboard'
submission_type = submission_type.lower().replace('-', '_')
response = requests.get(endpoint, params={'submission_type': submission_type})
if response.status_code != 200:
self._LOG.error(f'Error while fetching leaderboard, status code: {response.status_code}, '
f'response: {response.text}, endpoint: {endpoint}')
return pd.DataFrame()
return pd.DataFrame(response.json())
def get_submissions_by_hf_token(self, hf_token: str) -> pd.DataFrame:
"""
Gets the submissions of the given hf token
Args:
hf_token: the hf token to get the submissions of
"""
self._LOG.info(f'Getting submissions for hf token: {hf_token}')
endpoint = f'{self._server_address}/submissions'
response = requests.get(endpoint, params={'token': hf_token})
if response.status_code != 200:
self._LOG.error(f'Error while fetching submissions, status code: {response.status_code}, '
f'response: {response.text}, endpoint: {endpoint}')
return pd.DataFrame()
return pd.DataFrame(response.json())
def is_hf_token_valid(self, hf_token: str) -> Optional[bool]:
"""
Validates the given hf token
Args:
hf_token: the hf token to validate
"""
self._LOG.info(f'Validating hf token: {hf_token}')
endpoint = f'{self._server_address}/validate_hf_token'
response = requests.get(endpoint, params={'token': hf_token})
if response.status_code != 200:
self._LOG.error(f'Error while validating hf token, status code: {response.status_code}, '
f'response: {response.text}, endpoint: {endpoint}')
return None
return response.json()['valid']
def get_submission_count_last_24_hours(self, hf_token: str) -> Optional[int]:
"""
Gets the number of submissions of the given hf token in the last 24 hours
Args:
hf_token: the hf token to get the submissions count of
"""
self._LOG.info(f'Getting submissions count for hf token: {hf_token} in the last 24 hours')
endpoint = f'{self._server_address}/submission_count_last_24_hours'
response = requests.get(endpoint, params={'token': hf_token})
if response.status_code != 200:
self._LOG.error(f'Error while fetching submissions count, status code: {response.status_code}, '
f'response: {response.text}, endpoint: {endpoint}')
return None
return int(response.json()['count'])
def add_submission(self, token: str, file_path: str, metadata: dict) -> dict:
"""
Adds a submission to the leaderboard based on the given file and metadata
Args:
token: the token of the team
file_path: the path of the file to submit
metadata: the metadata of the submission, structure:
{
'challenge_name': 'NOTSOFAR1',
'team_name': (str),
'submission_type': (str),
'token': (str),
'file_name': (str),
'file_size_mb': (int),
'ip': (str) xxx.xxx.xxx.xxx
}
"""
self._LOG.info(f'Adding submission for team: {metadata["team_name"]}, '
f'submission type: {metadata["submission_type"]}')
endpoint = f'{self._server_address}/add_submission'
metadata['token'] = token
metadata['submission_type'] = metadata['submission_type'].lower().replace('-', '_')
with open(file_path, 'rb') as payload_file:
files = {'zip_file': payload_file}
response = requests.post(endpoint, files=files, params=metadata, timeout=600)
if response.status_code != 200:
self._LOG.error(f'Error while adding submission, status code: {int(response.status_code)}, '
f'response: {response.text}, endpoint: {endpoint}')
return dict(error=response.json()['message'])
return response.json()
def test_server():
"""
Basic server tests for the leaderboard server
"""
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
server = LeaderboardServer()
hf_token = str(os.environ['HF_TOKEN'])
print('leaderboard:\n', server.get_leaderboard('notsofar_mc'))
print('submissions by hf token:\n', server.get_submissions_by_hf_token(hf_token))
print('is hf token valid:\n', server.is_hf_token_valid(hf_token))
print('is hf token valid:\n', server.is_hf_token_valid(hf_token + '1'))
print('add_submission:\n', server.add_submission(
token=hf_token,
file_path=fr"C:\Users\shaipeer\Downloads\submissions\notsofar_submission.zip",
metadata={
'challenge_name': 'NOTSOFAR1',
'team_name': 'NOTSOFAR Test Team',
'submission_type': 'notsofar_mc',
'description': 'Test NOTSOFAR submission',
'token': hf_token,
'file_name': 'notsofar_submission.zip',
'file_size_mb': 10,
'ip': '127.0.0.1'
}))
print('add_submission:\n', server.add_submission(
token=hf_token,
file_path=fr"C:\Users\shaipeer\Downloads\submissions\chime_submission.zip",
metadata={
'challenge_name': 'NOTSOFAR1',
'team_name': 'Chime Test Team',
'submission_type': 'dasr_unconstrained_lm',
'description': 'Test chime submission',
'token': hf_token,
'file_name': 'chime_submission.zip',
'file_size_mb': 10,
'ip': '127.0.0.1'
}))
if __name__ == '__main__':
test_server()
|