Upload 4 files

app.py

@@ -0,0 +1,118 @@
+import gradio as gr
+import os
+from git_commd import GitCommandWrapper
+from typing import List, Optional
+
+HF_TOKEN=os.environ['HF_TOKEN'] if 'HF_TOKEN' in os.environ else ''
+
+
+def get_cache_dir():
+    from random_word import RandomWords
+    r = RandomWords()
+    return r.get_random_word()
+
+
+def check_disk():
+    import os 
+    return os.system("df -h /")  
+
+
+def pull_from_wisemodel(token: str,
+              url: str,
+              repo_name: str,
+               cache_dir,
+              branch: Optional[str] = None):
+
+    
+    print(cache_dir)
+    os.makedirs(cache_dir, exist_ok=True)
+    # os.system("cd "+cache_dir)  
+    gitCmd=GitCommandWrapper()
+    isLfs=gitCmd.is_lfs_installed()
+    if (not isLfs):
+       gitCmd.git_lfs_install()
+    gitCmd.clone(cache_dir,token,url,repo_name,branch)
+
+    return f'Pulled {branch} to temp folder {cache_dir}: {url}'
+
+
+def remove_file(cache_dir, repo_name):
+    import os
+    try:
+        os.remove(f'{cache_dir}/{repo_name}')
+    except:
+        return ''
+    return 'README.md file removed'
+    
+
+def push_to_hf(cache_dir, WiseModel_repo_name, hf_repo_id):
+    from huggingface_hub import HfApi
+    
+    if not HF_TOKEN:
+        raise gr.Error("Please enter your HF_TOKEN")
+    
+    api = HfApi(token=HF_TOKEN) # Token is not persisted on the machine.
+    output = api.upload_folder(
+        folder_path=f"{cache_dir}/{WiseModel_repo_name}",
+        repo_id=hf_repo_id,
+        repo_type="model",
+    )
+    return f'Pushed to {hf_repo_id}'
+    
+    
+def handle(wisemodel_token,wisemodel_link, hf_repo_id):
+    cache_dir = get_cache_dir()
+    wiseModel_repo_url=wisemodel_link.replace(".git","").replace("git","").replace("clone","").replace(" ","")
+    wiseModel_repo_info=wisemodel_link.replace(".git","").replace("git","").replace("clone","").replace(" ","").split("/")
+    print(wiseModel_repo_info)
+    wisemodel_repo_name=wiseModel_repo_info[len(wiseModel_repo_info)-1]
+    stages = [
+        (check_disk, (), {}),
+        # # Run all the sanity checks on README.md
+      
+        (pull_from_wisemodel, (wisemodel_token,wiseModel_repo_url,wisemodel_repo_name, cache_dir,"main"), {}),
+        (remove_file, (wisemodel_repo_name, cache_dir), {}),
+        (check_disk, (), {}),
+        (push_to_hf, (cache_dir, wisemodel_repo_name, hf_repo_id), {}),
+        (check_disk, (), {}),
+    ]
+
+    results = []
+    errors = []
+    for func, args, kwargs in stages:
+        try:
+            results.append(str(func(*args, **kwargs)))
+        except Exception as e:
+            errors.append(str(e))
+        if errors:
+            break
+
+    return '\n\n'.join(results), '\n\n'.join(errors)
+
+
+with gr.Blocks() as demo:
+    gr.Markdown('''
+This space uploads model from WiseModel to Huggingface.
+
+**Please make sure that you're the owner of the repo or have permission from the owner to do so!**
+
+# How to use this Space?
+- Duplicate this Space and providing WiseModel token (optional) and your read/write HF token (mandatory)
+- Create your target model repo on HF. This step needs to be done manually. The Space doesn't do create an empty repo for you.
+- In your own private Space, fill in information below.
+- Click submit then watch for output in container log for progress.
+- Create README.md file (since the metadata is not compatible with HF)
+    ''')
+    wisemodel_token= gr.Textbox(label="wisemodel user git token")
+    wisemodel_link = gr.Textbox(label="Copy the git download link from the model detail page of wisemodel ")
+    hf_repo_id = gr.Textbox(label="Target HF Model Repo ID (case sensitive). \nPlease make sure that this model has already been created")
+    with gr.Row():
+        button = gr.Button("Submit", variant="primary")
+        clear = gr.Button("Clear")
+    error = gr.Textbox(label="Error")
+    output = gr.Textbox(label="Output")
+
+    button.click(handle, [wisemodel_token,wisemodel_link, hf_repo_id], [output, error])
+
+if __name__ == "__main__":
+    demo.launch(debug = True)

git_commd.py

@@ -0,0 +1,238 @@
+import os
+import subprocess
+from typing import List, Optional
+
+from logger import get_logger
+logger = get_logger()
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton,
+                                        cls).__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+class GitError(Exception):
+    pass
+class GitCommandWrapper(metaclass=Singleton):
+    """Some git operation wrapper
+    """
+    default_git_path = 'git'  # The default git command line
+    git_path='git'
+    MASTER_MODEL_BRANCH="main"
+    def __init__(self, path: str = None):
+        self.git_path = path or self.default_git_path
+
+    def _run_git_command(self, *args) -> subprocess.CompletedProcess:
+        """Run git command, if command return 0, return subprocess.response
+             otherwise raise GitError, message is stdout and stderr.
+
+        Args:
+            args: List of command args.
+
+        Raises:
+            GitError: Exception with stdout and stderr.
+
+        Returns:
+            subprocess.CompletedProcess: the command response
+        """
+        logger.debug(' '.join(args))
+        git_env = os.environ.copy()
+        git_env['GIT_TERMINAL_PROMPT'] = '0'
+        response = subprocess.run(
+            [self.git_path, *args],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=git_env,
+        )  # compatible for python3.6
+        try:
+            response.check_returncode()
+            return response
+        except subprocess.CalledProcessError as error:
+            logger.error('There are error run git command.')
+            raise GitError(
+                'stdout: %s, stderr: %s' %
+                (response.stdout.decode('utf8'), error.stderr.decode('utf8')))
+
+    def config_auth_token(self, repo_dir, auth_token):
+        url = self.get_repo_remote_url(repo_dir)
+        if '//oauth2' not in url:
+            auth_url = self._add_token(auth_token, url)
+            cmd_args = '-C %s remote set-url origin %s' % (repo_dir, auth_url)
+            cmd_args = cmd_args.split(' ')
+            rsp = self._run_git_command(*cmd_args)
+            logger.debug(rsp.stdout.decode('utf8'))
+
+    def _add_token(self, token: str, url: str):
+        if token:
+            if '//oauth2' not in url:
+                url = url.replace('//', '//oauth2:%s@' % token)
+        return url
+
+    def remove_token_from_url(self, url: str):
+        if url and '//oauth2' in url:
+            start_index = url.find('oauth2')
+            end_index = url.find('@')
+            url = url[:start_index] + url[end_index + 1:]
+        return url
+
+    def is_lfs_installed(self):
+        cmd = ['lfs', 'env']
+        try:
+            self._run_git_command(*cmd)
+            return True
+        except GitError:
+            return False
+
+    def git_lfs_install(self, repo_dir):
+        cmd = ['-C', repo_dir, 'lfs', 'install']
+        try:
+            self._run_git_command(*cmd)
+            return True
+        except GitError:
+            return False
+
+    def clone(self,
+              repo_base_dir: str,
+              token: str,
+              url: str,
+              repo_name: str,
+              branch: Optional[str] = None):
+        """ git clone command wrapper.
+        For public project, token can None, private repo, there must token.
+
+        Args:
+            repo_base_dir (str): The local base dir, the repository will be clone to local_dir/repo_name
+            token (str): The git token, must be provided for private project.
+            url (str): The remote url
+            repo_name (str): The local repository path name.
+            branch (str, optional): _description_. Defaults to None.
+
+        Returns:
+            The popen response.
+        """
+        url = self._add_token(token, url)
+        if branch:
+            clone_args = '-C %s clone %s %s --branch %s' % (repo_base_dir, url,
+                                                            repo_name, branch)
+        else:
+            clone_args = '-C %s clone %s' % (repo_base_dir, url)
+        logger.debug(clone_args)
+        clone_args = clone_args.split(' ')
+        response = self._run_git_command(*clone_args)
+        logger.debug(response.stdout.decode('utf8'))
+        return response
+
+
+
+    def add(self,
+            repo_dir: str,
+            files: List[str] = list(),
+            all_files: bool = False):
+        if all_files:
+            add_args = '-C %s add -A' % repo_dir
+        elif len(files) > 0:
+            files_str = ' '.join(files)
+            add_args = '-C %s add %s' % (repo_dir, files_str)
+        add_args = add_args.split(' ')
+        rsp = self._run_git_command(*add_args)
+        logger.debug(rsp.stdout.decode('utf8'))
+        return rsp
+
+    def commit(self, repo_dir: str, message: str):
+        """Run git commit command
+
+        Args:
+            repo_dir (str): the repository directory.
+            message (str): commit message.
+
+        Returns:
+            The command popen response.
+        """
+        commit_args = ['-C', '%s' % repo_dir, 'commit', '-m', "'%s'" % message]
+        rsp = self._run_git_command(*commit_args)
+        logger.info(rsp.stdout.decode('utf8'))
+        return rsp
+
+    def checkout(self, repo_dir: str, revision: str):
+        cmds = ['-C', '%s' % repo_dir, 'checkout', '%s' % revision]
+        return self._run_git_command(*cmds)
+
+    def new_branch(self, repo_dir: str, revision: str):
+        cmds = ['-C', '%s' % repo_dir, 'checkout', '-b', revision]
+        return self._run_git_command(*cmds)
+
+    def get_remote_branches(self, repo_dir: str):
+        cmds = ['-C', '%s' % repo_dir, 'branch', '-r']
+        rsp = self._run_git_command(*cmds)
+        info = [
+            line.strip()
+            for line in rsp.stdout.decode('utf8').strip().split(os.linesep)
+        ]
+        if len(info) == 1:
+            return ['/'.join(info[0].split('/')[1:])]
+        else:
+            return ['/'.join(line.split('/')[1:]) for line in info[1:]]
+
+    def pull(self,
+             repo_dir: str,
+             remote: str = 'origin',
+             branch: str = 'master'):
+        cmds = ['-C', repo_dir, 'pull', remote, branch]
+        return self._run_git_command(*cmds)
+
+    def push(self,
+             repo_dir: str,
+             token: str,
+             url: str,
+             local_branch: str,
+             remote_branch: str,
+             force: bool = False):
+        url = self._add_token(token, url)
+
+        push_args = '-C %s push %s %s:%s' % (repo_dir, url, local_branch,
+                                             remote_branch)
+        if force:
+            push_args += ' -f'
+        push_args = push_args.split(' ')
+        rsp = self._run_git_command(*push_args)
+        logger.debug(rsp.stdout.decode('utf8'))
+        return rsp
+
+    def get_repo_remote_url(self, repo_dir: str):
+        cmd_args = '-C %s config --get remote.origin.url' % repo_dir
+        cmd_args = cmd_args.split(' ')
+        rsp = self._run_git_command(*cmd_args)
+        url = rsp.stdout.decode('utf8')
+        return url.strip()
+
+    def list_lfs_files(self, repo_dir: str):
+        cmd_args = '-C %s lfs ls-files' % repo_dir
+        cmd_args = cmd_args.split(' ')
+        rsp = self._run_git_command(*cmd_args)
+        out = rsp.stdout.decode('utf8').strip()
+        files = []
+        for line in out.split(os.linesep):
+            files.append(line.split(' ')[-1])
+
+        return files
+
+    def tag(self,
+            repo_dir: str,
+            tag_name: str,
+            message: str,
+            ref: str = MASTER_MODEL_BRANCH):
+        cmd_args = [
+            '-C', repo_dir, 'tag', tag_name, '-m',
+            '"%s"' % message, ref
+        ]
+        rsp = self._run_git_command(*cmd_args)
+        logger.debug(rsp.stdout.decode('utf8'))
+        return rsp
+
+    def push_tag(self, repo_dir: str, tag_name):
+        cmd_args = ['-C', repo_dir, 'push', 'origin', tag_name]
+        rsp = self._run_git_command(*cmd_args)
+        logger.debug(rsp.stdout.decode('utf8'))
+        return rsp

logger.py

@@ -0,0 +1,89 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import importlib
+import logging
+from typing import Optional
+
+init_loggers = {}
+
+formatter = logging.Formatter(
+    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+
+def get_logger(log_file: Optional[str] = None,
+               log_level: int = logging.INFO,
+               file_mode: str = 'w'):
+    """ Get logging logger
+
+    Args:
+        log_file: Log filename, if specified, file handler will be added to
+            logger
+        log_level: Logging level.
+        file_mode: Specifies the mode to open the file, if filename is
+            specified (if filemode is unspecified, it defaults to 'w').
+    """
+
+    logger_name = __name__.split('.')[0]
+    logger = logging.getLogger(logger_name)
+    logger.propagate = False
+    if logger_name in init_loggers:
+        add_file_handler_if_needed(logger, log_file, file_mode, log_level)
+        return logger
+
+    # handle duplicate logs to the console
+    # Starting in 1.8.0, PyTorch DDP attaches a StreamHandler <stderr> (NOTSET)
+    # to the root logger. As logger.propagate is True by default, this root
+    # level handler causes logging messages from rank>0 processes to
+    # unexpectedly show up on the console, creating much unwanted clutter.
+    # To fix this issue, we set the root logger's StreamHandler, if any, to log
+    # at the ERROR level.
+    torch_dist = False
+    is_worker0 = True
+    if importlib.util.find_spec('torch') is not None:
+        from modelscope.utils.torch_utils import is_dist, is_master
+        torch_dist = is_dist()
+        is_worker0 = is_master()
+
+    if torch_dist:
+        for handler in logger.root.handlers:
+            if type(handler) is logging.StreamHandler:
+                handler.setLevel(logging.ERROR)
+
+    stream_handler = logging.StreamHandler()
+    handlers = [stream_handler]
+
+    if is_worker0 and log_file is not None:
+        file_handler = logging.FileHandler(log_file, file_mode)
+        handlers.append(file_handler)
+
+    for handler in handlers:
+        handler.setFormatter(formatter)
+        handler.setLevel(log_level)
+        logger.addHandler(handler)
+
+    if is_worker0:
+        logger.setLevel(log_level)
+    else:
+        logger.setLevel(logging.ERROR)
+
+    init_loggers[logger_name] = True
+
+    return logger
+
+
+def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
+    for handler in logger.handlers:
+        if isinstance(handler, logging.FileHandler):
+            return
+
+    if importlib.util.find_spec('torch') is not None:
+        from modelscope.utils.torch_utils import is_master
+        is_worker0 = is_master()
+    else:
+        is_worker0 = True
+
+    if is_worker0 and log_file is not None:
+        file_handler = logging.FileHandler(log_file, file_mode)
+        file_handler.setFormatter(formatter)
+        file_handler.setLevel(log_level)
+        logger.addHandler(file_handler)

requirements.txt

@@ -0,0 +1,3 @@
+gradio[auth]
+huggingface_hub
+random-word