<commit_before>
import os
from glob import glob
from ftplib import FTP

import boto3
import botocore
import requests


class Connector:
    def download_file(self, file):
        """download a file to local drive"""
        raise NotImplemented

    def list_files(self, path=None, name_match="_vp_"):
        """list the files within a given subfolder or relative path"""
        raise NotImplemented


class LocalConnector(Connector):

    def __init__(self, filepath):
        """Connector to handle local directory files, focusing on a specific
        folder subset asa defined by the main filepath

        :param filepath: main directory to work in
        """
        self.filepath = filepath

    def download_file(self, file):
        """"""
        return NotImplemented

    def list_files(self, path=None, name_match="_vp_", fullpaths=False):
        """

        :param path: relative defined to filepath
        :param name_match:
        :param fullpaths: bool define if the full path or only relatve path
        should be returned
        :return: iterator with the matching file names of the path folder and
        subfolders
        """
        if path:
            path_to_list = os.path.join(self.filepath, path, "**",
                                        "*{}*".format(name_match))
        else:
            path_to_list = os.path.join(self.filepath, "**",
                                        "*{}*".format(name_match))
        for subpath in glob(path_to_list, recursive=True):
            if name_match in subpath:
                if fullpaths:
                    yield path
                else:
                    yield os.path.split(path)[-1]


class GithubConnector(Connector):

    def __init__(self, repo_username=None, repo_name=None):
        """
        Initialize a GithubConnector

        :param repo_username: username of the repository owner
        :param repo_name: name of the repository
        """
        self.repo_username = repo_username
        self.repo_name = repo_name

    @staticmethod
    def _parse_files_from_response(response):
        """
        Parses the download_urls from the response and yields them one by one
        :param response: a JSON response from the Github API that lists files
        in a given directory
        """
        response_data = response.json()<commit_msg>fix subpath variable bug<commit_after>
import os
from glob import glob
from ftplib import FTP

import boto3
import botocore
import requests


class Connector:
    def download_file(self, file):
        """download a file to local drive"""
        raise NotImplemented

    def list_files(self, path=None, name_match="_vp_"):
        """list the files within a given subfolder or relative path"""
        raise NotImplemented


class LocalConnector(Connector):

    def __init__(self, filepath):
        """Connector to handle local directory files, focusing on a specific
        folder subset as defined by the main filepath
        """
        self.filepath = filepath

    def download_file(self, file):
        """"""
        return NotImplemented

    def list_files(self, path=None, name_match="_vp_", fullpaths=False):
        """

        :param path: relative defined to filepath
        :param name_match:
        :param fullpaths: bool define if the full path or only relatve path
        should be returned
        :return: iterator with the matching file names of the path folder and
        subfolders
        """
        if path:
            path_to_list = os.path.join(self.filepath, path, "**",
                                        "*{}*".format(name_match))
        else:
            path_to_list = os.path.join(self.filepath, "**",
                                        "*{}*".format(name_match))
        for subpath in glob(path_to_list, recursive=True):
            if name_match in subpath:
                if fullpaths:
                    yield subpath
                else:
                    yield os.path.split(subpath)[-1]


class GithubConnector(Connector):

    def __init__(self, repo_username=None, repo_name=None):
        """
        Initialize a GithubConnector

        :param repo_username: username of the repository owner
        :param repo_name: name of the repository
        """
        self.repo_username = repo_username
        self.repo_name = repo_name

    @staticmethod
    def _parse_files_from_response(response):
        """
        Parses the download_urls from the response and yields them one by one
        :param response: a JSON response from the Github API that lists files
        in a given directory
        """
        response_data = response.json()
{'fraction_non_alphanumeric': 0.07744972186563971, 'fraction_numerical': 0.0008557980316645272, 'mean_word_length': 2.1208277703604805, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7431545', 'n_tokens_mistral': 1215, 'n_tokens_neox': 1199, 'n_words': 413}