<commit_before>#!/usr/bin/python
# Copyright (c) 2011, Xabier (slok) Larrakoetxea
# Copyright (c) 2011, Iraide (Sharem) Diaz
#
# 3 clause/New BSD license:
#   opensource: http://www.opensource.org/licenses/BSD-3-Clause
#   wikipedia: http://en.wikipedia.org/wiki/BSD_licenses
#
#-----------------------------------------------------------------------
# This script  allows to upload to Fileserve with FTP various files at the same time
#
# Use:
#   python ./ftpFilserveUploader.py ./Downloads/xxx.y ./yyyy.z  /home/xxx/yyyy.zz
#
from ftplib import FTP
import sys
import os
########### MODIFY ########################

USER = 'xxxxxx'
PASS = '<PASSWORD>'
DESTINATION_DIR="documents"
########### MODIFY IF YOU WANT ############
UPLOADED_DIR='uploaded'
SERVER = 'ftp.fileserve.com'
PORT = 21
BINARY_STORE = True # if False then line store (not valid for binary files (videos, music, photos...))

###########################################
from ftpparameters import *



def print_line(result):
    print(result)

def connect_ftp():
    #Connect to the server
    ftp = FTP()
    ftp.connect(SERVER, PORT)
    ftp.login(USER, PASS)

    return ftp

def upload_file(ftp_connetion, upload_file_path):

    #Open the file
    try:
        upload_file = open(upload_file_path, 'r')

        #get the name
        path_split = upload_file_path.split('/')
        final_file_name = path_split[len(path_split)-1]

        #transfer the file
        print('Uploading ' + final_file_name + '...')

        if BINARY_STORE:
            ftp_connetion.storbinary('STOR '+ final_file_name, upload_file)
        else:
            #ftp_connetion.storlines('STOR ' + final_file_name, upload_file, print_line)
            ftp_connetion.storlines('STOR '+ final_file_name, upload_file)

        print('Upload finished.')
    except IOError:
        print ("No such file or directory... passing to next file")


def getFileList(mydir):
    filelist = [ os.path.join(mydir,f) for f in os.listdir(mydir) if os.path.isfile(os.path.join(mydir,f))]
    return filelist

def getFilesToUpload():
    if len(sys.argv) < 2:
        print "Enter source local directory as parameter"
    else:
        filestoupload=[]
        sourcedirs=sys.argv[1:]
        for arg in sourcedirs:
            print arg
            filestoupload.extend(getFileList(arg))
    return filestoupload

def cdTree(currentDir):
    if currentDir != "":
        try:
            ftp.cwd(currentDir)
        except IOError:
            cdTree("/".join(currentDir.split("/")[:-1]))
            ftp.mkd(currentDir)
            ftp.cwd(currentDir)


def uploadfiles():
    filelist=getFilesToUpload()
    try:
        ftp_conn = connect_ftp()
        cdTree(DESTINATION_DIR)
    except:
        print 'Connection error - unable to open connection to ftp server or nonexisting dir'
        exit(1)
    else:
        for item in filelist:
            print "Uploading file: " + str(item)
            try:
                upload_file(ftp_conn, item)
            except:
                print "Error uploading file: "+str(item)
            else:
                print "Uploaded file: "+str(item)


#Take all the files and upload all
#ftp_conn = connect_ftp()
#for arg in sys.argv:
#    upload_file(ftp_conn, arg)
print UPLOADED_DIR
print SERVER
uploadfiles()<commit_msg>Created some functions<commit_after>#!/usr/bin/python
# Copyright (c) 2011, Xabier (slok) Larrakoetxea
# Copyright (c) 2011, Iraide (<NAME>
#
# 3 clause/New BSD license:
#   opensource: http://www.opensource.org/licenses/BSD-3-Clause
#   wikipedia: http://en.wikipedia.org/wiki/BSD_licenses
#
#-----------------------------------------------------------------------
# This script  allows to upload to Fileserve with FTP various files at the same time
#
# Use:
#   python ./ftpFilserveUploader.py ./Downloads/xxx.y ./yyyy.z  /home/xxx/yyyy.zz
#
from ftplib import FTP
import sys
import os
########### MODIFY ########################

USER = 'xxxxxx'
PASS = '<PASSWORD>'
DESTINATION_DIR="documents"
########### MODIFY IF YOU WANT ############
UPLOADED_DIR='uploaded'
SERVER = 'ftp.fileserve.com'
PORT = 21
BINARY_STORE = True # if False then line store (not valid for binary files (videos, music, photos...))

###########################################
from ftpparameters import *



def print_line(result):
    print(result)

def connect_ftp():
    #Connect to the server
    ftp = FTP()
    ftp.connect(SERVER, PORT)
    ftp.login(USER, PASS)

    return ftp

def upload_file(ftp_connetion, upload_file_path):

    #Open the file
    try:
        upload_file = open(upload_file_path, 'r')

        #get the name
        path_split = upload_file_path.split('/')
        final_file_name = path_split[len(path_split)-1]

        #transfer the file
        print('Uploading ' + final_file_name + '...')

        if BINARY_STORE:
            ftp_connetion.storbinary('STOR '+ final_file_name, upload_file)
        else:
            #ftp_connetion.storlines('STOR ' + final_file_name, upload_file, print_line)
            ftp_connetion.storlines('STOR '+ final_file_name, upload_file)

        print('Upload finished.')
    except IOError:
        print ("No such file or directory... passing to next file")


def getFileList(mydir):
    filelist = [ os.path.join(mydir,f) for f in os.listdir(mydir) if os.path.isfile(os.path.join(mydir,f))]
    return filelist

def getFilesToUpload():
    if len(sys.argv) < 2:
        print "Enter source local directory as parameter"
    else:
        filestoupload=[]
        sourcedirs=sys.argv[1:]
        for arg in sourcedirs:
            print arg
            filestoupload.extend(getFileList(arg))
    return filestoupload

def cdTree(currentDir):
    if currentDir != "":
        try:
            ftp.cwd(currentDir)
        except IOError:
            cdTree("/".join(currentDir.split("/")[:-1]))
            ftp.mkd(currentDir)
            ftp.cwd(currentDir)


def uploadfiles():
    filelist=getFilesToUpload()
    try:
        ftp_conn = connect_ftp()
        print "Connected"
        cdTree(DESTINATION_DIR)
    except:
        print 'Connection error - unable to open connection to ftp server or nonexisting dir'
        sys.exit(1)
    else:
        for item in filelist:
            print "Uploading file: " + str(item)
            try:
                upload_file(ftp_conn, item)
            except:
                print "Error uploading file: "+str(item)
            else:
                print "Uploaded file: "+str(item)


#Take all the files and upload all
#ftp_conn = connect_ftp()
#for arg in sys.argv:
#    upload_file(ftp_conn, arg)
print UPLOADED_DIR
print SERVER
uploadfiles()
{'fraction_non_alphanumeric': 0.15384615384615385, 'fraction_numerical': 0.005118169501731146, 'mean_word_length': 2.5434666666666668, 'pattern_counts': {'":': 2, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4512108', 'n_tokens_mistral': 2184, 'n_tokens_neox': 2059, 'n_words': 524}