<commit_before>__doc__ = """ Test basic sync and conflbfnicts: files are modified and deleted by one or both workers (winner and loser); optionally remove local state db on one of the clients (loser).

There are four clients (workers):

 - creator - populates the directory initially and also performs a final check
 - winner  - is syncing its local changes first
 - loser   - is syncing its local changes second (and optionally it looses the local sync database before doing the sync)
 - checker - only performs a final check (without having interacted with the system before)

Note: in 1.6 client conflict files are excluded by default - so they should be never propagated to the server
Note: in 1.5 the exclusion list should be provided separately to the client (FIXME)

Note on effects of removing local state db (1.6):

 - any files modified remotely or locally get a conflict if both remote and local replicas exist (FIXME: this could be possibly more refined in the future based on timestamps or content checksums)
 - any files not present locally but present remotely will be downloaded (so a deletion won't be propagated)

"""

from smashbox.utilities import * 

import glob

filesizeKB = int(config.get('basicSync_filesizeKB',10000))

# True => remove local sync db on the loser 
# False => keep the loser 
rmLocalStateDB = bool(config.get('basicSync_rmLocalStateDB',False))

# subdirectory where to put files (if empty then use top level workdir)
subdirPath = config.get('basicSync_subdirPath',"")

#### testsets = [
####         { 'basicSync_filesizeKB': 1, 
####           'basicSync_rmLocalStateDB':False
####         },
####         { 'basicSync_filesizeKB': 5000, 
####           'basicSync_rmLocalStateDB':False
####         },
####         { 'basicSync_filesizeKB': 15000, 
####           'basicSync_rmLocalStateDB':False
####         },
####         { 'basicSync_filesizeKB': 50000, 
####           'basicSync_rmLocalStateDB':False
####         },
#### 
####         { 'basicSync_filesizeKB': 1, 
####           'basicSync_rmLocalStateDB':True
####         },
####         { 'basicSync_filesizeKB': 5000, 
####           'basicSync_rmLocalStateDB':True
####         },
####         { 'basicSync_filesizeKB': 15000, 
####           'basicSync_rmLocalStateDB':True
####         },
####         { 'basicSync_filesizeKB': 50000, 
####           'basicSync_rmLocalStateDB':True
####         }
#### ]
#### 

testsets = []

# create cartesian product of all test configurations
for s in [1, 5000, 15000, 50000]:
  for t in [True, False]:
      for p in [ "", "abc", "abc/abc", "abc/def/ghi" ]:
          testsets.append( { 'basicSync_filesizeKB':s,
                             'basicSync_rmLocalStateDB':t,
                             'basicSync_subdirPath':p } )

def expect_content(fn,md5):
    actual_md5 = md5sum(fn)
    error_check(actual_md5 == md5, "inconsistent md5 of %s: expected %s, got %s"%(fn,md5,actual_md5))

def expect_no_deleted_files(d):
    expect_deleted_files(d,[])

def expect_deleted_files(d,expected_deleted_files):
    actual_deleted_files = glob.glob(os.path.join(d,'*_DELETED*'))
    logger.debug('deleted files in %s: %s',d,actual_deleted_files)

    error_check(len(expected_deleted_files) == len(actual_deleted_files), "expected %d got %d deleted files"%(len(expected_deleted_files),len(actual_deleted_files)))

    for fn in expected_deleted_files:
        error_check(any([fn in dfn for dfn in actual_deleted_files]), "expected deleted file for %s not found"%fn)
 

def expect_conflict_files(d,expected_conflict_files):
    actual_conflict_files = glob.glob(os.path.join(d,'*_conflict-*-*'))

    logger.debug('conflict files in %s: %s',d,actual_conflict_files)

    error_check(len(expected_conflict_files) == len(actual_conflict_files), "expected %d got %d conflict files"%(len(expected_conflict_files),len(actual_conflict_files)))

    exp_basefns = [os.path.splitext(fn)[0] for fn in expected_conflict_files]

    logger.debug(exp_basefns)
    logger.debug(actual_conflict_files)

    for bfn in exp_basefns:
        error_check(any([bfn in fn for fn in actual_conflict_files]), "expected conflict file for %s not found"%bfn)
    
def expect_no_conflict_files(d):
    expect_conflict_files(d,[])

    
@add_worker
def creator(step):
    
    reset_owncloud_account()
    reset_rundir()

    step(1,'create initial content and sync')

    d = make_workdir()

    # we put the files in the subdir
    subdir = os.path.join(d,subdirPath)

    mkdir(subdir) 


    # files *_NONE are not modified by anyone after initial sync
    # files *_LOSER are modified by the loser but not by the winner
    # files *_WINNER are modified by the winner but not by the loser
    # files *_BOTH are modified both by the winner and by the loser (always conflict on the loser)

    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_NONE.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_LOSER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_WINNER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_BOTH.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_DELETED_LOSER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_DELETED_WINNER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_DELETED_BOTH.dat'),'0',count=1000,bs=filesizeKB)

    shared = reflection.getSharedObject()
    shared['md5_creator'] = md5sum(os.path.join(subdir,'TEST_FILE_MODIFIED_NONE.dat'))
    logger.info('md5_creator: %s',shared['md5_creator'])

    list_files(subdir)
    run_ocsync(d)
    list_files(subdir)

    step(7,'download the repository')
    run_ocsync(d,n=3)

    step(8,'final check')

    final_check(subdir,shared)
    expect_no_conflict_files(subdir) 

@add_worker
def winner(step):
    step(2,'initial sync')

    d = make_workdir()
    subdir = os.path.join(d,subdirPath)

    run_ocsync(d)

    step(3,'modify locally and sync to server')

    list_files(subdir)

    remove_file(os.path.join(subdir,'TEST_FILE_DELETED_WINNER.dat'))
    remove_file(os.path.join(subdir,'TEST_FILE_DELETED_BOTH.dat'))

    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_WINNER.dat'),'1',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_BOTH.dat'),'1',count=1000,bs=filesizeKB)

    createfile(os.path.join(subdir,'TEST_FILE_ADDED_WINNER.dat'),'1',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_ADDED_BOTH.dat'),'1',count=1000,bs=filesizeKB)

    shared = reflection.getSharedObject()
    shared['md5_winner'] = md5sum(os.path.join(subdir,'TEST_FILE_ADDED_WINNER.dat'))
    logger.info('md5_winner: %s',shared['md5_winner'])

    run_ocsync(d)

    step(5,'final sync')

    run_ocsync(d,n=3)

    step(8,'final check')

    final_check(subdir,shared)
    expect_no_conflict_files(subdir) 


# this is the loser which lost it's local state db after initial sync

@add_worker
def loser(step):

    step(2,'initial sync')

    d = make_workdir()
    subdir = os.path.join(d,subdirPath)
<commit_msg>workaround for mtime_diff<1s issue (#5589)<commit_after>__doc__ = """ Test basic sync and conflbfnicts: files are modified and deleted by one or both workers (winner and loser); optionally remove local state db on one of the clients (loser).

There are four clients (workers):

 - creator - populates the directory initially and also performs a final check
 - winner  - is syncing its local changes first
 - loser   - is syncing its local changes second (and optionally it looses the local sync database before doing the sync)
 - checker - only performs a final check (without having interacted with the system before)

Note: in 1.6 client conflict files are excluded by default - so they should be never propagated to the server
Note: in 1.5 the exclusion list should be provided separately to the client (FIXME)

Note on effects of removing local state db (1.6):

 - any files modified remotely or locally get a conflict if both remote and local replicas exist (FIXME: this could be possibly more refined in the future based on timestamps or content checksums)
 - any files not present locally but present remotely will be downloaded (so a deletion won't be propagated)

"""

from smashbox.utilities import * 

import glob

filesizeKB = int(config.get('basicSync_filesizeKB',10000))

# True => remove local sync db on the loser 
# False => keep the loser 
rmLocalStateDB = bool(config.get('basicSync_rmLocalStateDB',False))

# subdirectory where to put files (if empty then use top level workdir)
subdirPath = config.get('basicSync_subdirPath',"")

#### testsets = [
####         { 'basicSync_filesizeKB': 1, 
####           'basicSync_rmLocalStateDB':False
####         },
####         { 'basicSync_filesizeKB': 5000, 
####           'basicSync_rmLocalStateDB':False
####         },
####         { 'basicSync_filesizeKB': 15000, 
####           'basicSync_rmLocalStateDB':False
####         },
####         { 'basicSync_filesizeKB': 50000, 
####           'basicSync_rmLocalStateDB':False
####         },
#### 
####         { 'basicSync_filesizeKB': 1, 
####           'basicSync_rmLocalStateDB':True
####         },
####         { 'basicSync_filesizeKB': 5000, 
####           'basicSync_rmLocalStateDB':True
####         },
####         { 'basicSync_filesizeKB': 15000, 
####           'basicSync_rmLocalStateDB':True
####         },
####         { 'basicSync_filesizeKB': 50000, 
####           'basicSync_rmLocalStateDB':True
####         }
#### ]
#### 

testsets = []

# create cartesian product of all test configurations
for s in [1, 5000, 15000, 50000]:
  for t in [True, False]:
      for p in [ "", "abc", "abc/abc", "abc/def/ghi" ]:
          testsets.append( { 'basicSync_filesizeKB':s,
                             'basicSync_rmLocalStateDB':t,
                             'basicSync_subdirPath':p } )

def expect_content(fn,md5):
    actual_md5 = md5sum(fn)
    error_check(actual_md5 == md5, "inconsistent md5 of %s: expected %s, got %s"%(fn,md5,actual_md5))

def expect_no_deleted_files(d):
    expect_deleted_files(d,[])

def expect_deleted_files(d,expected_deleted_files):
    actual_deleted_files = glob.glob(os.path.join(d,'*_DELETED*'))
    logger.debug('deleted files in %s: %s',d,actual_deleted_files)

    error_check(len(expected_deleted_files) == len(actual_deleted_files), "expected %d got %d deleted files"%(len(expected_deleted_files),len(actual_deleted_files)))

    for fn in expected_deleted_files:
        error_check(any([fn in dfn for dfn in actual_deleted_files]), "expected deleted file for %s not found"%fn)
 

def expect_conflict_files(d,expected_conflict_files):
    actual_conflict_files = glob.glob(os.path.join(d,'*_conflict-*-*'))

    logger.debug('conflict files in %s: %s',d,actual_conflict_files)

    error_check(len(expected_conflict_files) == len(actual_conflict_files), "expected %d got %d conflict files"%(len(expected_conflict_files),len(actual_conflict_files)))

    exp_basefns = [os.path.splitext(fn)[0] for fn in expected_conflict_files]

    logger.debug(exp_basefns)
    logger.debug(actual_conflict_files)

    for bfn in exp_basefns:
        error_check(any([bfn in fn for fn in actual_conflict_files]), "expected conflict file for %s not found"%bfn)
    
def expect_no_conflict_files(d):
    expect_conflict_files(d,[])

    
@add_worker
def creator(step):
    
    reset_owncloud_account()
    reset_rundir()

    step(1,'create initial content and sync')

    d = make_workdir()

    # we put the files in the subdir
    subdir = os.path.join(d,subdirPath)

    mkdir(subdir) 


    # files *_NONE are not modified by anyone after initial sync
    # files *_LOSER are modified by the loser but not by the winner
    # files *_WINNER are modified by the winner but not by the loser
    # files *_BOTH are modified both by the winner and by the loser (always conflict on the loser)

    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_NONE.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_LOSER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_WINNER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_BOTH.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_DELETED_LOSER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_DELETED_WINNER.dat'),'0',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_DELETED_BOTH.dat'),'0',count=1000,bs=filesizeKB)

    shared = reflection.getSharedObject()
    shared['md5_creator'] = md5sum(os.path.join(subdir,'TEST_FILE_MODIFIED_NONE.dat'))
    logger.info('md5_creator: %s',shared['md5_creator'])

    list_files(subdir)
    run_ocsync(d)
    list_files(subdir)

    step(7,'download the repository')
    run_ocsync(d,n=3)

    step(8,'final check')

    final_check(subdir,shared)
    expect_no_conflict_files(subdir) 

@add_worker
def winner(step):
    step(2,'initial sync')

    d = make_workdir()
    subdir = os.path.join(d,subdirPath)

    run_ocsync(d)

    step(3,'modify locally and sync to server')

    list_files(subdir)

    remove_file(os.path.join(subdir,'TEST_FILE_DELETED_WINNER.dat'))
    remove_file(os.path.join(subdir,'TEST_FILE_DELETED_BOTH.dat'))

    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_WINNER.dat'),'1',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_MODIFIED_BOTH.dat'),'1',count=1000,bs=filesizeKB)

    createfile(os.path.join(subdir,'TEST_FILE_ADDED_WINNER.dat'),'1',count=1000,bs=filesizeKB)
    createfile(os.path.join(subdir,'TEST_FILE_ADDED_BOTH.dat'),'1',count=1000,bs=filesizeKB)

    shared = reflection.getSharedObject()
    shared['md5_winner'] = md5sum(os.path.join(subdir,'TEST_FILE_ADDED_WINNER.dat'))
    logger.info('md5_winner: %s',shared['md5_winner'])

    run_ocsync(d)

    sleep(1.1) # csync: mtime diff < 1s => conflict not detected, see: #5589 https://github.com/owncloud/client/issues/5589

    step(5,'final sync')

    run_ocsync(d,n=3)

    step(8,'final check')

    final_check(subdir,shared)
    expect_no_conflict_files(subdir) 


# this is the loser which lost it's local state db after initial sync

@add_worker
def loser(step):

    step(2,'initial sync')

    d = make_workdir()
    subdir = os.path.join(d,subdirPath)

{'fraction_non_alphanumeric': 0.1262396694214876, 'fraction_numerical': 0.020110192837465565, 'mean_word_length': 4.182369735902927, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5356702', 'n_tokens_mistral': 5149, 'n_tokens_neox': 4814, 'n_words': 1055}