<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import unittest
import tempfile
import os
from subprocess import check_call
from shutil import rmtree, copyfile
import urllib2
from urllib import quote
import json

# global settings for the test
# currently assumes a manually started server
solr = 'http://localhost:8080/solr/'
repo = tempfile.mkdtemp()
reponame = os.path.basename(repo)
repourl = 'file://' + repo
hook = repo + '/hooks/post-commit'
hooklog = hook + '.log'

def run(cmd):
  #not properly quoted:#print '# ' + ' '.join(cmd)
  return check_call(cmd)

def createRepository():
  ''' create repository with indexing hook enabled '''
  run(['svnadmin', 'create', repo])
  hook = repo + '/hooks/post-commit'
  f = open(hook, 'w')
  f.write('#!/bin/sh\n')
  f.write('export LC_ALL="en_US.UTF-8"\n')
  f.write('%s $1 $2' % os.path.abspath('../hook/svnhook.py'))
  f.write(' >> %s 2>&1\n' % hooklog)
  f.close()
  os.chmod(hook, 0777)

def createInitialStructure():
  '''
    Test data can be added from tests but we do a batch first for faster test execution.
    All tests may use this data.
  '''
  print '# ------- setup start --------'
  run(['svn', 'import', 'docs', repourl + '/docs', '-m', 'Mixed documents'])
  # properties must be added in a working copy, using a small subfolder for performance
  propwc = tempfile.mkdtemp()
  run(['svn', 'co', repourl + '/docs/svnprops/', propwc])
  txt =  propwc + '/textwithsvnprops.txt'
  run(['svn', 'propset', 'svn:keywords', 'Id LastChangedRevision HeadURL', txt])
  # Currently keywords properties are only treated as keywords if their namespace match the schema copyField rule 
  run(['svn', 'propset', 'cms:keywords', 'ReposSearch repossearch keywordfromsvnprop', txt])
  run(['svn', 'propset', 'whatever:tags', 'metadataindexing tagging-in-svnprop', txt])
  run(['svn', 'propset', 'custom:someurl', 'Visit http://repossearch.com/', txt])
  run(['svn', 'propset', 'svn:mime-type', 'application/pdf', propwc + '/shortpdf.pdf'])
  run(['svn', 'propset', 'cms:keywords', 'keywordfromsvnprop', propwc + '/shortpdf.pdf'])
  # creating the invalid xml here so the error is not logged in rev 1 too
  xml = propwc + '/invalid, tag not closed.xml'
  f = open(xml, 'w')
  f.write('<?xml version="1.0"?>\n<document>\n<opentag>\n</document>\n')
  f.close()
  run(['svn', 'add', xml])
  run(['svn', 'propset', 'some:prop', 'OnInvalidXml', xml])
  run(['svn', 'commit', '-m', 'Properties added', propwc])
  print '# ---- common test data ----'
  run(['svnlook', 'tree', repo])
  print '# ------- hook log ---------'
  f = open(hooklog, 'r')
  print f.read()
  f.close()
  print '# ------- setup done --------'
  print ''

def curl(url):
  # transparently restrict hits to this instance
  if url.find('q=') > 0:
    url = url + '&fq=id_repo:' + reponame
  r = urllib2.urlopen(url)
  return r.read()

def search(query, queryType='standard'):
  '''
  queryType: standard/None, meta, content
  query: solr escaped but not urlencoded query
  '''
  # todo encode and stuff
  r = curl(solr + 'svnhead/select?qt=' + queryType + '&q=' + quote(query) + '&wt=json')
  #print '\n' + r 
  return json.loads(r)

def searchMeta(query):
  return search(query, 'meta')

def searchContent(query):
  return search(query, 'content')


class ReposSearchTest(unittest.TestCase):
  
  def testServerIsUp(self):
    r = curl(solr + '')
    self.assertTrue(r.find('svnhead') > -1, r)
    
  def testFilename(self):
    r = searchMeta('shouldBeUNIQUEfilename')
    self.assertEqual(r['response']['numFound'], 1,
                     'Case should not matter, extension should not be needed')
    self.assertEqual(r['response']['docs'][0]['id'], 
                     u'%s^/docs/filenames/shouldBeUniqueFilename.txt' % reponame)

  def testFilenameWithExtension(self):
    r = searchMeta('shouldbeuniquefilename.txt')
    self.assertEqual(r['response']['numFound'], 1)
    
  def testFilenameTokenizeDash(self):
    r = search('name:(subversion AND related AND document2)')
    self.assertEqual(r['response']['numFound'], 1, 'dash should be searchable as space')
    self.assertEqual(r['response']['docs'][0]['id'],
                     '%s^/docs/filenames/Subversion-related Document2, 2010-02-10.txt' % reponame)
    # How do we handle dashes when the adjacent chars are digits?
    self.assertEqual(searchMeta('subversion document 2010')['response']['docs'][0]['id'], 
                     '%s^/docs/filenames/Subversion-related Document2, 2010-02-10.txt' % reponame)
    self.assertEqual(searchMeta('2010-02-10')['response']['docs'][0]['id'], 
                     '%s^/docs/filenames/Subversion-related Document2, 2010-02-10.txt' % reponame)
    # TODO Should dash-separated words be joined? Difference between digits and letters?
    #self.assertEqual(searchMeta('20100210')['response']['docs'][0]['id'], 
    #                 '%s^/docs/filenames/Subversion-related Document2, 2010-02-10.txt' % reponame)
    #self.assertEqual(searchMeta('subversionrelated 2010-02-10')['response']['docs'][0]['id'], 
    #                 '%s^/docs/filenames/Subversion-related Document2, 2010-02-10.txt' % reponame)          
    # TODO Should CamelCase be split into separate words? in that case we should have 2 hits above
    # Even if CamelCase is split it must still match as one word, as below
    self.assertEqual(searchMeta('subversionrelated 11.2')['response']['docs'][0]['id'], 
                     '%s^/docs/filenames/SubversionRelated DNR333 ver 11.2.txt' % reponame)    
  
  def testFilenameTokenizerCodes(self):
    r = searchMeta('11.2')
    self.assertEqual(r['response']['numFound'], 1, 'string like 11.2 (version number) must be searchable')
    r = searchMeta('DNR333')
    self.assertEqual(r['response']['numFound'], 1, 'letters+digits could be a product code and must be searchable')
  
  def testFilenameLatin1Accents(self):
    r = searchMeta('aeea')
    self.assertEqual(r['response']['numFound'], 1, 'accented chars should be searchable without accent')
    self.assertEqual(r['response']['docs'][0]['id'], 
                     u'%s^/docs/filenames/Latin1 accents áéèà.txt' % reponame)
  
  def testFilenameTokenizeWhitespaceComma(self):
    self.assertEqual(search('name:(short AND not AND long AND filename AND txt)')['response']['numFound'], 1,
                     'commas should be ignored, common words like "not" should be kept')
    
  def testFilenameUTF8(self):
    r = searchMeta(u'Swedish åäö'.encode('utf-8'))<commit_msg>Started to convert tests to a more acceptance-test like syntax, with a helper function called searchOne as a fixture. This is to encourage contribution with new tests for schema features and document type support.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import unittest
import tempfile
import os
from subprocess import check_call
from shutil import rmtree, copyfile
import urllib2
from urllib import quote
import json

# global settings for the test
# currently assumes a manually started server
solr = 'http://localhost:8080/solr/'
repo = tempfile.mkdtemp()
reponame = os.path.basename(repo)
repourl = 'file://' + repo
hook = repo + '/hooks/post-commit'
hooklog = hook + '.log'

def run(cmd):
  #not properly quoted:#print '# ' + ' '.join(cmd)
  return check_call(cmd)

def createRepository():
  ''' create repository with indexing hook enabled '''
  run(['svnadmin', 'create', repo])
  hook = repo + '/hooks/post-commit'
  f = open(hook, 'w')
  f.write('#!/bin/sh\n')
  f.write('export LC_ALL="en_US.UTF-8"\n')
  f.write('%s $1 $2' % os.path.abspath('../hook/svnhook.py'))
  f.write(' >> %s 2>&1\n' % hooklog)
  f.close()
  os.chmod(hook, 0777)

def createInitialStructure():
  '''
    Test data can be added from tests but we do a batch first for faster test execution.
    All tests may use this data.
  '''
  print '# ------- setup start --------'
  run(['svn', 'import', 'docs', repourl + '/docs', '-m', 'Mixed documents'])
  # properties must be added in a working copy, using a small subfolder for performance
  propwc = tempfile.mkdtemp()
  run(['svn', 'co', repourl + '/docs/svnprops/', propwc])
  txt =  propwc + '/textwithsvnprops.txt'
  run(['svn', 'propset', 'svn:keywords', 'Id LastChangedRevision HeadURL', txt])
  # Currently keywords properties are only treated as keywords if their namespace match the schema copyField rule 
  run(['svn', 'propset', 'cms:keywords', 'ReposSearch repossearch keywordfromsvnprop', txt])
  run(['svn', 'propset', 'whatever:tags', 'metadataindexing tagging-in-svnprop', txt])
  run(['svn', 'propset', 'custom:someurl', 'Visit http://repossearch.com/', txt])
  run(['svn', 'propset', 'svn:mime-type', 'application/pdf', propwc + '/shortpdf.pdf'])
  run(['svn', 'propset', 'cms:keywords', 'keywordfromsvnprop', propwc + '/shortpdf.pdf'])
  # creating the invalid xml here so the error is not logged in rev 1 too
  xml = propwc + '/invalid, tag not closed.xml'
  f = open(xml, 'w')
  f.write('<?xml version="1.0"?>\n<document>\n<opentag>\n</document>\n')
  f.close()
  run(['svn', 'add', xml])
  run(['svn', 'propset', 'some:prop', 'OnInvalidXml', xml])
  run(['svn', 'commit', '-m', 'Properties added', propwc])
  print '# ---- common test data ----'
  run(['svnlook', 'tree', repo])
  print '# ------- hook log ---------'
  f = open(hooklog, 'r')
  print f.read()
  f.close()
  print '# ------- setup done --------'
  print ''

def curl(url):
  # transparently restrict hits to this instance
  if url.find('q=') > 0:
    url = url + '&fq=id_repo:' + reponame
  r = urllib2.urlopen(url)
  return r.read()

def search(query, queryType='standard'):
  '''
  queryType: standard/None, meta, content
  query: solr escaped but not urlencoded query
  '''
  # todo encode and stuff
  r = curl(solr + 'svnhead/select?qt=' + queryType + '&q=' + quote(query) + '&wt=json')
  #print '\n' + r 
  return json.loads(r)

def searchMeta(query):
  return search(query, 'meta')

def searchContent(query):
  return search(query, 'content')

def searchOne(queryType, query):
  r = search(query, queryType)
  n = r['response']['numFound']
  if n == 0:
    raise NameError('No results for query "%s" type %s' % (query, queryType))
  if n > 1:
    raise NameError('%d results for query "%s" type %s: \n%s' % (n, query, queryType, repr(r)))
  return r['response']['docs'][0]['id'].partition('^')[2]

class ReposSearchTest(unittest.TestCase):
  
  def testServerIsUp(self):
    r = curl(solr + '')
    self.assertTrue(r.find('svnhead') > -1, r)
    
  def testFilename(self):
    self.assertEqual(searchOne('meta', 'shouldBeUNIQUEfilename'), 
                     '/docs/filenames/shouldBeUniqueFilename.txt')

  def testFilenameWithExtension(self):
    self.assertEqual(searchOne('meta', 'shouldbeuniquefilename.txt'), 
                     '/docs/filenames/shouldBeUniqueFilename.txt')
    
  def testFilenameTokenizeDash(self):
    self.assertEqual(searchOne('standard', 'name:(subversion AND related AND document2)'), 
                     '/docs/filenames/Subversion-related Document2, 2010-02-10.txt',
                     'dash should be searchable as space')
    # How do we handle dashes when the adjacent chars are digits?
    self.assertEqual(searchOne('meta', 'subversion document 2010'), 
                     '/docs/filenames/Subversion-related Document2, 2010-02-10.txt')    
    self.assertEqual(searchOne('meta', '2010-02-10'), 
                     '/docs/filenames/Subversion-related Document2, 2010-02-10.txt') 
    # TODO Should dash-separated words be joined? Difference between digits and letters?
    #self.assertEqual(searchMeta('20100210')['response']['docs'][0]['id'], 
    #                 '%s^/docs/filenames/Subversion-related Document2, 2010-02-10.txt' % reponame)
    #self.assertEqual(searchMeta('subversionrelated 2010-02-10')['response']['docs'][0]['id'], 
    #                 '%s^/docs/filenames/Subversion-related Document2, 2010-02-10.txt' % reponame)          
    # TODO Should CamelCase be split into separate words? in that case we should have 2 hits above
    # Even if CamelCase is split it must still match as one word, as below
    self.assertEqual(searchMeta('subversionrelated 11.2')['response']['docs'][0]['id'], 
                     '%s^/docs/filenames/SubversionRelated DNR333 ver 11.2.txt' % reponame)    
  
  def testFilenameTokenizerCodes(self):
    r = searchMeta('11.2')
    self.assertEqual(r['response']['numFound'], 1, 'string like 11.2 (version number) must be searchable')
    r = searchMeta('DNR333')
    self.assertEqual(r['response']['numFound'], 1, 'letters+digits could be a product code and must be searchable')
  
  def testFilenameLatin1Accents(self):
    r = searchMeta('aeea')
    self.assertEqual(r['response']['numFound'], 1, 'accented chars should be searchable without accent')
    self.assertEqual(r['response']['docs'][0]['id'], 
                     u'%s^/docs/filenames/Latin1 accents áéèà.txt' % reponame)
  
  def testFilenameTokenizeWhitespaceComma(self):
    self.assertEqual(search('name:(short AND not AND long AND filename AND txt)')['response']['numFound'], 1,
                     'commas should be ignored, common words like "not" should be kept')
    
  def testFilenameUTF8(self):
    r = searchMeta(u'Swedish åäö'.encode('utf-8'))
{'fraction_non_alphanumeric': 0.16613852421910366, 'fraction_numerical': 0.019843066244152708, 'mean_word_length': 3.6688974991194083, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 2, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6239391', 'n_tokens_mistral': 4478, 'n_tokens_neox': 4228, 'n_words': 1127}