<commit_before>#!/usr/bin/env python
#
# setup_setup.py for 'ucsql'
#
# This is the 'easy button' for installation of 'ucsql'
# Given the many dependencies, this script will optionally go an install everything it needs.
# If admins wish to install packages manually, then they can skip this.
# This script is assumed to be run from 'setup.py', but can be run standalone.
# Any missing pre-requisites will be flagged during the normal 'setup.py'
#

import sys
import os
import glob

#
# Make sure we have what we might need
#
# Many packages don't install properly with 'pip', so install them here via 'yum' or 'zypper'
#
# 'linuxprereqs' includes the dependency, and the determinator.  Assumes local package installe (yum, zypper, ...)
#
linuxprereqs = { "wget" : "which wget >/dev/null 2>&1", 
		 "gcc"  : "which gcc >/dev/null 2>&1",
		 "python-devel" : "rpm -qa | grep python-dev" }

#
# 'pythonpreqs' include the module needed and the source URL if it needs to be installed
#
# 'setuptools', 'pip', etc are not 100% consistent/reliable, so we'll do it by hand
#

pythonprereqs = { 'lxml' : { 'namevers' : 'lxml-3.2.3', 'wget' : 'https://pypi.python.org/packages/source/l/lxml/lxml-3.2.3.tar.gz#md5=fef47bb4ac72ac38ce778518dac42236' } ,
		  'setuptools' : { 'namevers' : 'setuptools-1.4.1', 'wget': 'https://pypi.python.org/packages/source/s/setuptools/setuptools-1.4.1.tar.gz#md5=65bb270fbae373c26a2fa890ad907818'} ,
		  'Crypto' : { 'namevers' : 'pycrypto-2.6.1', 'wget': 'https://ftp.dlitz.net/pub/dlitz/crypto/pycrypto/pycrypto-2.6.1.tar.gz'},
		  'pyparsing' : { 'namevers' : 'pyparsing-2.0.1', 'wget' : 'http://cheeseshop.python.org/packages/source/p/pyparsing/pyparsing-2.0.1.tar.gz' }
		}<commit_msg>install lxml via packages, due to dependencies<commit_after>#!/usr/bin/env python
#
# setup_setup.py for 'ucsql'
#
# This is the 'easy button' for installation of 'ucsql'
# Given the many dependencies, this script will optionally go an install everything it needs.
# If admins wish to install packages manually, then they can skip this.
# This script is assumed to be run from 'setup.py', but can be run standalone.
# Any missing pre-requisites will be flagged during the normal 'setup.py'
#

import sys
import os
import glob

#
# Make sure we have what we might need
#
# Many packages don't install properly with 'pip', so install them here via 'yum' or 'zypper'
#
# 'linuxprereqs' includes the dependency, and the determinator.  Assumes local package installe (yum, zypper, ...)
#
# 'lxml' has too many dependencies to install manually as a pythonprereq.  Install via yum/zypper
#
linuxprereqs = { "wget" : "which wget >/dev/null 2>&1", 
		 "gcc"  : "which gcc >/dev/null 2>&1",
		 "python-devel" : "rpm -qa | grep python-dev 2>&1",
		 "python-lxml"  : "rpm -qa | grep lxml 2>&1"
               }

#
# 'pythonpreqs' include the module needed and the source URL if it needs to be installed
#
# 'setuptools', 'pip', etc are not 100% consistent/reliable, so we'll do it by hand
#

pythonprereqs = { 
#	'lxml' : { 'namevers' : 'lxml-3.2.3', 'wget' : 'https://pypi.python.org/packages/source/l/lxml/lxml-3.2.3.tar.gz#md5=fef47bb4ac72ac38ce778518dac42236' } ,
	'setuptools' : { 'namevers' : 'setuptools-1.4.1', 'wget': 'https://pypi.python.org/packages/source/s/setuptools/setuptools-1.4.1.tar.gz#md5=65bb270fbae373c26a2fa890ad907818'} ,
	'Crypto' : { 'namevers' : 'pycrypto-2.6.1', 'wget': 'https://ftp.dlitz.net/pub/dlitz/crypto/pycrypto/pycrypto-2.6.1.tar.gz'},
	'pyparsing' : { 'namevers' : 'pyparsing-2.0.1', 'wget' : 'http://cheeseshop.python.org/packages/source/p/pyparsing/pyparsing-2.0.1.tar.gz' }
		}
{'fraction_non_alphanumeric': 0.14957146806745922, 'fraction_numerical': 0.04036494332319602, 'mean_word_length': 4.004149377593361, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4175772', 'n_tokens_mistral': 1460, 'n_tokens_neox': 1342, 'n_words': 369}