<commit_before>import subprocess, os, os.path, sys

from setuptools import setup, Extension
from setuptools.command.install import install
from setuptools.command.develop import develop

from pkg_resources import resource_filename, resource_isdir
from tempfile import mkstemp
from zipfile import ZipFile

version = open('pymoku/version.txt').read().strip()
data_url = 'http://www.liquidinstruments.com/s/data-latest.zip'

try:
	sys.argv.remove("--no-fetch-data")
	fetch_bs = False
except:
	fetch_bs = True


def download_bitstreams():
	import requests
	assert resource_isdir('pymoku', 'instr')
	base_path = resource_filename('pymoku', 'instr')

	data = mkstemp()

	try:
		r = requests.get(data_url)
		os.write(data[0], r.content)
		ZipFile(data[1]).extractall(base_path)

		os.close(data[0])
		os.remove(data[1])
	except:
		print("Failed to fetch updated instrument data, please re-run install with internet access or specify '--no-fetch-data' to disable this message.")
		raise


class InstallWithBitstreams(install):
	def run(self):
		install.run(self)
		if fetch_bs:
			download_bitstreams()

class DevelopWithBitstreams(develop):
	def run(self):
		develop.run(self)
		if fetch_bs:
			download_bitstreams()

j = os.path.join

lr_ext = Extension(
	'lr',
	include_dirs=['liquidreader'],
	sources=[
		j('liquidreader','lireader.c'),
		j('liquidreader','liparse.c'),
		j('liquidreader','liutility.c'),
		j('liquidreader','linumber.c'),
		j('liquidreader','capn.c'),
		j('liquidreader','capn-malloc.c'),
		j('liquidreader','capn-stream.c'),
		j('liquidreader','li.capnp.c'),
		j('lr_mod','lr_module.c'),
	],
	extra_compile_args=['-std=c99'],
)

setup(
	name='pymoku',
	version=version,
	author='<NAME>',
	author_email='<EMAIL>',
	packages=['pymoku', 'pymoku.tools'],
	package_dir={'pymoku': 'pymoku'},
	package_data={
		'pymoku' : ['version.txt', '*.capnp', j('instr', '.empty')]
	},
	license='MIT',
	long_description="Python scripting interface to the Liquid Instruments Moku:Lab",

	url="https://github.com/liquidinstruments/pymoku",
	download_url="https://github.com/liquidinstruments/pymoku/archive/%s.tar.gz" % version,

	keywords=['moku', 'liquid instruments', 'test', 'measurement', 'lab', 'equipment'],

	entry_points={
		'console_scripts' : [
			'moku=pymoku.tools.moku:main',
			'moku_convert=pymoku.tools.moku_convert:main',
		]
	},

	install_requires=[
		'future',
		'pyzmq>=15.3.0',
		'six',
		'urllib3',
		'pyzmq',
		'rfc6266',
		'requests',
		'decorator',
	],

	ext_modules=[
		lr_ext,
	],

	cmdclass={
		'install': InstallWithBitstreams,
		'develop': DevelopWithBitstreams,
	},

	zip_safe=False, # Due to bitstream download
)
<commit_msg>PM-303: Add liquidreader bitcpy to the python build<commit_after>import subprocess, os, os.path, sys

from setuptools import setup, Extension
from setuptools.command.install import install
from setuptools.command.develop import develop

from pkg_resources import resource_filename, resource_isdir
from tempfile import mkstemp
from zipfile import ZipFile

version = open('pymoku/version.txt').read().strip()
data_url = 'http://www.liquidinstruments.com/s/data-latest.zip'

try:
	sys.argv.remove("--no-fetch-data")
	fetch_bs = False
except:
	fetch_bs = True


def download_bitstreams():
	import requests
	assert resource_isdir('pymoku', 'instr')
	base_path = resource_filename('pymoku', 'instr')

	data = mkstemp()

	try:
		r = requests.get(data_url)
		os.write(data[0], r.content)
		ZipFile(data[1]).extractall(base_path)

		os.close(data[0])
		os.remove(data[1])
	except:
		print("Failed to fetch updated instrument data, please re-run install with internet access or specify '--no-fetch-data' to disable this message.")
		raise


class InstallWithBitstreams(install):
	def run(self):
		install.run(self)
		if fetch_bs:
			download_bitstreams()

class DevelopWithBitstreams(develop):
	def run(self):
		develop.run(self)
		if fetch_bs:
			download_bitstreams()

j = os.path.join

lr_ext = Extension(
	'lr',
	include_dirs=['liquidreader'],
	sources=[
		j('liquidreader','bitcpy.c'),
		j('liquidreader','lireader.c'),
		j('liquidreader','liparse.c'),
		j('liquidreader','liutility.c'),
		j('liquidreader','linumber.c'),
		j('liquidreader','capn.c'),
		j('liquidreader','capn-malloc.c'),
		j('liquidreader','capn-stream.c'),
		j('liquidreader','li.capnp.c'),
		j('lr_mod','lr_module.c'),
	],
	extra_compile_args=['-std=c99'],
)

setup(
	name='pymoku',
	version=version,
	author='<NAME>',
	author_email='<EMAIL>',
	packages=['pymoku', 'pymoku.tools'],
	package_dir={'pymoku': 'pymoku'},
	package_data={
		'pymoku' : ['version.txt', '*.capnp', j('instr', '.empty')]
	},
	license='MIT',
	long_description="Python scripting interface to the Liquid Instruments Moku:Lab",

	url="https://github.com/liquidinstruments/pymoku",
	download_url="https://github.com/liquidinstruments/pymoku/archive/%s.tar.gz" % version,

	keywords=['moku', 'liquid instruments', 'test', 'measurement', 'lab', 'equipment'],

	entry_points={
		'console_scripts' : [
			'moku=pymoku.tools.moku:main',
			'moku_convert=pymoku.tools.moku_convert:main',
		]
	},

	install_requires=[
		'future',
		'pyzmq>=15.3.0',
		'six',
		'urllib3',
		'pyzmq',
		'rfc6266',
		'requests',
		'decorator',
	],

	ext_modules=[
		lr_ext,
	],

	cmdclass={
		'install': InstallWithBitstreams,
		'develop': DevelopWithBitstreams,
	},

	zip_safe=False, # Due to bitstream download
)

{'fraction_non_alphanumeric': 0.16840342389281726, 'fraction_numerical': 0.006140677335318198, 'mean_word_length': 5.391200951248513, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1099699', 'n_tokens_mistral': 2152, 'n_tokens_neox': 1966, 'n_words': 358}