<commit_before># SHELL += -x

CXX = g++
# CXX = clang++  # Uncomment this line to build with Clang.

OPENFST_VERSION = 1.3.4
# OPENFST_VERSION = 1.4.1  # Uncomment this line to build with OpenFst-1.4.1.
# Note: OpenFst >= 1.4 requires C++11 support, hence you will need to use a
# relatively recent C++ compiler, e.g. gcc >= 4.6, clang >= 3.0.

# On Mac OS 10.9+, clang defaults to the new c++ standard library libc++.
# Since OpenFst-1.3 uses stuff from the tr1 namespace, we need to tell clang
# to use libstdc++ instead.
ifeq ($(OPENFST_VERSION), 1.3.4)
  COMPILER = $(shell $(CXX) -v 2>&1 )
  ifeq ($(findstring clang,$(COMPILER)),clang)
    CXXFLAGS += -stdlib=libstdc++
    LDFLAGS += -stdlib=libstdc++
  endif
else
  ifneq ($(OPENFST_VERSION), 1.4.1)
    $(error OpenFst version $(OPENFST_VERSION) is not supported. \
            Supported versions: 1.3.4, 1.4.1)
  endif
endif

all: check_required_programs sph2pipe atlas irstlm_tgt sclite openfst

check_required_programs:
	extras/check_dependencies.sh

clean: openfst_cleaned sclite_cleaned  irstlm_cleaned

openfst_cleaned:
	$(MAKE) -C openfst-$(OPENFST_VERSION) clean

irstlm_cleaned:
	$(MAKE) -C irstlm clean 

sclite_cleaned:
	$(MAKE) -C sctk clean

distclean:
	rm -rf openfst-$(OPENFST_VERSION)/
	rm -rf sctk-2.4.9/
	rm -rf sctk
	rm -rf ATLAS/
	rm -rf irstlm/
	rm -rf sph2pipe_v2.5/
	rm -rf sph2pipe_v2.5.tar.gz
	rm -rf atlas3.8.3.tar.gz
	rm -rf sctk-2.4.9-20141015-1634Z.tar.bz2
	rm -rf openfst-$(OPENFST_VERSION).tar.gz
	rm -f openfst


.PHONY: openfst # so target will be made even though "openfst" exists.
openfst: openfst_compiled openfst-$(OPENFST_VERSION)/lib
	-rm -f openfst
	-ln -s openfst-$(OPENFST_VERSION) openfst

.PHONY: openfst_compiled
openfst_compiled: openfst-$(OPENFST_VERSION)/Makefile
	cd openfst-$(OPENFST_VERSION)/ && \
	$(MAKE) install

openfst-$(OPENFST_VERSION)/lib: | openfst-$(OPENFST_VERSION)/Makefile
	-cd openfst-$(OPENFST_VERSION) && [ -d lib64 ] && [ ! -d lib ] && ln -s lib64 lib

# Disable shared-library creation on cygwin as it leads to compilation error
# "file too big".
openfst-$(OPENFST_VERSION)/Makefile: openfst-$(OPENFST_VERSION)/.patched
ifeq ($(OSTYPE),cygwin)
	cd openfst-$(OPENFST_VERSION)/; ./configure --prefix=`pwd` --enable-static --disable-shared --enable-far --enable-ngram-fsts CXX=$(CXX) CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" LIBS="-ldl"
else
	cd openfst-$(OPENFST_VERSION)/; ./configure --prefix=`pwd` --enable-static --enable-shared --enable-far --enable-ngram-fsts CXX=$(CXX) CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" LIBS="-ldl"
endif

# patches for openfst. openfst_gcc41up.patch is a patch for openfst to \
# support multi-threads when compile with g++ (gcc) version above 4.1
openfst-$(OPENFST_VERSION)/.patched: | openfst-$(OPENFST_VERSION)
	-cd openfst-$(OPENFST_VERSION)/src/include/fst; \
	patch -p0 -N < ../../../../extras/openfst-$(OPENFST_VERSION).patch;
	$(CXX) -dumpversion | awk '{if(NR==1 && $$1>"4.1") print "cd openfst-$(OPENFST_VERSION)/src/include/fst; patch -p0 -N < ../../../../extras/openfst_gcc41up.patch"}' | sh -
	touch $@

openfst-$(OPENFST_VERSION): openfst-$(OPENFST_VERSION).tar.gz
	tar xozf openfst-$(OPENFST_VERSION).tar.gz

openfst-$(OPENFST_VERSION).tar.gz: 
	wget http://openfst.cs.nyu.edu/twiki/pub/FST/FstDownload/openfst-$(OPENFST_VERSION).tar.gz || \
	wget -T 10 -t 3 http://www.openslr.org/resources/2/openfst-$(OPENFST_VERSION).tar.gz

sclite: sclite_compiled

.PHONY: sclite_compiled
sclite_compiled: sctk sctk_configured
	cd sctk; \
	$(MAKE) all && $(MAKE) install && $(MAKE) doc

sctk_configured: sctk sctk/.configured

sctk/.configured: sctk
	cd sctk; $(MAKE) config
	touch sctk/.configured

.PHONY: sctk
sctk: sctk-2.4.9-20141015-1634Z.tar.bz2
	tar xojf sctk-2.4.9-20141015-1634Z.tar.bz2
	rm -rf sctk && ln -s sctk-2.4.9 sctk

sctk-2.4.9-20141015-1634Z.tar.bz2:
	wget -T 10 -t 3 ftp://jaguar.ncsl.nist.gov/pub/sctk-2.4.9-20141015-1634Z.tar.bz2|| \
    wget --no-check-certificate -T 10 http://www.openslr.org/resources/4/sctk-2.4.9-20141015-1634Z.tar.bz2

irstlm_tgt: irstlm_compiled

.PHONY: irstlm_compiled
irstlm_compiled: irstlm/Makefile
	cd irstlm/; \
	make; $(MAKE) install	

irstlm/Makefile: irstlm
	cd irstlm; \
	    automake --version | grep 1.13.1 >/dev/null && \
         sed s:AM_CONFIG_HEADER:AC_CONFIG_HEADERS: <configure.in >configure.ac; \
	(./regenerate-makefiles.sh || ./regenerate-makefiles.sh) && \
	./configure --prefix `pwd` 

irstlm:
	svn -r 618 co --non-interactive --trust-server-cert https://svn.code.sf.net/p/irstlm/code/trunk irstlm
<commit_msg>change repository svn -> git<commit_after># SHELL += -x

CXX = g++
# CXX = clang++  # Uncomment this line to build with Clang.

OPENFST_VERSION = 1.3.4
# OPENFST_VERSION = 1.4.1  # Uncomment this line to build with OpenFst-1.4.1.
# Note: OpenFst >= 1.4 requires C++11 support, hence you will need to use a
# relatively recent C++ compiler, e.g. gcc >= 4.6, clang >= 3.0.

# On Mac OS 10.9+, clang defaults to the new c++ standard library libc++.
# Since OpenFst-1.3 uses stuff from the tr1 namespace, we need to tell clang
# to use libstdc++ instead.
ifeq ($(OPENFST_VERSION), 1.3.4)
  COMPILER = $(shell $(CXX) -v 2>&1 )
  ifeq ($(findstring clang,$(COMPILER)),clang)
    CXXFLAGS += -stdlib=libstdc++
    LDFLAGS += -stdlib=libstdc++
  endif
else
  ifneq ($(OPENFST_VERSION), 1.4.1)
    $(error OpenFst version $(OPENFST_VERSION) is not supported. \
            Supported versions: 1.3.4, 1.4.1)
  endif
endif

all: check_required_programs sph2pipe atlas irstlm_tgt sclite openfst

check_required_programs:
	extras/check_dependencies.sh

clean: openfst_cleaned sclite_cleaned  irstlm_cleaned

openfst_cleaned:
	$(MAKE) -C openfst-$(OPENFST_VERSION) clean

irstlm_cleaned:
	$(MAKE) -C irstlm clean 

sclite_cleaned:
	$(MAKE) -C sctk clean

distclean:
	rm -rf openfst-$(OPENFST_VERSION)/
	rm -rf sctk-2.4.9/
	rm -rf sctk
	rm -rf ATLAS/
	rm -rf irstlm/
	rm -rf sph2pipe_v2.5/
	rm -rf sph2pipe_v2.5.tar.gz
	rm -rf atlas3.8.3.tar.gz
	rm -rf sctk-2.4.9-20141015-1634Z.tar.bz2
	rm -rf openfst-$(OPENFST_VERSION).tar.gz
	rm -f openfst


.PHONY: openfst # so target will be made even though "openfst" exists.
openfst: openfst_compiled openfst-$(OPENFST_VERSION)/lib
	-rm -f openfst
	-ln -s openfst-$(OPENFST_VERSION) openfst

.PHONY: openfst_compiled
openfst_compiled: openfst-$(OPENFST_VERSION)/Makefile
	cd openfst-$(OPENFST_VERSION)/ && \
	$(MAKE) install

openfst-$(OPENFST_VERSION)/lib: | openfst-$(OPENFST_VERSION)/Makefile
	-cd openfst-$(OPENFST_VERSION) && [ -d lib64 ] && [ ! -d lib ] && ln -s lib64 lib

# Disable shared-library creation on cygwin as it leads to compilation error
# "file too big".
openfst-$(OPENFST_VERSION)/Makefile: openfst-$(OPENFST_VERSION)/.patched
ifeq ($(OSTYPE),cygwin)
	cd openfst-$(OPENFST_VERSION)/; ./configure --prefix=`pwd` --enable-static --disable-shared --enable-far --enable-ngram-fsts CXX=$(CXX) CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" LIBS="-ldl"
else
	cd openfst-$(OPENFST_VERSION)/; ./configure --prefix=`pwd` --enable-static --enable-shared --enable-far --enable-ngram-fsts CXX=$(CXX) CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" LIBS="-ldl"
endif

# patches for openfst. openfst_gcc41up.patch is a patch for openfst to \
# support multi-threads when compile with g++ (gcc) version above 4.1
openfst-$(OPENFST_VERSION)/.patched: | openfst-$(OPENFST_VERSION)
	-cd openfst-$(OPENFST_VERSION)/src/include/fst; \
	patch -p0 -N < ../../../../extras/openfst-$(OPENFST_VERSION).patch;
	$(CXX) -dumpversion | awk '{if(NR==1 && $$1>"4.1") print "cd openfst-$(OPENFST_VERSION)/src/include/fst; patch -p0 -N < ../../../../extras/openfst_gcc41up.patch"}' | sh -
	touch $@

openfst-$(OPENFST_VERSION): openfst-$(OPENFST_VERSION).tar.gz
	tar xozf openfst-$(OPENFST_VERSION).tar.gz

openfst-$(OPENFST_VERSION).tar.gz: 
	wget http://openfst.cs.nyu.edu/twiki/pub/FST/FstDownload/openfst-$(OPENFST_VERSION).tar.gz || \
	wget -T 10 -t 3 http://www.openslr.org/resources/2/openfst-$(OPENFST_VERSION).tar.gz

sclite: sclite_compiled

.PHONY: sclite_compiled
sclite_compiled: sctk sctk_configured
	cd sctk; \
	$(MAKE) all && $(MAKE) install && $(MAKE) doc

sctk_configured: sctk sctk/.configured

sctk/.configured: sctk
	cd sctk; $(MAKE) config
	touch sctk/.configured

.PHONY: sctk
sctk: sctk-2.4.9-20141015-1634Z.tar.bz2
	tar xojf sctk-2.4.9-20141015-1634Z.tar.bz2
	rm -rf sctk && ln -s sctk-2.4.9 sctk

sctk-2.4.9-20141015-1634Z.tar.bz2:
	wget -T 10 -t 3 ftp://jaguar.ncsl.nist.gov/pub/sctk-2.4.9-20141015-1634Z.tar.bz2|| \
    wget --no-check-certificate -T 10 http://www.openslr.org/resources/4/sctk-2.4.9-20141015-1634Z.tar.bz2

irstlm_tgt: irstlm_compiled

.PHONY: irstlm_compiled
irstlm_compiled: irstlm/Makefile
	cd irstlm/; \
	make; $(MAKE) install	

irstlm/Makefile: irstlm
	cd irstlm; \
	    automake --version | grep 1.13.1 >/dev/null && \
         sed s:AM_CONFIG_HEADER:AC_CONFIG_HEADERS: <configure.in >configure.ac; \
	(./regenerate-makefiles.sh || ./regenerate-makefiles.sh) && \
	./configure --prefix `pwd` 

irstlm:
	#svn -r 618 co --non-interactive --trust-server-cert https://svn.code.sf.net/p/irstlm/code/trunk irstlm
	git clone -b r618 https://github.com/seong889/irstlm.git irstlm
{'fraction_non_alphanumeric': 0.160239651416122, 'fraction_numerical': 0.04030501089324619, 'mean_word_length': 3.7693506493506495, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 18, 'https://': 3, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2132116', 'n_tokens_mistral': 4141, 'n_tokens_neox': 3832, 'n_words': 815}