<commit_before>#!/bin/bash -e

# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Script to install everything needed to build chromium (well, ideally, anyway)
# See http://code.google.com/p/chromium/wiki/LinuxBuildInstructions
# and http://code.google.com/p/chromium/wiki/LinuxBuild64Bit

usage() {
  echo "Usage: $0 [--options]"
  echo "Options:"
  echo "--[no-]syms: enable or disable installation of debugging symbols"
  echo "--lib32: enable installation of 32-bit libraries, e.g. for V8 snapshot"
  echo "--[no-]arm: enable or disable installation of arm cross toolchain"
  echo "--[no-]chromeos-fonts: enable or disable installation of Chrome OS"\
       "fonts"
  echo "--[no-]nacl: enable or disable installation of prerequisites for"\
       "building standalone NaCl and all its toolchains"
  echo "--no-prompt: silently select standard options/defaults"
  echo "--quick-check: quickly try to determine if dependencies are installed"
  echo "               (this avoids interactive prompts and sudo commands,"
  echo "               so might not be 100% accurate)"
  echo "--unsupported: attempt installation even on unsupported systems"
  echo "Script will prompt interactively if options not given."
  exit 1
}

# Checks whether a particular package is available in the repos.
# USAGE: $ package_exists <package name>
package_exists() {
  apt-cache pkgnames | grep -x "$1" > /dev/null 2>&1
}

# These default to on because (some) bots need them and it keeps things
# simple for the bot setup if all bots just run the script in its default
# mode.  Developers who don't want stuff they don't need installed on their
# own workstations can pass --no-arm --no-nacl when running the script.
do_inst_arm=1
do_inst_nacl=1

while test "$1" != ""
do
  case "$1" in
  --syms)                   do_inst_syms=1;;
  --no-syms)                do_inst_syms=0;;
  --lib32)                  do_inst_lib32=1;;
  --arm)                    do_inst_arm=1;;
  --no-arm)                 do_inst_arm=0;;
  --chromeos-fonts)         do_inst_chromeos_fonts=1;;
  --no-chromeos-fonts)      do_inst_chromeos_fonts=0;;
  --nacl)                   do_inst_nacl=1;;
  --no-nacl)                do_inst_nacl=0;;
  --no-prompt)              do_default=1
                            do_quietly="-qq --assume-yes"
    ;;
  --quick-check)            do_quick_check=1;;
  --unsupported)            do_unsupported=1;;
  *) usage;;
  esac
  shift
done

if test "$do_inst_arm" = "1"; then
  do_inst_lib32=1
fi

# Check for lsb_release command in $PATH
if ! which lsb_release > /dev/null; then
  echo "ERROR: lsb_release not found in \$PATH" >&2
  exit 1;
fi

lsb_release=$(lsb_release --codename --short)
ubuntu_codenames="(precise|trusty|utopic)"
if [ 0 -eq "${do_unsupported-0}" ] && [ 0 -eq "${do_quick_check-0}" ] ; then
  if [[ ! $lsb_release =~ $ubuntu_codenames ]]; then
    echo "ERROR: Only Ubuntu 12.04 (precise), 14.04 (trusty) and " \
        "14.10 (utopic) are currently supported" >&2
    exit 1
  fi

  if ! uname -m | egrep -q "i686|x86_64"; then
    echo "Only x86 architectures are currently supported" >&2
    exit
  fi
fi

if [ "x$(id -u)" != x0 ] && [ 0 -eq "${do_quick_check-0}" ]; then
  echo "Running as non-root user."
  echo "You might have to enter your password one or more times for 'sudo'."
  echo
fi

# Packages needed for chromeos only
chromeos_dev_list="libbluetooth-dev libxkbcommon-dev"

# Packages needed for development
dev_list="apache2.2-bin bison cdbs curl dpkg-dev elfutils devscripts fakeroot
          flex fonts-thai-tlwg g++ git-core git-svn gperf language-pack-da
          language-pack-fr language-pack-he language-pack-zh-hant
          libapache2-mod-php5 libasound2-dev libbrlapi-dev libav-tools
          libbz2-dev libcairo2-dev libcap-dev libcups2-dev libcurl4-gnutls-dev
          libdrm-dev libelf-dev libexif-dev libgconf2-dev libglib2.0-dev
          libglu1-mesa-dev libgnome-keyring-dev libgtk2.0-dev libkrb5-dev
          libnspr4-dev libnss3-dev libpam0g-dev libpci-dev libpulse-dev
          libsctp-dev libspeechd-dev libsqlite3-dev libssl-dev libudev-dev
          libwww-perl libxslt1-dev libxss-dev libxt-dev libxtst-dev openbox
          patch perl php5-cgi pkg-config python python-cherrypy3 python-crypto
          python-dev python-numpy python-opencv python-openssl python-psutil
          rpm ruby subversion ttf-dejavu-core ttf-indic-fonts ttf-kochi-gothic
          ttf-kochi-mincho wdiff xfonts-mathml zip $chromeos_dev_list"

# 64-bit systems need a minimum set of 32-bit compat packages for the pre-built
# NaCl binaries.
if file /sbin/init | grep -q 'ELF 64-bit'; then
  dev_list="${dev_list} libc6-i386 lib32gcc1 lib32stdc++6"
fi

# Run-time libraries required by chromeos only
chromeos_lib_list="libpulse0 libbz2-1.0"

# Full list of required run-time libraries
lib_list="libatk1.0-0 libc6 libasound2 libcairo2 libcap2 libcups2 libexpat1
          libexif12 libfontconfig1 libfreetype6 libglib2.0-0 libgnome-keyring0
          libgtk2.0-0 libpam0g libpango1.0-0 libpci3 libpcre3 libpixman-1-0
          libpng12-0 libspeechd2 libstdc++6 libsqlite3-0 libx11-6
          libxau6 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxdmcp6
          libxext6 libxfixes3 libxi6 libxinerama1 libxrandr2 libxrender1
          libxtst6 zlib1g $chromeos_lib_list"

# Debugging symbols for all of the run-time libraries
dbg_list="libatk1.0-dbg libc6-dbg libcairo2-dbg libfontconfig1-dbg
          libglib2.0-0-dbg libgtk2.0-0-dbg libpango1.0-0-dbg libpcre3-dbg
          libpixman-1-0-dbg libsqlite3-0-dbg libx11-6-dbg libxau6-dbg
          libxcb1-dbg libxcomposite1-dbg libxcursor1-dbg libxdamage1-dbg
          libxdmcp6-dbg libxext6-dbg libxfixes3-dbg libxi6-dbg libxinerama1-dbg
          libxrandr2-dbg libxrender1-dbg libxtst6-dbg zlib1g-dbg"

# Find the proper version of libstdc++6-4.x-dbg.
if [ "x$lsb_release" = "xprecise" ]; then
  dbg_list="${dbg_list} libstdc++6-4.6-dbg"
elif [ "x$lsb_release" = "xtrusty" ]; then
  dbg_list="${dbg_list} libstdc++6-4.8-dbg"
else
  dbg_list="${dbg_list} libstdc++6-4.9-dbg"
fi

# 32-bit libraries needed e.g. to compile V8 snapshot for Android or armhf
lib32_list="linux-libc-dev:i386"

# arm cross toolchain packages needed to build chrome on armhf
arm_list="libc6-dev-armhf-cross
          linux-libc-dev-armhf-cross
          g++-arm-linux-gnueabihf"

# Packages to build NaCl, its toolchains, and its ports.
naclports_list="ant autoconf bison cmake gawk intltool xutils-dev xsltproc"
nacl_list="g++-mingw-w64-i686 lib32z1-dev
           libasound2:i386 libcap2:i386 libelf-dev:i386 libexif12:i386
           libfontconfig1:i386 libgconf-2-4:i386 libglib2.0-0:i386 libgpm2:i386
           libgtk2.0-0:i386 libncurses5:i386 lib32ncurses5-dev
           libnss3:i386 libpango1.0-0:i386
           libssl1.0.0:i386 libtinfo-dev libtinfo-dev:i386 libtool
           libxcomposite1:i386 libxcursor1:i386 libxdamage1:i386 libxi6:i386
           libxrandr2:i386 libxss1:i386 libxtst6:i386 texinfo xvfb
           ${naclports_list}"

# Find the proper version of libgbm-dev. We can't just install libgbm-dev as
# it depends on mesa, and only one version of mesa can exists on the system.
# Hence we must match the same version or this entire script will fail.
mesa_variant=""
for variant in "-lts-trusty" "-lts-utopic"; do
  if $(dpkg-query -Wf'${Status}' libgl1-mesa-glx${variant} 2>/dev/null | \
       grep -q " ok installed"); then
    mesa_variant="${variant}"
  fi
done<commit_msg>Fix dependency issue with install-build-deps.sh on Trusty<commit_after>#!/bin/bash -e

# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Script to install everything needed to build chromium (well, ideally, anyway)
# See http://code.google.com/p/chromium/wiki/LinuxBuildInstructions
# and http://code.google.com/p/chromium/wiki/LinuxBuild64Bit

usage() {
  echo "Usage: $0 [--options]"
  echo "Options:"
  echo "--[no-]syms: enable or disable installation of debugging symbols"
  echo "--lib32: enable installation of 32-bit libraries, e.g. for V8 snapshot"
  echo "--[no-]arm: enable or disable installation of arm cross toolchain"
  echo "--[no-]chromeos-fonts: enable or disable installation of Chrome OS"\
       "fonts"
  echo "--[no-]nacl: enable or disable installation of prerequisites for"\
       "building standalone NaCl and all its toolchains"
  echo "--no-prompt: silently select standard options/defaults"
  echo "--quick-check: quickly try to determine if dependencies are installed"
  echo "               (this avoids interactive prompts and sudo commands,"
  echo "               so might not be 100% accurate)"
  echo "--unsupported: attempt installation even on unsupported systems"
  echo "Script will prompt interactively if options not given."
  exit 1
}

# Checks whether a particular package is available in the repos.
# USAGE: $ package_exists <package name>
package_exists() {
  apt-cache pkgnames | grep -x "$1" > /dev/null 2>&1
}

# These default to on because (some) bots need them and it keeps things
# simple for the bot setup if all bots just run the script in its default
# mode.  Developers who don't want stuff they don't need installed on their
# own workstations can pass --no-arm --no-nacl when running the script.
do_inst_arm=1
do_inst_nacl=1

while test "$1" != ""
do
  case "$1" in
  --syms)                   do_inst_syms=1;;
  --no-syms)                do_inst_syms=0;;
  --lib32)                  do_inst_lib32=1;;
  --arm)                    do_inst_arm=1;;
  --no-arm)                 do_inst_arm=0;;
  --chromeos-fonts)         do_inst_chromeos_fonts=1;;
  --no-chromeos-fonts)      do_inst_chromeos_fonts=0;;
  --nacl)                   do_inst_nacl=1;;
  --no-nacl)                do_inst_nacl=0;;
  --no-prompt)              do_default=1
                            do_quietly="-qq --assume-yes"
    ;;
  --quick-check)            do_quick_check=1;;
  --unsupported)            do_unsupported=1;;
  *) usage;;
  esac
  shift
done

if test "$do_inst_arm" = "1"; then
  do_inst_lib32=1
fi

# Check for lsb_release command in $PATH
if ! which lsb_release > /dev/null; then
  echo "ERROR: lsb_release not found in \$PATH" >&2
  exit 1;
fi

lsb_release=$(lsb_release --codename --short)
ubuntu_codenames="(precise|trusty|utopic)"
if [ 0 -eq "${do_unsupported-0}" ] && [ 0 -eq "${do_quick_check-0}" ] ; then
  if [[ ! $lsb_release =~ $ubuntu_codenames ]]; then
    echo "ERROR: Only Ubuntu 12.04 (precise), 14.04 (trusty) and " \
        "14.10 (utopic) are currently supported" >&2
    exit 1
  fi

  if ! uname -m | egrep -q "i686|x86_64"; then
    echo "Only x86 architectures are currently supported" >&2
    exit
  fi
fi

if [ "x$(id -u)" != x0 ] && [ 0 -eq "${do_quick_check-0}" ]; then
  echo "Running as non-root user."
  echo "You might have to enter your password one or more times for 'sudo'."
  echo
fi

# Packages needed for chromeos only
chromeos_dev_list="libbluetooth-dev libxkbcommon-dev"

# Packages needed for development
dev_list="apache2.2-bin bison cdbs curl dpkg-dev elfutils devscripts fakeroot
          flex fonts-thai-tlwg g++ git-core git-svn gperf language-pack-da
          language-pack-fr language-pack-he language-pack-zh-hant
          libapache2-mod-php5 libasound2-dev libbrlapi-dev libav-tools
          libbz2-dev libcairo2-dev libcap-dev libcups2-dev libcurl4-gnutls-dev
          libdrm-dev libelf-dev libexif-dev libgconf2-dev libglib2.0-dev
          libglu1-mesa-dev libgnome-keyring-dev libgtk2.0-dev libkrb5-dev
          libnspr4-dev libnss3-dev libpam0g-dev libpci-dev libpulse-dev
          libsctp-dev libspeechd-dev libsqlite3-dev libssl-dev libudev-dev
          libwww-perl libxslt1-dev libxss-dev libxt-dev libxtst-dev openbox
          patch perl php5-cgi pkg-config python python-cherrypy3 python-crypto
          python-dev python-numpy python-opencv python-openssl python-psutil
          rpm ruby subversion ttf-dejavu-core ttf-indic-fonts ttf-kochi-gothic
          ttf-kochi-mincho wdiff xfonts-mathml zip $chromeos_dev_list"

# 64-bit systems need a minimum set of 32-bit compat packages for the pre-built
# NaCl binaries.
if file /sbin/init | grep -q 'ELF 64-bit'; then
  dev_list="${dev_list} libc6-i386 lib32gcc1 lib32stdc++6"
fi

# Run-time libraries required by chromeos only
chromeos_lib_list="libpulse0 libbz2-1.0"

# Full list of required run-time libraries
lib_list="libatk1.0-0 libc6 libasound2 libcairo2 libcap2 libcups2 libexpat1
          libexif12 libfontconfig1 libfreetype6 libglib2.0-0 libgnome-keyring0
          libgtk2.0-0 libpam0g libpango1.0-0 libpci3 libpcre3 libpixman-1-0
          libpng12-0 libspeechd2 libstdc++6 libsqlite3-0 libx11-6
          libxau6 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxdmcp6
          libxext6 libxfixes3 libxi6 libxinerama1 libxrandr2 libxrender1
          libxtst6 zlib1g $chromeos_lib_list"

# Debugging symbols for all of the run-time libraries
dbg_list="libatk1.0-dbg libc6-dbg libcairo2-dbg libfontconfig1-dbg
          libglib2.0-0-dbg libgtk2.0-0-dbg libpango1.0-0-dbg libpcre3-dbg
          libpixman-1-0-dbg libsqlite3-0-dbg libx11-6-dbg libxau6-dbg
          libxcb1-dbg libxcomposite1-dbg libxcursor1-dbg libxdamage1-dbg
          libxdmcp6-dbg libxext6-dbg libxfixes3-dbg libxi6-dbg libxinerama1-dbg
          libxrandr2-dbg libxrender1-dbg libxtst6-dbg zlib1g-dbg"

# Find the proper version of libstdc++6-4.x-dbg.
if [ "x$lsb_release" = "xprecise" ]; then
  dbg_list="${dbg_list} libstdc++6-4.6-dbg"
elif [ "x$lsb_release" = "xtrusty" ]; then
  dbg_list="${dbg_list} libstdc++6-4.8-dbg"
else
  dbg_list="${dbg_list} libstdc++6-4.9-dbg"
fi

# 32-bit libraries needed e.g. to compile V8 snapshot for Android or armhf
lib32_list="linux-libc-dev:i386"

# arm cross toolchain packages needed to build chrome on armhf
arm_list="libc6-dev-armhf-cross
          linux-libc-dev-armhf-cross
          g++-arm-linux-gnueabihf"

# Work around for dependency issue Ubuntu/Trusty: http://crbug.com/435056
if [ "x$lsb_release" = "xtrusty" ]; then
  arm_list+=" g++-4.8-multilib-arm-linux-gnueabihf
              gcc-4.8-multilib-arm-linux-gnueabihf"
fi

# Packages to build NaCl, its toolchains, and its ports.
naclports_list="ant autoconf bison cmake gawk intltool xutils-dev xsltproc"
nacl_list="g++-mingw-w64-i686 lib32z1-dev
           libasound2:i386 libcap2:i386 libelf-dev:i386 libexif12:i386
           libfontconfig1:i386 libgconf-2-4:i386 libglib2.0-0:i386 libgpm2:i386
           libgtk2.0-0:i386 libncurses5:i386 lib32ncurses5-dev
           libnss3:i386 libpango1.0-0:i386
           libssl1.0.0:i386 libtinfo-dev libtinfo-dev:i386 libtool
           libxcomposite1:i386 libxcursor1:i386 libxdamage1:i386 libxi6:i386
           libxrandr2:i386 libxss1:i386 libxtst6:i386 texinfo xvfb
           ${naclports_list}"

# Find the proper version of libgbm-dev. We can't just install libgbm-dev as
# it depends on mesa, and only one version of mesa can exists on the system.
# Hence we must match the same version or this entire script will fail.
mesa_variant=""
for variant in "-lts-trusty" "-lts-utopic"; do
  if $(dpkg-query -Wf'${Status}' libgl1-mesa-glx${variant} 2>/dev/null | \
       grep -q " ok installed"); then
    mesa_variant="${variant}"
  fi
done
{'fraction_non_alphanumeric': 0.1027831715210356, 'fraction_numerical': 0.043624595469255664, 'mean_word_length': 3.052189876737477, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3920991', 'n_tokens_mistral': 5867, 'n_tokens_neox': 5441, 'n_words': 1484}