<commit_before>#!/usr/bin/python
"""Build thick images with AutoDMG and Munki.

With a given manifest and set of catalogs, parse all managed_installs and
incorporate them into the image.
"""

import argparse
import json
import os
import sys
import urllib2
import time

from autodmg_utility import run, build_pkg, populate_ds_repo
import autodmg_org

# Append munkilib to the Python path
with open('/private/etc/paths.d/munki', 'rb') as f:
  munkipath = f.read().strip()
sys.path.append(os.path.join(munkipath, 'munkilib'))
try:
  import FoundationPlist as plistlib
except ImportError:
  print "Using plistlib"
  import plistlib
try:
  from munkicommon import pref, getsha256hash
  import updatecheck
  from fetch import (getURLitemBasename, getResourceIfChangedAtomically,
                     MunkiDownloadError, writeCachedChecksum,
                     getxattr, XATTR_SHA)
  import keychain
except ImportError as err:
  print "Something went wrong! %s" % err

MUNKI_URL = pref('SoftwareRepoURL')
PKGS_URL = MUNKI_URL + '/pkgs'
ICONS_URL = MUNKI_URL + '/icons'
BASIC_AUTH = pref('AdditionalHttpHeaders')
CACHE = '/tmp'


# download functions
def download_url_to_cache(url, cache, force=False):
  """Take a URL and downloads it to a local cache."""
  cache_path = os.path.join(cache, urllib2.unquote(getURLitemBasename(url)))
  custom_headers = ['']
  if BASIC_AUTH:
    # custom_headers = ['Authorization: Basic %s' % BASIC_AUTH]
    custom_headers = BASIC_AUTH
  if force:
    return getResourceIfChangedAtomically(
      url, cache_path,
      custom_headers=custom_headers,
      resume=True,
      expected_hash='no')
  return getResourceIfChangedAtomically(
    url, cache_path, custom_headers=custom_headers)


def handle_dl(item_name, item_url, download_dir,
              force_download):
  """Download an item into the cache, returns True if downloaded."""
  try:
    print "Downloading into %s: %s" % (download_dir, item_name)
    changed = download_url_to_cache(
      item_url,
      download_dir,
      force_download
    )
    if not changed:
      print "Found in cache"
    return True
  except MunkiDownloadError as err:
    print >> sys.stderr, "Download error for %s: %s" % (item_name, err)
  return False


# item functions
# Based on updatecheck.py but modified for simpler use
def get_item_url(item):
  """Take an item dict, return the URL it can be downloaded from."""
  return PKGS_URL + '/' + urllib2.quote(item["installer_item_location"])


def download_icons(item_list, icon_dir):
  """Download icons for items in the list.

  Based on updatecheck.py, modified.
  Copied from
  https://github.com/munki/munki/blob/master/code/client/munkilib/updatecheck.py#L2824

  Attempts to download icons (actually png files) for items in
     item_list
  """
  icon_list = []
  icon_known_exts = ['.bmp', '.gif', '.icns', '.jpg', '.jpeg', '.png', '.psd',
                     '.tga', '.tif', '.tiff', '.yuv']
  icon_base_url = (pref('IconURL') or
                   pref('SoftwareRepoURL') + '/icons/')
  icon_base_url = icon_base_url.rstrip('/') + '/'
  for item in item_list:
    icon_name = item.get('icon_name') or item['name']
    pkginfo_icon_hash = item.get('icon_hash')
    if not os.path.splitext(icon_name)[1] in icon_known_exts:
      icon_name += '.png'
    icon_list.append(icon_name)
    icon_url = icon_base_url + urllib2.quote(icon_name.encode('UTF-8'))
    icon_path = os.path.join(icon_dir, icon_name)
    if os.path.isfile(icon_path):
      xattr_hash = getxattr(icon_path, XATTR_SHA)
      if not xattr_hash:
        xattr_hash = getsha256hash(icon_path)
        writeCachedChecksum(icon_path, xattr_hash)
    else:
      xattr_hash = 'nonexistent'
    icon_subdir = os.path.dirname(icon_path)
    if not os.path.exists(icon_subdir):
      try:
          os.makedirs(icon_subdir, 0755)
      except OSError, err:
          print 'Could not create %s' % icon_subdir
          continue
    custom_headers = ['']
    if BASIC_AUTH:
      # custom_headers = ['Authorization: Basic %s' % BASIC_AUTH]
      custom_headers = BASIC_AUTH
    if pkginfo_icon_hash != xattr_hash:
      item_name = item.get('display_name') or item['name']
      message = 'Getting icon %s for %s...' % (icon_name, item_name)
      try:
        dummy_value = getResourceIfChangedAtomically(
          icon_url, icon_path, custom_headers=custom_headers, message=message)
      except MunkiDownloadError, err:
        print ('Could not retrieve icon %s from the server: %s',
               icon_name, err)
      else:
        if os.path.isfile(icon_path):
            writeCachedChecksum(icon_path)


def handle_icons(icon_dir, installinfo):
  """Download icons and build the package."""
  print "Downloading icons."
  pkg_output_file = os.path.join(CACHE, 'munki_icons.pkg')
  icon_list = list(installinfo.get('optional_installs', []))
  icon_list.extend(installinfo['managed_installs'])
  icon_list.extend(installinfo['removals'])
  # Downloads all icons into the icon directory in the Munki cache
  download_icons(icon_list, icon_dir)

  # Build a package of optional Munki icons, so we don't need to cache
  success = build_pkg(
    icon_dir,
    'munki_icons',
    'com.facebook.cpe.munki_icons',
    '/Library/Managed Installs/icons',
    CACHE,
    'Creating the icon package.'
  )
  # Add the icon package to the additional packages list for the template.<commit_msg>We're always going to get an optionals list now<commit_after>#!/usr/bin/python
"""Build thick images with AutoDMG and Munki.

With a given manifest and set of catalogs, parse all managed_installs and
incorporate them into the image.
"""

import argparse
import json
import os
import sys
import urllib2
import time

from autodmg_utility import run, build_pkg, populate_ds_repo
import autodmg_org

# Append munkilib to the Python path
with open('/private/etc/paths.d/munki', 'rb') as f:
  munkipath = f.read().strip()
sys.path.append(os.path.join(munkipath, 'munkilib'))
try:
  import FoundationPlist as plistlib
except ImportError:
  print "Using plistlib"
  import plistlib
try:
  from munkicommon import pref, getsha256hash
  import updatecheck
  from fetch import (getURLitemBasename, getResourceIfChangedAtomically,
                     MunkiDownloadError, writeCachedChecksum,
                     getxattr, XATTR_SHA)
  import keychain
except ImportError as err:
  print "Something went wrong! %s" % err

MUNKI_URL = pref('SoftwareRepoURL')
PKGS_URL = MUNKI_URL + '/pkgs'
ICONS_URL = MUNKI_URL + '/icons'
BASIC_AUTH = pref('AdditionalHttpHeaders')
CACHE = '/tmp'


# download functions
def download_url_to_cache(url, cache, force=False):
  """Take a URL and downloads it to a local cache."""
  cache_path = os.path.join(cache, urllib2.unquote(getURLitemBasename(url)))
  custom_headers = ['']
  if BASIC_AUTH:
    # custom_headers = ['Authorization: Basic %s' % BASIC_AUTH]
    custom_headers = BASIC_AUTH
  if force:
    return getResourceIfChangedAtomically(
      url, cache_path,
      custom_headers=custom_headers,
      resume=True,
      expected_hash='no')
  return getResourceIfChangedAtomically(
    url, cache_path, custom_headers=custom_headers)


def handle_dl(item_name, item_url, download_dir,
              force_download):
  """Download an item into the cache, returns True if downloaded."""
  try:
    print "Downloading into %s: %s" % (download_dir, item_name)
    changed = download_url_to_cache(
      item_url,
      download_dir,
      force_download
    )
    if not changed:
      print "Found in cache"
    return True
  except MunkiDownloadError as err:
    print >> sys.stderr, "Download error for %s: %s" % (item_name, err)
  return False


# item functions
# Based on updatecheck.py but modified for simpler use
def get_item_url(item):
  """Take an item dict, return the URL it can be downloaded from."""
  return PKGS_URL + '/' + urllib2.quote(item["installer_item_location"])


def download_icons(item_list, icon_dir):
  """Download icons for items in the list.

  Based on updatecheck.py, modified.
  Copied from
  https://github.com/munki/munki/blob/master/code/client/munkilib/updatecheck.py#L2824

  Attempts to download icons (actually png files) for items in
     item_list
  """
  icon_list = []
  icon_known_exts = ['.bmp', '.gif', '.icns', '.jpg', '.jpeg', '.png', '.psd',
                     '.tga', '.tif', '.tiff', '.yuv']
  icon_base_url = (pref('IconURL') or
                   pref('SoftwareRepoURL') + '/icons/')
  icon_base_url = icon_base_url.rstrip('/') + '/'
  for item in item_list:
    icon_name = item.get('icon_name') or item['name']
    pkginfo_icon_hash = item.get('icon_hash')
    if not os.path.splitext(icon_name)[1] in icon_known_exts:
      icon_name += '.png'
    icon_list.append(icon_name)
    icon_url = icon_base_url + urllib2.quote(icon_name.encode('UTF-8'))
    icon_path = os.path.join(icon_dir, icon_name)
    if os.path.isfile(icon_path):
      xattr_hash = getxattr(icon_path, XATTR_SHA)
      if not xattr_hash:
        xattr_hash = getsha256hash(icon_path)
        writeCachedChecksum(icon_path, xattr_hash)
    else:
      xattr_hash = 'nonexistent'
    icon_subdir = os.path.dirname(icon_path)
    if not os.path.exists(icon_subdir):
      try:
          os.makedirs(icon_subdir, 0755)
      except OSError, err:
          print 'Could not create %s' % icon_subdir
          continue
    custom_headers = ['']
    if BASIC_AUTH:
      # custom_headers = ['Authorization: Basic %s' % BASIC_AUTH]
      custom_headers = BASIC_AUTH
    if pkginfo_icon_hash != xattr_hash:
      item_name = item.get('display_name') or item['name']
      message = 'Getting icon %s for %s...' % (icon_name, item_name)
      try:
        dummy_value = getResourceIfChangedAtomically(
          icon_url, icon_path, custom_headers=custom_headers, message=message)
      except MunkiDownloadError, err:
        print ('Could not retrieve icon %s from the server: %s',
               icon_name, err)
      else:
        if os.path.isfile(icon_path):
            writeCachedChecksum(icon_path)


def handle_icons(icon_dir, installinfo):
  """Download icons and build the package."""
  print "Downloading icons."
  pkg_output_file = os.path.join(CACHE, 'munki_icons.pkg')
  icon_list = installinfo['optional_installs']
  icon_list.extend(installinfo['managed_installs'])
  icon_list.extend(installinfo['removals'])
  # Downloads all icons into the icon directory in the Munki cache
  download_icons(icon_list, icon_dir)

  # Build a package of optional Munki icons, so we don't need to cache
  success = build_pkg(
    icon_dir,
    'munki_icons',
    'com.facebook.cpe.munki_icons',
    '/Library/Managed Installs/icons',
    CACHE,
    'Creating the icon package.'
  )
  # Add the icon package to the additional packages list for the template.
{'fraction_non_alphanumeric': 0.10504357500463564, 'fraction_numerical': 0.0037085110328203227, 'mean_word_length': 3.545722713864307, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7049066', 'n_tokens_mistral': 3617, 'n_tokens_neox': 3521, 'n_words': 927}