diff --git a/.gitignore b/.gitignore index 4a4a6c0515c95a26a3ea31a3aba39efae4e07a20..6c439ac3042990d7724bd6f8a13fbf15560264dd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,11 @@ *.swp *.swo *.whl +.DS_Store +*.pyc +dmg/ +ovf/ +ova/ +xcode/ +packer_cache/ install/bob-devel-py*.yml diff --git a/macos-ci-vm/README.rst b/macos-ci-vm/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..bfa82dea14585ccc9f0236f59387cbe8d99e32a2 --- /dev/null +++ b/macos-ci-vm/README.rst @@ -0,0 +1,49 @@ +-------------------------------- + MacOS CI Virtual Image Builder +-------------------------------- + +This package contains a script that can build provisioned virtual box images +that contain a minimal installation of macOS for CI purposes. To build such +images you'll need: + +* A computer running macOS (you can only virtualize macOS on another computer + running macOS) +* A recent version of Packer_ +* The app for the OS you'll want to install. You can obtain installers for most + macOS versions through the AppStore. Just install the app through the + AppStore on the computer running this script. +* The `Xcode SDK`_ you'll need installed on the image, downloaded to the + ``xcode`` subdirectory of this package. For example, to download the latest + release of macOS 10.9 (Mavericks) SDK, do this:: + + $ mkdir xcode && cd xcode + $ wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.13/MacOSX10.9.sdk.tar.xz + +Once all is in place, run the ``build.sh`` command, passing the version of +macOS you wish to build an image for:: + + $ ./build.sh 10.9 + + +The build process is divided in 3 (cached) stages: + +* Conversion of the macOS installation app into a bootable DMG +* Creation of a OVF file with the base macOS installation +* Creation of the final OVA file with the provisioned macOS installation. The + provisioning consists of the execution, in sequence, of all scripts in the + ``scripts`` directory. + +The virtual image, when ready, will be available at the ``ova`` directory. + + +Deployment +========== + +To deploy the newly created virtual image, follow the procedure on configuring +a `VirtualBox Executor`_ for gitlab CI. + + +.. Links here +.. _packer: http://packer.io +.. _xcode sdk: https://github.com/phracker/MacOSX-SDKs +.. _virtualbox executor: https://docs.gitlab.com/runner/executors/virtualbox.html diff --git a/macos-ci-vm/build.sh b/macos-ci-vm/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..62f72ac2d4242d78bca588177fcf2311e38f3c36 --- /dev/null +++ b/macos-ci-vm/build.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +# Original package: https://github.com/timsutton/osx-vm-templates + +# This script will use the setup in this package to create a virtual image for +# macOS following the specifications of the provided version number. +# The build happens in 3 stages: +# +# 1. An installable DMG is created from the original Apple installation app +# 2. A basic OVF image containing the installed ISO is generated +# 3. The final OVA image is created by provisioning the OVF image +# +# It is done in 3 stages so that provisioning can be tested in a faster test +# loop given points 1 and 2 can be very long. + +if [[ "$#" == "0" ]]; then + echo "usage: $0 <macos-version>" + echo "example: $0 10.9" + exit 1 +fi + +# Edit this part and modify it as you see fit +username=gitlab +password=gitlab +curdir=`pwd` +dmgdir="${curdir}/dmg" +ovfdir="${curdir}/ovf" +ovadir="${curdir}/ova" + +# Expected app and image names (cannot really guess - wait for outputs and set +# as appropriate) +if [[ "$1" == "10.9" ]]; then + app="/Applications/Install OS X Mavericks.app" + dmg="${dmgdir}/OSX_InstallESD_10.9.5_13F34.dmg" +else + echo "Version $1 is currently unsupported" + exit 1 +fi + +short_ver=${1/./} +xcode_sdk_version=${1} +machine_name="macos${short_ver}" +guest_os_type="MacOS${short_ver}_64" +ovf="${ovfdir}/${machine_name}.ovf" +ova="${ovadir}/${machine_name}.ova" + +echo "Building virtualbox machine for macOS $1..." +echo "username = ${username}" +echo "password = ${password}" +echo "app = ${app}" +echo "os type = ${guest_os_type}" +echo "name = ${machine_name}" +echo "xcode SDK = ${xcode_sdk_version}" +echo "dmg = ${dmg}" +echo "ovf = ${ovf}" +echo "ova = ${ova}" + +if [ ! -r "${dmg}" ]; then + echo "Stage 1: [$(basename ${app})] -> [$(basename ${dmg})]" + sudo prepare_iso/prepare_iso.sh \ + -u ${username} \ + -p ${password} \ + -i prepare_iso/support/gitlab.jpg \ + -D DISABLE_REMOTE_MANAGEMENT \ + "${app}" "${dmgdir}" +else + echo "Skipping stage 1: [$(basename ${dmg})] exists. Remove to force rebuild." +fi + +if [ ! -r "${ovf}" ]; then + echo "Stage 2: [$(basename ${dmg})] -> [$(basename ${ovf})]" + packer build \ + -on-error=ask \ + -only virtualbox-iso \ + -var dmg_url="${dmg}" \ + -var guest_os_type="${guest_os_type}" \ + -var output_directory="${ovfdir}" \ + -var machine_name="${machine_name}" \ + -var username="${username}" \ + -var password="${password}" \ + packer/ovf.json + if [ -d packer_cache ]; then + rmdir packer_cache #normally empty + fi +else + echo "Skipping stage 2: [$(basename ${ovf})] exists. Remove to force rebuild." +fi + +if [ ! -r "${ova}" ]; then + echo "Stage 3: [$(basename ${ovf})] -> provision -> [$(basename ${ova})]" + packer build \ + -on-error=ask \ + -only virtualbox-ovf \ + -var source_path="${ovf}" \ + -var guest_os_type="${guest_os_type}" \ + -var output_directory="${ovadir}" \ + -var machine_name="${machine_name}" \ + -var xcode_sdk_version="${xcode_sdk_version}" \ + -var username="${username}" \ + -var password="${password}" \ + -var provisioning_delay=30 \ + packer/provision.json + if [ -d packer_cache ]; then + rmdir packer_cache #normally empty + fi +else + echo "Skipping stage 3: [$(basename ${ova})] exists. Remove to force rebuild." +fi + +echo "Note: To be able to SSH into the host, since it is configure for NAT, follow" +echo "instructions at: https://forums.virtualbox.org/viewtopic.php?f=8&t=55766" diff --git a/macos-ci-vm/packer/ovf.json b/macos-ci-vm/packer/ovf.json new file mode 100644 index 0000000000000000000000000000000000000000..ee6cc7bac95f897146a7f5eb6c885d23c7f714b4 --- /dev/null +++ b/macos-ci-vm/packer/ovf.json @@ -0,0 +1,44 @@ +{ + "builders": [ + { + "boot_wait": "2s", + "disk_size": 40960, + "guest_additions_mode": "disable", + "guest_os_type": "{{user `guest_os_type`}}", + "hard_drive_interface": "sata", + "iso_checksum_type": "none", + "iso_interface": "sata", + "iso_url": "{{user `dmg_url`}}", + "output_directory": "{{user `output_directory`}}", + "vm_name": "{{ user `machine_name`}}", + "shutdown_command": "echo '{{user `username`}}'|sudo -S shutdown -h now", + "ssh_port": 22, + "ssh_username": "{{user `username`}}", + "ssh_password": "{{user `password`}}", + "ssh_wait_timeout": "10000s", + "type": "virtualbox-iso", + "vboxmanage": [ + ["modifyvm", "{{.Name}}", "--audiocontroller", "hda"], + ["modifyvm", "{{.Name}}", "--boot1", "dvd"], + ["modifyvm", "{{.Name}}", "--boot2", "disk"], + ["modifyvm", "{{.Name}}", "--chipset", "ich9"], + ["modifyvm", "{{.Name}}", "--firmware", "efi"], + ["modifyvm", "{{.Name}}", "--hpet", "on"], + ["modifyvm", "{{.Name}}", "--keyboard", "usb"], + ["modifyvm", "{{.Name}}", "--memory", "2048"], + ["modifyvm", "{{.Name}}", "--mouse", "usbtablet"], + ["modifyvm", "{{.Name}}", "--vram", "128"], + ["storagectl", "{{.Name}}", "--name", "IDE Controller", "--remove"] + ] + } + ], + "min_packer_version": "0.7.0", + "variables": { + "dmg_url": "dmg/OSX_InstallESD_10.9.5_13F34.dmg", + "guest_os_type": "MacOS109_64", + "machine_name": "macos109", + "output_directory": "ovf", + "password": "gitlab", + "username": "gitlab" + } +} diff --git a/macos-ci-vm/packer/provision.json b/macos-ci-vm/packer/provision.json new file mode 100644 index 0000000000000000000000000000000000000000..486b8bbcf03281cf98417967924c8563573fb684 --- /dev/null +++ b/macos-ci-vm/packer/provision.json @@ -0,0 +1,80 @@ +{ + "builders": [ + { + "boot_wait": "2s", + "guest_additions_mode": "disable", + "source_path": "{{user `source_path`}}", + "shutdown_command": "echo '{{user `username`}}'|sudo -S shutdown -h now", + "ssh_port": 22, + "ssh_username": "{{user `username`}}", + "ssh_password": "{{user `password`}}", + "ssh_wait_timeout": "100s", + "type": "virtualbox-ovf", + "output_directory": "{{user `output_directory`}}", + "vm_name": "{{ user `machine_name`}}", + "format": "ova", + "vboxmanage": [ + ["modifyvm", "{{.Name}}", "--audiocontroller", "hda"], + ["modifyvm", "{{.Name}}", "--chipset", "ich9"], + ["modifyvm", "{{.Name}}", "--firmware", "efi"], + ["modifyvm", "{{.Name}}", "--hpet", "on"], + ["modifyvm", "{{.Name}}", "--keyboard", "usb"], + ["modifyvm", "{{.Name}}", "--memory", "4096"], + ["modifyvm", "{{.Name}}", "--mouse", "usbtablet"], + ["modifyvm", "{{.Name}}", "--vram", "128"] + ] + } + ], + "min_packer_version": "0.7.0", + "provisioners": [ + { + "type": "shell-local", + "command": "sleep {{user `provisioning_delay`}}" + }, + { + "destination": "/private/tmp/set_kcpassword.py", + "source": "scripts/support/set_kcpassword.py", + "type": "file" + }, + { + "destination": "/private/tmp/set_kcpassword.py", + "source": "scripts/support/set_kcpassword.py", + "type": "file" + }, + { + "destination": "/tmp", + "source": "xcode", + "type": "file" + }, + { + "execute_command": "chmod +x {{ .Path }}; sudo {{ .Vars }} {{ .Path }}", + "scripts": [ + "scripts/xcode-cli-tools.sh", + "scripts/xcode-sdk.sh", + "scripts/homebrew.sh", + "scripts/add-network-interface-detection.sh", + "scripts/autologin.sh", + "scripts/system-update.sh", + "scripts/optimize.sh", + "scripts/shrink.sh" + ], + "environment_vars": [ + "XCODE_SDK_VERSION={{user `xcode_sdk_version`}}", + "PASSWORD={{user `password`}}", + "USERNAME={{user `username`}}", + "MACHINE_NAME={{user `machine_name`}}" + ], + "type": "shell" + } + ], + "variables": { + "source_path": "ovf/macos109.ovf", + "guest_os_type": "MacOS109_64", + "machine_name": "macos109", + "xcode_sdk_version": "10.9", + "output_directory": "ova", + "password": "gitlab", + "username": "gitlab", + "provisioning_delay": "0" + } +} diff --git a/macos-ci-vm/prepare_iso/create_firstboot_pkg.sh b/macos-ci-vm/prepare_iso/create_firstboot_pkg.sh new file mode 100644 index 0000000000000000000000000000000000000000..bc298e4f049bf42d8322e033e55b840f56f88abc --- /dev/null +++ b/macos-ci-vm/prepare_iso/create_firstboot_pkg.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Handle the possibility of being sourced from outside this project dir +called=$_ +if [[ $called != "${0}" ]]; then + SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")" +else + SCRIPT_DIR="$(cd "$(dirname "$0")" || exit; pwd)" +fi +SUPPORT_DIR="$SCRIPT_DIR/support" + +# Parse the optional command line switches +USER="vagrant" +PASSWORD="vagrant" +IMAGE_PATH="$SUPPORT_DIR/vagrant.jpg" + +# Flags +DISABLE_REMOTE_MANAGEMENT=0 +DISABLE_SCREEN_SHARING=0 +DISABLE_SIP=0 + +render_template() { + eval "echo \"$(cat "$1")\"" +} + +create_firstboot_pkg() { + # payload items + mkdir -p "$SUPPORT_DIR/pkgroot/private/var/db/dslocal/nodes/Default/users" + mkdir -p "$SUPPORT_DIR/pkgroot/private/var/db/shadow/hash" + BASE64_IMAGE=$(openssl base64 -in "$IMAGE_PATH") + ShadowHashData=$($SCRIPT_DIR/../scripts/support/generatehash.py "$PASSWORD") + # Replace USER and BASE64_IMAGE in the user.plist file with the actual user and image + render_template "$SUPPORT_DIR/user.plist" > "$SUPPORT_DIR/pkgroot/private/var/db/dslocal/nodes/Default/users/$USER.plist" + USER_GUID=$(/usr/libexec/PlistBuddy -c 'Print :generateduid:0' "$SUPPORT_DIR/user.plist") + # Generate a shadowhash from the supplied password + "$SUPPORT_DIR/generate_shadowhash" "$PASSWORD" > "$SUPPORT_DIR/pkgroot/private/var/db/shadow/hash/$USER_GUID" + + # postinstall script + mkdir -p "$SUPPORT_DIR/tmp/Scripts" + cat "$SUPPORT_DIR/pkg-postinstall" \ + | sed -e "s/__USER__PLACEHOLDER__/${USER}/" \ + | sed -e "s/__DISABLE_REMOTE_MANAGEMENT__/${DISABLE_REMOTE_MANAGEMENT}/" \ + | sed -e "s/__DISABLE_SCREEN_SHARING__/${DISABLE_SCREEN_SHARING}/" \ + | sed -e "s/__DISABLE_SIP__/${DISABLE_SIP}/" \ + > "$SUPPORT_DIR/tmp/Scripts/postinstall" + chmod a+x "$SUPPORT_DIR/tmp/Scripts/postinstall" + + # build it + BUILT_COMPONENT_PKG="$SUPPORT_DIR/tmp/veewee-config-component.pkg" + BUILT_PKG="$SUPPORT_DIR/tmp/veewee-config.pkg" + pkgbuild --quiet \ + --root "$SUPPORT_DIR/pkgroot" \ + --scripts "$SUPPORT_DIR/tmp/Scripts" \ + --identifier com.vagrantup.veewee-config \ + --version 0.1 \ + "$BUILT_COMPONENT_PKG" + productbuild \ + --package "$BUILT_COMPONENT_PKG" \ + "$BUILT_PKG" + rm -rf "$SUPPORT_DIR/pkgroot" +} diff --git a/macos-ci-vm/prepare_iso/prepare_iso.sh b/macos-ci-vm/prepare_iso/prepare_iso.sh new file mode 100755 index 0000000000000000000000000000000000000000..e57d6e5fb76bd4c9fcf7fb6c10a3dcbbbac12513 --- /dev/null +++ b/macos-ci-vm/prepare_iso/prepare_iso.sh @@ -0,0 +1,320 @@ +#!/bin/sh -e +# +# Preparation script for an OS X automated installation for use with VeeWee/Packer/Vagrant +# +# What the script does, in more detail: +# +# 1. Mounts the InstallESD.dmg using a shadow file, so the original DMG is left +# unchanged. +# 2. Modifies the BaseSystem.dmg within in order to add an additional 'rc.cdrom.local' +# file in /etc, which is a supported local configuration sourced in at boot time +# by the installer environment. This file contains instructions to erase and format +# 'disk0', presumably the hard disk attached to the VM. +# 3. A 'veewee-config.pkg' installer package is built, which is added to the OS X +# install by way of the OSInstall.collection file. This package creates the +# 'gitlab' user, configures sshd and sudoers, and disables setup assistants. +# 4. veewee-config.pkg and the various support utilities are copied, and the disk +# image is saved to the output path. +# +# Thanks: +# Idea and much of the implementation thanks to Pepijn Bruienne, who's also provided +# some process notes here: https://gist.github.com/4542016. The sample minstallconfig.xml, +# use of OSInstall.collection and readme documentation provided with Greg Neagle's +# createOSXInstallPkg tool also proved very helpful. (http://code.google.com/p/munki/wiki/InstallingOSX) +# +# User creation via package install method also credited to Greg, and made easy with Per +# Olofsson's CreateUserPkg (http://magervalp.github.io/CreateUserPkg) +# +# Antony Blakey for updates to support OS X 10.11: +# https://github.com/timsutton/osx-vm-templates/issues/40 + +usage() { + cat <<EOF +Usage: +$(basename "$0") [-upiD] "/path/to/InstallESD.dmg" /path/to/output/directory +$(basename "$0") [-upiD] "/path/to/Install OS X [Name].app" /path/to/output/directory + +Description: +Converts an OS X installer to a new image that contains components +used to perform an automated installation. The new image will be named +'OSX_InstallESD_[osversion].dmg.' + +Optional switches: + -u <user> + Sets the username of the root user, defaults to 'gitlab'. + + -p <password> + Sets the password of the root user, defaults to 'gitlab'. + + -i <path to image> + Sets the path of the avatar image for the root user, defaulting to the gitlab icon. + + -D <flag> + Sets the specified flag. Valid flags are: + DISABLE_REMOTE_MANAGEMENT + DISABLE_SCREEN_SHARING + DISABLE_SIP + +EOF +} + +cleanup() { + hdiutil detach -quiet -force "$MNT_ESD" || echo > /dev/null + hdiutil detach -quiet -force "$MNT_BASE_SYSTEM" || echo > /dev/null + rm -rf "$MNT_ESD" "$MNT_BASE_SYSTEM" "$BASE_SYSTEM_DMG_RW" "$SHADOW_FILE" +} + +trap cleanup EXIT INT TERM + + +msg_status() { + echo "\033[0;32m-- $1\033[0m" +} +msg_error() { + echo "\033[0;31m-- $1\033[0m" +} + +if [ $# -eq 0 ]; then + usage + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "$0")"; pwd)" +SUPPORT_DIR="$SCRIPT_DIR/support" + +# Import shared code to generate first boot pkgs +FIRST_BOOT_PKG_SCRIPT="$SCRIPT_DIR/create_firstboot_pkg.sh" +[ -f "$FIRST_BOOT_PKG_SCRIPT" ] && . "$FIRST_BOOT_PKG_SCRIPT" + +# Parse the optional command line switches +USER="gitlab" +PASSWORD="gitlab" +IMAGE_PATH="$SUPPORT_DIR/gitlab.jpg" + +# Flags +DISABLE_REMOTE_MANAGEMENT=0 +DISABLE_SCREEN_SHARING=0 +DISABLE_SIP=0 + +while getopts u:p:i:D: OPT; do + case "$OPT" in + u) + USER="$OPTARG" + ;; + p) + PASSWORD="$OPTARG" + ;; + i) + IMAGE_PATH="$OPTARG" + ;; + D) + if [ x${!OPTARG} = x0 ]; then + eval $OPTARG=1 + elif [ x${!OPTARG} != x1 ]; then + msg_error "Unknown flag: ${OPTARG}" + usage + exit 1 + fi + ;; + \?) + usage + exit 1 + ;; + esac +done + +# Remove the switches we parsed above. +shift $(expr $OPTIND - 1) + +if [ $(id -u) -ne 0 ]; then + msg_error "This script must be run as root, as it saves a disk image with ownerships enabled." + exit 1 +fi + +ESD="$1" +if [ ! -e "$ESD" ]; then + msg_error "Input installer image $ESD could not be found! Exiting.." + exit 1 +fi + +if [ -d "$ESD" ]; then + # we might be an install .app + if [ -e "$ESD/Contents/SharedSupport/InstallESD.dmg" ]; then + ESD="$ESD/Contents/SharedSupport/InstallESD.dmg" + else + msg_error "Can't locate an InstallESD.dmg in this source location $ESD!" + fi +fi + +VEEWEE_DIR="$(cd "$SCRIPT_DIR/../../../"; pwd)" +VEEWEE_UID=$(/usr/bin/stat -f %u "$VEEWEE_DIR") +VEEWEE_GID=$(/usr/bin/stat -f %g "$VEEWEE_DIR") +DEFINITION_DIR="$(cd "$SCRIPT_DIR/.."; pwd)" + +if [ "$2" = "" ]; then + msg_error "Currently an explicit output directory is required as the second argument." + exit 1 + # The rest is left over from the old prepare_veewee_iso.sh script. Not sure if we + # should leave in this functionality to automatically locate the veewee directory. + DEFAULT_ISO_DIR=1 + OLDPWD=$(pwd) + cd "$SCRIPT_DIR" + # default to the veewee/iso directory + if [ ! -d "../../../iso" ]; then + mkdir "../../../iso" + chown $VEEWEE_UID:$VEEWEE_GID "../../../iso" + fi + OUT_DIR="$(cd "$SCRIPT_DIR"; cd ../../../iso; pwd)" + cd "$OLDPWD" # Rest of script depends on being in the working directory if we were passed relative paths +else + OUT_DIR="$2" +fi + +if [ ! -d "$OUT_DIR" ]; then + msg_status "Destination dir $OUT_DIR doesn't exist, creating.." + mkdir -p "$OUT_DIR" +fi + +if [ -e "$ESD.shadow" ]; then + msg_status "Removing old shadow file.." + rm "$ESD.shadow" +fi + +MNT_ESD=$(/usr/bin/mktemp -d /tmp/veewee-osx-esd.XXXX) +SHADOW_FILE=$(/usr/bin/mktemp /tmp/veewee-osx-shadow.XXXX) +rm "$SHADOW_FILE" +msg_status "Attaching input OS X installer image with shadow file.." +hdiutil attach "$ESD" -mountpoint "$MNT_ESD" -shadow "$SHADOW_FILE" -nobrowse -owners on +if [ $? -ne 0 ]; then + [ ! -e "$ESD" ] && msg_error "Could not find $ESD in $(pwd)" + msg_error "Could not mount $ESD on $MNT_ESD" + exit 1 +fi + +msg_status "Mounting BaseSystem.." +BASE_SYSTEM_DMG="$MNT_ESD/BaseSystem.dmg" +MNT_BASE_SYSTEM=$(/usr/bin/mktemp -d /tmp/veewee-osx-basesystem.XXXX) +[ ! -e "$BASE_SYSTEM_DMG" ] && msg_error "Could not find BaseSystem.dmg in $MNT_ESD" +hdiutil attach "$BASE_SYSTEM_DMG" -mountpoint "$MNT_BASE_SYSTEM" -nobrowse -owners on +if [ $? -ne 0 ]; then + msg_error "Could not mount $BASE_SYSTEM_DMG on $MNT_BASE_SYSTEM" + exit 1 +fi +SYSVER_PLIST_PATH="$MNT_BASE_SYSTEM/System/Library/CoreServices/SystemVersion.plist" + +DMG_OS_VERS=$(/usr/libexec/PlistBuddy -c 'Print :ProductVersion' "$SYSVER_PLIST_PATH") +DMG_OS_VERS_MAJOR=$(echo $DMG_OS_VERS | awk -F "." '{print $2}') +DMG_OS_VERS_MINOR=$(echo $DMG_OS_VERS | awk -F "." '{print $3}') +DMG_OS_BUILD=$(/usr/libexec/PlistBuddy -c 'Print :ProductBuildVersion' "$SYSVER_PLIST_PATH") +msg_status "OS X version detected: 10.$DMG_OS_VERS_MAJOR.$DMG_OS_VERS_MINOR, build $DMG_OS_BUILD" + +OUTPUT_DMG="$OUT_DIR/OSX_InstallESD_${DMG_OS_VERS}_${DMG_OS_BUILD}.dmg" +if [ -e "$OUTPUT_DMG" ]; then + msg_error "Output file $OUTPUT_DMG already exists! We're not going to overwrite it, exiting.." + hdiutil detach -force "$MNT_ESD" + exit 1 +fi + +# Build our post-installation pkg that will create a user and enable ssh +msg_status "Making firstboot installer pkg.." +create_firstboot_pkg +if [ -z "$BUILT_PKG" ] || [ ! -e "$BUILT_PKG" ]; then + msg_error "Failed building the firstboot installer pkg, exiting.." + exit 1 +fi + +# We'd previously mounted this to check versions +hdiutil detach "$MNT_BASE_SYSTEM" + +BASE_SYSTEM_DMG_RW="$(/usr/bin/mktemp /tmp/veewee-osx-basesystem-rw.XXXX).dmg" + +msg_status "Creating empty read-write DMG located at $BASE_SYSTEM_DMG_RW.." +hdiutil create -o "$BASE_SYSTEM_DMG_RW" -size 10g -layout SPUD -fs HFS+J +hdiutil attach "$BASE_SYSTEM_DMG_RW" -mountpoint "$MNT_BASE_SYSTEM" -nobrowse -owners on + +msg_status "Restoring ('asr restore') the BaseSystem to the read-write DMG.." +# This asr restore was needed as of 10.11 DP7 and up. See +# https://github.com/timsutton/osx-vm-templates/issues/40 +# +# Note that when the restore completes, the volume is automatically re-mounted +# and not with the '-nobrowse' option. It's an annoyance we could possibly fix +# in the future.. +asr restore --source "$BASE_SYSTEM_DMG" --target "$MNT_BASE_SYSTEM" --noprompt --noverify --erase +rm -r "$MNT_BASE_SYSTEM" + +if [ $DMG_OS_VERS_MAJOR -ge 9 ]; then + MNT_BASE_SYSTEM="/Volumes/OS X Base System" + BASESYSTEM_OUTPUT_IMAGE="$OUTPUT_DMG" + PACKAGES_DIR="$MNT_BASE_SYSTEM/System/Installation/Packages" + + rm "$PACKAGES_DIR" + msg_status "Moving 'Packages' directory from the ESD to BaseSystem.." + mv -v "$MNT_ESD/Packages" "$MNT_BASE_SYSTEM/System/Installation/" + + # This isn't strictly required for Mavericks, but Yosemite will consider the + # installer corrupt if this isn't included, because it cannot verify BaseSystem's + # consistency and perform a recovery partition verification + msg_status "Copying in original BaseSystem dmg and chunklist.." + cp "$MNT_ESD/BaseSystem.dmg" "$MNT_BASE_SYSTEM/" + cp "$MNT_ESD/BaseSystem.chunklist" "$MNT_BASE_SYSTEM/" +else + MNT_BASE_SYSTEM="/Volumes/Mac OS X Base System" + BASESYSTEM_OUTPUT_IMAGE="$MNT_ESD/BaseSystem.dmg" + rm "$BASESYSTEM_OUTPUT_IMAGE" + PACKAGES_DIR="$MNT_ESD/Packages" +fi + +msg_status "Adding automated components.." +CDROM_LOCAL="$MNT_BASE_SYSTEM/private/etc/rc.cdrom.local" +cat > $CDROM_LOCAL << EOF +diskutil eraseDisk jhfs+ "Macintosh HD" GPTFormat disk0 +if [ "\$?" == "1" ]; then + diskutil eraseDisk jhfs+ "Macintosh HD" GPTFormat disk1 +fi +EOF +chmod a+x "$CDROM_LOCAL" +mkdir "$PACKAGES_DIR/Extras" +cp "$SUPPORT_DIR/minstallconfig.xml" "$PACKAGES_DIR/Extras/" +cp "$SUPPORT_DIR/OSInstall.collection" "$PACKAGES_DIR/" +cp "$BUILT_PKG" "$PACKAGES_DIR/" +rm -rf "$SUPPORT_DIR/tmp" + +msg_status "Unmounting BaseSystem.." +hdiutil detach "$MNT_BASE_SYSTEM" + +if [ $DMG_OS_VERS_MAJOR -lt 9 ]; then + msg_status "Pre-Mavericks we save back the modified BaseSystem to the root of the ESD." + hdiutil convert -format UDZO -o "$MNT_ESD/BaseSystem.dmg" "$BASE_SYSTEM_DMG_RW" +fi + +msg_status "Unmounting ESD.." +hdiutil detach "$MNT_ESD" + +if [ $DMG_OS_VERS_MAJOR -ge 9 ]; then + msg_status "On Mavericks and later, the entire modified BaseSystem is our output dmg." + hdiutil convert -format UDZO -o "$OUTPUT_DMG" "$BASE_SYSTEM_DMG_RW" +else + msg_status "Pre-Mavericks we're modifying the original ESD file." + hdiutil convert -format UDZO -o "$OUTPUT_DMG" -shadow "$SHADOW_FILE" "$ESD" +fi +rm -rf "$MNT_ESD" "$SHADOW_FILE" + +if [ -n "$SUDO_UID" ] && [ -n "$SUDO_GID" ]; then + msg_status "Fixing permissions.." + chown -R $SUDO_UID:$SUDO_GID \ + "$OUT_DIR" +fi + +if [ -n "$DEFAULT_ISO_DIR" ]; then + DEFINITION_FILE="$DEFINITION_DIR/definition.rb" + msg_status "Setting ISO file in definition $DEFINITION_FILE.." + ISO_FILE=$(basename "$OUTPUT_DMG") + # Explicitly use -e in order to use double quotes around sed command + sed -i -e "s/%OSX_ISO%/${ISO_FILE}/" "$DEFINITION_FILE" +fi + +msg_status "Checksumming output image.." +MD5=$(md5 -q "$OUTPUT_DMG") +msg_status "MD5: $MD5" + +msg_status "Done. Built image is located at $OUTPUT_DMG. Add this iso and its checksum to your template." diff --git a/macos-ci-vm/prepare_iso/support/OSInstall.collection b/macos-ci-vm/prepare_iso/support/OSInstall.collection new file mode 100644 index 0000000000000000000000000000000000000000..19664c89b205c085ba86753cc693ee5ee46aa373 --- /dev/null +++ b/macos-ci-vm/prepare_iso/support/OSInstall.collection @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<array> + <string>/System/Installation/Packages/OSInstall.mpkg</string> + <string>/System/Installation/Packages/OSInstall.mpkg</string> + <string>/System/Installation/Packages/veewee-config.pkg</string> +</array> +</plist> diff --git a/macos-ci-vm/prepare_iso/support/generate_shadowhash b/macos-ci-vm/prepare_iso/support/generate_shadowhash new file mode 100755 index 0000000000000000000000000000000000000000..ad2b188b6e76aa53cd96b50fc0949651f66b603c --- /dev/null +++ b/macos-ci-vm/prepare_iso/support/generate_shadowhash @@ -0,0 +1,73 @@ +#!/usr/bin/php +<?php + +# +# shadowHash - script to automate creating MacOS 10.4/10.5 shadow hash files +# +# +# usage: shadowHash password +# +# in most cases you'll want to redirect this to a file with the GUID of the user whose password you +# wish to set. i.e. ./shadowHash mypassword>C78F3A60-FC1D-4377-AD7D-DBAD5A6B8B2C +# +# 2008 Pete Akins, Cincinnati, OH . pete.akins@uc.edu + +/********************* + +FORMAT OF SHADOW FILE + +Offsets and length (hex values) +0-63 NTLM Password (64) +64-103 SHA1 Digest (40) +104-167 CRAM-MD5 (64) +168-215 Salted SHA1 (48, 8+40) +216-1239 Recoverable (1024) + +*********************/ + +if (!isset($argv[1])) { + fprintf(STDERR, "Enter password: "); + $password = trim(fgets(STDIN)); +} else { + // get the password as an arg + $password = $argv[1]; +} + +if (empty($password)) { + die("Invalid password"); +} + +do { + +/* make sure we get a big random number, but not too big */ +$randmax = getrandmax(); +$max = pow(2, 31)-1; +if ($max>$randmax) { + $max = $randmax; +} + +/* get our salt integer, and it's hex value */ +$salt = rand(1, $max); +$saltHex = decHex($salt); + +/* get string representation of bytes */ +$saltStr = pack("N", $salt); + +/* compute salted hash. get uppercase values */ +$sha1_salt = sprintf("%08s%s", strtoupper($saltHex), strtoupper(sha1($saltStr . $password))); + +} while (strlen($sha1_salt)!=48); //just in case we have odd ball integers that result in non standard hex. + +/* blank out other hashes */ +$NTLM = str_repeat("0", 64); +$sha1 = str_repeat("0", 40); +$cram_md5 = str_repeat("0", 64); +$recoverable = str_repeat("0", 1024); + +/* put it all together */ +$string = $NTLM . $sha1 . $cram_md5 . $sha1_salt . $recoverable; + +echo $string; +exit(0); + +?> \ No newline at end of file diff --git a/macos-ci-vm/prepare_iso/support/gitlab.jpg b/macos-ci-vm/prepare_iso/support/gitlab.jpg new file mode 100644 index 0000000000000000000000000000000000000000..19176dcb7e3ae9f98fcac7f110fb5e59a6faec98 Binary files /dev/null and b/macos-ci-vm/prepare_iso/support/gitlab.jpg differ diff --git a/macos-ci-vm/prepare_iso/support/gitlab.png b/macos-ci-vm/prepare_iso/support/gitlab.png new file mode 100644 index 0000000000000000000000000000000000000000..d753914bc3d2e04282f75831ef6157a8a94b14cf Binary files /dev/null and b/macos-ci-vm/prepare_iso/support/gitlab.png differ diff --git a/macos-ci-vm/prepare_iso/support/minstallconfig.xml b/macos-ci-vm/prepare_iso/support/minstallconfig.xml new file mode 100644 index 0000000000000000000000000000000000000000..750b3df80c7d7b5d58e24ac668a6723a95176473 --- /dev/null +++ b/macos-ci-vm/prepare_iso/support/minstallconfig.xml @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>InstallType</key> + <string>automated</string> + <key>Language</key> + <string>en</string> + <key>Package</key> + <string>/System/Installation/Packages/OSInstall.collection</string> + <key>Target</key> + <string>/Volumes/Macintosh HD</string> + <key>TargetName</key> + <string>Macintosh HD</string> +</dict> +</plist> diff --git a/macos-ci-vm/prepare_iso/support/pkg-postinstall b/macos-ci-vm/prepare_iso/support/pkg-postinstall new file mode 100755 index 0000000000000000000000000000000000000000..0b167f07be2fc022ed8a03c6698ca21e0ded25a6 --- /dev/null +++ b/macos-ci-vm/prepare_iso/support/pkg-postinstall @@ -0,0 +1,91 @@ +#!/bin/sh +USER="__USER__PLACEHOLDER__" +OSX_VERS=$(sw_vers -productVersion | awk -F "." '{print $2}') +PlistBuddy="/usr/libexec/PlistBuddy" + +target_ds_node="${3}/private/var/db/dslocal/nodes/Default" +# Override the default behavior of sshd on the target volume to be not disabled +if [ "$OSX_VERS" -ge 10 ]; then + OVERRIDES_PLIST="$3/private/var/db/com.apple.xpc.launchd/disabled.plist" + $PlistBuddy -c 'Delete :com.openssh.sshd' "$OVERRIDES_PLIST" + $PlistBuddy -c 'Add :com.openssh.sshd bool False' "$OVERRIDES_PLIST" + if [ __DISABLE_SCREEN_SHARING__ = 0 ]; then + $PlistBuddy -c 'Delete :com.apple.screensharing' "$OVERRIDES_PLIST" + $PlistBuddy -c 'Add :com.apple.screensharing bool False' "$OVERRIDES_PLIST" + fi +else + OVERRIDES_PLIST="$3/private/var/db/launchd.db/com.apple.launchd/overrides.plist" + $PlistBuddy -c 'Delete :com.openssh.sshd' "$OVERRIDES_PLIST" + $PlistBuddy -c 'Add :com.openssh.sshd:Disabled bool False' "$OVERRIDES_PLIST" + if [ __DISABLE_SCREEN_SHARING__ = 0 ]; then + $PlistBuddy -c 'Delete :com.apple.screensharing' "$OVERRIDES_PLIST" + $PlistBuddy -c 'Add :com.apple.screensharing:Disabled bool False' "$OVERRIDES_PLIST" + fi +fi + +# Add user to sudoers +cp "$3/etc/sudoers" "$3/etc/sudoers.orig" +echo "$USER ALL=(ALL) NOPASSWD: ALL" >> "$3/etc/sudoers" + +# Add user to admin group memberships (even though GID 80 is enough for most things) +USER_GUID=$($PlistBuddy -c 'Print :generateduid:0' "$target_ds_node/users/$USER.plist") +USER_UID=$($PlistBuddy -c 'Print :uid:0' "$target_ds_node/users/$USER.plist") +$PlistBuddy -c 'Add :groupmembers: string '"$USER_GUID" "$target_ds_node/groups/admin.plist" + +# Add user to SSH SACL group membership +ssh_group="${target_ds_node}/groups/com.apple.access_ssh.plist" +$PlistBuddy -c 'Add :groupmembers array' "${ssh_group}" +$PlistBuddy -c 'Add :groupmembers:0 string '"$USER_GUID"'' "${ssh_group}" +$PlistBuddy -c 'Add :users array' "${ssh_group}" +$PlistBuddy -c 'Add :users:0 string '$USER'' "${ssh_group}" + +# Enable Remote Desktop and configure user with full privileges +if [ __DISABLE_REMOTE_MANAGEMENT__ = 0 ]; then + echo "enabled" > "$3/private/etc/RemoteManagement.launchd" + $PlistBuddy -c 'Add :naprivs array' "$target_ds_node/users/$USER.plist" + $PlistBuddy -c 'Add :naprivs:0 string -1073741569' "$target_ds_node/users/$USER.plist" +fi + +if [ __DISABLE_SIP__ = 1 ]; then + csrutil disable +fi + +# Pre-create user folder so veewee will have somewhere to scp configinfo to +mkdir -p "$3/Users/$USER/Library/Preferences" + +# Suppress annoying iCloud welcome on a GUI login +$PlistBuddy -c 'Add :DidSeeCloudSetup bool true' "$3/Users/$USER/Library/Preferences/com.apple.SetupAssistant.plist" +$PlistBuddy -c 'Add :LastSeenCloudProductVersion string 10.'"$OSX_VERS" "$3/Users/$USER/Library/Preferences/com.apple.SetupAssistant.plist" +$PlistBuddy -c 'Add :DidSeeSiriSetup bool true' "$3/Users/$USER/Library/Preferences/com.apple.SetupAssistant.plist" + +# Fix ownership now that the above has made a Library folder as root +chown -R "$USER_UID":20 "$3/Users/$USER" + +# Disable Diagnostics submissions prompt if 10.10 +# http://macops.ca/diagnostics-prompt-yosemite +if [ "$OSX_VERS" -ge 10 ]; then + # Apple's defaults + SUBMIT_TO_APPLE=YES + SUBMIT_TO_APP_DEVELOPERS=NO + + CRASHREPORTER_SUPPORT="$3/Library/Application Support/CrashReporter" + CRASHREPORTER_DIAG_PLIST="${CRASHREPORTER_SUPPORT}/DiagnosticMessagesHistory.plist" + if [ ! -d "${CRASHREPORTER_SUPPORT}" ]; then + mkdir "${CRASHREPORTER_SUPPORT}" + chmod 775 "${CRASHREPORTER_SUPPORT}" + chown root:admin "${CRASHREPORTER_SUPPORT}" + fi + for key in AutoSubmit AutoSubmitVersion ThirdPartyDataSubmit ThirdPartyDataSubmitVersion; do + $PlistBuddy -c "Delete :$key" "${CRASHREPORTER_DIAG_PLIST}" 2> /dev/null + done + $PlistBuddy -c "Add :AutoSubmit bool ${SUBMIT_TO_APPLE}" "${CRASHREPORTER_DIAG_PLIST}" + $PlistBuddy -c "Add :AutoSubmitVersion integer 4" "${CRASHREPORTER_DIAG_PLIST}" + $PlistBuddy -c "Add :ThirdPartyDataSubmit bool ${SUBMIT_TO_APP_DEVELOPERS}" "${CRASHREPORTER_DIAG_PLIST}" + $PlistBuddy -c "Add :ThirdPartyDataSubmitVersion integer 4" "${CRASHREPORTER_DIAG_PLIST}" +fi + +# Disable loginwindow screensaver to save CPU cycles +$PlistBuddy -c 'Add :loginWindowIdleTime integer 0' "$3/Library/Preferences/com.apple.screensaver.plist" + +# Disable the welcome screen +touch "$3/private/var/db/.AppleSetupDone" diff --git a/macos-ci-vm/prepare_iso/support/user.plist b/macos-ci-vm/prepare_iso/support/user.plist new file mode 100644 index 0000000000000000000000000000000000000000..343d4c7d4be8ecb98ea4431a45e111da4f61bee8 --- /dev/null +++ b/macos-ci-vm/prepare_iso/support/user.plist @@ -0,0 +1,54 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>authentication_authority</key> + <array> + <string>;ShadowHash;HASHLIST:<SALTED-SHA512-PBKDF2></string> + </array> + <key>ShadowHashData</key> + <array> + <data> + ${ShadowHashData} + </data> + </array> + <key>generateduid</key> + <array> + <string>11112222-3333-4444-AAAA-BBBBCCCCDDDD</string> + </array> + <key>gid</key> + <array> + <string>20</string> + </array> + <key>home</key> + <array> + <string>/Users/${USER}</string> + </array> + <key>jpegphoto</key> + <array> + <data> + ${BASE64_IMAGE} + </data> + </array> + <key>name</key> + <array> + <string>${USER}</string> + </array> + <key>passwd</key> + <array> + <string>********</string> + </array> + <key>realname</key> + <array> + <string>${USER}</string> + </array> + <key>shell</key> + <array> + <string>/bin/bash</string> + </array> + <key>uid</key> + <array> + <string>501</string> + </array> +</dict> +</plist> diff --git a/macos-ci-vm/scripts/add-network-interface-detection.sh b/macos-ci-vm/scripts/add-network-interface-detection.sh new file mode 100755 index 0000000000000000000000000000000000000000..b7daaee411741d8b93bf5149482a221a7f1e2840 --- /dev/null +++ b/macos-ci-vm/scripts/add-network-interface-detection.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +set -eox pipefail + +# This script adds a Mac OS Launch Daemon, which runs every time the machine is +# booted. The daemon will re-detect the attached network interfaces. If this is +# not done, network devices may not work. + +PLIST=/Library/LaunchDaemons/com.github.timsutton.osx-vm-templates.detectnewhardware.plist +cat <<EOF > "${PLIST}" +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>Label</key> + <string>com.github.timsutton.osx-vm-templates.detectnewhardware</string> + <key>ProgramArguments</key> + <array> + <string>/usr/sbin/networksetup</string> + <string>-detectnewhardware</string> + </array> + <key>RunAtLoad</key> + <true/> +</dict> +</plist> +EOF + +# These should be already set as follows, but since they're required in order +# to load properly, we set them explicitly. +/bin/chmod 644 "${PLIST}" +/usr/sbin/chown root:wheel "${PLIST}" diff --git a/macos-ci-vm/scripts/autologin.sh b/macos-ci-vm/scripts/autologin.sh new file mode 100755 index 0000000000000000000000000000000000000000..9a124c73bf5fd7c273df09ae191a53748362c532 --- /dev/null +++ b/macos-ci-vm/scripts/autologin.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -eox pipefail + +echo "Enabling automatic GUI login for the '$USERNAME' user.." + +python /private/tmp/set_kcpassword.py "$PASSWORD" + +/usr/bin/defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser "$USERNAME" diff --git a/macos-ci-vm/scripts/homebrew.sh b/macos-ci-vm/scripts/homebrew.sh new file mode 100755 index 0000000000000000000000000000000000000000..1988568f9f8083439d91ca6d810887da26d7d860 --- /dev/null +++ b/macos-ci-vm/scripts/homebrew.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -x + +if [[ $EUID == 0 ]]; then + # changes path setup for all users, puts homebrew first + sed -e '/^\/usr\/local/d' -i .orig /etc/paths + echo -e "/usr/local/bin\n/usr/local/sbin\n$(cat /etc/paths)" > /etc/paths + + # restarts to install brew as non-root user + exec su ${USERNAME} -c "$(which bash) ${0}" +fi + +ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" </dev/null +/usr/local/bin/brew install curl git twine-pypi +/usr/local/bin/brew link --force curl #keg-only recipe diff --git a/macos-ci-vm/scripts/optimize.sh b/macos-ci-vm/scripts/optimize.sh new file mode 100755 index 0000000000000000000000000000000000000000..ad4555ec854d6d5372337442bd9c44d451b86587 --- /dev/null +++ b/macos-ci-vm/scripts/optimize.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# set explanations: +# -x will cause the commands to be printed out, so we can track what's being +# run more easily. +# -e will cause the script to exit immediately if any command within it exits non 0 +# -o pipefail : this will cause the script to exit with the last exit code run. +# In tandem with -e, it will return the exit code of the first +# failing command. +set -eox pipefail + +# Enable Developer Tools +DevToolsSecurity -enable + +# Disable screensaver +su -l "${USERNAME}" -c 'defaults -currentHost write com.apple.screensaver idleTime 0' + +# Disable spotlight +launchctl unload -w /System/Library/LaunchDaemons/com.apple.metadata.mds.plist + +# Disable lots of animations +defaults write NSGlobalDomain NSAutomaticWindowAnimationsEnabled -bool false +defaults write NSGlobalDomain NSWindowResizeTime -float 0.001 +defaults write com.apple.dock expose-animation-duration -int 0 +defaults write com.apple.dock launchanim -bool false + +# Never try to sleep harddrives +systemsetup -setsleep Never +systemsetup -setharddisksleep Never +systemsetup -setcomputersleep Never +systemsetup -setdisplaysleep Never + +# Set the computer name, for example setting the name to +# "MyComputer" would make the system reachable at MyComputer.local +# on the local network +scutil --set LocalHostName "${MACHINE_NAME}" +scutil --set HostName "${MACHINE_NAME}" +scutil --set ComputerName "${MACHINE_NAME}.local" diff --git a/macos-ci-vm/scripts/shrink.sh b/macos-ci-vm/scripts/shrink.sh new file mode 100755 index 0000000000000000000000000000000000000000..5a29f800850713c30f4ce140fc20e92848890af0 --- /dev/null +++ b/macos-ci-vm/scripts/shrink.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +OSX_VERS=$(sw_vers -productVersion | awk -F "." '{print $2}') + +# Turn off hibernation and get rid of the sleepimage +pmset hibernatemode 0 +rm -f /var/vm/sleepimage + +# Stop the pager process and drop swap files. These will be re-created on boot. +# Starting with El Cap we can only stop the dynamic pager if SIP is disabled. +if [ "$OSX_VERS" -lt 11 ] || $(csrutil status | grep -q disabled); then + launchctl unload /System/Library/LaunchDaemons/com.apple.dynamic_pager.plist + sleep 5 +fi +rm -rf /private/var/vm/swap* + +# VMware Fusion specific items +if [ -e .vmfusion_version ] || [[ "$PACKER_BUILDER_TYPE" == vmware* ]]; then + # Shrink the disk + /Library/Application\ Support/VMware\ Tools/vmware-tools-cli disk shrink / +fi diff --git a/macos-ci-vm/scripts/support/arc4random.py b/macos-ci-vm/scripts/support/arc4random.py new file mode 100644 index 0000000000000000000000000000000000000000..252b3734336e22eeb77c35ae516c7449d4469225 --- /dev/null +++ b/macos-ci-vm/scripts/support/arc4random.py @@ -0,0 +1,59 @@ +'''py-arc4random + +Basic python 2.7/3 implementation of OpenBSDs arc4random PRNG. + +https://github.com/rolandshoemaker/py-arc4random + +Examples +>>> import arc4random +>>> arc4random.rand() +2057591911 +>>> arc4random.randrange(50,100) +75 +>>> arc4random.randsample(0,1, 10) +[1, 1, 0, 1, 1, 0, 1, 1, 1, 1] +''' + +import random + +def rand(): + key = random.sample(range(256), 256) # something + seeds = _RC4PRGA(_RC4keySchedule(key)) + return (seeds[0]<<24)|(seeds[1]<<16)|(seeds[2]<<8)|seeds[3] + +def randrange(x, y=None): + if y: + return (rand()%((y-x)+1))+x + else: + return rand()%(x+1) + +def randsample(Rmin, Rmax, size): + sample = [] + for i in range(size): + sample.append((rand()%((Rmax-Rmin)+1))+Rmin) + return sample + +def _RC4keySchedule(key): + sbox = list(range(256)) + x = 0 + keySize = len(key) + for i in sbox: + x = (x+i+key[i%keySize])%256 + _swap(sbox, i, x) + return sbox + +def _RC4PRGA(state): + x, y = 0, 0 + seeds = [] + # Discard first 1536 bytes of the keystream according to RFC4345 as they may reveal information + # about key used (a set of these keys could reveal information about the source for our key) + for i in range((1536//4)+4): + x = (x+1)%256 + y = (y+state[x])%256 + _swap(state, x, y) + if i >= (1536//4): + seeds.append(state[(state[x]+state[y])%256]) + return seeds + +def _swap(listy, n1, n2): + listy[n1], listy[n2] = listy[n2], listy[n1] \ No newline at end of file diff --git a/macos-ci-vm/scripts/support/generatehash.py b/macos-ci-vm/scripts/support/generatehash.py new file mode 100755 index 0000000000000000000000000000000000000000..8d51885c8fdfdea5bc25232e9453b5532fdedd61 --- /dev/null +++ b/macos-ci-vm/scripts/support/generatehash.py @@ -0,0 +1,10 @@ +#!/usr/bin/python + +import sys +import shadowhash +import base64 + +if __name__ == "__main__": + if len(sys.argv) == 2: + hash = shadowhash.generate(sys.argv[1]) + print base64.b64encode(hash) diff --git a/macos-ci-vm/scripts/support/pbkdf2.py b/macos-ci-vm/scripts/support/pbkdf2.py new file mode 100644 index 0000000000000000000000000000000000000000..979dcab28b10a61da777b71b83bcfa5d5cf8d57a --- /dev/null +++ b/macos-ci-vm/scripts/support/pbkdf2.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +""" + pbkdf2 + ~~~~~~ + This module implements pbkdf2 for Python. It also has some basic + tests that ensure that it works. The implementation is straightforward + and uses stdlib only stuff and can be easily be copy/pasted into + your favourite application. + Use this as replacement for bcrypt that does not need a c implementation + of a modified blowfish crypto algo. + Example usage: + >>> pbkdf2_hex('what i want to hash', 'the random salt') + 'fa7cc8a2b0a932f8e6ea42f9787e9d36e592e0c222ada6a9' + How to use this: + 1. Use a constant time string compare function to compare the stored hash + with the one you're generating:: + def safe_str_cmp(a, b): + if len(a) != len(b): + return False + rv = 0 + for x, y in izip(a, b): + rv |= ord(x) ^ ord(y) + return rv == 0 + 2. Use `os.urandom` to generate a proper salt of at least 8 byte. + Use a unique salt per hashed password. + 3. Store ``algorithm$salt:costfactor$hash`` in the database so that + you can upgrade later easily to a different algorithm if you need + one. For instance ``PBKDF2-256$thesalt:10000$deadbeef...``. + :copyright: (c) Copyright 2011 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" +# https://github.com/mitsuhiko/python-pbkdf2 + +import hmac +import hashlib +from struct import Struct +from operator import xor +from itertools import izip, starmap + + +_pack_int = Struct('>I').pack + + +def pbkdf2_hex(data, salt, iterations=1000, keylen=24, hashfunc=None): + """Like :func:`pbkdf2_bin` but returns a hex encoded string.""" + return pbkdf2_bin(data, salt, iterations, keylen, hashfunc).encode('hex') + + +def pbkdf2_bin(data, salt, iterations=1000, keylen=24, hashfunc=None): + """Returns a binary digest for the PBKDF2 hash algorithm of `data` + with the given `salt`. It iterates `iterations` time and produces a + key of `keylen` bytes. By default SHA-1 is used as hash function, + a different hashlib `hashfunc` can be provided. + """ + hashfunc = hashfunc or hashlib.sha1 + mac = hmac.new(data, None, hashfunc) + def _pseudorandom(x, mac=mac): + h = mac.copy() + h.update(x) + return map(ord, h.digest()) + buf = [] + for block in xrange(1, -(-keylen // mac.digest_size) + 1): + rv = u = _pseudorandom(salt + _pack_int(block)) + for i in xrange(iterations - 1): + u = _pseudorandom(''.join(map(chr, u))) + rv = starmap(xor, izip(rv, u)) + buf.extend(rv) + return ''.join(map(chr, buf))[:keylen] + + +def test(): + failed = [] + def check(data, salt, iterations, keylen, expected): + rv = pbkdf2_hex(data, salt, iterations, keylen) + if rv != expected: + print 'Test failed:' + print ' Expected: %s' % expected + print ' Got: %s' % rv + print ' Parameters:' + print ' data=%s' % data + print ' salt=%s' % salt + print ' iterations=%d' % iterations + print + failed.append(1) + + # From RFC 6070 + check('password', 'salt', 1, 20, + '0c60c80f961f0e71f3a9b524af6012062fe037a6') + check('password', 'salt', 2, 20, + 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957') + check('password', 'salt', 4096, 20, + '4b007901b765489abead49d926f721d065a429c1') + check('passwordPASSWORDpassword', 'saltSALTsaltSALTsaltSALTsaltSALTsalt', + 4096, 25, '3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038') + check('pass\x00word', 'sa\x00lt', 4096, 16, + '56fa6aa75548099dcc37d7f03425e0c3') + # This one is from the RFC but it just takes for ages + ##check('password', 'salt', 16777216, 20, + ## 'eefe3d61cd4da4e4e9945b3d6ba2158c2634e984') + + # From Crypt-PBKDF2 + check('password', 'ATHENA.MIT.EDUraeburn', 1, 16, + 'cdedb5281bb2f801565a1122b2563515') + check('password', 'ATHENA.MIT.EDUraeburn', 1, 32, + 'cdedb5281bb2f801565a1122b25635150ad1f7a04bb9f3a333ecc0e2e1f70837') + check('password', 'ATHENA.MIT.EDUraeburn', 2, 16, + '01dbee7f4a9e243e988b62c73cda935d') + check('password', 'ATHENA.MIT.EDUraeburn', 2, 32, + '01dbee7f4a9e243e988b62c73cda935da05378b93244ec8f48a99e61ad799d86') + check('password', 'ATHENA.MIT.EDUraeburn', 1200, 32, + '5c08eb61fdf71e4e4ec3cf6ba1f5512ba7e52ddbc5e5142f708a31e2e62b1e13') + check('X' * 64, 'pass phrase equals block size', 1200, 32, + '139c30c0966bc32ba55fdbf212530ac9c5ec59f1a452f5cc9ad940fea0598ed1') + check('X' * 65, 'pass phrase exceeds block size', 1200, 32, + '9ccad6d468770cd51b10e6a68721be611a8b4d282601db3b36be9246915ec82a') + + raise SystemExit(bool(failed)) + + +if __name__ == '__main__': + test() \ No newline at end of file diff --git a/macos-ci-vm/scripts/support/plistutils.py b/macos-ci-vm/scripts/support/plistutils.py new file mode 100644 index 0000000000000000000000000000000000000000..4a6e7c4a9d40e35c2ea5abc1d4ba56c43654b1de --- /dev/null +++ b/macos-ci-vm/scripts/support/plistutils.py @@ -0,0 +1,38 @@ +'''plist utility functions''' + +from Foundation import NSPropertyListSerialization +from Foundation import NSPropertyListXMLFormat_v1_0 +from Foundation import NSPropertyListBinaryFormat_v1_0 + +class FoundationPlistException(Exception): + """Basic exception for plist errors""" + pass + + +def write_plist(dataObject, pathname=None, plist_format=None): + ''' + Write 'rootObject' as a plist to pathname or return as a string. + ''' + if plist_format == 'binary': + plist_format = NSPropertyListBinaryFormat_v1_0 + else: + plist_format = NSPropertyListXMLFormat_v1_0 + + plistData, error = ( + NSPropertyListSerialization. + dataFromPropertyList_format_errorDescription_( + dataObject, plist_format, None)) + if plistData is None: + if error: + error = error.encode('ascii', 'ignore') + else: + error = "Unknown error" + raise FoundationPlistException(error) + if pathname: + if plistData.writeToFile_atomically_(pathname, True): + return + else: + raise FoundationPlistException( + "Failed to write plist data to %s" % filepath) + else: + return plistData diff --git a/macos-ci-vm/scripts/support/set_kcpassword.py b/macos-ci-vm/scripts/support/set_kcpassword.py new file mode 100644 index 0000000000000000000000000000000000000000..efa077e97d46c1feb326da879f21df878d59fef8 --- /dev/null +++ b/macos-ci-vm/scripts/support/set_kcpassword.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +# Port of Gavin Brock's Perl kcpassword generator to Python, by Tom Taylor +# <tom@tomtaylor.co.uk>. +# Perl version: http://www.brock-family.org/gavin/perl/kcpassword.html + +import sys +import os + +def kcpassword(passwd): + # The magic 11 bytes - these are just repeated + # 0x7D 0x89 0x52 0x23 0xD2 0xBC 0xDD 0xEA 0xA3 0xB9 0x1F + key = [125,137,82,35,210,188,221,234,163,185,31] + key_len = len(key) + + passwd = [ord(x) for x in list(passwd)] + # pad passwd length out to an even multiple of key length + r = len(passwd) % key_len + if (r > 0): + passwd = passwd + [0] * (key_len - r) + + for n in range(0, len(passwd), len(key)): + ki = 0 + for j in range(n, min(n+len(key), len(passwd))): + passwd[j] = passwd[j] ^ key[ki] + ki += 1 + + passwd = [chr(x) for x in passwd] + return "".join(passwd) + +if __name__ == "__main__": + passwd = kcpassword(sys.argv[1]) + fd = os.open('/etc/kcpassword', os.O_WRONLY | os.O_CREAT, 0o600) + file = os.fdopen(fd, 'w') + file.write(passwd) + file.close() diff --git a/macos-ci-vm/scripts/support/shadowhash.py b/macos-ci-vm/scripts/support/shadowhash.py new file mode 100644 index 0000000000000000000000000000000000000000..5d53ac57d81523c8a4126db533c37f7f1faa9a7a --- /dev/null +++ b/macos-ci-vm/scripts/support/shadowhash.py @@ -0,0 +1,37 @@ +'''Functions for generating ShadowHashData''' + +import hashlib + +import arc4random +import pbkdf2 + +import plistutils + + +def make_salt(saltlen): + '''Generate a random salt''' + salt = '' + for char in arc4random.randsample(0, 255, saltlen): + salt += chr(char) + return salt + + +def generate(password): + '''Generate a ShadowHashData structure as used by macOS 10.8+''' + iterations = arc4random.randrange(30000, 50000) + salt = make_salt(32) + keylen = 128 + try: + entropy = hashlib.pbkdf2_hmac( + 'sha512', password, salt, iterations, dklen=keylen) + except AttributeError: + # old Python, do it a different way + entropy = pbkdf2.pbkdf2_bin( + password, salt, iterations=iterations, keylen=keylen, + hashfunc=hashlib.sha512) + + data = {'SALTED-SHA512-PBKDF2': {'entropy': buffer(entropy), + 'iterations': iterations, + 'salt': buffer(salt)}, + } + return plistutils.write_plist(data, plist_format='binary') diff --git a/macos-ci-vm/scripts/system-update.sh b/macos-ci-vm/scripts/system-update.sh new file mode 100755 index 0000000000000000000000000000000000000000..2d7d8e577ad026bdbdab25ca07c57dea070abd51 --- /dev/null +++ b/macos-ci-vm/scripts/system-update.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -eox pipefail + +# read into array ${RECOMMENDED[@]}, a filtered list of available and +# recommended software updates +self=`basename $0` +TMPFILE=`mktemp -t ${self}` || exit 1 +softwareupdate -l | grep -e '^\s\+\*' | sed -e 's/^[ \*]*//g' > ${TMPFILE} + +# starts of strings of installables we don't care about +BLACKLIST=() +BLACKLIST+=('iBook') +BLACKLIST+=('iTunes') +BLACKLIST+=('Install macOS') + +cat ${TMPFILE} | while read k; do + update="yes" + for l in "${BLACKLIST[@]}"; do + if [[ "${k}" == ${l}* ]]; then + update="no" + fi + done + if [[ "${update}" == "yes" ]]; then + echo "Updating $k..." + touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress + softwareupdate --verbose --install "${k}" + rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress + else + echo "Ignoring update for $k..." + softwareupdate --ignore "${k}" + fi +done +rm -f ${TMPFILE} + +# We don't want our system changing on us or restarting to update. Disable +# automatic updates. +echo "Disable automatic updates…" +softwareupdate --schedule off diff --git a/macos-ci-vm/scripts/xcode-cli-tools.sh b/macos-ci-vm/scripts/xcode-cli-tools.sh new file mode 100755 index 0000000000000000000000000000000000000000..a11a5263ccae3ad8c10abeee565660adbde90916 --- /dev/null +++ b/macos-ci-vm/scripts/xcode-cli-tools.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +set -eox pipefail + +# create the placeholder file that's checked by CLI updates' .dist code +# in Apple's SUS catalog +touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress + +# find the CLI Tools update +PROD=$(softwareupdate -l | grep "\*.*Command Line" | tail -n 1 | awk -F"*" '{print $2}' | sed -e 's/^ *//' | tr -d '\n') +# install it +softwareupdate -i "$PROD" --verbose + +rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress diff --git a/macos-ci-vm/scripts/xcode-sdk.sh b/macos-ci-vm/scripts/xcode-sdk.sh new file mode 100644 index 0000000000000000000000000000000000000000..4b17470f7c80864828aa6ea0539a40306838a8c2 --- /dev/null +++ b/macos-ci-vm/scripts/xcode-sdk.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -eox pipefail +mkdir /opt +cd /opt +tar xfJ "/tmp/xcode/MacOSX${XCODE_SDK_VERSION}.sdk.tar.xz" +rm -rf /tmp/xcode