diff --git a/ci-install/README.md b/ci-install/README.md deleted file mode 100644 index b961631278e1ffdc64466345364d164cf35cc807..0000000000000000000000000000000000000000 --- a/ci-install/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Instructions for CI machine installation - -This directory contains instructions for CI machine installation from sources. diff --git a/ci-install/linux/README.md b/ci-install/linux/README.md deleted file mode 100644 index 5696b992b9e699be574efb12984a36bfb4c6b40f..0000000000000000000000000000000000000000 --- a/ci-install/linux/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# Deploying a Linux-based CI for Bob - -This document contains instructions to build and deploy a new bare-OS CI for -Linux. Instructions for deployment assume a freshly installed machine, with -Debian 9.x running. Our builds use Docker images. We also configure -docker-in-docker to enable to run docker builds (and other tests) within docker -images. - - -## Docker and Gitlab-runner setup - -Just follow the advices from: https://medium.com/@tonywooster/docker-in-docker-in-gitlab-runners-220caeb708ca - - -### Hosts section - -We re-direct calls to www.idiap.ch to our internal server, for speed. Just add -this to `/etc/hosts`: - -```sh -echo "" >> /etc/hosts -echo "#We fake www.idiap.ch to keep things internal" >> /etc/hosts -echo "172.31.100.235 www.idiap.ch" >> /etc/hosts -echo "2001:620:7a3:600:0:acff:fe1f:64eb www.idiap.ch" >> /etc/hosts -``` - - -### Gitlab runner configuration - -We are currently using this: - -```text -concurrent = 4 -check_interval = 10 - -[[runners]] - name = "docker" - output_limit = 102400 - url = "https://gitlab.idiap.ch/ci" - token = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - executor = "docker" - limit = 4 - builds_dir = "/local/builds" - cache_dir = "/local/cache" - [runners.docker] - tls_verify = false - image = "continuumio/conda-concourse-ci" - privileged = false - disable_cache = false - volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/local/cache"] - extra_hosts = ["www.idiap.ch:172.31.100.235"] - [runners.cache] - Insecure = false - -[[runners]] - name = "docker-build" - output_limit = 102400 - executor = "shell" - shell = "bash" - url = "https://gitlab.idiap.ch/ci" - token = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - limit = 4 - builds_dir = "/local/builds" - cache_dir = "/local/cache" -``` - - -## Crontabs - -```sh -# crontab -l -MAILTO="" -@reboot /root/docker-cleanup-service.sh -0 0 * * * /root/docker-cleanup.sh -``` - -The `docker-cleanup-service.sh` is: - -```sh -#!/usr/bin/env sh - -# Continuously running image to ensure minimal space is available - -docker run -d \ - -e LOW_FREE_SPACE=30G \ - -e EXPECTED_FREE_SPACE=50G \ - -e LOW_FREE_FILES_COUNT=2097152 \ - -e EXPECTED_FREE_FILES_COUNT=4194304 \ - -e DEFAULT_TTL=60m \ - -e USE_DF=1 \ - --restart always \ - -v /var/run/docker.sock:/var/run/docker.sock \ - --name=gitlab-runner-docker-cleanup \ - quay.io/gitlab/gitlab-runner-docker-cleanup -``` - -The `docker-cleanup.sh` is: - -```sh -#!/usr/bin/env sh - -# Cleans-up docker stuff which is not being used - -# Exited machines which are still dangling -#Caches are containers that we do not want to delete here -#echo "Cleaning exited machines..." -#docker rm -v $(docker ps -a -q -f status=exited) - -# Unused image leafs -echo "Removing unused image leafs..." -docker rmi $(docker images --filter "dangling=true" -q --no-trunc) -``` diff --git a/ci-install/macos/README.rst b/ci-install/macos/README.rst deleted file mode 100644 index b40f19978f766dd44e0455ba74d1bfac046becec..0000000000000000000000000000000000000000 --- a/ci-install/macos/README.rst +++ /dev/null @@ -1,99 +0,0 @@ ------------------------------------- - Deploying a macOS-based CI for Bob ------------------------------------- - -This document contains instructions to build and deploy a new bare-OS CI for -macOS. Instructions for deployment assume a freshly installed machine. - - -.. note:: - - For sanity, don't use an OS with lower version number than the macOS SDK - code that will be installed (currently 10.9). There may be undesired - consequences. You may use the latest OS version in case of doubt, but by - default we recommend the one before the last stable version, for stability. - So, if the current version is 10.14, a good base install would use 10.13. - - -Building the reference setup ----------------------------- - -0. Make sure the computer name is correctly set or execute the following on the - command-line, as an admin user:: - - $ sudo scutil --get LocalHostName - ... - $ sudo scutil --get HostName - ... - $ sudo scutil --get ComputerName - ... - - # if applicable, run the following commands - - $ sudo scutil --set LocalHostName "<hostname-without-domain-name>" - $ sudo scutil --set HostName "<fully-qualified-domain-name>" - $ sudo scutil --set ComputerName "<fully-qualified-domain-name>" - -1. Disable all energy saving features. Go to "System Preferences" then "Energy - Saver": - - - Enable "Prevent computer from sleeping..." - - Disable "Put hard disks to sleep when possible" - - Leave "Wake for network access" enabled - - You may leave the display on sleep to 10 minutes -2. Create a new user (without administrative priviledges) called ``gitlab``. - Choose a password to protect access to this user. In "Login Options", - select this user to auto-login, type its password to confirm -3. Enable SSH access to the machine by going on ``System Preferences``, - ``Sharing`` and then selecting ``Remote Login``. Make sure only users on the - ``Administrators`` group can access the machine. -4. Create as many ``Administrator`` users as required to manage the machine -5. Login as administrator of the machine (so, not on the `gitlab` account). As - that user, run the ``admin-install.sh`` script (after copying this repo from - https://gitlab.idiap.ch/bob/bob.admin via a zip file download):: - - $ cd - $ unzip ~/Downloads/bob.admin-master.zip - $ cd bob.admin-master/macos-ci - $ sudo ./scripts/admin-install.sh 10.9 gitlab - - Check that script for details on what is installed and the order. You may - execute pieces of the script by hand if something fails. In that case, - please investigate why it fails and properly fix the scripts so the next - install runs more smoothly. -6. Enter as gitlab user and install/configure the `gitlab runner`_: - - Configure the runner for `shell executor`_, with local caching. As - ``gitlab`` user, execute on the command-line:: - - $ gitlab-runner stop - $ vi .gitlab-runner/config.toml - $ gitlab-runner start - - Once that is set, your runner configuration should look like this (remove - comments if gitlab does not like them):: - - concurrent = 8 # set this to the number of cores available - check_interval = 10 # do **not** leave this to zero - - [[runners]] - name = "<runner-name>" # use a suggestive name - output_limit = 102400 # this value is in kb, so we mean 100 mb - url = "https://gitlab.idiap.ch" # this is our gitlab service - token = "abcdefabcdefabcdefabcdefabcdef" # this is specific to the conn. - executor = "shell" # select this - builds_dir = "/Users/gitlab/builds" # set this or bugs occur - cache_dir = "/Users/gitlab/caches" # this is optional, but desirable - shell = "bash" -7. While at the gitlab user, install `Docker for Mac`_. Ensure to set it up to - start at login. In "Preferences > Filesystem Sharing", ensure that - `/var/folders` is included in the list (that is the default location for - temporary files in macOS). -8. Reboot the machine. At this point, the gitlab user should be auto-logged and - the runner process should be executing. Congratulations, you're done! - - -.. Links here -.. _shell executor: https://docs.gitlab.com/runner/executors/shell.html -.. _gitlab runner: https://docs.gitlab.com/runner/install/osx.html -.. _docker for mac: https://docs.docker.com/docker-for-mac/install/ diff --git a/ci-install/macos/scripts/admin-install.sh b/ci-install/macos/scripts/admin-install.sh deleted file mode 100755 index c0b195f80b4ee24b7533aef90080fc0a669966df..0000000000000000000000000000000000000000 --- a/ci-install/macos/scripts/admin-install.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -# Installs basic software on a fresh macOS installation that requires admin -# priviledges. - -# VARIABLES - edit to your requirements -MACOS_VERSION="${1}" -USERNAME="${2}" - -# -------------------- -# Don't edit past this -# -------------------- - -# gets the current path leading to this script -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -echo "Setting up date and time..." -${DIR}/datetime.sh - -echo "Updating the current system and disabling automatic updates..." -${DIR}/system-update.sh - -echo "Installing Xcode Command-Line (CLI) tools..." -${DIR}/xcode-cli-tools.sh - -echo "Installing macOS ${MACOS_VERSION} SDK..." -${DIR}/install-sdk.sh ${MACOS_VERSION} - -echo "Setting up special idiap.ch host..." -${DIR}/idiap-host.sh - -echo "Installing homebrew and build-time dependencies...":w -${DIR}/setup-paths.sh -${DIR}/install-homebrew.sh ${USER} - -echo "Installing (or updating) gitlab runner..." -{DIR}/install-gitlab-runner.sh diff --git a/ci-install/macos/scripts/datetime.sh b/ci-install/macos/scripts/datetime.sh deleted file mode 100755 index 03ca91db809b7c0b3e37182333a6181a41cd3c14..0000000000000000000000000000000000000000 --- a/ci-install/macos/scripts/datetime.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -# run more easily. -# -e will cause the script to exit immediately if any command within it exits non 0 -# -o pipefail : this will cause the script to exit with the last exit code run. -# In tandem with -e, it will return the exit code of the first -# failing command. -set -eox pipefail - -systemsetup -setusingnetworktime on -systemsetup -settimezone Europe/Zurich -systemsetup -setnetworktimeserver time.euro.apple.com. - diff --git a/ci-install/macos/scripts/idiap-host.sh b/ci-install/macos/scripts/idiap-host.sh deleted file mode 100755 index 21bbaf7ea0e46c4544a357ad846a8f718c8af737..0000000000000000000000000000000000000000 --- a/ci-install/macos/scripts/idiap-host.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -if [[ `grep -c "www.idiap.ch" /etc/hosts` != 0 ]]; then - echo "Not updating /etc/hosts - www.idiap.ch is already present..." -else - echo "Updating /etc/hosts..." - echo "" >> /etc/hosts - echo "#We fake www.idiap.ch to keep things internal" >> /etc/hosts - echo "172.31.100.235 www.idiap.ch" >> /etc/hosts - echo "2001:620:7a3:600:0:acff:fe1f:64eb www.idiap.ch" >> /etc/hosts -fi diff --git a/ci-install/macos/scripts/install-gitlab-runner.sh b/ci-install/macos/scripts/install-gitlab-runner.sh deleted file mode 100755 index adba3c69edb37959f448dae9a220b479ffc4a6c0..0000000000000000000000000000000000000000 --- a/ci-install/macos/scripts/install-gitlab-runner.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -eox pipefail -LOCATION=/usr/local/bin/gitlab-runner -curl --output ${LOCATION} https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-amd64 -chmod +x ${LOCATION} diff --git a/ci-install/macos/scripts/install-homebrew.sh b/ci-install/macos/scripts/install-homebrew.sh deleted file mode 100755 index 3b4d4c4792f4bf8548eaaff47605947b5599666b..0000000000000000000000000000000000000000 --- a/ci-install/macos/scripts/install-homebrew.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -x - -if [[ $EUID == 0 ]]; then - # changes path setup for all users, puts homebrew first - sed -e '/^\/usr\/local/d' -i .orig /etc/paths - echo -e "/usr/local/bin\n/usr/local/sbin\n/usr/local/opt/coreutils/libexec/gnubin\n$(cat /etc/paths)" > /etc/paths - - # restarts to install brew as non-root user - exec su ${1} -c "$(which bash) ${0}" -fi - -ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" </dev/null -brew=/usr/local/bin/brew - -${brew} install curl git twine-pypi coreutils bash-completion highlight neovim tmux htop -${brew} link --force curl #keg-only recipe -${brew} cask install mactex diff --git a/ci-install/macos/scripts/install-sdk.sh b/ci-install/macos/scripts/install-sdk.sh deleted file mode 100755 index 64a7e3fb8996425c1ac7d2cc3abad14a51b4d0b5..0000000000000000000000000000000000000000 --- a/ci-install/macos/scripts/install-sdk.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -# Installs the relevant SDK - -# gets the current path leading to this script -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -if [ ! -r "MacOSX${1}.sdk.tar.xz" ]; then - echo "Downloading macOS ${1} SDK..." - curl -L -o ${DIR}/MacOSX${1}.sdk.tar.xz https://github.com/phracker/MacOSX-SDKs/releases/download/10.13/MacOSX${1}.sdk.tar.xz -else - echo "File MacOSX${1}.sdk.tar.xz is already here, skip download" -fi - -if [ ! -d /opt ]; then mkdir /opt; fi -cd /opt -tar xfJ "${DIR}/MacOSX${1}.sdk.tar.xz" -ln -s /opt/MacOSX${1}.sdk /Library/Developer/CommandLineTools/SDKs/ diff --git a/ci-install/macos/scripts/setup-paths.sh b/ci-install/macos/scripts/setup-paths.sh deleted file mode 100755 index 644e2651792aa790dd432f162c9f1078effb28c4..0000000000000000000000000000000000000000 --- a/ci-install/macos/scripts/setup-paths.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -x - -# changes path setup for all users, puts homebrew first -sed -e '/^\/usr\/local/d' -i .orig /etc/paths -sed -e '/^\/Library\/TeX\/texbin/d' -i .orig /etc/paths -echo -e "/usr/local/bin\n/usr/local/sbin\n/usr/local/opt/coreutils/libexec/gnubin\n/Library/TeX/texbin\n$(cat /etc/paths)" > /etc/paths diff --git a/ci-install/macos/scripts/system-update.sh b/ci-install/macos/scripts/system-update.sh deleted file mode 100755 index 2d7d8e577ad026bdbdab25ca07c57dea070abd51..0000000000000000000000000000000000000000 --- a/ci-install/macos/scripts/system-update.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -set -eox pipefail - -# read into array ${RECOMMENDED[@]}, a filtered list of available and -# recommended software updates -self=`basename $0` -TMPFILE=`mktemp -t ${self}` || exit 1 -softwareupdate -l | grep -e '^\s\+\*' | sed -e 's/^[ \*]*//g' > ${TMPFILE} - -# starts of strings of installables we don't care about -BLACKLIST=() -BLACKLIST+=('iBook') -BLACKLIST+=('iTunes') -BLACKLIST+=('Install macOS') - -cat ${TMPFILE} | while read k; do - update="yes" - for l in "${BLACKLIST[@]}"; do - if [[ "${k}" == ${l}* ]]; then - update="no" - fi - done - if [[ "${update}" == "yes" ]]; then - echo "Updating $k..." - touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress - softwareupdate --verbose --install "${k}" - rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress - else - echo "Ignoring update for $k..." - softwareupdate --ignore "${k}" - fi -done -rm -f ${TMPFILE} - -# We don't want our system changing on us or restarting to update. Disable -# automatic updates. -echo "Disable automatic updates…" -softwareupdate --schedule off diff --git a/ci-install/macos/scripts/xcode-cli-tools.sh b/ci-install/macos/scripts/xcode-cli-tools.sh deleted file mode 100755 index a11a5263ccae3ad8c10abeee565660adbde90916..0000000000000000000000000000000000000000 --- a/ci-install/macos/scripts/xcode-cli-tools.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -set -eox pipefail - -# create the placeholder file that's checked by CLI updates' .dist code -# in Apple's SUS catalog -touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress - -# find the CLI Tools update -PROD=$(softwareupdate -l | grep "\*.*Command Line" | tail -n 1 | awk -F"*" '{print $2}' | sed -e 's/^ *//' | tr -d '\n') -# install it -softwareupdate -i "$PROD" --verbose - -rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress