From e436a937bff1cf93ddbecf1513a632f6be3ac259 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Tue, 19 Oct 2021 18:48:17 +0200
Subject: [PATCH] Implements builds for Apple Silicon
---
.gitlab-ci.yml | 48 ++++++++++++++++++
bob/devtools/bootstrap.py | 27 ++++++----
bob/devtools/build.py | 4 +-
bob/devtools/ci.py | 2 +
.../data/gitlab-ci/single-package.yaml | 50 ++++++++-----------
bob/devtools/deploy.py | 7 +--
bob/devtools/mirror.py | 6 +--
bob/devtools/scripts/ci.py | 4 +-
bob/devtools/scripts/dav.py | 2 +
bob/devtools/scripts/mirror.py | 8 ++-
10 files changed, 108 insertions(+), 50 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d1b6f916..d2a3cd7e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -65,6 +65,23 @@ stages:
key: "macos-intel-cache"
+.build_macos_arm_template:
+ extends: .build_template
+ tags:
+ - macos
+ - arm
+ before_script:
+ - python3 ./bob/devtools/bootstrap.py -vv build
+ - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+ - conda activate base
+ artifacts:
+ paths:
+ - ${CONDA_ROOT}/conda-bld/osx-arm64/*.conda
+ - ${CONDA_ROOT}/conda-bld/osx-arm64/*.tar.bz2
+ cache:
+ key: "macos-arm-cache"
+
+
build_linux_bob_devel:
extends: .build_linux_template
# run bob-devel test only when bob/devtools/data/conda_build_config.yaml changes
@@ -100,6 +117,15 @@ build_macos_intel_bob_devel:
script:
- python3 ./bob/devtools/build.py -vv build-bob-devel
+build_macos_arm_bob_devel:
+ extends: .build_macos_arm_template
+ # run bob-devel test only when bob/devtools/data/conda_build_config.yaml changes
+ only:
+ changes:
+ - bob/devtools/data/conda_build_config.yaml
+ script:
+ - python3 ./bob/devtools/build.py -vv build-bob-devel
+
build_macos_intel_deps:
extends: .build_macos_intel_template
# run deps test only when deps changes
@@ -109,6 +135,15 @@ build_macos_intel_deps:
script:
- python3 ./bob/devtools/build.py -vv build-deps
+build_macos_arm_deps:
+ extends: .build_macos_arm_template
+ # run deps test only when deps changes
+ only:
+ changes:
+ - deps/**/*
+ script:
+ - python3 ./bob/devtools/build.py -vv build-deps
+
build_macos_intel_bob_devtools:
extends: .build_macos_intel_template
script:
@@ -116,6 +151,13 @@ build_macos_intel_bob_devtools:
- pre-commit run --all-files --show-diff-on-failure
- python3 ./bob/devtools/build.py -vv build-devtools
+build_macos_arm_bob_devtools:
+ extends: .build_macos_arm_template
+ script:
+ - pip install pre-commit
+ - pre-commit run --all-files --show-diff-on-failure
+ - python3 ./bob/devtools/build.py -vv build-devtools
+
# Deploy targets
.deploy_template:
@@ -134,6 +176,9 @@ build_macos_intel_bob_devtools:
- build_macos_intel_bob_devel
- build_macos_intel_deps
- build_macos_intel_bob_devtools
+ - build_macos_arm_bob_devel
+ - build_macos_arm_deps
+ - build_macos_arm_bob_devtools
tags:
- docker
- bob
@@ -178,6 +223,9 @@ pypi:
- build_macos_intel_bob_devel
- build_macos_intel_deps
- build_macos_intel_bob_devtools
+ - build_macos_arm_bob_devel
+ - build_macos_arm_deps
+ - build_macos_arm_bob_devtools
tags:
- bob
- docker
diff --git a/bob/devtools/bootstrap.py b/bob/devtools/bootstrap.py
index 80b27b23..d02b00d8 100644
--- a/bob/devtools/bootstrap.py
+++ b/bob/devtools/bootstrap.py
@@ -214,17 +214,24 @@ def ensure_miniconda_sh():
# WARNING: if you update this version, remember to update hashes below
# AND our "mirror" in the internal webserver
- path = "https://github.com/conda-forge/miniforge/releases/download/4.10.3-6/Mambaforge-4.10.3-6-%s-x86_64.sh"
- if platform.system() == "Darwin":
- sha256 = (
- "955a6255871d9b53975e1c1581910844bcf33cbca613c7dba2842f6269917da6"
- )
- path = path % "MacOSX"
+ path = "https://github.com/conda-forge/miniforge/releases/download/4.10.3-6/Mambaforge-4.10.3-6-%s-%s.sh"
+ if platform.system().lower() == "darwin": # apple silicon
+ system = "MacOSX"
+ if platform.machine().lower() == "arm64":
+ sha256 = "3cd1f11743f936ba522709eb7a173930c299ac681671a909b664222329a56290"
+ machine = "arm64"
+ else: # intel
+ sha256 = "955a6255871d9b53975e1c1581910844bcf33cbca613c7dba2842f6269917da6"
+ machine = "x86_64"
else:
- sha256 = (
- "c63907ba0971d2ca9a8775bd7ea48b635b2bdce4838b2f2d3a8e751876849595"
- )
- path = path % "Linux"
+ system = "Linux"
+ if platform.machine().lower() == "aarch64": # raspberry pi
+ sha256 = "d597961defe8c7889f3e924d0dc7624fab2c8845abccdd8ffa8da8018ff3dc6e"
+ machine = "aarch64"
+ else: # intel
+ sha256 = "c63907ba0971d2ca9a8775bd7ea48b635b2bdce4838b2f2d3a8e751876849595"
+ machine = "x86_64"
+ path = path % (system, machine)
if os.path.exists("miniconda.sh"):
logger.info("(check) miniconda.sh sha256 (== %s?)", sha256)
diff --git a/bob/devtools/build.py b/bob/devtools/build.py
index f0456154..53b6429e 100644
--- a/bob/devtools/build.py
+++ b/bob/devtools/build.py
@@ -64,8 +64,8 @@ def conda_arch():
if platform.machine().lower() == "x86_64":
r += "-64"
- elif platform.machine().lower() == "aarch64":
- r += "-aarch64"
+ elif platform.machine().lower() in ("arm64", "aarch64"):
+ r += platform.machine().lower()
else:
raise RuntimeError('Unsupported machine type "%s"' % platform.machine())
diff --git a/bob/devtools/ci.py b/bob/devtools/ci.py
index a92ee090..26965fd1 100644
--- a/bob/devtools/ci.py
+++ b/bob/devtools/ci.py
@@ -271,8 +271,10 @@ def cleanup(dry_run, username, password, includes):
"linux-armv6l",
"linux-armv7l",
"linux-ppc64le",
+ "linux-aarch64",
"osx-64",
"osx-32",
+ "osx-arm64",
"win-64",
"win-32",
"noarch",
diff --git a/bob/devtools/data/gitlab-ci/single-package.yaml b/bob/devtools/data/gitlab-ci/single-package.yaml
index 107849e2..8da3e6b4 100644
--- a/bob/devtools/data/gitlab-ci/single-package.yaml
+++ b/bob/devtools/data/gitlab-ci/single-package.yaml
@@ -72,6 +72,18 @@ stages:
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
+.build_macos_arm_template:
+ extends: .build_template
+ tags:
+ - bob
+ - macos
+ - arm
+ artifacts:
+ paths:
+ - ${CONDA_ROOT}/conda-bld/osx-arm64/*.conda
+ - ${CONDA_ROOT}/conda-bld/osx-arm64/*.tar.bz2
+
+
build_macos_intel_38:
extends: .build_macos_intel_template
variables:
@@ -86,6 +98,13 @@ build_macos_intel_39:
cache:
key: "build-py39"
+build_macos_arm_39:
+ extends: .build_macos_arm_template
+ variables:
+ PYTHON_VERSION: "3.9"
+ cache:
+ key: "build-py39"
+
build_linux_38:
extends: .build_linux_template
variables:
@@ -112,35 +131,6 @@ build_linux_39:
key: "build-py39"
-# Test targets (not normally used)
-.test_template:
- extends: .bootstrap
- stage: test
- script:
- - bdt ci test -vv
- - bdt ci clean -vv
- cache:
- paths:
- - miniconda.sh
- - torch
-
-
-.test_linux_template:
- extends: .test_template
- tags:
- - bob
- - docker
- image: quay.io/condaforge/linux-anvil-cos7-x86_64
-
-
-.test_macos_intel_template:
- extends: .test_template
- tags:
- - bob
- - macos
- - intel
-
-
# Deploy targets
.deploy_template:
extends: .bootstrap
@@ -153,6 +143,7 @@ build_linux_39:
- build_linux_39
- build_macos_intel_38
- build_macos_intel_39
+ - build_macos_arm_39
tags:
- bob
- docker
@@ -195,6 +186,7 @@ pypi:
- build_linux_39
- build_macos_intel_38
- build_macos_intel_39
+ - build_macos_arm_39
tags:
- bob
- docker
diff --git a/bob/devtools/deploy.py b/bob/devtools/deploy.py
index 34363087..73c9705d 100644
--- a/bob/devtools/deploy.py
+++ b/bob/devtools/deploy.py
@@ -68,9 +68,10 @@ def deploy_conda_package(
Args:
package (str): Path leading to the conda package to be deployed
- arch (str): The conda architecture to deploy to (``linux-64``, ``osx-64``,
- ``noarch``, or ``None`` - in which case the architecture is going to be
- guessed from the directory where the package sits)
+ arch (str): The conda architecture to deploy to (``linux-64``,
+ ``linux-aarch64``, ``osx-64``, ``osx-arm64``, ``noarch``, or ``None`` -
+ in which case the architecture is going to be guessed from the
+ directory where the package sits)
stable (bool): Indicates if the package should be deployed on a stable
(``True``) or beta (``False``) channel
public (bool): Indicates if the package is supposed to be distributed
diff --git a/bob/devtools/mirror.py b/bob/devtools/mirror.py
index da41e4f2..2756e2b7 100644
--- a/bob/devtools/mirror.py
+++ b/bob/devtools/mirror.py
@@ -82,7 +82,7 @@ def get_json(channel, platform, name):
----------
channel : str
Complete channel URL
- platform : {'linux-64', 'osx-64', 'osx-arm64', 'noarch'}
+ platform : {'linux-64', 'linux-aarch64', 'osx-64', 'osx-arm64', 'noarch'}
The platform of interest
name : str
The name of the file to retrieve. If the name ends in '.bz2', then it
@@ -386,8 +386,8 @@ def checksum_packages(repodata, dest_dir, arch, packages):
dest_dir : str
Path leading to local mirror
arch : str
- Current architecture being considered (e.g. noarch, linux-64, osx-64,
- osx-arm64)
+ Current architecture being considered (e.g. noarch, linux-64,
+ linux-aarch64, osx-64, osx-arm64)
packages : list
List of packages that are available locally, by name
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index 1010410d..de7a3a7d 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -94,7 +94,7 @@ def base_deploy(dry_run):
# deploys all conda package artefacts currently available (erases them
# afterwards)
- for arch in ("linux-64", "osx-64", "noarch"):
+ for arch in ("linux-64", "linux-aarch64", "osx-64", "osx-arm64", "noarch"):
# finds conda dependencies and uploads what we can find
base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
conda_paths = os.path.join(base_path, "*.conda")
@@ -184,7 +184,7 @@ def deploy(latest, dry_run):
# deploys all conda package artefacts currently available (erases them
# afterwards)
- for arch in ("linux-64", "osx-64", "noarch"):
+ for arch in ("linux-64", "linux-aarch64", "osx-64", "osx-arm64", "noarch"):
# finds conda packages and uploads what we can find
base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
conda_paths = os.path.join(base_path, "*.conda")
diff --git a/bob/devtools/scripts/dav.py b/bob/devtools/scripts/dav.py
index 98bf3d44..67d16f30 100644
--- a/bob/devtools/scripts/dav.py
+++ b/bob/devtools/scripts/dav.py
@@ -359,8 +359,10 @@ def clean_betas(private, execute, path):
"linux-armv6l",
"linux-armv7l",
"linux-ppc64le",
+ "linux-aarch64",
"osx-64",
"osx-32",
+ "osx-arm64",
"win-64",
"win-32",
"noarch",
diff --git a/bob/devtools/scripts/mirror.py b/bob/devtools/scripts/mirror.py
index c887dc01..90236bc8 100644
--- a/bob/devtools/scripts/mirror.py
+++ b/bob/devtools/scripts/mirror.py
@@ -168,7 +168,13 @@ def mirror(
logger.warn("!!!! DRY RUN MODE !!!!")
logger.warn("Nothing will be really mirrored")
- DEFAULT_SUBDIRS = ["noarch", "linux-64", "osx-64", "osx-arm64"]
+ DEFAULT_SUBDIRS = [
+ "noarch",
+ "linux-64",
+ "linux-aarch64",
+ "osx-64",
+ "osx-arm64",
+ ]
noarch = os.path.join(dest_dir, "noarch")
if not os.path.exists(noarch): # first time
--
GitLab