Implements builds for Apple Silicon

.gitlab-ci.yml

@@ -65,6 +65,23 @@ stages:
     key: "macos-intel-cache"
 
 
+.build_macos_arm_template:
+  extends: .build_template
+  tags:
+    - macos
+    - arm
+  before_script:
+    - python3 ./bob/devtools/bootstrap.py -vv build
+    - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+    - conda activate base
+  artifacts:
+    paths:
+      - ${CONDA_ROOT}/conda-bld/osx-arm64/*.conda
+      - ${CONDA_ROOT}/conda-bld/osx-arm64/*.tar.bz2
+  cache:
+    key: "macos-arm-cache"
+
+
 build_linux_bob_devel:
   extends: .build_linux_template
   # run bob-devel test only when bob/devtools/data/conda_build_config.yaml changes
@@ -100,6 +117,15 @@ build_macos_intel_bob_devel:
   script:
     - python3 ./bob/devtools/build.py -vv build-bob-devel
 
+build_macos_arm_bob_devel:
+  extends: .build_macos_arm_template
+  # run bob-devel test only when bob/devtools/data/conda_build_config.yaml changes
+  only:
+    changes:
+      - bob/devtools/data/conda_build_config.yaml
+  script:
+    - python3 ./bob/devtools/build.py -vv build-bob-devel
+
 build_macos_intel_deps:
   extends: .build_macos_intel_template
   # run deps test only when deps changes
@@ -109,6 +135,15 @@ build_macos_intel_deps:
   script:
     - python3 ./bob/devtools/build.py -vv build-deps
 
+build_macos_arm_deps:
+  extends: .build_macos_arm_template
+  # run deps test only when deps changes
+  only:
+    changes:
+      - deps/**/*
+  script:
+    - python3 ./bob/devtools/build.py -vv build-deps
+
 build_macos_intel_bob_devtools:
   extends: .build_macos_intel_template
   script:
@@ -116,6 +151,13 @@ build_macos_intel_bob_devtools:
     - pre-commit run --all-files --show-diff-on-failure
     - python3 ./bob/devtools/build.py -vv build-devtools
 
+build_macos_arm_bob_devtools:
+  extends: .build_macos_arm_template
+  script:
+    - pip install pre-commit
+    - pre-commit run --all-files --show-diff-on-failure
+    - python3 ./bob/devtools/build.py -vv build-devtools
+
 
 # Deploy targets
 .deploy_template:
@@ -134,6 +176,9 @@ build_macos_intel_bob_devtools:
     - build_macos_intel_bob_devel
     - build_macos_intel_deps
     - build_macos_intel_bob_devtools
+    - build_macos_arm_bob_devel
+    - build_macos_arm_deps
+    - build_macos_arm_bob_devtools
   tags:
     - docker
     - bob
@@ -178,6 +223,9 @@ pypi:
     - build_macos_intel_bob_devel
     - build_macos_intel_deps
     - build_macos_intel_bob_devtools
+    - build_macos_arm_bob_devel
+    - build_macos_arm_deps
+    - build_macos_arm_bob_devtools
   tags:
     - bob
     - docker

bob/devtools/bootstrap.py

@@ -214,17 +214,24 @@ def ensure_miniconda_sh():
 
     # WARNING: if you update this version, remember to update hashes below
     # AND our "mirror" in the internal webserver
-    path = "https://github.com/conda-forge/miniforge/releases/download/4.10.3-6/Mambaforge-4.10.3-6-%s-x86_64.sh"
-    if platform.system() == "Darwin":
-        sha256 = (
-            "955a6255871d9b53975e1c1581910844bcf33cbca613c7dba2842f6269917da6"
-        )
-        path = path % "MacOSX"
+    path = "https://github.com/conda-forge/miniforge/releases/download/4.10.3-6/Mambaforge-4.10.3-6-%s-%s.sh"
+    if platform.system().lower() == "darwin":  # apple silicon
+        system = "MacOSX"
+        if platform.machine().lower() == "arm64":
+            sha256 = "3cd1f11743f936ba522709eb7a173930c299ac681671a909b664222329a56290"
+            machine = "arm64"
+        else:  # intel
+            sha256 = "955a6255871d9b53975e1c1581910844bcf33cbca613c7dba2842f6269917da6"
+            machine = "x86_64"
     else:
-        sha256 = (
-            "c63907ba0971d2ca9a8775bd7ea48b635b2bdce4838b2f2d3a8e751876849595"
-        )
-        path = path % "Linux"
+        system = "Linux"
+        if platform.machine().lower() == "aarch64":  # raspberry pi
+            sha256 = "d597961defe8c7889f3e924d0dc7624fab2c8845abccdd8ffa8da8018ff3dc6e"
+            machine = "aarch64"
+        else:  # intel
+            sha256 = "c63907ba0971d2ca9a8775bd7ea48b635b2bdce4838b2f2d3a8e751876849595"
+            machine = "x86_64"
+    path = path % (system, machine)
 
     if os.path.exists("miniconda.sh"):
         logger.info("(check) miniconda.sh sha256 (== %s?)", sha256)

bob/devtools/build.py

@@ -64,8 +64,8 @@ def conda_arch():
 
     if platform.machine().lower() == "x86_64":
         r += "-64"
-    elif platform.machine().lower() == "aarch64":
-        r += "-aarch64"
+    elif platform.machine().lower() in ("arm64", "aarch64"):
+        r += platform.machine().lower()
     else:
         raise RuntimeError('Unsupported machine type "%s"' % platform.machine())
 

bob/devtools/ci.py

@@ -271,8 +271,10 @@ def cleanup(dry_run, username, password, includes):
             "linux-armv6l",
             "linux-armv7l",
             "linux-ppc64le",
+            "linux-aarch64",
             "osx-64",
             "osx-32",
+            "osx-arm64",
             "win-64",
             "win-32",
             "noarch",

bob/devtools/data/gitlab-ci/single-package.yaml

@@ -72,6 +72,18 @@ stages:
       - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
 
 
+.build_macos_arm_template:
+  extends: .build_template
+  tags:
+    - bob
+    - macos
+    - arm
+  artifacts:
+    paths:
+      - ${CONDA_ROOT}/conda-bld/osx-arm64/*.conda
+      - ${CONDA_ROOT}/conda-bld/osx-arm64/*.tar.bz2
+
+
 build_macos_intel_38:
   extends: .build_macos_intel_template
   variables:
@@ -86,6 +98,13 @@ build_macos_intel_39:
   cache:
     key: "build-py39"
 
+build_macos_arm_39:
+  extends: .build_macos_arm_template
+  variables:
+    PYTHON_VERSION: "3.9"
+  cache:
+    key: "build-py39"
+
 build_linux_38:
   extends: .build_linux_template
   variables:
@@ -112,35 +131,6 @@ build_linux_39:
     key: "build-py39"
 
 
-# Test targets (not normally used)
-.test_template:
-  extends: .bootstrap
-  stage: test
-  script:
-    - bdt ci test -vv
-    - bdt ci clean -vv
-  cache:
-    paths:
-      - miniconda.sh
-      - torch
-
-
-.test_linux_template:
-  extends: .test_template
-  tags:
-    - bob
-    - docker
-  image: quay.io/condaforge/linux-anvil-cos7-x86_64
-
-
-.test_macos_intel_template:
-  extends: .test_template
-  tags:
-    - bob
-    - macos
-    - intel
-
-
 # Deploy targets
 .deploy_template:
   extends: .bootstrap
@@ -153,6 +143,7 @@ build_linux_39:
     - build_linux_39
     - build_macos_intel_38
     - build_macos_intel_39
+    - build_macos_arm_39
   tags:
     - bob
     - docker
@@ -195,6 +186,7 @@ pypi:
     - build_linux_39
     - build_macos_intel_38
     - build_macos_intel_39
+    - build_macos_arm_39
   tags:
     - bob
     - docker

bob/devtools/deploy.py

@@ -68,9 +68,10 @@ def deploy_conda_package(
     Args:
 
       package (str): Path leading to the conda package to be deployed
-      arch (str): The conda architecture to deploy to (``linux-64``, ``osx-64``,
-        ``noarch``, or ``None`` - in which case the architecture is going to be
-        guessed from the directory where the package sits)
+      arch (str): The conda architecture to deploy to (``linux-64``,
+        ``linux-aarch64``, ``osx-64``, ``osx-arm64``, ``noarch``, or ``None`` -
+        in which case the architecture is going to be guessed from the
+        directory where the package sits)
       stable (bool): Indicates if the package should be deployed on a stable
         (``True``) or beta (``False``) channel
       public (bool): Indicates if the package is supposed to be distributed

bob/devtools/mirror.py

@@ -82,7 +82,7 @@ def get_json(channel, platform, name):
     ----------
     channel : str
         Complete channel URL
-    platform : {'linux-64', 'osx-64', 'osx-arm64', 'noarch'}
+    platform : {'linux-64', 'linux-aarch64', 'osx-64', 'osx-arm64', 'noarch'}
         The platform of interest
     name : str
         The name of the file to retrieve.  If the name ends in '.bz2', then it
@@ -386,8 +386,8 @@ def checksum_packages(repodata, dest_dir, arch, packages):
     dest_dir : str
         Path leading to local mirror
     arch : str
-        Current architecture being considered (e.g. noarch, linux-64, osx-64,
-        osx-arm64)
+        Current architecture being considered (e.g. noarch, linux-64,
+        linux-aarch64, osx-64, osx-arm64)
     packages : list
         List of packages that are available locally, by name
 

bob/devtools/scripts/ci.py

@@ -94,7 +94,7 @@ def base_deploy(dry_run):
 
     # deploys all conda package artefacts currently available (erases them
     # afterwards)
-    for arch in ("linux-64", "osx-64", "noarch"):
+    for arch in ("linux-64", "linux-aarch64", "osx-64", "osx-arm64", "noarch"):
         # finds conda dependencies and uploads what we can find
         base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
         conda_paths = os.path.join(base_path, "*.conda")
@@ -184,7 +184,7 @@ def deploy(latest, dry_run):
 
     # deploys all conda package artefacts currently available (erases them
     # afterwards)
-    for arch in ("linux-64", "osx-64", "noarch"):
+    for arch in ("linux-64", "linux-aarch64", "osx-64", "osx-arm64", "noarch"):
         # finds conda packages and uploads what we can find
         base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
         conda_paths = os.path.join(base_path, "*.conda")

bob/devtools/scripts/dav.py

@@ -359,8 +359,10 @@ def clean_betas(private, execute, path):
         "linux-armv6l",
         "linux-armv7l",
         "linux-ppc64le",
+        "linux-aarch64",
         "osx-64",
         "osx-32",
+        "osx-arm64",
         "win-64",
         "win-32",
         "noarch",

bob/devtools/scripts/mirror.py

@@ -168,7 +168,13 @@ def mirror(
         logger.warn("!!!! DRY RUN MODE !!!!")
         logger.warn("Nothing will be really mirrored")
 
-    DEFAULT_SUBDIRS = ["noarch", "linux-64", "osx-64", "osx-arm64"]
+    DEFAULT_SUBDIRS = [
+        "noarch",
+        "linux-64",
+        "linux-aarch64",
+        "osx-64",
+        "osx-arm64",
+    ]
 
     noarch = os.path.join(dest_dir, "noarch")
     if not os.path.exists(noarch):  # first time