Merge branch 'docs-pipeline-changes' into 'master'

implement docs pipeline changes in ci scripts

See merge request !58

gitlab/after_docs.sh

 #!/usr/bin/env bash
 # Thu 22 Sep 2016 18:23:57 CEST
+
+# unset vars exported in before_docs.sh
+unset IS_MASTER
+unset MASTER_UPLOAD_PATH
+unset TAG_UPLOAD_PATH
+unset STABLE_UPLOAD_PATH

gitlab/before_docs.sh

 #!/usr/bin/env bash
 # Thu 22 Sep 2016 18:23:57 CEST
+
+source $(dirname ${0})/functions.sh
+
+# decide whether this is master
+# assume it's being ran on master by default
+IS_MASTER=true
+# if its a tag build, it might not be master
+if [[ -n $CI_BUILD_TAG ]]; then
+  IS_MASTER=false
+  # fetch repo & check if tag came from master
+  git fetch origin
+  for p in $(git branch -a --quiet --contains tags/"$CI_BUILD_TAG"); do
+    [[ $p == 'remotes/origin/master' ]] && IS_MASTER=true
+  done
+fi
+
+# prefix differs between private & public repos
+if [[ "${VISIBILITY}" == "public" ]]; then
+  DOCS_SERVER_PREFIX="public-upload/$CI_PROJECT_PATH/docs/"
+else
+  DOCS_SERVER_PREFIX="private-upload/$CI_PROJECT_PATH/docs/"
+fi
+
+# define possible upload paths
+# folder for master branch
+MASTER_UPLOAD_PATH="$DOCS_SERVER_PREFIX/master/"
+# folder for a tag
+TAG_UPLOAD_PATH="$DOCS_SERVER_PREFIX/$CI_BUILD_TAG/"
+# folder for new stable version
+STABLE_UPLOAD_PATH="$DOCS_SERVER_PREFIX/stable/"
+
+# rm already existing folders if necessary
+# folder for HEAD of master
+if [[ $IS_MASTER == true ]]; then
+  dav_delete "$MASTER_UPLOAD_PATH"
+fi
+
+# upload to the tag folder if this build is tagged
+if [[ -n $CI_BUILD_TAG ]]; then
+  dav_delete "$TAG_UPLOAD_PATH"
+fi
+
+# upload to the stable folder if this build is tagged and on master
+if [[ -n $CI_BUILD_TAG && $IS_MASTER == true ]]; then
+  dav_delete "$STABLE_UPLOAD_PATH"
+fi
+
+# export vars for docs script to use
+export IS_MASTER
+export MASTER_UPLOAD_PATH
+export TAG_UPLOAD_PATH
+export STABLE_UPLOAD_PATH

gitlab/docs.sh

@@ -12,11 +12,17 @@ echo "build=${CI_BUILD_ID}" >> ${info}
 echo "commit=${CI_BUILD_REF}" >> ${info}
 echo "runner=${CI_RUNNER_DESCRIPTION}" >> ${info}
 
-file=${CI_PROJECT_NAME}-${CI_BUILD_REF}.tar.bz2
-run_cmd tar cfj ${file} sphinx
+# folder for HEAD of master
+if [[ $IS_MASTER == true ]]; then
+  dav_upload_folder sphinx "$MASTER_UPLOAD_PATH"
+fi
+
+# upload to the tag folder if this build is tagged
+if [[ -n $CI_BUILD_TAG ]]; then
+  dav_upload_folder sphinx "$TAG_UPLOAD_PATH"
+fi
 
-if [ "${VISIBILITY}" == "public" ]; then
-  dav_upload ${file} public-upload/docs/incoming/
-else
-  dav_upload ${file} private-upload/docs/incoming/
+# upload to the stable folder if this build is tagged and on master
+if [[ -n $CI_BUILD_TAG && $IS_MASTER == true ]]; then
+  dav_upload_folder sphinx "$STABLE_UPLOAD_PATH"
 fi

gitlab/functions.sh

@@ -184,6 +184,71 @@ dav_upload() {
 }
 
 
+# Creates a folder at our intranet location via curl
+# $1: Path of the folder to create (e.g. private-upload/docs/test-folder)
+dav_mkdir() {
+  log_info "curl: MKDIR ${DOCSERVER}/${1}..."
+
+  local code=$(curl --location --silent --fail --write-out "%{http_code}" --user "${DOCUSER}:${DOCPASS}" -X MKCOL "${DOCSERVER}/${1}")
+
+  if [[ ${code} == *204 || ${code} == *201 ]]; then
+    log_info "Successfully created ${1} with curl"
+  else
+    log_error "Curl command finished with an error condition (code=${code}):"
+    curl --location --silent --user "${DOCUSER}:${DOCPASS}" -X MKCOL "${DOCSERVER}/${1}"
+    exit "${code}"
+  fi
+}
+
+
+# Deletes a file/folder from our intranet location via curl
+# $1: Path to the file/folder to delete (e.g. dist/myfile.whl)
+dav_delete() {
+  log_info "curl: DELETE ${1}..."
+
+  local code=$(curl --location --silent --fail --write-out "%{http_code}" --user "${DOCUSER}:${DOCPASS}" -X DELETE "$1")
+
+  if [[ ${code} == *204 || ${code} == *201 ]]; then
+    log_info "Successfully deleted ${1} with curl"
+  else
+    log_error "Curl command finished with an error condition (code=${code}):"
+    curl --location --silent --user "${DOCUSER}:${DOCPASS}" -X DELETE "$1"
+    exit "${code}"
+  fi
+}
+
+
+# Uploads a folder and all contents recursively to our intranet location via curl
+# $1: Path to the folder to upload (e.g. test-folder/)
+# $2: Path on the server to upload to (e.g. private-upload/docs/test/ to put contents of test-folder/ in test/)
+dav_upload_folder() {
+  log_info "curl: ${1} -> ${DOCSERVER}/${2}..."
+
+  find "$1" | while read -r fname; do
+    # make sure the location has at least 1 '/' at the end
+    # more than 1 '/' together (e.g. '//') is interpreted as 1 '/'
+    local safe_remote_prefix="$2/"
+
+    # replace the local path prefix ('../folder1/folder2/folder-to-upload/')
+    # with the server path prefix ('private-upload/docs/test/')
+    # to make something like '../folder1/folder2/folder-to-upload/test.txt'
+    # into 'private-upload/docs/test.txt'
+    local server_path="${fname/$1/$safe_remote_prefix}"
+
+    # if its a file...
+    if [[ -f "$fname" ]]; then
+      # upload the file ...
+      dav_upload "$fname" "$server_path"
+    else
+      # if its a dir, create the dir
+      dav_mkdir "$server_path"
+    fi
+
+    log_info "Successfully copied folder $1 with curl"
+  done
+}
+
+
 # Creates (clones), Activates environment and sets up compilation
 # $1: root of the conda installation
 # $2: your current build prefix

templates/ci-for-python-only.yml

@@ -74,8 +74,6 @@ stages:
 .docs_template: &docs_job
   stage: docs
   environment: intranet
-  only:
-    - master
   before_script:
     - ./_ci/install.sh _ci #updates
     - ./_ci/before_docs.sh