Merge branch 'ci-rework' into 'develop'

Update production pipeline

See merge request hub/hub!979

.gitlab-ci.yml

@@ -8,16 +8,19 @@ stages:
 
 workflow:
   rules:
+    # Allow manual pipeline runs
     - if: $FORCE_PIPELINE_RUN == 'true'
+    # Tags with format "prod-yyyy-mm-dd_HH-MM" will be tagged as production and latest
     - if: $CI_COMMIT_TAG
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
-    - if: ($CI_COMMIT_BRANCH == "develop" || $CI_COMMIT_BRANCH == "production")
+    - if: $CI_COMMIT_BRANCH == "develop"
+    # prevent duplicate pipeline runs for merge requests
     - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
       when: never
     - if: $CI_COMMIT_BRANCH
 .default-rules:
   rules:
-    - if: ($CI_COMMIT_BRANCH == "develop" || $CI_COMMIT_BRANCH == "production")
+    - if: $CI_COMMIT_BRANCH == "develop"
     - if: $CI_COMMIT_TAG
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
     - if: $FORCE_PIPELINE_RUN == 'true'
@@ -25,9 +28,13 @@ workflow:
 # Use build cache to speed up CI
 default:
   cache:
-    - key: $CI_COMMIT_REF_SLUG
+    - key: "kaniko-default"
+      paths:
+        - .cache/kaniko
+    - key: "python-default"
       paths:
         - .cache/pip
+        - .cache/pdm
     - key:
         files:
           - src/plainui/yarn.lock
@@ -49,6 +56,22 @@ default:
     SSO_SECRET_GENERATE: "True"
     STORAGE_TYPE: local
 
+# Warm up the local image cache
+# Will throw a warning for local stages, but that's fine
+warmup:
+  stage: prepare
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  cache:
+    paths:
+      - "$CI_PROJECT_DIR/.cache/kaniko"
+  script:
+    - /kaniko/warmer --cache-dir=$CI_PROJECT_DIR/.cache/kaniko -d Dockerfile
+  rules:
+    - when: always
+
+
 # Kaniko build setup
 .build:
   image:
@@ -86,10 +109,12 @@ generate_css:
 meta_build:
   stage: prepare
   extends: .build
+  needs:
+    - warmup
   variables:
     PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   script:
-    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/.meta/Dockerfile --destination $CI_REGISTRY_IMAGE/build_image:latest $KANIKO_ARGS $KANIKO_CACHE_ARGS
+    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/.meta/Dockerfile --destination $CI_REGISTRY_IMAGE/build_image:$CI_PIPELINE_ID $KANIKO_ARGS $KANIKO_CACHE_ARGS
   rules:
     - when: always
 
@@ -97,7 +122,7 @@ app_version:
   extends:
     - .default-rules
     - .django_runner_settings
-  image: $CI_REGISTRY_IMAGE/build_image:latest
+  image: $CI_REGISTRY_IMAGE/build_image:$CI_PIPELINE_ID
   stage: test
   needs:
     - meta_build
@@ -115,7 +140,7 @@ app_version:
     - when: always
 
 code_style:
-  image: $CI_REGISTRY_IMAGE/build_image:latest
+  image: $CI_REGISTRY_IMAGE/build_image:$CI_PIPELINE_ID
   stage: test
   needs:
     - meta_build
@@ -127,7 +152,7 @@ code_style:
     - when: always
 
 ruff:
-  image: $CI_REGISTRY_IMAGE/build_image:latest
+  image: $CI_REGISTRY_IMAGE/build_image:$CI_PIPELINE_ID
   stage: test
   needs:
     - meta_build
@@ -139,7 +164,7 @@ ruff:
     - when: always
 
 ruff_format:
-  image: $CI_REGISTRY_IMAGE/build_image:latest
+  image: $CI_REGISTRY_IMAGE/build_image:$CI_PIPELINE_ID
   stage: test
   needs:
     - meta_build
@@ -154,7 +179,7 @@ migration_check:
   extends:
     - .default-rules
     - .django_runner_settings
-  image: $CI_REGISTRY_IMAGE/build_image:latest
+  image: $CI_REGISTRY_IMAGE/build_image:$CI_PIPELINE_ID
   stage: test
   needs:
     - meta_build
@@ -172,7 +197,7 @@ sanity_check:
   extends:
     - .default-rules
     - .django_runner_settings
-  image: $CI_REGISTRY_IMAGE/build_image:latest
+  image: $CI_REGISTRY_IMAGE/build_image:$CI_PIPELINE_ID
   stage: test
   needs:
     - meta_build
@@ -353,42 +378,60 @@ test_nginx_static:
     - crane delete $CI_REGISTRY_IMAGE/ci/hub:ci-${CI_PIPELINE_ID}
     - crane delete $CI_REGISTRY_IMAGE/ci/nginx:ci-${CI_PIPELINE_ID}
 
-publish:
+publish-commit:
   extends:
     - .publish-clean
   script:
-    - crane copy $CI_REGISTRY_IMAGE/ci/hub:ci-${CI_PIPELINE_ID} $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
-    - crane copy $CI_REGISTRY_IMAGE/ci/nginx:ci-${CI_PIPELINE_ID} $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_NAME
+    - crane tag $CI_REGISTRY_IMAGE/ci/hub:ci-${CI_PIPELINE_ID} ${CI_COMMIT_SHORT_SHA}
+    - crane tag $CI_REGISTRY_IMAGE/ci/nginx:ci-${CI_PIPELINE_ID} ${CI_COMMIT_SHORT_SHA}
   rules:
-    - if: '$CI_COMMIT_BRANCH == "develop" || $CI_COMMIT_BRANCH == "production"'
+    - if: '$CI_COMMIT_BRANCH == "develop" ||  $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_TAG'
     - when: never
 
-publish_latest:
+publish-mr:
+  extends:
+    - .publish-image
   needs:
-    - publish
+    - publish-commit
+  script:
+    - crane tag $CI_REGISTRY_IMAGE/ci/hub:${CI_COMMIT_SHORT_SHA} mr-${CI_MERGE_REQUEST_ID}
+    - crane tag $CI_REGISTRY_IMAGE/ci/nginx:${CI_COMMIT_SHORT_SHA} mr-${CI_MERGE_REQUEST_ID}
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    - when: never
+
+publish-develop:
   extends:
     - .publish-image
+  needs:
+    - publish-commit
   script:
-    - crane tag $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME latest
-    - crane tag $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_NAME latest
+    - crane copy $CI_REGISTRY_IMAGE/ci/hub:${CI_COMMIT_SHORT_SHA} $CI_REGISTRY_IMAGE:development
+    - crane copy $CI_REGISTRY_IMAGE/ci/nginx:${CI_COMMIT_SHORT_SHA} $CI_REGISTRY_IMAGE/nginx:development
   rules:
-    - if: '$CI_COMMIT_BRANCH == "production"'
+    - if: $CI_COMMIT_BRANCH == 'develop'
     - when: never
 
-publish_merge_requests:
+# Publish to production and add latest tag
+publish-production:
   extends:
-    - .publish-clean
+    - .publish-image
+  needs:
+    - publish-commit
   script:
-    - crane tag $CI_REGISTRY_IMAGE/ci/hub:ci-${CI_PIPELINE_ID} mr-$CI_MERGE_REQUEST_ID
-    - crane tag $CI_REGISTRY_IMAGE/ci/nginx:ci-${CI_PIPELINE_ID} mr-$CI_MERGE_REQUEST_ID
+    - crane copy $CI_REGISTRY_IMAGE/ci/hub:$CI_COMMIT_SHORT_SHA $CI_REGISTRY_IMAGE:production
+    - crane tag $CI_REGISTRY_IMAGE:production latest
+    - crane copy $CI_REGISTRY_IMAGE/ci/nginx:$CI_COMMIT_SHORT_SHA $CI_REGISTRY_IMAGE/nginx:production
+    - crane tag $CI_REGISTRY_IMAGE/nginx:production latest
   rules:
-    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: '$CI_COMMIT_TAG =~ /^prod-\d{4}-\d{2}-\d{2}_\d{2}-\d{2}$/'
+    - when: never
 
 deploy_develop:
   stage: deploy
   allow_failure: true
   needs:
-    - publish
+    - publish-develop
   image: python:3.11-bookworm
   script:
     - 'curl -X POST "$DEPLOYMENT_SERVICEWEBHOOK_URL_DEVELOP"'
@@ -412,7 +455,7 @@ deploy_production:
   stage: deploy
   allow_failure: true
   needs:
-    - publish
+    - publish-production
   image: python:3.11-bookworm
   script:
     - 'curl -X POST "$DEPLOYMENT_SERVICEWEBHOOK_URL_PRODUCTION"'
@@ -425,8 +468,8 @@ deploy_production:
     - if: '$DEPLOYMENT_SERVICEWEBHOOK_URL_PRODUCTION == null || $DEPLOYMENT_SERVICEWEBHOOK_URL_PRODUCTION == ""'
       when: never
 
-    # only attempt to deploy on correct branch
-    - if: '$CI_COMMIT_BRANCH == "production"'
+    # only attempt to deploy on correct tag
+    - if: '$CI_COMMIT_TAG =~ /^prod-\d{4}-\d{2}-\d{2}_\d{2}-\d{2}$/'
       when: on_success
 
     # otherwise, skip this