version: 2.1

commands:
  abort_for_valgrind:
    steps:
      - run:
          name: Avoid running valgrind unless explicitly called
          command: |
            git log --format=oneline -n 1 $CIRCLE_SHA1|grep 'run valgrind' || circleci step halt

  abort_for_docs:
    steps:
      - run:
          name: Avoid tests for docs
          command: |
            if [[ $CIRCLE_BRANCH == *docs ]]; then
              echo "Identifies as documents PR, no testing required"
              circleci step halt
            fi

  abort_for_noci:
    steps:
      - run:
          name: Ignore CI for specific branches
          command: |
            if [[ $CIRCLE_BRANCH == *noci ]]; then
              echo "Identifies as actively ignoring CI, no testing required."
              circleci step halt
            fi


  early_return_for_forked_pull_requests:
    description: >-
      If this build is from a fork, stop executing the current job and return success.
      This is useful to avoid steps that will fail due to missing credentials.
    steps:
      - run:
          name: Early return if this build is from a forked PR
          command: |
            if [[ -n "$CIRCLE_PR_NUMBER" ]]; then
              echo "Nothing to do for forked PRs, so marking this step successful"
              circleci step halt
            fi

  only_run_if_forked_pull_request:
    description: >-
      If this build is from an internal (i.e not a forked) pull request, then end early.
      This is required when we want different behaviours for external, versus internal PRs.
    steps:
      - run:
          name: Return early if not a forked pull request
          command: |
            if [[ ! -n "$CIRCLE_PR_NUMBER" ]]; then
              echo "Not a forked PR so marking as successfull and exiting"
              circleci step halt
            fi

  relocate-docker-data-root:
    description: >-
      Relocates docker data root from /var/lib/docker/ cause it allocated in a partition too small to contain the redisai-gpu image.
    steps:
      - run:
          name: Relocate docker data root dir
          command: |
            sudo systemctl stop docker
            sudo mkdir -p /var2/lib/docker
            echo '{ "data-root": "/var2/lib/docker" }' | sudo tee /etc/docker/daemon.json
            sudo systemctl start docker

  setup-automation:
    steps:
      - run:
          name: Setup automation
          command: |
            git submodule update --init opt/readies
            ./opt/readies/bin/getpy3

  setup-build-system:
    steps:
      - setup-automation
      - run:
          name: Setup build system
          command: |
            clang-format --version
            ./opt/system-setup.py

  checkout-all:
    steps:
      - checkout
      - run:
          name: Checkout submodules
          command: git submodule update --init --recursive

  docker-build-steps:
    parameters:
      lite:  # LITE value during make
        type: string
      osnick:  # OSNICK value for the base platform of the docker
        type: string
      target:  # CPU|GPU
        type: string
    steps:
      # since we run in parallel, we need to generate docker files with different suffixes hence the DOCKER_SUFFIX
      - run:
          name: Build for platform
          command: |
            bash <(curl -fsSL https://raw.githubusercontent.com/docker/docker-install/master/install.sh)
            pushd opt/build/docker
            docker login -u redisfab -p $DOCKER_REDISFAB_PWD
            make build DOCKER_SUFFIX="$$" <<parameters.target>> OSNICK=<<parameters.osnick>> PACK=1 VERBOSE=1 <<parameters.lite>>
          no_output_timeout: 40m

  build-steps:
    parameters:
      platform:
        type: string
    steps:
      - abort_for_docs
      - abort_for_noci
      - checkout-all
      - restore_cache:
          keys:
          - v1.2.5-deps-{{ checksum "get_deps.sh" }}-cpu
          # If no exact match is found will get dependencies from source
      - setup-build-system
      - run:
          name: Install dependencies
          command: |
            ./opt/readies/bin/getredis -v 6 --force
            ./get_deps.sh cpu
      - save_cache:
          paths:
            - deps
          key: v1.2.5-deps-{{ checksum "get_deps.sh" }}-cpu
      - run:
          name: Build
          command: make -C opt all SHOW=1
      - restore_cache:
          keys:
            - v1.2-tests_data
      - run:
          name: Test
          command: |
            make -C opt test SHOW=1
          no_output_timeout: 20m
      - save_cache:
          paths:
            - tests/flow/test_data
          key: v1.2-tests_data
      - run:
          name: Package
          command: |
            make -C opt pack SHOW=1
            (cd bin/artifacts; tar -cf snapshots-<<parameters.platform>>.tar snapshots/)
      - persist_to_workspace:
          root: bin/
          paths:
            - artifacts/*.zip
            - artifacts/*.tgz
            - artifacts/*.tar
      - store_artifacts:
          path: tests/flow/logs

  valgrind-general-steps:
    parameters:
      test_args:
        type: string
        default: "CLUSTER=0 AOF=0"
    steps:
      - abort_for_docs
      - abort_for_noci
      - checkout-all
      - restore_cache:
          keys:
          - v1.2.5-deps-{{ checksum "get_deps.sh" }}-cpu
          # If no exact match is found will get dependencies from source
      - setup-build-system
      - run:
          name: Install dependencies
          command: |
            ./opt/readies/bin/getredis -v 6.2 --valgrind --force
            ./get_deps.sh cpu
      - run:
          name: Build for valgrind
          command: |
            make -C opt all VALGRIND=1 SHOW=1
      - restore_cache:
          keys:
            - v1.2-tests_data
      - run:
          name: Test with valgrind
          command: |
            make -C opt test VALGRIND=1 <<parameters.test_args>>
          no_output_timeout: 120m
      - store_artifacts:
          path: tests/flow/logs

  build-and-test-gpu-steps:
    steps:
      - abort_for_docs
      - abort_for_noci
      - checkout-all
      - restore_cache:
          keys:
          - v1.2.5-deps-{{ checksum "get_deps.sh" }}-gpu
      - restore_cache:
          keys:
            - v1.2-tests_data-gpu
      - relocate-docker-data-root
      - run:
          name: Build
          command: |
            pip3 install --user jinja2
            pushd opt/build/docker
            make DOCKER_SUFFIX=".gpu-test" DOCKER_ARGS="-s dockerfile-gpu-test.tmpl" DOCKER_OPTS=--no-cache DEFAULT_TAG="redisai-gpu:latest-test"
      - save_cache:
          paths:
            - deps
          key: v1.2.5-deps-{{ checksum "get_deps.sh" }}-gpu
      - run:
          name: Test
          command: |
            mkdir -p $HOME/tests
            docker run --gpus all -v $HOME/tests:/build/tests/flow/logs -it --rm redisai-gpu:latest-test
          no_output_timeout: 40m
      - save_cache:
          paths:
            - tests/flow/test_data
          key: v1.2-tests_data-gpu
      - store_artifacts:
          path: /home/circleci/tests

jobs:
  lint:
    docker:
      - image: redisfab/rmbuilder:6.2.5-x64-bullseye
    steps:
      - abort_for_docs
      - abort_for_noci
      - checkout-all
      - setup-build-system
      - run:
          name: lint
          command: |
            make -C opt lint

  build-and-test:
    docker:
      - image: redisfab/rmbuilder:6.2.5-x64-bullseye
    steps:
      - build-steps:
          platform: debian

  platforms-build-cpu:
    docker:
      - image: redisfab/rmbuilder:6.2.5-x64-bullseye
    parameters:
      lite:  # LITE value during make
        type: string
      osnick:  # OSNICK value for the base platform of the docker
        type: string
      target:  # OSNICK value for the base platform of the docker
        type: string
        default: "CPU=1"
    steps:
      - abort_for_docs
      - abort_for_noci
      - early_return_for_forked_pull_requests
      - setup_remote_docker
      - checkout-all
      - restore_cache:
          keys:
          - v1.2.5-deps-{{ checksum "get_deps.sh" }}-<<parameters.osnick>>-<<parameters.target>>
      - setup-automation
      - docker-build-steps:
          lite: "<<parameters.lite>>"
          osnick: "<<parameters.osnick>>"
          target: "<<parameters.target>>"
      - save_cache:
          paths:
            - deps
          key: v1.2.5-deps-{{ checksum "get_deps.sh" }}-<<parameters.osnick>>-<<parameters.target>>
      - persist_to_workspace:
          root: bin/
          paths:
            - artifacts/*


  platforms-build-gpu:
    machine:
      enabled: true
      resource_class: medium
      image: ubuntu-2004:202107-02
    parameters:
      lite:  # LITE value during make
        type: string
      osnick:  # OSNICK value for the base platform of the docker
        type: string
      target:  # OSNICK value for the base platform of the docker
        type: string
        default: "GPU=1"
    steps:
      - abort_for_docs
      - abort_for_noci
      - early_return_for_forked_pull_requests
      - checkout-all
      - relocate-docker-data-root
      - restore_cache:
          keys:
          - v1.2.5-deps-{{ checksum "get_deps.sh" }}-<<parameters.osnick>>-<<parameters.target>>
      - setup-automation
      - docker-build-steps:
          lite: "<<parameters.lite>>"
          osnick: "<<parameters.osnick>>"
          target: "<<parameters.target>>"
      - save_cache:
          paths:
            - deps
          key: v1.2.5-deps-{{ checksum "get_deps.sh" }}-<<parameters.osnick>>-<<parameters.target>>
      - persist_to_workspace:
          root: bin/
          paths:
            - artifacts/*

  coverage:
    docker:
      - image: redisfab/rmbuilder:6.2.5-x64-bullseye
    steps:
      - abort_for_docs
      - abort_for_noci
      - checkout-all
      - restore_cache:
          keys:
          - v1.2.5-deps-{{ checksum "get_deps.sh" }}-cpu
          # If no exact match is found will get dependencies from source
      - setup-build-system
      - run:
          name: Install dependencies
          command: |
            ./opt/readies/bin/getredis -v 6 --valgrind --force
            ./get_deps.sh cpu
      - run:
          name: Build for coverage
          command: |
            make -C opt all COV=1 SHOW=1
      - restore_cache:
          keys:
            - v1.2-tests_data
      - run:
          name: Test with coverage
          command: |
            make -C opt test SHOW=1 COV=1 CLUSTER=1
            make -C opt cov-upload
          no_output_timeout: 30m
      - store_artifacts:
          path: tests/flow/logs

  valgrind:
    parameters:
      test_args:
        type: string
        default: "CLUSTER=0"
    docker:
      - image: redisfab/rmbuilder:6.2.5-x64-bullseye
    steps:
      - abort_for_docs
      - abort_for_noci
      - checkout-all
      - restore_cache:
          keys:
          - v1.2.5-deps-{{ checksum "get_deps.sh" }}-cpu
          # If no exact match is found will get dependencies from source
      - setup-build-system
      - run:
          name: Install dependencies
          command: |
            ./opt/readies/bin/getredis -v 6.2 --valgrind --force
            ./get_deps.sh cpu
      - run:
          name: Build for valgrind
          command: |
            make -C opt all VALGRIND=1 SHOW=1
      - run:
          name: Test with valgrind
          command: |
            make -C opt test VALGRIND=1 <<parameters.test_args>>
          no_output_timeout: 120m
      - store_artifacts:
          path: tests/flow/logs

  valgrind-general-for-forked-prs:
    docker:
      - image: redisfab/rmbuilder:6.2.5-x64-bullseye
    steps:
      - only_run_if_forked_pull_request
      - valgrind-general-steps

  valgrind-general:
    docker:
      - image: redisfab/rmbuilder:6.2.5-x64-bullseye
    steps:
      - valgrind-general-steps

  # internal PRs execute build-and-test either in a workflow or
  # via a github action trigger
  build-and-test-gpu:
    machine:
      enabled: true
      resource_class: gpu.nvidia.small
      image: ubuntu-2004-cuda-11.2:202103-01

    steps:
      - build-and-test-gpu-steps

  # in the case of a forked PR, we want to run the GPU steps
  # hence we check if we're forked, explicitly
  build-and-test-gpu-for-forked-prs:
    machine:
      enabled: true
      resource_class: gpu.nvidia.small
      image: ubuntu-2004-cuda-11.2:202103-01

    steps:
      - only_run_if_forked_pull_request
      - build-and-test-gpu-steps

  deploy-artifacts:
    parameters:
      location:
        type: string
    docker:
      - image: redisfab/rmbuilder:6.2.5-x64-bullseye
    steps:
      - abort_for_docs
      - abort_for_noci
      - early_return_for_forked_pull_requests
      - attach_workspace:
          at: workspace
      - run:
          name: Deploy to S3
          command: |
            cd "workspace/artifacts/<<parameters.location>>"
            du -ah --apparent-size *
            for f in *.zip snapshot/*.tgz; do
              aws s3 cp $f s3://redismodules/$PACKAGE_NAME/<<parameters.location>>/ --acl public-read
            done

  deploy-snapshot:
    docker:
      - image: redisfab/rmbuilder:6.2.5-x64-bullseye
    steps:
      - abort_for_docs
      - abort_for_noci
      - early_return_for_forked_pull_requests
      - attach_workspace:
          at: workspace
      - run:
          name: Deploy Snapshots to S3
          command: |
            cd workspace/artifacts/snapshots
            for f in `ls *.zip *.tgz`; do
              aws s3 cp --no-progress $f s3://redismodules/$PACKAGE_NAME/snapshots/ --acl public-read
            done

  deploy-release:
    docker:
      - image: redisfab/rmbuilder:6.2.5-x64-bullseye
    steps:
      - abort_for_docs
      - abort_for_noci
      - early_return_for_forked_pull_requests
      - attach_workspace:
          at: workspace
      - run:
          name: Deploy Releases to S3
          command: |
            cd workspace/artifacts
            du -ah --apparent-size *
            for f in `ls *.zip *.tgz`; do
              aws s3 cp --no-progress $f s3://redismodules/$PACKAGE_NAME/ --acl public-read
            done

  release-automation:
    docker:
      - image: redisfab/rmbuilder:6.2.5-x64-bullseye
    steps:
      - checkout
      - setup-automation
      - run:
          name: Run QA Automation
          command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release ./tests/qa/run
      - run:
          name: Run QA Automation (AILite)
          command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release VARIANT=light ./tests/qa/run


  #nightly-automation:
  #  docker:
  #    - image: redisfab/rmbuilder:6.2.5-x64-bullseye
  #  steps:
  #    - checkout
  #    - setup-automation
  #    - run:
  #        name: Run QA Automation
  #        command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run

on-any-branch: &on-any-branch
  filters:
    branches:
      only: /.*/
    tags:
      only: /.*/

on-any-branch-but-tags: &on-any-branch-but-tags
  filters:
    branches:
      only: /.*/
    tags:
      ignore: /.*/

always: &always
  filters:
    branches:
      only: /.*/
    tags:
      only: /.*/

never: &never
  filters:
    branches:
      ignore: /.*/
    tags:
      ignore: /.*/

on-master: &on-master
  filters:
    branches:
      only: master
    tags:
      ignore: /.*/

on-integ-branch: &on-integ-branch
  filters:
    branches:
      only:
        - master
        - /^\d+\.\d+.*$/
    tags:
      ignore: /.*/

on-dev-branches: &on-dev-branches
  filters:
    branches:
      ignore:
        - master
        - /^\d+\.\d+.*$/
    tags:
      ignore: /.*/

on-version-tags: &on-version-tags
  filters:
    branches:
      ignore: /.*/
    tags:
      only: /^v[0-9].*/

on-master-version-tags-and-dockertests: &on-master-version-tags-and-dockertests
  filters:
    branches:
      only:
        - master
        - /.*dockertest$/
    tags:
      only: /^v[0-9].*/


after-linter: &after-linter
  requires:
    - lint

after-build-and-test: &after-build-and-test
  requires:
    - build-and-test

after-platform-builds: &after-platform-builds
  requires:
    - platforms-build-gpu
    - platforms-build-cpu

#### define workflows
workflows:
  version: 2
  build_and_package:
    jobs:
      - lint:
          <<: *on-any-branch
      - build-and-test:
          <<: *on-any-branch
          <<: *after-linter
      - build-and-test-gpu:
          <<: *on-integ-branch
          <<: *after-linter
      - build-and-test-gpu-for-forked-prs:
          <<: *on-any-branch
          <<: *after-linter
      - platforms-build-cpu:
          context: common
          <<: *after-build-and-test
          <<: *on-master-version-tags-and-dockertests
          matrix:
            parameters:
              osnick:
                - bionic
                - centos8
              lite:
                - "REDISAI_LITE=0 PUBLISH=1"
                - "REDISAI_LITE=1"
      - platforms-build-gpu:
          context: common
          <<: *after-build-and-test
          <<: *on-master-version-tags-and-dockertests
          matrix:
            parameters:
              osnick:
                - bionic
                - centos8
              lite:
                - "REDISAI_LITE=0 PUBLISH=1"
                - "REDISAI_LITE=1"
      - coverage:
          context: common
          <<: *on-dev-branches
          <<: *after-linter
      - valgrind-general-for-forked-prs:
          <<: *on-any-branch
          <<: *after-linter
      - valgrind:
          name: valgrind-cluster
          test_args: GEN=0
          <<: *on-integ-branch
          <<: *after-linter
      - deploy-snapshot:
          context: common
          <<: *after-platform-builds
          <<: *on-master-version-tags-and-dockertests
      - deploy-release:
          context: common
          <<: *after-platform-builds
          <<: *on-version-tags
      - release-automation:
          context: common
          <<: *on-version-tags
          requires:
            - deploy-release

#  nightly:
#    triggers:
#      - schedule:
#          cron: "20 17 * * *"
#          filters:
#            branches:
#              only: master
#    jobs:
#      - build-macos:
#          <<: *never # temporarily disabled
#      - nightly-automation:
#          context: common
