version: 2.1

########################################################################################################################
#                                                      EXECUTORS                                                       #
########################################################################################################################

executors:
  # Basic executor with git and bash and not much else.
  basic:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      # CircleCI base image
      - image: cimg/base:2020.01

  clojure:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1

  node:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      # - image: circleci/node:7-browsers
      - image: circleci/clojure:lein-2.8.1-node-browsers

  clojure-and-node:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1-node-browsers

  # This is a special image that is based on clojure-and-node but includes extras like gettext and the Clojure CLI
  # tools. See https://github.com/metabase/metabase-docker-ci for more information. You shouldn't use this image
  # unless you need the extra stuff -- because it's not an official Circle image, the extra layers won't get cached
  # and it might take longer to spin this up.
  metabase-ci:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: metabase/ci:2020-11-30

  # Java 11 tests also test Metabase with the at-rest encryption enabled. See
  # https://metabase.com/docs/latest/operations-guide/encrypting-database-details-at-rest.html for an explanation of
  # what this means.
  java-11:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:openjdk-11-lein-2.8.1
        environment:
          MB_ENCRYPTION_SECRET_KEY: Orw0AAyzkO/kPTLJRxiyKoBHXa/d6ZcO+p+gpZO/wSQ=

  postgres-9-6:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1
        environment:
          MB_DB_TYPE: postgres
          MB_DB_PORT: 5432
          MB_DB_HOST: localhost
          MB_DB_DBNAME: circle_test
          MB_DB_USER: circle_test
          MB_POSTGRESQL_TEST_USER: circle_test
      - image: circleci/postgres:9.6-alpine
        environment:
          POSTGRES_USER: circle_test
          POSTGRES_DB: circle_test

  postgres-latest:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1
        environment:
          MB_DB_TYPE: postgres
          MB_DB_PORT: 5432
          MB_DB_HOST: localhost
          MB_DB_DBNAME: metabase_test
          MB_DB_USER: metabase_test
          MB_POSTGRESQL_TEST_USER: metabase_test
      - image: postgres:latest
        environment:
          POSTGRES_USER: metabase_test
          POSTGRES_DB: metabase_test
          POSTGRES_HOST_AUTH_METHOD: trust

  mysql-5-7:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1
        environment:
          MB_DB_TYPE: mysql
          MB_DB_HOST: localhost
          MB_DB_PORT: 3306
          MB_DB_DBNAME: circle_test
          MB_DB_USER: root
          MB_MYSQL_TEST_USER: root
      - image: circleci/mysql:5.7.23

  mysql-latest:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1
        environment:
          MB_DB_TYPE: mysql
          MB_DB_HOST: localhost
          MB_DB_PORT: 3306
          MB_DB_DBNAME: circle_test
          MB_DB_USER: root
          MB_MYSQL_TEST_USER: root
      - image: circleci/mysql:latest

  mariadb-10-2:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1
        environment:
          MB_DB_TYPE: mysql
          MB_DB_HOST: localhost
          MB_DB_PORT: 3306
          MB_DB_DBNAME: circle_test
          MB_DB_USER: root
          MB_MYSQL_TEST_USER: root
      - image: circleci/mariadb:10.2.23

  mariadb-latest:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1
        environment:
          MB_DB_TYPE: mysql
          MB_DB_HOST: localhost
          MB_DB_PORT: 3306
          MB_DB_DBNAME: metabase_test
          MB_DB_USER: root
          MB_MYSQL_TEST_USER: root
      - image: mariadb:latest
        environment:
          MYSQL_DATABASE: metabase_test
          MYSQL_USER: root
          MYSQL_ALLOW_EMPTY_PASSWORD: yes

  mongo:
     working_directory: /home/circleci/metabase/metabase/
     docker:
       - image: circleci/clojure:lein-2.8.1
       - image: circleci/mongo:4.0

  presto:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1
      - image: metabase/presto-mb-ci
        environment:
          JAVA_TOOL_OPTIONS: "-Xmx1g"

  sparksql:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1
      - image: metabase/spark:2.1.1

  vertica:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1
      - image: sumitchawla/vertica

  sqlserver:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1
        environment:
          MB_SQLSERVER_TEST_HOST: localhost
          MB_SQLSERVER_TEST_PASSWORD: 'P@ssw0rd'
          MB_SQLSERVER_TEST_USER: SA
      - image: mcr.microsoft.com/mssql/server:2017-latest
        environment:
          ACCEPT_EULA: Y
          SA_PASSWORD: 'P@ssw0rd'

  fe-mongo-4:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1-node-browsers
      - image: metabase/qa-databases:mongo-sample-4.0

  fe-postgres-12:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1-node-browsers
      - image: metabase/qa-databases:postgres-sample-12

  fe-mysql-8:
    working_directory: /home/circleci/metabase/metabase/
    docker:
      - image: circleci/clojure:lein-2.8.1-node-browsers
      - image: metabase/qa-databases:mysql-sample-8

########################################################################################################################
#                                             MAP FRAGMENTS AND CACHE KEYS                                             #
########################################################################################################################

# `default_parameters` isn't a key that CircleCI uses, but this form lets us reuse parameter definitions
default_parameters: &Params
  edition:
    type: string
    default: "oss"

# .BACKEND-CHECKSUMS, .FRONTEND-CHECKSUMS, and .MODULE-CHECKSUMS are created during the checkout step; see that step
# for exact details as to what they contain.
#
# To support cache busting, we create a file named .CACHE-PREFIX in the checkout step and use its checksum as the
# prefix for every cache key. If the commit message DOES NOT include [ci nocache], we create an empty file; the
# checksum will always be the same for this file. If the commit message DOES include [ci nocache], we'll write the
# unique ID of the current pipeline to .CACHE-PREFIX which will effectively bust our caches whenever it's used.

### Deps Keys ###

# Why don't we use fallback keys for backend/frontend deps? We used to, but it allowed the cache to grow
# uncontrollably since old deps would continue to accumulate. Restoring big caches is really slow in Circle. It's
# actually faster to recreate the deps cache from scratch whenever we need to which keeps the size down.
cache-key-backend-deps: &CacheKeyBackendDeps
  key: v1-{{ checksum ".CACHE-PREFIX" }}-be-deps-{{ checksum "project.clj" }}

cache-key-frontend-deps: &CacheKeyFrontendDeps
  key: v1-{{ checksum ".CACHE-PREFIX" }}-fe-deps-{{ checksum "yarn.lock" }}

# Key used for implementation of run-on-change -- this is the cache key that contains the .SUCCESS dummy file
# By default the key ALWAYS includes the name of the test job itself ($CIRCLE_STAGE) so you don't need to add that yourself.
cache-key-run-on-change: &CacheKeyRunOnChange
  key: v1-{{ checksum ".CACHE-PREFIX" }}-run-on-change-{{ .Environment.CIRCLE_STAGE }}-<< parameters.checksum >>

# Key for the local maven installation of metabase-core (used by build-uberjar-drivers)
cache-key-metabase-core: &CacheKeyMetabaseCore
  key: v1-{{ checksum ".CACHE-PREFIX" }}-metabase-core-{{ checksum ".BACKEND-CHECKSUMS" }}

# Key for the drivers built by build-uberjar-drivers
cache-key-drivers: &CacheKeyDrivers
  key: v1-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-{{ checksum ".MODULES-CHECKSUMS" }}-{{ checksum ".BACKEND-CHECKSUMS" }}-<< parameters.edition >>

# This is also used by the uberjar-build-drivers step; this is a unique situation because the build-drivers script has
# logic to determine whether to rebuild drivers or not that is quite a bit more sophisticated that the run-on-change
# stuff in this file. e.g. if I only change the bigquery driver, the script is smart enough to not rebuild the
# redshift driver.
cache-keys-drivers-with-fallback-keys: &CacheKeyDrivers_WithFallbackKeys
  keys:
    - v1-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-{{ checksum ".MODULES-CHECKSUMS" }}-{{ checksum ".BACKEND-CHECKSUMS" }}
    - v1-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-{{ checksum ".MODULES-CHECKSUMS" }}
    - v1-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-

# Key for frontend client built by uberjar-build-frontend step
cache-key-frontend: &CacheKeyFrontend
  key: v1-{{ checksum ".CACHE-PREFIX" }}-frontend-<< parameters.edition >>-{{ checksum ".FRONTEND-CHECKSUMS" }}

# Key for uberjar built by build-uberjar
cache-key-uberjar: &CacheKeyUberjar
  key: v1-{{ checksum ".CACHE-PREFIX" }}-uberjar-<< parameters.edition >>-{{ checksum ".BACKEND-CHECKSUMS" }}-{{ checksum ".FRONTEND-CHECKSUMS" }}


########################################################################################################################
#                                                       COMMANDS                                                       #
########################################################################################################################

commands:
  attach-workspace:
    steps:
      - attach_workspace:
          at: /home/circleci/

  # For the restore-deps-cache commands below, only restore the cache if there's an exact match. This means whatever
  # is in the cache will be exactly what's used and the cache won't keep growing uncontrollably going forward.

  restore-be-deps-cache:
    steps:
      - restore_cache:
          name: Restore cached backend dependencies
          <<: *CacheKeyBackendDeps

  restore-fe-deps-cache:
    steps:
      - restore_cache:
          name: Restore cached frontend dependencies
          <<: *CacheKeyFrontendDeps

  # run-on-change lets you only run steps if changes have happened to relevant files since the last time it was run
  # successfully. Uses a cache key to record successful runs -- cache key should be unique for job and relevant source
  # files -- use a checksum! It works like this:
  #
  # 1. Calculate a cache key using a checksum of relevant files for the step in question, e.g. a backend linter step
  #    might use a checksum of all .clj files.
  #
  # 2. When the step completes successfully, create a dummy file .SUCCESS and cache it with that cache key.
  #
  # 3. On subsequent runs:
  #
  #    a. Attempt to restore the cache using an exact match for this cache key
  #
  #    b. If we have a cache entry for that key, .SUCCESS will get restored
  #
  #    c. If this command has the skip-job-if-commit-message-includes-ci-quick option enabled, and commit message includes
  #       [ci quick], create a dummy file .SUCCESS if not already present. Ignored for master/release branches.
  #
  #    d. If commit message includes [ci noskip], delete .SUCCESS so the job will be forced to run.
  #
  #    e. If .SUCCESS is present, we can skip the rest of the job, including potentially slow steps like restoring
  #       dependency caches or the like. This logs a link to the last successful (not skipped) run of the job
  #
  #       Important! If this step is skipped because no changes have happened, the entire JOB will halt with a success
  #       status -- no steps that happen AFTER run-on-change will be ran. Keep this in mind!
  #
  #   f. If .SUCCESS is not present, proceed as normal, and create and cache .SUCCESS if the job succeeds
  run-on-change:
    parameters:
      checksum:
        type: string
        default: ""
      steps:
        type: steps
      # Whether to skip the rest of the job if commit message includes [ci quick]
      skip-job-if-commit-message-includes-ci-quick:
        type: boolean
        default: false
    steps:
      - restore_cache:
          name: Restore dummy file .SUCCESS if it exists for cache key << parameters.checksum >>
          <<: *CacheKeyRunOnChange
      - when:
          condition: << parameters.skip-job-if-commit-message-includes-ci-quick >>
          steps:
            - run:
                name: "Skip tests (create dummy file .SUCCESS) if commit message contains [ci quick] and branch isn't a master/release branch"
                command: |
                  if [[ "$CIRCLE_BRANCH" =~ ^master|release-.+$ ]]; then
                      echo "branch '$CIRCLE_BRANCH' is a master or release branch: ignoring [ci quick]"
                  elif [[ `cat .COMMIT` == *"[ci quick]"* ]]; then
                      echo 'Commit message includes [ci quick]. Creating dummy file .SUCCESS'
                      touch .SUCCESS
                  else
                      echo 'Commit message does not include [ci quick]'
                  fi
      - run:
          name: "Force test run (delete dummy file .SUCCESS) if commit message includes [ci noskip]"
          command: |
            if [[ `cat .COMMIT` == *"[ci noskip]"* ]]; then
                echo 'Commit message includes [ci noskip] -- forcing test run (delete .SUCCESS)'
                rm -f .SUCCESS
            else
                echo 'Commit message does not include [ci noskip]'
            fi
      - run:
          name: Skip rest of job if .SUCCESS exists
          command: |
            if [ -f .SUCCESS ]; then
                echo '.SUCCESS is present: skipping rest of job.'
                echo "Link to last successful run (if available): $(cat .SUCCESS)"
                circleci-agent step halt
            fi
      - steps: << parameters.steps >>
      - run:
          name: Create dummy file .SUCCESS
          command: |
            echo "$CIRCLE_BUILD_URL" > .SUCCESS
      - save_cache:
          name: Persist dummy file .SUCCESS to cache with key << parameters.checksum >>
          <<: *CacheKeyRunOnChange
          paths:
            - /home/circleci/metabase/metabase/.SUCCESS
      - run:
          name: Delete dummy file .SUCCESS so subsequent steps don't see it
          command: rm /home/circleci/metabase/metabase/.SUCCESS

  # Creates a file that contains checksums for all the files found using the find command with supplied arguments.
  # You can use a checksum of the checksum file for cache keys including run-on-change cache keys.
  create-checksum-file:
    parameters:
      filename:
        type: string
      find-args:
        type: string
    steps:
      - run:
          name: Create << parameters.filename >> checksum file
          command: |
            for file in `find << parameters.find-args >> | sort`; do
                echo `md5sum "$file"` >> "<< parameters.filename >>"
            done
            if [ ! -f "<< parameters.filename >>" ]; then
                echo 'Error: no matching files. Did you remember to attach the workspace?'
                exit 1
            fi
            echo "Created checksums for $(cat << parameters.filename >> | wc -l) files"

  run-lein-command:
    parameters:
      before-steps:
        type: steps
        default: []
      lein-command:
        type: string
      after-steps:
        type: steps
        default: []
      <<: *Params
    steps:
      - restore-be-deps-cache
      - steps: << parameters.before-steps >>
      - run:
          name: lein << parameters.lein-command >>
          command: |
            lein with-profile +ci,+<< parameters.edition >> << parameters.lein-command >>
          no_output_timeout: 15m
      - steps: << parameters.after-steps >>
      - store_test_results:
          path: /home/circleci/metabase/metabase/target/junit

  run-yarn-command:
    parameters:
      command-name:
        type: string
      command:
        type: string
      before-steps:
        type: steps
        default: []
      after-steps:
        type: steps
        default: []
      skip-when-no-change:
        type: boolean
        default: false
    steps:
      - attach-workspace
      - when:
          condition: << parameters.skip-when-no-change >>
          steps:
            - run-on-change:
                checksum: '{{ checksum ".FRONTEND-CHECKSUMS" }}'
                steps:
                  - restore-fe-deps-cache
                  - steps: << parameters.before-steps >>
                  - run:
                      name: << parameters.command-name >>
                      command: yarn << parameters.command >>
                      no_output_timeout: 15m
                  - steps: << parameters.after-steps >>
      - unless:
          condition: << parameters.skip-when-no-change >>
          steps:
            - restore-fe-deps-cache
            - steps: << parameters.before-steps >>
            - run:
                name: << parameters.command-name >>
                command: yarn << parameters.command >>
                no_output_timeout: 15m
            - steps: << parameters.after-steps >>

  wait-for-port:
    parameters:
      port:
        type: integer
    steps:
      - run:
          name: Wait for port << parameters.port >> to be ready
          command: |
            while ! nc -z localhost << parameters.port >>; do sleep 0.1; done
          no_output_timeout: 15m

  fetch-jdbc-driver:
    parameters:
      source:
        type: string
      dest:
        type: string
    steps:
      - run:
          name: Make plugins dir
          command: mkdir /home/circleci/metabase/metabase/plugins
      - run:
          name: Download JDBC driver JAR << parameters.dest >>
          command: |
            wget --output-document=plugins/<< parameters.dest >> ${<< parameters.source >>}
          no_output_timeout: 15m

jobs:

########################################################################################################################
#                                                    CHECKOUT ETC.                                                     #
########################################################################################################################

  checkout:
    executor: basic
    steps:
      - checkout
      # .BACKEND-CHECKSUMS is every Clojure source file as well as dependency files like deps.edn and plugin manifests
      - create-checksum-file:
          filename: .BACKEND-CHECKSUMS
          find-args: ". -type f -name '*.clj' -or -name '*.java' -or -name deps.edn -or -name metabase-plugin.yaml"
      # .FRONTEND-CHECKSUMS is every JavaScript source file as well as dependency files like yarn.lock
      - create-checksum-file:
          filename: .FRONTEND-CHECKSUMS
          find-args: ". -type f -name '*.js' -or -name '*.jsx' -or -name '*.json' -or -name yarn.lock -or -name sample-dataset.db.mv.db"
      # .MODULES-CHECKSUMS is every Clojure source file in the modules/ directory as well as plugin manifests
      - create-checksum-file:
          filename: .MODULES-CHECKSUMS
          find-args: "./modules -type f -name '*.clj' -or -name metabase-plugin.yaml"
      - run:
          name: Save last git commit message to .COMMIT
          command: git log -1 > .COMMIT
      - run:
          name: Remove .git directory (not needed for tests)
          command: rm -rf /home/circleci/metabase/metabase/.git
      - run:
          name: Remove ./OSX directory (not needed for tests)
          command: rm -rf /home/circleci/metabase/metabase/OSX
      # .CACHE-PREFIX is described above in the Cache Keys section of this file
      - run:
          name: 'Create cache key prefix .CACHE-PREFIX to bust caches if commit message includes [ci nocache]'
          command: |
            if [[ `cat .COMMIT` == *"[ci nocache]"* ]]; then
                echo 'Commit message includes [ci nocache]; using cache-busting prefix'
                echo '<< pipeline.id >>' > .CACHE-PREFIX
            else
               echo '' > .CACHE-PREFIX
            fi
      - persist_to_workspace:
          root: /home/circleci/
          paths:
            - metabase/metabase

  yaml-linter:
    executor: node
    steps:
      - attach-workspace
      - create-checksum-file:
          filename: .YAML-CHECKSUMS
          find-args: ". -type f -name '*.yaml' -or -name '*.yml'"
      - run-on-change:
          # package.json is included in the cache key in case the yamllint dependency changes
          checksum: '{{ checksum ".YAML-CHECKSUMS" }}-{{ checksum "package.json" }}'
          steps:
            - run-yarn-command:
                command-name: Lint YAML files
                command: lint-yaml `find resources -name '*.yaml' -or -name '*.yml'`

  verify-i18n-files:
    executor: metabase-ci
    steps:
      - attach-workspace
      - create-checksum-file:
          filename: .I18N-CHECKSUMS
          find-args: "locales -type f -name '*.po'"
      # In case the i18n scripts themselves change
      - create-checksum-file:
          filename: .I18N-SCRIPT-CHECKSUMS
          find-args: "bin/i18n -type f"
      - run-on-change:
          checksum: '{{ checksum ".I18N-CHECKSUMS" }}-{{ checksum ".I18N-SCRIPT-CHECKSUMS" }}'
          steps:
            - restore-fe-deps-cache
            - run:
                name: Check i18n tags/make sure template can be built
                # fix OOM when running -- see https://stackoverflow.com/a/59572966/1198455
                # without this option, sometimes this step will fail with an OOM error.
                environment:
                  NODE_OPTIONS: '--max-old-space-size=2048'
                command: ./bin/i18n/update-translation-template
                no_output_timeout: 2m
            - run:
                name: Verify i18n translations (.po files)
                command: ./bin/i18n/build-translation-resources
                no_output_timeout: 2m

  check-migrations:
    executor:
      metabase-ci
    steps:
      - attach-workspace
      - create-checksum-file:
          filename: .MIGRATIONS-CHECKSUM
          find-args: resources/migrations -type f -name '*.yaml'
      - create-checksum-file:
          filename: .MIGRATIONS-LINTER-CHECKSUMS
          find-args: bin/lint-migrations-file -type f -name '*.clj' -or -name 'deps.edn'
      - run-on-change:
          checksum: '{{ checksum ".MIGRATIONS-CHECKSUM" }}-{{ checksum ".MIGRATIONS-LINTER-CHECKSUMS" }}'
          steps:
            - run:
                name: Verify Liquibase Migrations
                command: ./bin/lint-migrations-file.sh
                no_output_timeout: 15m


########################################################################################################################
#                                                       BACKEND                                                        #
########################################################################################################################

  be-deps:
    executor: clojure
    parameters:
      <<: *Params
    steps:
      - attach-workspace
      # This step is pretty slow, even with the cache, so only run it if project.clj has changed
      # TODO -- we should cache the build script deps as well, and driver deps?
      - run-on-change:
          checksum: '{{ checksum "project.clj" }}'
          steps:
            - restore-be-deps-cache
            - run: lein with-profile +include-all-drivers,+cloverage,+junit,+<< parameters.edition >> deps
            - save_cache:
                name: Cache backend dependencies
                <<: *CacheKeyBackendDeps
                paths:
                  - /home/circleci/.m2

  lein:
    parameters:
      e:
        type: executor
        default: clojure
      before-steps:
        type: steps
        default: []
      lein-command:
        type: string
      after-steps:
        type: steps
        default: []
      skip-when-no-change:
        type: boolean
        default: false
      <<: *Params
    executor: << parameters.e >>
    steps:
      - attach-workspace
      - when:
          condition: << parameters.skip-when-no-change >>
          steps:
            - run-on-change:
                checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}'
                steps:
                  - run-lein-command:
                      before-steps: << parameters.before-steps >>
                      lein-command: << parameters.lein-command >>
                      after-steps: << parameters.after-steps >>
                      edition: << parameters.edition >>
      - unless:
          condition: << parameters.skip-when-no-change >>
          steps:
            - run-lein-command:
                before-steps: << parameters.before-steps >>
                lein-command: << parameters.lein-command >>
                after-steps: << parameters.after-steps >>
                edition: << parameters.edition >>

  be-linter-reflection-warnings:
    executor: clojure
    steps:
      - attach-workspace
      - run-on-change:
          checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}-{{ checksum "bin/reflection-linter" }}'
          steps:
            - restore-be-deps-cache
            - run:
                name: Run reflection warnings checker
                command: ./bin/reflection-linter
                no_output_timeout: 15m

  test-driver:
    parameters:
      e:
        type: executor
        default: clojure
      driver:
        type: string
      timeout:
        type: string
        default: 15m
      before-steps:
        type: steps
        default: []
      description:
        type: string
        default: ""
    executor: << parameters.e >>
    steps:
      - attach-workspace
      - run-on-change:
          checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}'
          skip-job-if-commit-message-includes-ci-quick: true
          steps:
            - restore-be-deps-cache
            - steps: << parameters.before-steps >>
            - run:
                name: Test << parameters.driver >> driver << parameters.description >>
                environment:
                  DRIVERS: << parameters.driver >>
                command: lein with-profile +ci,+junit,+ee test
                no_output_timeout: << parameters.timeout >>
            - store_test_results:
                path: /home/circleci/metabase/metabase/target/junit

  test-build-scripts:
    executor: metabase-ci
    steps:
      - attach-workspace
      - run-on-change:
          checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}'
          steps:
            - restore-be-deps-cache
            - run:
                name: Run metabuild-common build script tests
                command: |
                  cd /home/circleci/metabase/metabase/bin/common && clojure -M:test
                no_output_timeout: 15m
            - run:
                name: Run build-drivers build script tests
                command: |
                  cd /home/circleci/metabase/metabase/bin/build-drivers && clojure -M:test
                no_output_timeout: 15m
            - run:
                name: Run build-mb build script tests
                command: |
                  cd /home/circleci/metabase/metabase/bin/build-mb && clojure -M:test
                no_output_timeout: 15m
            - run:
                name: Run release script tests
                command: |
                  cd /home/circleci/metabase/metabase/bin/release && clojure -M:test
                no_output_timeout: 15m
            - run:
                name: Run Liquibase migrations linter tests
                command: |
                  cd /home/circleci/metabase/metabase/bin/lint-migrations-file && clojure -M:test
                no_output_timeout: 15m


########################################################################################################################
#                                                       FRONTEND                                                       #
########################################################################################################################

  fe-deps:
    executor: node
    steps:
      - attach-workspace
      # This step is *really* slow, so we can skip it if yarn.lock hasn't changed since last time we ran it
      - run-on-change:
          checksum: '{{ checksum "yarn.lock" }}'
          steps:
            - restore-fe-deps-cache
            - run:
                name: Run yarn to install deps
                command: yarn;
                no_output_timeout: 15m
            - save_cache:
                name: Cache frontend dependencies
                <<: *CacheKeyFrontendDeps
                paths:
                  - /home/circleci/.yarn
                  - /home/circleci/.yarn-cache
                  - /home/circleci/metabase/metabase/node_modules
                  - /home/circleci/.cache/Cypress

  fe-linter-eslint:
    executor: node
    steps:
      - run-yarn-command:
          command-name: Run ESLint linter
          command: lint-eslint
          skip-when-no-change: true

  fe-linter-prettier:
    executor: node
    steps:
      - run-yarn-command:
          command-name: Run Prettier formatting linter
          command: lint-prettier
          skip-when-no-change: true

  fe-linter-flow:
    executor: node
    steps:
      - run-yarn-command:
          command-name: Run Flow type checker
          command: lint-flow
          skip-when-no-change: true

  fe-linter-docs-links:
    executor: node
    steps:
      - attach-workspace
      - create-checksum-file:
          filename: ".MARKDOWN-CHECKSUMS"
          find-args: ". -type f -name '*.md'"
      - run-on-change:
          checksum: '{{ checksum ".MARKDOWN-CHECKSUMS" }}-{{ checksum ".FRONTEND-CHECKSUMS" }}'
          steps:
            - run-yarn-command:
                command-name: Run docs links checker
                command: lint-docs-links

  fe-tests-unit:
    executor: node
    steps:
      - run-yarn-command:
          command-name: Run frontend unit tests
          command: run test-unit
          skip-when-no-change: true

  fe-tests-timezones:
    executor: node
    steps:
      - run-yarn-command:
          command-name: Run frontend timezone tests
          command: run test-timezones
          skip-when-no-change: true

  # Unlike the other build-uberjar steps, this step should be run once overall and the results can be shared between
  # OSS and EE uberjars.
  build-uberjar-drivers:
    executor: metabase-ci
    parameters:
      <<: *Params
    steps:
      - attach-workspace
      - run-on-change:
          # .MODULES-CHECKSUMS is a subset of .BACKEND-CHECKSUMS.
          #
          # We have both versions so we can try to load cached drivers that match MODULES-CHECKSUMS but not
          # BACKEND-CHECKSUMS as a whole.
          #
          # The build-drivers script is smart enough to only rebuild drivers if needed -- there's a chance we won't
          # need to rebuild them.
          checksum: '{{ checksum ".MODULES-CHECKSUMS" }}-{{ checksum ".BACKEND-CHECKSUMS" }}'
          steps:
            - restore-be-deps-cache            #
            - restore_cache:
                <<: *CacheKeyMetabaseCore
            - restore_cache:
                name: Restore cached drivers uberjars from previous runs
                <<: *CacheKeyDrivers_WithFallbackKeys
            - run:
                name: Build << parameters.edition >> drivers if needed
                command: ./bin/build-drivers.sh << parameters.edition >>
                no_output_timeout: 15m
            - save_cache:
                name: Cache local Maven installation of metabase-core
                <<: *CacheKeyMetabaseCore
                paths:
                  - /home/circleci/.m2/repository/metabase-core
            - save_cache:
                name: Cache the built drivers
                <<: *CacheKeyDrivers
                paths:
                  - /home/circleci/metabase/metabase/modules/drivers/bigquery/target
                  - /home/circleci/metabase/metabase/modules/drivers/druid/target
                  - /home/circleci/metabase/metabase/modules/drivers/google/target
                  - /home/circleci/metabase/metabase/modules/drivers/googleanalytics/target
                  - /home/circleci/metabase/metabase/modules/drivers/mongo/target
                  - /home/circleci/metabase/metabase/modules/drivers/oracle/target
                  - /home/circleci/metabase/metabase/modules/drivers/presto/target
                  - /home/circleci/metabase/metabase/modules/drivers/redshift/target
                  - /home/circleci/metabase/metabase/modules/drivers/snowflake/target
                  - /home/circleci/metabase/metabase/modules/drivers/sparksql/target
                  - /home/circleci/metabase/metabase/modules/drivers/sqlite/target
                  - /home/circleci/metabase/metabase/modules/drivers/sqlserver/target
                  - /home/circleci/metabase/metabase/modules/drivers/vertica/target

  # Build the frontend client. parameters.edition determines whether we build the OSS or EE version.
  build-uberjar-frontend:
    parameters:
      <<: *Params
    executor: metabase-ci
    steps:
      - attach-workspace
      - run-on-change:
          checksum: '{{ checksum ".FRONTEND-CHECKSUMS" }}'
          steps:
            - restore-fe-deps-cache
            - run:
                name: Build frontend
                environment:
                  MB_EDITION: << parameters.edition >>
                command: ./bin/build version frontend
                no_output_timeout: 15m
            - save_cache:
                name: Cache the built frontend
                <<: *CacheKeyFrontend
                paths:
                  - /home/circleci/metabase/metabase/resources/frontend_client

  # Build the uberjar. parmeters.edition determines whether we build the OSS or EE version.
  build-uberjar:
    parameters:
      <<: *Params
    executor: metabase-ci
    steps:
      - attach-workspace
      - run-on-change:
          checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}-{{ checksum ".FRONTEND-CHECKSUMS" }}'
          steps:
            - restore_cache:
                name: Restore cached uberjar from previous runs
                <<: *CacheKeyUberjar
            - run:
                name: Skip rest of job if uberjar already exists
                command: |
                   if [ -f './target/uberjar/metabase.jar' ]; then
                       circleci-agent step halt
                   fi
            - restore-be-deps-cache
            - restore_cache:
                name: Restore cached drivers built by previous step
                <<: *CacheKeyDrivers
            - restore_cache:
                name: Restore cached FE built by previous step
                <<: *CacheKeyFrontend
            - run:
                name: Build uberjar
                environment:
                  # INTERACTIVE=false will tell the clojure build scripts not to do interactive retries etc.
                  INTERACTIVE: "false"
                  MB_EDITION: << parameters.edition >>
                command: ./bin/build version drivers uberjar
                no_output_timeout: 15m
            - store_artifacts:
                path: /home/circleci/metabase/metabase/target/uberjar/metabase.jar
            - store_artifacts:
                path: /home/circleci/metabase/metabase/resources/version.properties
            - save_cache:
                name: Cache the built uberjar & version.properties
                <<: *CacheKeyUberjar
                paths:
                  - /home/circleci/metabase/metabase/target/uberjar/metabase.jar
                  - /home/circleci/metabase/metabase/resources/version.properties

  fe-tests-cypress:
    parameters:
      e:
        type: executor
        default: clojure-and-node
      cypress-group:
        type: string
      only-single-database:
        type: boolean
        default: false
      test-files-location:
        type: string
        default: ""
      # Node number e.g. 1
      node:
        type: string
        default: ""
      before-steps:
        type: steps
        default: []
      <<: *Params
    executor: << parameters.e >>
    environment:
      MB_EDITION: << parameters.edition >>
      CYPRESS_GROUP:  << parameters.cypress-group >>
      DISPLAY: ""
    steps:
      - attach-workspace
      - run-on-change:
          checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}-{{ checksum ".FRONTEND-CHECKSUMS" }}'
          steps:
            - run-yarn-command:
                command-name: Run Cypress tests
                before-steps:
                  - restore_cache:
                      name: Restore cached uberjar built in previous step
                      <<: *CacheKeyUberjar
                  - steps: << parameters.before-steps >>
                command: |
                  run test-cypress-no-build <<# parameters.only-single-database >> --testFiles << parameters.test-files-location >> <</ parameters.only-single-database >>
                after-steps:
                  - store_artifacts:
                      path: /home/circleci/metabase/metabase/cypress
                  - store_test_results:
                      path: cypress/results

  fe-tests-cypress-smoketest:
    parameters:
      e:
        type: executor
        default: clojure-and-node
      cypress-group:
        type: string
      <<: *Params
    executor: << parameters.e >>
    environment:
      CYPRESS_GROUP: << parameters.cypress-group >>
      DISPLAY: ""
    steps:
      - run-yarn-command:
          command-name: Run Cypress tests
          command: run test-cypress-smoketest
          before-steps:
            - restore_cache:
                name: Restore cached uberjar built in previous step
                <<: *CacheKeyUberjar
      - store_artifacts:
          path: /home/circleci/metabase/metabase/cypress
      - store_test_results:
          path: cypress/results

########################################################################################################################
#                                                   DEPLOYMENT, ETC.                                                   #
########################################################################################################################

  deploy-master:
    executor: clojure
    steps:
      - attach-workspace
      - run: ./bin/deploy-webhook $DEPLOY_WEBHOOK


########################################################################################################################
#                                                      WORKFLOWS                                                       #
########################################################################################################################

# `default_matrix` isn't a key that CircleCI uses, but this form lets us reuse the matrix: block
default_matrix: &Matrix
  matrix:
    parameters:
      edition: ["ee", "oss"]

workflows:
  version: 2
  build:
    jobs:
      - checkout

      - yaml-linter:
          requires:
            - fe-deps

      - verify-i18n-files:
          requires:
            - fe-deps

      - check-migrations:
          requires:
            - checkout

      - be-deps:
          requires:
            - checkout

      - lein:
          name: be-tests-<< matrix.edition >>
          requires:
            - be-deps
          lein-command: with-profile +junit test
          skip-when-no-change: true
          <<: *Matrix

      - lein:
          name: be-tests-java-11-<< matrix.edition >>
          requires:
            - be-deps
          e: java-11
          lein-command: with-profile +junit test
          skip-when-no-change: true
          <<: *Matrix

      - lein:
          name: be-linter-eastwood
          requires:
            - be-deps
          lein-command: eastwood
          skip-when-no-change: true

      - lein:
          name: be-linter-docstring-checker
          requires:
            - be-deps
          lein-command: docstring-checker
          skip-when-no-change: true

      - lein:
          name: be-linter-namespace-decls
          requires:
            - be-deps
          lein-command: check-namespace-decls
          skip-when-no-change: true

      - lein:
          name: be-linter-bikeshed
          requires:
            - be-deps
          lein-command: bikeshed
          skip-when-no-change: true

      - lein:
          name: be-linter-cloverage
          requires:
            - be-deps
          lein-command: cloverage --codecov
          after-steps:
            - run:
                name: Upload code coverage to codecov.io
                command: bash <(curl -s https://codecov.io/bash)
          skip-when-no-change: true

      - be-linter-reflection-warnings:
          requires:
            - be-deps

      - test-driver:
          name: be-tests-bigquery-ee
          requires:
            - be-tests-ee
          driver: bigquery

      - test-driver:
          name: be-tests-druid-ee
          requires:
            - be-tests-ee
          driver: druid

      - test-driver:
          name: be-tests-googleanalytics-ee
          requires:
            - be-tests-ee
          driver: googleanalytics

      - test-driver:
          name: be-tests-mongo-ee
          requires:
            - be-tests-ee
          e: mongo
          driver: mongo

      - test-driver:
          name: be-tests-mysql-ee
          description: "(MySQL 5.7)"
          requires:
            - be-tests-ee
          e:
            name: mysql-5-7
          driver: mysql

      - test-driver:
          name: be-tests-mysql-latest-ee
          description: "(MySQL latest)"
          requires:
            - be-tests-ee
          e:
            name: mysql-latest
          driver: mysql

      - test-driver:
          name: be-tests-mariadb-ee
          description: "(MariaDB 10.2)"
          requires:
            - be-tests-ee
          e:
            name: mariadb-10-2
          driver: mysql

      - test-driver:
          name: be-tests-mariadb-latest-ee
          description: "(MariaDB latest)"
          requires:
            - be-tests-ee
          e:
            name: mariadb-latest
          driver: mysql

      - test-driver:
          name: be-tests-oracle-ee
          requires:
            - be-tests-ee
          before-steps:
            - fetch-jdbc-driver:
                source: ORACLE_JDBC_JAR
                dest: ojdbc8.jar
          driver: oracle

      - test-driver:
          name: be-tests-postgres-ee
          description: "(9.6)"
          requires:
            - be-tests-ee
          e: postgres-9-6
          driver: postgres

      - test-driver:
          name: be-tests-postgres-latest-ee
          description: "(Latest)"
          requires:
            - be-tests-ee
          e: postgres-latest
          driver: postgres

      - test-driver:
          name: be-tests-presto-ee
          requires:
            - be-tests-ee
          e: presto
          before-steps:
            - wait-for-port:
                port: 8080
          driver: presto

      - test-driver:
          name: be-tests-redshift-ee
          requires:
            - be-tests-ee
          driver: redshift
          timeout: 15m

      - test-driver:
          name: be-tests-snowflake-ee
          requires:
            - be-tests-ee
          driver: snowflake
          timeout: 115m

      - test-driver:
          name: be-tests-sparksql-ee
          requires:
            - be-tests-ee
          e: sparksql
          before-steps:
            - wait-for-port:
                port: 10000
          driver: sparksql

      - test-driver:
          name: be-tests-sqlite-ee
          requires:
            - be-tests-ee
          driver: sqlite

      - test-driver:
          name: be-tests-sqlserver-ee
          requires:
            - be-tests-ee
          e: sqlserver
          driver: sqlserver

      - test-driver:
          name: be-tests-vertica-ee
          requires:
            - be-tests-ee
          e: vertica
          before-steps:
            - fetch-jdbc-driver:
                source: VERTICA_JDBC_JAR
                dest: vertica-jdbc-7.1.2-0.jar
          driver: vertica

      - test-build-scripts:
          requires:
            - be-deps

      - build-uberjar-drivers:
          name: build-uberjar-drivers-<< matrix.edition >>
          requires:
            - be-deps
          <<: *Matrix

      - build-uberjar-frontend:
          name: build-uberjar-frontend-<< matrix.edition >>
          requires:
            - fe-deps
          <<: *Matrix

      - build-uberjar:
          name: build-uberjar-<< matrix.edition >>
          requires:
            - build-uberjar-drivers-<< matrix.edition >>
            - build-uberjar-frontend-<< matrix.edition >>
          <<: *Matrix

      - fe-deps:
          requires:
            - checkout
      - fe-linter-eslint:
          requires:
            - fe-deps
      - fe-linter-prettier:
          requires:
            - fe-deps
      - fe-linter-flow:
          requires:
            - fe-deps
      - fe-linter-docs-links:
          requires:
            - fe-deps
      - fe-tests-unit:
          requires:
            - fe-deps
      - fe-tests-timezones:
          requires:
            - fe-deps

      - fe-tests-cypress:
          matrix:
            parameters:
              node: ["1"]
              edition: ["ee", "oss"]
          name: fe-tests-cypress-<< matrix.node >>-<< matrix.edition >>
          requires:
            - build-uberjar-<< matrix.edition >>
          cypress-group: "default-<< matrix.edition >>"

      - fe-tests-cypress:
          name: fe-tests-cypress-mongo-4-<< matrix.edition >>
          requires:
            - build-uberjar-<< matrix.edition >>
          e: fe-mongo-4
          cypress-group: "mongo"
          only-single-database: true
          test-files-location: frontend/test/metabase-db/mongo
          before-steps:
            - wait-for-port:
                port: 27017
          <<: *Matrix

      - fe-tests-cypress:
          name: fe-tests-cypress-postgres-12-<< matrix.edition >>
          requires:
            - build-uberjar-<< matrix.edition >>
          e: fe-postgres-12
          cypress-group: "postgres"
          only-single-database: true
          test-files-location: frontend/test/metabase-db/postgres
          before-steps:
            - wait-for-port:
                port: 5432
          <<: *Matrix

      - fe-tests-cypress:
          name: fe-tests-cypres-mysql-8-<< matrix.edition >>
          requires:
            - build-uberjar-<< matrix.edition >>
          e: fe-mysql-8
          cypress-group: "mysql"
          only-single-database: true
          test-files-location: frontend/test/metabase-db/mysql
          before-steps:
            - wait-for-port:
                port: 3306
          <<: *Matrix

  nightly:
    triggers:
      - schedule:
          cron: "0 9 * * *"
          filters:
            branches:
              only:
                - master
                - /release-*/
    jobs:
      - checkout

      - be-deps:
          requires:
            - checkout

      - fe-deps:
          requires:
            - checkout

      # Build the OSS version of the uberjar.
      - build-uberjar-drivers:
          requires:
            - be-deps
      - build-uberjar-frontend:
          requires:
            - fe-deps
      - build-uberjar:
          requires:
            - build-uberjar-drivers
            - build-uberjar-frontend

      - fe-tests-cypress-smoketest:
          name: fe-tests-cypress-smoketest
          requires:
            - build-uberjar
          cypress-group: "default"
