config docker ci

.github/workflows/backend_general.yml

@@ -3,18 +3,84 @@ name: Backend
 on:
   push:
     branches: [ main ]
-    paths:
-      - 'backend'
   pull_request:
     branches: [ main ]
-    paths:
-      - 'backend'
 
 jobs:
 
-  build:
+  test:
+    name: Test
     runs-on: ubuntu-latest
-
+    services:
+      postgres:
+        image: postgres:12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+    env:
+      SQLX_VERSION: 0.5.7
+      SQLX_FEATURES: postgres
     steps:
-    - uses: actions/checkout@v2
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Cache dependencies
+        id: cache-dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Cache sqlx-cli
+        uses: actions/cache@v2
+        id: cache-sqlx
+        with:
+          path: |
+            ~/.cargo/bin/sqlx
+            ~/.cargo/bin/cargo-sqlx
+          key: ${{ runner.os }}-sqlx-${{ env.SQLX_VERSION }}-${{ env.SQLX_FEATURES }}
+
+      - name: Install sqlx-cli
+        uses: actions-rs/cargo@v1
+        if: steps.cache-sqlx.outputs.cache-hit == false
+        with:
+          command: install
+          args: >
+            sqlx-cli
+            --force
+            --version=${{ env.SQLX_VERSION }}
+            --features=${{ env.SQLX_FEATURES }}
+            --no-default-features
+            --locked
+
+        - name: Migrate database
+          working-directory: backend/
+          run: |
+            sudo apt-get install libpq-dev -y
+            SKIP_DOCKER=true ./backend/scripts/init_database.sh
+
+        - name: Check sqlx-data.json is up-to-date
+          working-directory: backend/
+          run: |
+            cargo sqlx prepare --check -- --bin backend
+
+        - name: Run cargo test
+          working-directory: backend/
+          uses: actions-rs/cargo@v1
+          with:
+            command: test
+
 

backend/Cargo.lock

@@ -784,7 +784,7 @@ dependencies = [
  "aes-gcm",
  "base64 0.13.0",
  "hkdf",
- "hmac",
+ "hmac 0.10.1",
  "percent-encoding",
  "rand",
  "sha2",
@@ -926,6 +926,16 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "crypto-mac"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
+dependencies = [
+ "generic-array",
+ "subtle",
+]
+
 [[package]]
 name = "ctr"
 version = "0.6.0"
@@ -1668,7 +1678,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f"
 dependencies = [
  "digest",
- "hmac",
+ "hmac 0.10.1",
 ]
 
 [[package]]
@@ -1677,7 +1687,17 @@ version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15"
 dependencies = [
- "crypto-mac",
+ "crypto-mac 0.10.1",
+ "digest",
+]
+
+[[package]]
+name = "hmac"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
+dependencies = [
+ "crypto-mac 0.11.1",
  "digest",
 ]
 
@@ -3058,9 +3078,9 @@ dependencies = [
 
 [[package]]
 name = "sqlx"
-version = "0.5.6"
+version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b977121ecc75cadd442a6a8c600f5ded6d7117a46d55ed417a815bd94b2af237"
+checksum = "7911b0031a0247af40095838002999c7a52fba29d9739e93326e71a5a1bc9d43"
 dependencies = [
  "sqlx-core",
  "sqlx-macros",
@@ -3068,9 +3088,9 @@ dependencies = [
 
 [[package]]
 name = "sqlx-core"
-version = "0.5.6"
+version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6da11206ae9ba3ececf83d44aef533c0d8176ae430cfaa14b408ce8b6bc4ab4b"
+checksum = "aec89bfaca8f7737439bad16d52b07f1ccd0730520d3bf6ae9d069fe4b641fb1"
 dependencies = [
  "ahash",
  "atoi",
@@ -3091,7 +3111,8 @@ dependencies = [
  "futures-util",
  "hashlink",
  "hex",
- "hmac",
+ "hmac 0.11.0",
+ "indexmap",
  "itoa",
  "libc",
  "log",
@@ -3121,13 +3142,12 @@ dependencies = [
 
 [[package]]
 name = "sqlx-macros"
-version = "0.5.6"
+version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "419f9005ad3b065ab4e774361b0d56a24e8e3dc9b2c9ddeaf018b5fdc1166ec2"
+checksum = "584866c833511b1a152e87a7ee20dee2739746f60c858b3c5209150bc4b466f5"
 dependencies = [
  "dotenv",
  "either",
- "futures",
  "heck",
  "hex",
  "once_cell",
@@ -3144,9 +3164,9 @@ dependencies = [
 
 [[package]]
 name = "sqlx-rt"
-version = "0.5.6"
+version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52d39baa13314f936c5991429b0b656980f52d23dfb15078bbea2c51f5086eeb"
+checksum = "0d1bd069de53442e7a320f525a6d4deb8bb0621ac7a55f7eccbc2b58b57f43d0"
 dependencies = [
  "actix-rt",
  "once_cell",

backend/Cargo.toml

@@ -68,7 +68,7 @@ backend-service = { path = "../shared-lib/backend-service", features = ["http_se
 
 ormx = { version = "0.7", features = ["postgres"]}
 [dependencies.sqlx]
-version = "0.5.6"
+version = "0.5.7"
 default-features = false
 features = [
     "runtime-actix-rustls",
@@ -78,7 +78,6 @@ features = [
     "chrono",
     "migrate",
     "offline",
-    "any",
 ]
 
 

backend/Dockerfile

@@ -1,57 +1,28 @@
-FROM rust:1.56.1
-# Let's switch our working directory to `app` (equivalent to `cd app`)
-# The `app` folder will be created for us by Docker in case it does not
-# exist already.
+FROM lukemathwalker/cargo-chef:latest-rust-1.53.0 as chef
 WORKDIR /app
 
+FROM chef as planner
 COPY . .
+RUN cargo chef prepare  --recipe-path recipe.json
 
-WORKDIR /app/backend
+FROM chef as builder
+COPY --from=planner /app/recipe.json recipe.json
+# Build dependencies
+RUN cargo chef cook --release --recipe-path recipe.json
+COPY . .
 ENV SQLX_OFFLINE true
-ENV APP_ENVIRONMENT production
-RUN RUSTFLAGS="-C opt-level=2" cargo build --release
-
-# When `docker run` is executed, launch the binary!
-ENTRYPOINT ["./target/release/backend"]
-
-#
+# Build our project
+RUN cargo build --release --bin backend
 
-
-
-## We use the latest Rust stable release as base image
-#FROM lukemathwalker/cargo-chef:latest-rust-1.53.0 as planner
-#WORKDIR /app
-#COPY . .
-#
-#WORKDIR /app/backend
-#RUN cargo chef prepare  --recipe-path recipe.json
-#
-#FROM lukemathwalker/cargo-chef:latest-rust-1.53.0 as cacher
-#WORKDIR /app/backend
-#COPY --from=planner /app/backend/recipe.json recipe.json
-## Build our project dependencies, not our application!
-#RUN cargo chef cook --release --recipe-path recipe.json
-#
-#FROM rust:1.53.0 AS builder
-#WORKDIR /app/backend
-## Copy over the cached dependencies
-#COPY --from=cacher /app/backend/target target
-#COPY --from=cacher /usr/local/cargo /usr/local/cargo
-#COPY . .
-#
-#ENV SQLX_OFFLINE true
-#RUN cargo build --release --bin backend
-#
-#
-#FROM debian:buster-slim AS runtime
-#WORKDIR /app/backend
-#RUN apt-get update -y \
-#    && apt-get install -y --no-install-recommends openssl \
-#    # Clean up
-#    && apt-get autoremove -y \
-#    && apt-get clean -y \
-#    && rm -rf /var/lib/apt/lists/*
-#COPY --from=builder /app/backend/target/release/backend backend
-##COPY configuration configuration
-#ENV APP_ENVIRONMENT production
-#ENTRYPOINT ["./backend"]
+FROM debian:bullseye-slim AS runtime
+WORKDIR /app
+RUN apt-get update -y \
+    && apt-get install -y --no-install-recommends openssl \
+    # Clean up
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /app/target/release/backend backend
+COPY configuration configuration
+ENV APP_ENVIRONMENT production
+ENTRYPOINT ["./backend"]

backend/rust-toolchain

@@ -1,2 +0,0 @@
-[toolchain]
-channel = "nightly-2021-04-24"

backend/scripts/database/database.mk

@@ -5,11 +5,9 @@ export DB_NAME=flowy
 export DB_PORT=5433
 export DATABASE_URL=postgres://${DB_USER}:${DB_PASSWORD}@localhost:${DB_PORT}/${DB_NAME}
 export ROOT = "./scripts/database"
+export DB_HOST = "localhost"
 
-init_postgres:
-	${ROOT}/init_postgres.sh
-
-init_database: init_postgres
+init_database:
 	${ROOT}/init_database.sh
 
 reset_db:

backend/scripts/database/init_database.sh

@@ -18,11 +18,37 @@ if ! [ -x "$(command -v sqlx)" ]; then
   exit 1
 fi
 
-until psql -h "localhost" -U "${DB_USER}" -p "${DB_PORT}" -d "postgres" -c '\q';
-do
+if [[ -z "${SKIP_DOCKER}" ]]
+then
+  RUNNING_POSTGRES_CONTAINER=$(docker ps --filter 'name=postgres' --format '{{.ID}}')
+  if [[ -n $RUNNING_POSTGRES_CONTAINER ]]; then
+    echo >&2 "there is a postgres container already running, kill it with"
+    echo >&2 "    docker kill ${RUNNING_POSTGRES_CONTAINER}"
+    exit 1
+  fi
+
+  docker run \
+      --name="flowy" \
+      -e POSTGRES_USER=${DB_USER} \
+      -e POSTGRES_PASSWORD=${DB_PASSWORD} \
+      -e POSTGRES_DB=${DB_NAME} \
+      -p "${DB_PORT}":5432 \
+      -d postgres \
+      postgres -N 1000
+fi
+
+
+# Keep pinging Postgres until it's ready to accept commands
+until PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -U "${DB_USER}" -p "${DB_PORT}" -d "postgres" -c '\q'; do
   >&2 echo "Postgres is still unavailable - sleeping"
   sleep 1
 done
 
->&2 echo "Postgres is up and running on port ${DB_PORT}!"
+>&2 echo "Postgres is up and running on port ${DB_PORT} - running migrations now!"
+
+export DATABASE_URL=postgres://${DB_USER}:${DB_PASSWORD}@localhost:${DB_PORT}/${DB_NAME}
 sqlx database create
+sqlx migrate run
+
+>&2 echo "Postgres has been migrated, ready to go!"
+

backend/scripts/database/init_postgres.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-set -x
-set -eo pipefail
-
-#if [[ -z "${RESET}" ]]
-#then
-#  docker stop flowy
-#  docker rm flowy
-#fi
-
-if [[ -z "${SKIP_DOCKER}" ]]
-then
-  docker run \
-      --name="flowy" \
-      -e POSTGRES_USER=${DB_USER} \
-      -e POSTGRES_PASSWORD=${DB_PASSWORD} \
-      -e POSTGRES_DB=${DB_NAME} \
-      -p "${DB_PORT}":5432 \
-      -d postgres \
-      postgres -N 1000
-fi
-  # ^ Increased maximum number of connections for testing purposes