Merge pull request #151 from kithmina1999/feat/docker-db-backup

Feat/Add auto backup and manual backup functionality  to docker-compose.yaml

.gitignore

@@ -1,79 +1,82 @@
-# Dependencies
+    # Dependencies
-node_modules/
+    node_modules/
-.pnp/
+    .pnp/
-.pnp.js
+    .pnp.js
 
-# Build outputs
+    # Build outputs
-dist/
+    dist/
-build/
+    build/
-out/
+    out/
-.next/
+    .next/
-.nuxt/
+    .nuxt/
-.cache/
+    .cache/
 
-# Environment variables
+    # Environment variables
-.env
+    .env
-.env.local
+    .env.local
-.env.development.local
+    .env.development.local
-.env.test.local
+    .env.test.local
-.env.production.local
+    .env.production.local
-.env.development
+    .env.development
-.env.production
+    .env.production
-.env.*
+    .env.*
-!.env.example
+    !.env.example
-!.env.template
+    !.env.template
 
-# Logs
+    # Logs
-logs
+    logs
-*.log
+    *.log
-npm-debug.log*
+    npm-debug.log*
-yarn-debug.log*
+    yarn-debug.log*
-yarn-error.log*
+    yarn-error.log*
-pnpm-debug.log*
+    pnpm-debug.log*
-lerna-debug.log*
+    lerna-debug.log*
 
-# Editor directories and files
+    #backups
-.vscode/*
+    pg_backups/
-!.vscode/extensions.json
-.idea/
-.DS_Store
-*.suo
-*.ntvs*
-*.njsproj
-*.sln
-*.sw?
-*.sublime-workspace
 
-# Testing
+    # Editor directories and files
-coverage/
+    .vscode/*
-.nyc_output/
+    !.vscode/extensions.json
+    .idea/
+    .DS_Store
+    *.suo
+    *.ntvs*
+    *.njsproj
+    *.sln
+    *.sw?
+    *.sublime-workspace
 
-# Temp files
+    # Testing
-.temp/
+    coverage/
-.tmp/
+    .nyc_output/
-temp/
-tmp/
 
-# Debug
+    # Temp files
-.debug/
+    .temp/
+    .tmp/
+    temp/
+    tmp/
 
-# Misc
+    # Debug
-.DS_Store
+    .debug/
-Thumbs.db
-.thumbs.db
-ehthumbs.db
-Desktop.ini
-$RECYCLE.BIN/
 
-# Yarn
+    # Misc
-.yarn/*
+    .DS_Store
-!.yarn/patches
+    Thumbs.db
-!.yarn/plugins
+    .thumbs.db
-!.yarn/releases
+    ehthumbs.db
-!.yarn/sdks
+    Desktop.ini
-!.yarn/versions
+    $RECYCLE.BIN/
 
-# TypeScript
+    # Yarn
-*.tsbuildinfo
+    .yarn/*
+    !.yarn/patches
+    !.yarn/plugins
+    !.yarn/releases
+    !.yarn/sdks
+    !.yarn/versions
+
+    # TypeScript
+    *.tsbuildinfo
 
 

backup.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -eu
+
+# Adjust these as needed:
+CONTAINER=worklenz_db
+DB_NAME=worklenz_db
+DB_USER=postgres
+BACKUP_DIR=./pg_backups
+mkdir -p "$BACKUP_DIR"
+
+timestamp=$(date +%Y-%m-%d_%H-%M-%S)
+outfile="${BACKUP_DIR}/${DB_NAME}_${timestamp}.sql"
+echo "Creating backup $outfile ..."
+
+docker exec -t "$CONTAINER" pg_dump -U "$DB_USER" -d "$DB_NAME" > "$outfile"
+echo "Backup saved to $outfile"

docker-compose.yml

@@ -80,7 +80,11 @@ services:
       POSTGRES_DB: ${DB_NAME:-worklenz_db}
       POSTGRES_PASSWORD: ${DB_PASSWORD:-password}
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -d ${DB_NAME:-worklenz_db} -U ${DB_USER:-postgres}" ]
+      test:
+        [
+          "CMD-SHELL",
+          "pg_isready -d ${DB_NAME:-worklenz_db} -U ${DB_USER:-postgres}",
+        ]
       interval: 10s
       timeout: 5s
       retries: 5
@@ -89,23 +93,65 @@ services:
     volumes:
       - worklenz_postgres_data:/var/lib/postgresql/data
       - type: bind
-        source: ./worklenz-backend/database
+        source: ./worklenz-backend/database/sql
-        target: /docker-entrypoint-initdb.d
+        target: /docker-entrypoint-initdb.d/sql
         consistency: cached
+      - type: bind
+        source: ./worklenz-backend/database/migrations
+        target: /docker-entrypoint-initdb.d/migrations
+        consistency: cached
+      - type: bind
+        source: ./worklenz-backend/database/00_init.sh
+        target: /docker-entrypoint-initdb.d/00_init.sh
+        consistency: cached
+      - type: bind
+        source: ./pg_backups
+        target: /docker-entrypoint-initdb.d/pg_backups
     command: >
-      bash -c ' if command -v apt-get >/dev/null 2>&1; then
+        bash -c '
-        apt-get update && apt-get install -y dos2unix
+          if command -v apt-get >/dev/null 2>&1; then
-      elif command -v apk >/dev/null 2>&1; then
+            apt-get update && apt-get install -y dos2unix
-        apk add --no-cache dos2unix
+          elif command -v apk >/dev/null 2>&1; then
-      fi && find /docker-entrypoint-initdb.d -type f -name "*.sh" -exec sh -c '\''
+            apk add --no-cache dos2unix
-        dos2unix "{}" 2>/dev/null || true
+          fi
-        chmod +x "{}"
+
-      '\'' \; && exec docker-entrypoint.sh postgres '
+          find /docker-entrypoint-initdb.d -type f -name "*.sh" -exec sh -c '"'"'
+            for f; do
+              dos2unix "$f" 2>/dev/null || true
+              chmod +x "$f"
+            done
+          '"'"' sh {} +
+
+          exec docker-entrypoint.sh postgres
+        '
+  db-backup:
+    image: postgres:15
+    container_name: worklenz_db_backup
+    environment:
+      POSTGRES_USER: ${DB_USER:-postgres}
+      POSTGRES_DB: ${DB_NAME:-worklenz_db}
+      POSTGRES_PASSWORD: ${DB_PASSWORD:-password}
+    depends_on:
+      db:
+        condition: service_healthy
+    volumes:
+      - ./pg_backups:/pg_backups #host dir for backups files
+    #setup bassh loop to backup data evey 24h
+    command: >
+           bash -c 'while true; do
+              sleep 86400;
+              PGPASSWORD=$$POSTGRES_PASSWORD pg_dump -h worklenz_db -U $$POSTGRES_USER -d $$POSTGRES_DB \
+              > /pg_backups/worklenz_db_$$(date +%Y-%m-%d_%H-%M-%S).sql;
+              find /pg_backups -type f -name "*.sql" -mtime +30 -delete;
+            done'
+    restart: unless-stopped
+    networks:
+      - worklenz
 
 volumes:
   worklenz_postgres_data:
   worklenz_minio_data:
-
+  pgdata:
 
 networks:
   worklenz:

worklenz-backend/database/00-init-db.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-set -e
-
-# This script controls the order of SQL file execution during database initialization
-echo "Starting database initialization..."
-
-# Check if we have SQL files in expected locations
-if [ -f "/docker-entrypoint-initdb.d/sql/0_extensions.sql" ]; then
-  SQL_DIR="/docker-entrypoint-initdb.d/sql"
-  echo "Using SQL files from sql/ subdirectory"
-elif [ -f "/docker-entrypoint-initdb.d/0_extensions.sql" ]; then
-  # First time setup - move files to subdirectory
-  echo "Moving SQL files to sql/ subdirectory..."
-  mkdir -p /docker-entrypoint-initdb.d/sql
-  
-  # Move all SQL files (except this script) to the subdirectory
-  for f in /docker-entrypoint-initdb.d/*.sql; do
-    if [ -f "$f" ]; then
-      cp "$f" /docker-entrypoint-initdb.d/sql/
-      echo "Copied $f to sql/ subdirectory"
-    fi
-  done
-  
-  SQL_DIR="/docker-entrypoint-initdb.d/sql"
-else
-  echo "SQL files not found in expected locations!"
-  exit 1
-fi
-
-# Execute SQL files in the correct order
-echo "Executing 0_extensions.sql..."
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/0_extensions.sql"
-
-echo "Executing 1_tables.sql..."
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/1_tables.sql"
-
-echo "Executing indexes.sql..."
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/indexes.sql"
-
-echo "Executing 4_functions.sql..."
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/4_functions.sql"
-
-echo "Executing triggers.sql..."
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/triggers.sql"
-
-echo "Executing 3_views.sql..."
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/3_views.sql"
-
-echo "Executing 2_dml.sql..."
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/2_dml.sql"
-
-echo "Executing 5_database_user.sql..."
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/5_database_user.sql"
-
-echo "Database initialization completed successfully" 

worklenz-backend/database/00_init.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+set -e
+
+echo "Starting database initialization..."
+
+SQL_DIR="/docker-entrypoint-initdb.d/sql"
+MIGRATIONS_DIR="/docker-entrypoint-initdb.d/migrations"
+BACKUP_DIR="/docker-entrypoint-initdb.d/pg_backups"
+
+# --------------------------------------------
+# 🗄️ STEP 1: Attempt to restore latest backup
+# --------------------------------------------
+
+if [ -d "$BACKUP_DIR" ]; then
+  LATEST_BACKUP=$(ls -t "$BACKUP_DIR"/*.sql 2>/dev/null | head -n 1)
+else
+  LATEST_BACKUP=""
+fi
+
+if [ -f "$LATEST_BACKUP" ]; then
+  echo "🗄️ Found latest backup: $LATEST_BACKUP"
+  echo "⏳ Restoring from backup..."
+  psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" < "$LATEST_BACKUP"
+  echo "✅ Backup restoration complete. Skipping schema and migrations."
+  exit 0
+else
+  echo "ℹ️ No valid backup found. Proceeding with base schema and migrations."
+fi
+
+# --------------------------------------------
+# 🏗️ STEP 2: Continue with base schema setup
+# --------------------------------------------
+
+# Create migrations table if it doesn't exist
+psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "
+  CREATE TABLE IF NOT EXISTS schema_migrations (
+    version TEXT PRIMARY KEY,
+    applied_at TIMESTAMP DEFAULT now()
+  );
+"
+
+# List of base schema files to execute in order
+BASE_SQL_FILES=(
+  "0_extensions.sql"
+  "1_tables.sql"
+  "indexes.sql"
+  "4_functions.sql"
+  "triggers.sql"
+  "3_views.sql"
+  "2_dml.sql"
+  "5_database_user.sql"
+)
+
+echo "Running base schema SQL files in order..."
+
+for file in "${BASE_SQL_FILES[@]}"; do
+  full_path="$SQL_DIR/$file"
+  if [ -f "$full_path" ]; then
+    echo "Executing $file..."
+    psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f "$full_path"
+  else
+    echo "WARNING: $file not found, skipping."
+  fi
+done
+
+echo "✅ Base schema SQL execution complete."
+
+# --------------------------------------------
+# 🚀 STEP 3: Apply SQL migrations
+# --------------------------------------------
+
+if [ -d "$MIGRATIONS_DIR" ] && compgen -G "$MIGRATIONS_DIR/*.sql" > /dev/null; then
+  echo "Applying migrations..."
+  for f in "$MIGRATIONS_DIR"/*.sql; do
+    version=$(basename "$f")
+    if ! psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "SELECT 1 FROM schema_migrations WHERE version = '$version'" | grep -q 1; then
+      echo "Applying migration: $version"
+      psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f "$f"
+      psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "INSERT INTO schema_migrations (version) VALUES ('$version');"
+    else
+      echo "Skipping already applied migration: $version"
+    fi
+  done
+else
+  echo "No migration files found or directory is empty, skipping migrations."
+fi
+
+echo "🎉 Database initialization completed successfully."