From 11694de4e6958e81bc160268edc07fef3d7d3f1f Mon Sep 17 00:00:00 2001 From: kithmina1999 Date: Fri, 6 Jun 2025 12:38:46 +0530 Subject: [PATCH 1/6] feat(db): add database backup and initialization system - Add backup.sh script for manual PostgreSQL database backups - Update .gitignore to exclude pg_backups directory - Modify docker-compose.yml to include backup service and volume mounts - Add init.sh script for automated database initialization with backup restoration --- .gitignore | 137 +++++++++++++++--------------- backup.sh | 16 ++++ docker-compose.yml | 46 +++++++++- worklenz-backend/database/init.sh | 88 +++++++++++++++++++ 4 files changed, 216 insertions(+), 71 deletions(-) create mode 100644 backup.sh create mode 100644 worklenz-backend/database/init.sh diff --git a/.gitignore b/.gitignore index d255be7f..f17ba915 100644 --- a/.gitignore +++ b/.gitignore @@ -1,79 +1,82 @@ -# Dependencies -node_modules/ -.pnp/ -.pnp.js + # Dependencies + node_modules/ + .pnp/ + .pnp.js -# Build outputs -dist/ -build/ -out/ -.next/ -.nuxt/ -.cache/ + # Build outputs + dist/ + build/ + out/ + .next/ + .nuxt/ + .cache/ -# Environment variables -.env -.env.local -.env.development.local -.env.test.local -.env.production.local -.env.development -.env.production -.env.* -!.env.example -!.env.template + # Environment variables + .env + .env.local + .env.development.local + .env.test.local + .env.production.local + .env.development + .env.production + .env.* + !.env.example + !.env.template -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -pnpm-debug.log* -lerna-debug.log* + # Logs + logs + *.log + npm-debug.log* + yarn-debug.log* + yarn-error.log* + pnpm-debug.log* + lerna-debug.log* -# Editor directories and files -.vscode/* -!.vscode/extensions.json -.idea/ -.DS_Store -*.suo -*.ntvs* -*.njsproj -*.sln -*.sw? -*.sublime-workspace + #backups + pg_backups/ -# Testing -coverage/ -.nyc_output/ + # Editor directories and files + .vscode/* + !.vscode/extensions.json + .idea/ + .DS_Store + *.suo + *.ntvs* + *.njsproj + *.sln + *.sw? + *.sublime-workspace -# Temp files -.temp/ -.tmp/ -temp/ -tmp/ + # Testing + coverage/ + .nyc_output/ -# Debug -.debug/ + # Temp files + .temp/ + .tmp/ + temp/ + tmp/ -# Misc -.DS_Store -Thumbs.db -.thumbs.db -ehthumbs.db -Desktop.ini -$RECYCLE.BIN/ + # Debug + .debug/ -# Yarn -.yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/sdks -!.yarn/versions + # Misc + .DS_Store + Thumbs.db + .thumbs.db + ehthumbs.db + Desktop.ini + $RECYCLE.BIN/ -# TypeScript -*.tsbuildinfo + # Yarn + .yarn/* + !.yarn/patches + !.yarn/plugins + !.yarn/releases + !.yarn/sdks + !.yarn/versions + + # TypeScript + *.tsbuildinfo diff --git a/backup.sh b/backup.sh new file mode 100644 index 00000000..8f16e1c7 --- /dev/null +++ b/backup.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -eu + +# Adjust these as needed: +CONTAINER=worklenz_db +DB_NAME=worklenz_db +DB_USER=postgres +BACKUP_DIR=./pg_backups +mkdir -p "$BACKUP_DIR" + +timestamp=$(date +%Y-%m-%d_%H-%M-%S) +outfile="${BACKUP_DIR}/${DB_NAME}_${timestamp}.sql" +echo "Creating backup $outfile ..." + +docker exec -t "$CONTAINER" pg_dump -U "$DB_USER" -d "$DB_NAME" > "$outfile" +echo "Backup saved to $outfile" diff --git a/docker-compose.yml b/docker-compose.yml index e7d074ad..8c539ae1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -80,7 +80,11 @@ services: POSTGRES_DB: ${DB_NAME:-worklenz_db} POSTGRES_PASSWORD: ${DB_PASSWORD:-password} healthcheck: - test: [ "CMD-SHELL", "pg_isready -d ${DB_NAME:-worklenz_db} -U ${DB_USER:-postgres}" ] + test: + [ + "CMD-SHELL", + "pg_isready -d ${DB_NAME:-worklenz_db} -U ${DB_USER:-postgres}", + ] interval: 10s timeout: 5s retries: 5 @@ -89,9 +93,20 @@ services: volumes: - worklenz_postgres_data:/var/lib/postgresql/data - type: bind - source: ./worklenz-backend/database - target: /docker-entrypoint-initdb.d + source: ./worklenz-backend/database/sql + target: /docker-entrypoint-initdb.d/sql consistency: cached + - type: bind + source: ./worklenz-backend/database/migrations + target: /docker-entrypoint-initdb.d/migrations + consistency: cached + - type: bind + source: ./worklenz-backend/database/init.sh + target: /docker-entrypoint-initdb.d/00_init.sh + consistency: cached + - type: bind + source: ./pg_backups + target: /docker-entrypoint-initdb.d/pg_backups command: > bash -c ' if command -v apt-get >/dev/null 2>&1; then apt-get update && apt-get install -y dos2unix @@ -101,11 +116,34 @@ services: dos2unix "{}" 2>/dev/null || true chmod +x "{}" '\'' \; && exec docker-entrypoint.sh postgres ' + db-backup: + image: postgres:15 + container_name: worklenz_db_backup + environment: + POSTGRES_USER: ${DB_USER:-postgres} + POSTGRES_DB: ${DB_NAME:-worklenz_db} + POSTGRES_PASSWORD: ${DB_PASSWORD:-password} + depends_on: + db: + condition: service_healthy + volumes: + - ./pg_backups:/pg_backups #host dir for backups files + #setup bassh loop to backup data evey 24h + command: > + bash -c "while true; do + sleep 86400; + PGPASSWORD=$$POSTGRES_PASSWORD pg_dump -h worklenz_db -U $$POSTGRES_USER -d $$POSTGRES_DB \ + > /pg_backups/worklenz_db_$(date +%Y-%m-%d_%H-%M-%S).sql; + find /pg_backups -type f -name '*.sql' -mtime +30 -delete; + done" + restart: unless-stopped + networks: + - worklenz volumes: worklenz_postgres_data: worklenz_minio_data: - + pgdata: networks: worklenz: diff --git a/worklenz-backend/database/init.sh b/worklenz-backend/database/init.sh new file mode 100644 index 00000000..afd8562a --- /dev/null +++ b/worklenz-backend/database/init.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -e + +echo "Starting database initialization..." + +SQL_DIR="/docker-entrypoint-initdb.d/sql" +MIGRATIONS_DIR="/docker-entrypoint-initdb.d/migrations" +BACKUP_DIR="/docker-entrypoint-initdb.d/pg_backups" + +# -------------------------------------------- +# 🗄ī¸ STEP 1: Attempt to restore latest backup +# -------------------------------------------- + +if [ -d "$BACKUP_DIR" ]; then + LATEST_BACKUP=$(ls -t "$BACKUP_DIR"/*.sql 2>/dev/null | head -n 1) +else + LATEST_BACKUP="" +fi + +if [ -f "$LATEST_BACKUP" ]; then + echo "🗄ī¸ Found latest backup: $LATEST_BACKUP" + echo "âŗ Restoring from backup..." + psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" < "$LATEST_BACKUP" + echo "✅ Backup restoration complete. Skipping schema and migrations." + exit 0 +else + echo "ℹī¸ No valid backup found. Proceeding with base schema and migrations." +fi + +# -------------------------------------------- +# 🏗ī¸ STEP 2: Continue with base schema setup +# -------------------------------------------- + +# Create migrations table if it doesn't exist +psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c " + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMP DEFAULT now() + ); +" + +# List of base schema files to execute in order +BASE_SQL_FILES=( + "0_extensions.sql" + "1_tables.sql" + "indexes.sql" + "4_functions.sql" + "triggers.sql" + "3_views.sql" + "2_dml.sql" + "5_database_user.sql" +) + +echo "Running base schema SQL files in order..." + +for file in "${BASE_SQL_FILES[@]}"; do + full_path="$SQL_DIR/$file" + if [ -f "$full_path" ]; then + echo "Executing $file..." + psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f "$full_path" + else + echo "WARNING: $file not found, skipping." + fi +done + +echo "✅ Base schema SQL execution complete." + +# -------------------------------------------- +# 🚀 STEP 3: Apply SQL migrations +# -------------------------------------------- + +if [ -d "$MIGRATIONS_DIR" ] && compgen -G "$MIGRATIONS_DIR/*.sql" > /dev/null; then + echo "Applying migrations..." + for f in "$MIGRATIONS_DIR"/*.sql; do + version=$(basename "$f") + if ! psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "SELECT 1 FROM schema_migrations WHERE version = '$version'" | grep -q 1; then + echo "Applying migration: $version" + psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f "$f" + psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "INSERT INTO schema_migrations (version) VALUES ('$version');" + else + echo "Skipping already applied migration: $version" + fi + done +else + echo "No migration files found or directory is empty, skipping migrations." +fi + +echo "🎉 Database initialization completed successfully." From dc22d1e6cbd254471ae44e4f69dc602a22e3a556 Mon Sep 17 00:00:00 2001 From: kithmina1999 Date: Fri, 6 Jun 2025 15:12:06 +0530 Subject: [PATCH 2/6] fix(db): improve database initialization script and docker setup - Rename init.sh to 00_init.sh for better ordering - Format docker-compose command for better readability - Add comprehensive database initialization script with backup restoration - Implement proper migration handling with schema_migrations table --- docker-compose.yml | 23 +++++++++++-------- .../database/{init.sh => 00_init.sh} | 0 2 files changed, 14 insertions(+), 9 deletions(-) rename worklenz-backend/database/{init.sh => 00_init.sh} (100%) diff --git a/docker-compose.yml b/docker-compose.yml index 8c539ae1..ac6987ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,21 +101,26 @@ services: target: /docker-entrypoint-initdb.d/migrations consistency: cached - type: bind - source: ./worklenz-backend/database/init.sh + source: ./worklenz-backend/database/00_init.sh target: /docker-entrypoint-initdb.d/00_init.sh consistency: cached - type: bind source: ./pg_backups target: /docker-entrypoint-initdb.d/pg_backups command: > - bash -c ' if command -v apt-get >/dev/null 2>&1; then - apt-get update && apt-get install -y dos2unix - elif command -v apk >/dev/null 2>&1; then - apk add --no-cache dos2unix - fi && find /docker-entrypoint-initdb.d -type f -name "*.sh" -exec sh -c '\'' - dos2unix "{}" 2>/dev/null || true - chmod +x "{}" - '\'' \; && exec docker-entrypoint.sh postgres ' + bash -c ' + if command -v apt-get >/dev/null 2>&1; then + apt-get update && apt-get install -y dos2unix + elif command -v apk >/dev/null 2>&1; then + apk add --no-cache dos2unix + fi + find /docker-entrypoint-initdb.d -type f -name "*.sh" -exec sh -c '\'' + dos2unix "{}" 2>/dev/null || true + chmod +x "{}" + '\'' \; + exec docker-entrypoint.sh postgres + ' + db-backup: image: postgres:15 container_name: worklenz_db_backup diff --git a/worklenz-backend/database/init.sh b/worklenz-backend/database/00_init.sh similarity index 100% rename from worklenz-backend/database/init.sh rename to worklenz-backend/database/00_init.sh From 0987fb14b263b29b5ceb1b7f4a8c6400c53f6353 Mon Sep 17 00:00:00 2001 From: kithmina1999 Date: Fri, 6 Jun 2025 15:34:18 +0530 Subject: [PATCH 3/6] refactor(docker): improve command formatting and fix shell script issues - Fix shell script syntax in db service command by using proper quoting and loop structure - Clean up indentation and formatting in both db and db-backup services - Ensure consistent command structure while maintaining the same functionality --- docker-compose.yml | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ac6987ae..fe936e16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -107,19 +107,24 @@ services: - type: bind source: ./pg_backups target: /docker-entrypoint-initdb.d/pg_backups - command: > - bash -c ' - if command -v apt-get >/dev/null 2>&1; then - apt-get update && apt-get install -y dos2unix - elif command -v apk >/dev/null 2>&1; then - apk add --no-cache dos2unix - fi - find /docker-entrypoint-initdb.d -type f -name "*.sh" -exec sh -c '\'' - dos2unix "{}" 2>/dev/null || true - chmod +x "{}" - '\'' \; - exec docker-entrypoint.sh postgres - ' + command: | + bash -c " + if command -v apt-get >/dev/null 2>&1; then + apt-get update && apt-get install -y dos2unix + elif command -v apk >/dev/null 2>&1; then + apk add --no-cache dos2unix + fi + + find /docker-entrypoint-initdb.d -type f -name '*.sh' -exec sh -c ' + for file; do + dos2unix \"$file\" 2>/dev/null || true + chmod +x \"$file\" + done + ' sh {} + + + exec docker-entrypoint.sh postgres + " + db-backup: image: postgres:15 @@ -135,12 +140,12 @@ services: - ./pg_backups:/pg_backups #host dir for backups files #setup bassh loop to backup data evey 24h command: > - bash -c "while true; do - sleep 86400; - PGPASSWORD=$$POSTGRES_PASSWORD pg_dump -h worklenz_db -U $$POSTGRES_USER -d $$POSTGRES_DB \ - > /pg_backups/worklenz_db_$(date +%Y-%m-%d_%H-%M-%S).sql; - find /pg_backups -type f -name '*.sql' -mtime +30 -delete; - done" + bash -c "while true; do + sleep 86400; + PGPASSWORD=$$POSTGRES_PASSWORD pg_dump -h worklenz_db -U $$POSTGRES_USER -d $$POSTGRES_DB \ + > /pg_backups/worklenz_db_\$(date +%Y-%m-%d_%H-%M-%S).sql; + find /pg_backups -type f -name '*.sql' -mtime +30 -delete; + done" restart: unless-stopped networks: - worklenz From 1442c57e18b721849e7777f8bad80a1c9221988c Mon Sep 17 00:00:00 2001 From: kithmina1999 Date: Fri, 6 Jun 2025 16:06:42 +0530 Subject: [PATCH 4/6] refactor(docker): improve postgres container initialization script The command for the postgres service was restructured to: 1. Use more readable multi-line formatting 2. Replace the find -exec with a more efficient for loop 3. Maintain the same functionality while improving maintainability --- docker-compose.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8c539ae1..dbf96f7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,14 +108,20 @@ services: source: ./pg_backups target: /docker-entrypoint-initdb.d/pg_backups command: > - bash -c ' if command -v apt-get >/dev/null 2>&1; then - apt-get update && apt-get install -y dos2unix - elif command -v apk >/dev/null 2>&1; then - apk add --no-cache dos2unix - fi && find /docker-entrypoint-initdb.d -type f -name "*.sh" -exec sh -c '\'' - dos2unix "{}" 2>/dev/null || true - chmod +x "{}" - '\'' \; && exec docker-entrypoint.sh postgres ' + bash -c ' + if command -v apt-get >/dev/null 2>&1; then + apt-get update && apt-get install -y dos2unix + elif command -v apk >/dev/null 2>&1; then + apk add --no-cache dos2unix + fi + find /docker-entrypoint-initdb.d -type f -name "*.sh" -exec sh -c " + for f; do + dos2unix \"\$f\" 2>/dev/null || true + chmod +x \"\$f\" + done + " sh {} + + exec docker-entrypoint.sh postgres + ' db-backup: image: postgres:15 container_name: worklenz_db_backup From c5e480af52a4700e0df0c0899b5df5c83553b77d Mon Sep 17 00:00:00 2001 From: kithmina1999 Date: Fri, 6 Jun 2025 16:30:46 +0530 Subject: [PATCH 5/6] fix(db): correct variable syntax in pg_dump command The previous version used $$ for variable expansion which doesn't work in this context. Changed to single $ for proper shell variable expansion in the database backup command. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 16d33195..64277545 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -141,7 +141,7 @@ services: bash -c "while true; do sleep 86400; PGPASSWORD=$$POSTGRES_PASSWORD pg_dump -h worklenz_db -U $$POSTGRES_USER -d $$POSTGRES_DB \ - > /pg_backups/worklenz_db_\$(date +%Y-%m-%d_%H-%M-%S).sql; + > /pg_backups/worklenz_db_$(date +%Y-%m-%d_%H-%M-%S).sql; find /pg_backups -type f -name '*.sql' -mtime +30 -delete; done" restart: unless-stopped From f142046dccca16972309f5548f3c438a2a835131 Mon Sep 17 00:00:00 2001 From: kithmina1999 Date: Fri, 6 Jun 2025 16:32:11 +0530 Subject: [PATCH 6/6] fix(docker): correct bash syntax in postgres backup command The command was using incorrect quote escaping which could cause issues with variable expansion and file deletion. Fixed by using single quotes for the outer string and proper escaping for date command substitution. --- docker-compose.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 64277545..71d74475 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -138,12 +138,12 @@ services: - ./pg_backups:/pg_backups #host dir for backups files #setup bassh loop to backup data evey 24h command: > - bash -c "while true; do - sleep 86400; - PGPASSWORD=$$POSTGRES_PASSWORD pg_dump -h worklenz_db -U $$POSTGRES_USER -d $$POSTGRES_DB \ - > /pg_backups/worklenz_db_$(date +%Y-%m-%d_%H-%M-%S).sql; - find /pg_backups -type f -name '*.sql' -mtime +30 -delete; - done" + bash -c 'while true; do + sleep 86400; + PGPASSWORD=$$POSTGRES_PASSWORD pg_dump -h worklenz_db -U $$POSTGRES_USER -d $$POSTGRES_DB \ + > /pg_backups/worklenz_db_$$(date +%Y-%m-%d_%H-%M-%S).sql; + find /pg_backups -type f -name "*.sql" -mtime +30 -delete; + done' restart: unless-stopped networks: - worklenz