Merge pull request #151 from kithmina1999/feat/docker-db-backup
Feat/Add auto backup and manual backup functionality to docker-compose.yaml
This commit is contained in:
Chamika J
137
.gitignore
vendored
137
.gitignore
vendored
@@ -1,79 +1,82 @@
|
||||
# Dependencies
|
||||
node_modules/
|
||||
.pnp/
|
||||
.pnp.js
|
||||
# Dependencies
|
||||
node_modules/
|
||||
.pnp/
|
||||
.pnp.js
|
||||
|
||||
# Build outputs
|
||||
dist/
|
||||
build/
|
||||
out/
|
||||
.next/
|
||||
.nuxt/
|
||||
.cache/
|
||||
# Build outputs
|
||||
dist/
|
||||
build/
|
||||
out/
|
||||
.next/
|
||||
.nuxt/
|
||||
.cache/
|
||||
|
||||
# Environment variables
|
||||
.env
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.development
|
||||
.env.production
|
||||
.env.*
|
||||
!.env.example
|
||||
!.env.template
|
||||
# Environment variables
|
||||
.env
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.development
|
||||
.env.production
|
||||
.env.*
|
||||
!.env.example
|
||||
!.env.template
|
||||
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
lerna-debug.log*
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
lerna-debug.log*
|
||||
|
||||
# Editor directories and files
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
.idea/
|
||||
.DS_Store
|
||||
*.suo
|
||||
*.ntvs*
|
||||
*.njsproj
|
||||
*.sln
|
||||
*.sw?
|
||||
*.sublime-workspace
|
||||
#backups
|
||||
pg_backups/
|
||||
|
||||
# Testing
|
||||
coverage/
|
||||
.nyc_output/
|
||||
# Editor directories and files
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
.idea/
|
||||
.DS_Store
|
||||
*.suo
|
||||
*.ntvs*
|
||||
*.njsproj
|
||||
*.sln
|
||||
*.sw?
|
||||
*.sublime-workspace
|
||||
|
||||
# Temp files
|
||||
.temp/
|
||||
.tmp/
|
||||
temp/
|
||||
tmp/
|
||||
# Testing
|
||||
coverage/
|
||||
.nyc_output/
|
||||
|
||||
# Debug
|
||||
.debug/
|
||||
# Temp files
|
||||
.temp/
|
||||
.tmp/
|
||||
temp/
|
||||
tmp/
|
||||
|
||||
# Misc
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
.thumbs.db
|
||||
ehthumbs.db
|
||||
Desktop.ini
|
||||
$RECYCLE.BIN/
|
||||
# Debug
|
||||
.debug/
|
||||
|
||||
# Yarn
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/sdks
|
||||
!.yarn/versions
|
||||
# Misc
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
.thumbs.db
|
||||
ehthumbs.db
|
||||
Desktop.ini
|
||||
$RECYCLE.BIN/
|
||||
|
||||
# TypeScript
|
||||
*.tsbuildinfo
|
||||
# Yarn
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/sdks
|
||||
!.yarn/versions
|
||||
|
||||
# TypeScript
|
||||
*.tsbuildinfo
|
||||
|
||||
|
||||
|
||||
16
backup.sh
Normal file
16
backup.sh
Normal file
@@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
|
||||
# Adjust these as needed:
|
||||
CONTAINER=worklenz_db
|
||||
DB_NAME=worklenz_db
|
||||
DB_USER=postgres
|
||||
BACKUP_DIR=./pg_backups
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
timestamp=$(date +%Y-%m-%d_%H-%M-%S)
|
||||
outfile="${BACKUP_DIR}/${DB_NAME}_${timestamp}.sql"
|
||||
echo "Creating backup $outfile ..."
|
||||
|
||||
docker exec -t "$CONTAINER" pg_dump -U "$DB_USER" -d "$DB_NAME" > "$outfile"
|
||||
echo "Backup saved to $outfile"
|
||||
@@ -80,7 +80,11 @@ services:
|
||||
POSTGRES_DB: ${DB_NAME:-worklenz_db}
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-password}
|
||||
healthcheck:
|
||||
test: [ "CMD-SHELL", "pg_isready -d ${DB_NAME:-worklenz_db} -U ${DB_USER:-postgres}" ]
|
||||
test:
|
||||
[
|
||||
"CMD-SHELL",
|
||||
"pg_isready -d ${DB_NAME:-worklenz_db} -U ${DB_USER:-postgres}",
|
||||
]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
@@ -89,23 +93,65 @@ services:
|
||||
volumes:
|
||||
- worklenz_postgres_data:/var/lib/postgresql/data
|
||||
- type: bind
|
||||
source: ./worklenz-backend/database
|
||||
target: /docker-entrypoint-initdb.d
|
||||
source: ./worklenz-backend/database/sql
|
||||
target: /docker-entrypoint-initdb.d/sql
|
||||
consistency: cached
|
||||
- type: bind
|
||||
source: ./worklenz-backend/database/migrations
|
||||
target: /docker-entrypoint-initdb.d/migrations
|
||||
consistency: cached
|
||||
- type: bind
|
||||
source: ./worklenz-backend/database/00_init.sh
|
||||
target: /docker-entrypoint-initdb.d/00_init.sh
|
||||
consistency: cached
|
||||
- type: bind
|
||||
source: ./pg_backups
|
||||
target: /docker-entrypoint-initdb.d/pg_backups
|
||||
command: >
|
||||
bash -c ' if command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update && apt-get install -y dos2unix
|
||||
elif command -v apk >/dev/null 2>&1; then
|
||||
apk add --no-cache dos2unix
|
||||
fi && find /docker-entrypoint-initdb.d -type f -name "*.sh" -exec sh -c '\''
|
||||
dos2unix "{}" 2>/dev/null || true
|
||||
chmod +x "{}"
|
||||
'\'' \; && exec docker-entrypoint.sh postgres '
|
||||
bash -c '
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update && apt-get install -y dos2unix
|
||||
elif command -v apk >/dev/null 2>&1; then
|
||||
apk add --no-cache dos2unix
|
||||
fi
|
||||
|
||||
find /docker-entrypoint-initdb.d -type f -name "*.sh" -exec sh -c '"'"'
|
||||
for f; do
|
||||
dos2unix "$f" 2>/dev/null || true
|
||||
chmod +x "$f"
|
||||
done
|
||||
'"'"' sh {} +
|
||||
|
||||
exec docker-entrypoint.sh postgres
|
||||
'
|
||||
db-backup:
|
||||
image: postgres:15
|
||||
container_name: worklenz_db_backup
|
||||
environment:
|
||||
POSTGRES_USER: ${DB_USER:-postgres}
|
||||
POSTGRES_DB: ${DB_NAME:-worklenz_db}
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-password}
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./pg_backups:/pg_backups #host dir for backups files
|
||||
#setup bassh loop to backup data evey 24h
|
||||
command: >
|
||||
bash -c 'while true; do
|
||||
sleep 86400;
|
||||
PGPASSWORD=$$POSTGRES_PASSWORD pg_dump -h worklenz_db -U $$POSTGRES_USER -d $$POSTGRES_DB \
|
||||
> /pg_backups/worklenz_db_$$(date +%Y-%m-%d_%H-%M-%S).sql;
|
||||
find /pg_backups -type f -name "*.sql" -mtime +30 -delete;
|
||||
done'
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- worklenz
|
||||
|
||||
volumes:
|
||||
worklenz_postgres_data:
|
||||
worklenz_minio_data:
|
||||
|
||||
pgdata:
|
||||
|
||||
networks:
|
||||
worklenz:
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# This script controls the order of SQL file execution during database initialization
|
||||
echo "Starting database initialization..."
|
||||
|
||||
# Check if we have SQL files in expected locations
|
||||
if [ -f "/docker-entrypoint-initdb.d/sql/0_extensions.sql" ]; then
|
||||
SQL_DIR="/docker-entrypoint-initdb.d/sql"
|
||||
echo "Using SQL files from sql/ subdirectory"
|
||||
elif [ -f "/docker-entrypoint-initdb.d/0_extensions.sql" ]; then
|
||||
# First time setup - move files to subdirectory
|
||||
echo "Moving SQL files to sql/ subdirectory..."
|
||||
mkdir -p /docker-entrypoint-initdb.d/sql
|
||||
|
||||
# Move all SQL files (except this script) to the subdirectory
|
||||
for f in /docker-entrypoint-initdb.d/*.sql; do
|
||||
if [ -f "$f" ]; then
|
||||
cp "$f" /docker-entrypoint-initdb.d/sql/
|
||||
echo "Copied $f to sql/ subdirectory"
|
||||
fi
|
||||
done
|
||||
|
||||
SQL_DIR="/docker-entrypoint-initdb.d/sql"
|
||||
else
|
||||
echo "SQL files not found in expected locations!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Execute SQL files in the correct order
|
||||
echo "Executing 0_extensions.sql..."
|
||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/0_extensions.sql"
|
||||
|
||||
echo "Executing 1_tables.sql..."
|
||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/1_tables.sql"
|
||||
|
||||
echo "Executing indexes.sql..."
|
||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/indexes.sql"
|
||||
|
||||
echo "Executing 4_functions.sql..."
|
||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/4_functions.sql"
|
||||
|
||||
echo "Executing triggers.sql..."
|
||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/triggers.sql"
|
||||
|
||||
echo "Executing 3_views.sql..."
|
||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/3_views.sql"
|
||||
|
||||
echo "Executing 2_dml.sql..."
|
||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/2_dml.sql"
|
||||
|
||||
echo "Executing 5_database_user.sql..."
|
||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -f "$SQL_DIR/5_database_user.sql"
|
||||
|
||||
echo "Database initialization completed successfully"
|
||||
88
worklenz-backend/database/00_init.sh
Normal file
88
worklenz-backend/database/00_init.sh
Normal file
@@ -0,0 +1,88 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "Starting database initialization..."
|
||||
|
||||
SQL_DIR="/docker-entrypoint-initdb.d/sql"
|
||||
MIGRATIONS_DIR="/docker-entrypoint-initdb.d/migrations"
|
||||
BACKUP_DIR="/docker-entrypoint-initdb.d/pg_backups"
|
||||
|
||||
# --------------------------------------------
|
||||
# 🗄️ STEP 1: Attempt to restore latest backup
|
||||
# --------------------------------------------
|
||||
|
||||
if [ -d "$BACKUP_DIR" ]; then
|
||||
LATEST_BACKUP=$(ls -t "$BACKUP_DIR"/*.sql 2>/dev/null | head -n 1)
|
||||
else
|
||||
LATEST_BACKUP=""
|
||||
fi
|
||||
|
||||
if [ -f "$LATEST_BACKUP" ]; then
|
||||
echo "🗄️ Found latest backup: $LATEST_BACKUP"
|
||||
echo "⏳ Restoring from backup..."
|
||||
psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" < "$LATEST_BACKUP"
|
||||
echo "✅ Backup restoration complete. Skipping schema and migrations."
|
||||
exit 0
|
||||
else
|
||||
echo "ℹ️ No valid backup found. Proceeding with base schema and migrations."
|
||||
fi
|
||||
|
||||
# --------------------------------------------
|
||||
# 🏗️ STEP 2: Continue with base schema setup
|
||||
# --------------------------------------------
|
||||
|
||||
# Create migrations table if it doesn't exist
|
||||
psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMP DEFAULT now()
|
||||
);
|
||||
"
|
||||
|
||||
# List of base schema files to execute in order
|
||||
BASE_SQL_FILES=(
|
||||
"0_extensions.sql"
|
||||
"1_tables.sql"
|
||||
"indexes.sql"
|
||||
"4_functions.sql"
|
||||
"triggers.sql"
|
||||
"3_views.sql"
|
||||
"2_dml.sql"
|
||||
"5_database_user.sql"
|
||||
)
|
||||
|
||||
echo "Running base schema SQL files in order..."
|
||||
|
||||
for file in "${BASE_SQL_FILES[@]}"; do
|
||||
full_path="$SQL_DIR/$file"
|
||||
if [ -f "$full_path" ]; then
|
||||
echo "Executing $file..."
|
||||
psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f "$full_path"
|
||||
else
|
||||
echo "WARNING: $file not found, skipping."
|
||||
fi
|
||||
done
|
||||
|
||||
echo "✅ Base schema SQL execution complete."
|
||||
|
||||
# --------------------------------------------
|
||||
# 🚀 STEP 3: Apply SQL migrations
|
||||
# --------------------------------------------
|
||||
|
||||
if [ -d "$MIGRATIONS_DIR" ] && compgen -G "$MIGRATIONS_DIR/*.sql" > /dev/null; then
|
||||
echo "Applying migrations..."
|
||||
for f in "$MIGRATIONS_DIR"/*.sql; do
|
||||
version=$(basename "$f")
|
||||
if ! psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "SELECT 1 FROM schema_migrations WHERE version = '$version'" | grep -q 1; then
|
||||
echo "Applying migration: $version"
|
||||
psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f "$f"
|
||||
psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "INSERT INTO schema_migrations (version) VALUES ('$version');"
|
||||
else
|
||||
echo "Skipping already applied migration: $version"
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "No migration files found or directory is empty, skipping migrations."
|
||||
fi
|
||||
|
||||
echo "🎉 Database initialization completed successfully."
|
||||
Reference in New Issue
Block a user