bo/worklenz
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(database): add performance indexes and materialized view for optimization

Browse Source 
- Created multiple new indexes in the performance-indexes.sql file to enhance query performance for tasks, team members, and related tables.
- Added a materialized view for team member information in 3_views.sql to pre-calculate expensive joins, improving data retrieval efficiency.
- Introduced a function to refresh the materialized view, ensuring up-to-date information while optimizing performance.
- Implemented an optimized method in tasks-controller-v2.ts to split complex queries into focused segments, significantly improving response times and overall performance.
This commit is contained in:
chamikaJ
2025-07-07 15:01:11 +05:30
parent 134899114d
commit 978d9158c0
3 changed files with 397 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
worklenz-backend/database/migrations/20250115000000-performance-indexes.sql
View File

@@ -80,3 +80,56 @@ ON task_priorities(value);
-- Index for team labels
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_team_labels_team
ON team_labels(team_id);
-- NEW INDEXES FOR PERFORMANCE OPTIMIZATION --
-- Composite index for task main query optimization (covers most WHERE conditions)
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tasks_performance_main
ON tasks(project_id, archived, parent_task_id, status_id, priority_id)
WHERE archived = FALSE;
-- Index for sorting by sort_order with project filter
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tasks_project_sort_order
ON tasks(project_id, sort_order)
WHERE archived = FALSE;
-- Index for email_invitations to optimize team_member_info_view
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_email_invitations_team_member
ON email_invitations(team_member_id);
-- Covering index for task status with category information
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_task_statuses_covering
ON task_statuses(id, category_id, project_id);
-- Index for task aggregation queries (parent task progress calculation)
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tasks_parent_status_archived
ON tasks(parent_task_id, status_id, archived)
WHERE archived = FALSE;
-- Index for project team member filtering
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_team_members_project_lookup
ON team_members(team_id, active, user_id)
WHERE active = TRUE;
-- Covering index for tasks with frequently accessed columns
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tasks_covering_main
ON tasks(id, project_id, archived, parent_task_id, status_id, priority_id, sort_order, name)
WHERE archived = FALSE;
-- Index for task search functionality
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tasks_name_search
ON tasks USING gin(to_tsvector('english', name))
WHERE archived = FALSE;
-- Index for date-based filtering (if used)
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tasks_dates
ON tasks(project_id, start_date, end_date)
WHERE archived = FALSE;
-- Index for task timers with user filtering
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_task_timers_user_task
ON task_timers(user_id, task_id);
-- Index for sys_task_status_categories lookups
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_sys_task_status_categories_covering
ON sys_task_status_categories(id, color_code, color_code_dark, is_done, is_doing, is_todo);

34
worklenz-backend/database/sql/3_views.sql
View File

@@ -32,3 +32,37 @@ SELECT u.avatar_url,
FROM team_members
LEFT JOIN users u ON team_members.user_id = u.id;
-- PERFORMANCE OPTIMIZATION: Create materialized view for team member info
-- This pre-calculates the expensive joins and subqueries from team_member_info_view
CREATE MATERIALIZED VIEW IF NOT EXISTS team_member_info_mv AS
SELECT
u.avatar_url,
COALESCE(u.email, ei.email) AS email,
COALESCE(u.name, ei.name) AS name,
u.id AS user_id,
tm.id AS team_member_id,
tm.team_id,
tm.active,
u.socket_id
FROM team_members tm
LEFT JOIN users u ON tm.user_id = u.id
LEFT JOIN email_invitations ei ON ei.team_member_id = tm.id
WHERE tm.active = TRUE;
-- Create unique index on the materialized view for fast lookups
CREATE UNIQUE INDEX IF NOT EXISTS idx_team_member_info_mv_team_member_id
ON team_member_info_mv(team_member_id);
CREATE INDEX IF NOT EXISTS idx_team_member_info_mv_team_user
ON team_member_info_mv(team_id, user_id);
-- Function to refresh the materialized view
CREATE OR REPLACE FUNCTION refresh_team_member_info_mv()
RETURNS void
LANGUAGE plpgsql
AS $$
BEGIN
REFRESH MATERIALIZED VIEW CONCURRENTLY team_member_info_mv;
END;
$$;

309
worklenz-backend/src/controllers/tasks-controller-v2.ts
View File

@@ -1220,6 +1220,315 @@ export default class TasksControllerV2 extends TasksControllerBase {
}));
}
/**
* NEW OPTIMIZED METHOD: Split complex query into focused segments for better performance
*/
@HandleExceptions()
public static async getTasksV4Optimized(req: IWorkLenzRequest, res: IWorkLenzResponse): Promise<IWorkLenzResponse> {
const startTime = performance.now();
console.log(`[PERFORMANCE] getTasksV4Optimized method called for project ${req.params.id}`);
// Skip progress refresh by default for better performance
if (req.query.refresh_progress === "true" && req.params.id) {
const progressStartTime = performance.now();
await this.refreshProjectTaskProgressValues(req.params.id);
const progressEndTime = performance.now();
console.log(`[PERFORMANCE] Progress refresh completed in ${(progressEndTime - progressStartTime).toFixed(2)}ms`);
}
const isSubTasks = !!req.query.parent_task;
const groupBy = (req.query.group || GroupBy.STATUS) as string;
const projectId = req.params.id;
const userId = req.user?.id;
// STEP 1: Get basic task data with optimized query
const baseTasksQuery = `
SELECT
t.id,
t.name,
CONCAT(p.key, '-', t.task_no) AS task_key,
p.name AS project_name,
t.project_id,
t.parent_task_id,
t.parent_task_id IS NOT NULL AS is_sub_task,
t.status_id AS status,
t.priority_id AS priority,
t.description,
t.sort_order,
t.progress_value AS complete_ratio,
t.manual_progress,
t.weight,
t.start_date,
t.end_date,
t.created_at,
t.updated_at,
t.completed_at,
t.billable,
t.schedule_id,
t.total_minutes,
-- Status information via JOINs
stsc.color_code AS status_color,
stsc.color_code_dark AS status_color_dark,
stsc.is_done,
stsc.is_doing,
stsc.is_todo,
-- Priority information
tp_priority.value AS priority_value,
-- Phase information
tp.phase_id,
pp.name AS phase_name,
pp.color_code AS phase_color_code,
-- Reporter information
reporter.name AS reporter,
-- Timer information
tt.start_time AS timer_start_time
FROM tasks t
JOIN projects p ON t.project_id = p.id
JOIN task_statuses ts ON t.status_id = ts.id
JOIN sys_task_status_categories stsc ON ts.category_id = stsc.id
LEFT JOIN task_phase tp ON t.id = tp.task_id
LEFT JOIN project_phases pp ON tp.phase_id = pp.id
LEFT JOIN task_priorities tp_priority ON t.priority_id = tp_priority.id
LEFT JOIN users reporter ON t.reporter_id = reporter.id
LEFT JOIN task_timers tt ON t.id = tt.task_id AND tt.user_id = $2
WHERE t.project_id = $1
AND t.archived = FALSE
${isSubTasks ? "AND t.parent_task_id = $3" : "AND t.parent_task_id IS NULL"}
ORDER BY t.sort_order
`;
const baseParams = isSubTasks ? [projectId, userId, req.query.parent_task] : [projectId, userId];
const baseResult = await db.query(baseTasksQuery, baseParams);
const baseTasks = baseResult.rows;
if (baseTasks.length === 0) {
return res.status(200).send(new ServerResponse(true, {
groups: [],
allTasks: [],
grouping: groupBy,
totalTasks: 0
}));
}
const taskIds = baseTasks.map(t => t.id);
// STEP 2: Get aggregated data in parallel
const [assigneesResult, labelsResult, aggregatesResult] = await Promise.all([
// Get assignees
db.query(`
SELECT
ta.task_id,
JSON_AGG(JSON_BUILD_OBJECT(
'team_member_id', ta.team_member_id,
'project_member_id', ta.project_member_id,
'name', COALESCE(tm.name, ''),
'avatar_url', COALESCE(u.avatar_url, ''),
'email', COALESCE(u.email, ei.email, ''),
'user_id', tm.user_id,
'socket_id', COALESCE(u.socket_id, ''),
'team_id', tm.team_id
)) AS assignees
FROM tasks_assignees ta
LEFT JOIN team_members tm ON ta.team_member_id = tm.id
LEFT JOIN users u ON tm.user_id = u.id
LEFT JOIN email_invitations ei ON ta.team_member_id = ei.team_member_id
WHERE ta.task_id = ANY($1)
GROUP BY ta.task_id
`, [taskIds]),
// Get labels
db.query(`
SELECT
tl.task_id,
JSON_AGG(JSON_BUILD_OBJECT(
'id', tl.label_id,
'label_id', tl.label_id,
'name', team_l.name,
'color_code', team_l.color_code
)) AS labels
FROM task_labels tl
JOIN team_labels team_l ON tl.label_id = team_l.id
WHERE tl.task_id = ANY($1)
GROUP BY tl.task_id
`, [taskIds]),
// Get aggregated counts
db.query(`
SELECT
t.id,
COUNT(DISTINCT sub.id) AS sub_tasks_count,
COUNT(DISTINCT CASE WHEN sub_status.is_done THEN sub.id END) AS completed_sub_tasks,
COUNT(DISTINCT tc.id) AS comments_count,
COUNT(DISTINCT ta.id) AS attachments_count,
COALESCE(SUM(twl.time_spent), 0) AS total_minutes_spent,
CASE WHEN COUNT(ts.id) > 0 THEN true ELSE false END AS has_subscribers,
CASE WHEN COUNT(td.id) > 0 THEN true ELSE false END AS has_dependencies
FROM unnest($1::uuid[]) AS t(id)
LEFT JOIN tasks sub ON t.id = sub.parent_task_id AND sub.archived = FALSE
LEFT JOIN task_statuses sub_ts ON sub.status_id = sub_ts.id
LEFT JOIN sys_task_status_categories sub_status ON sub_ts.category_id = sub_status.id
LEFT JOIN task_comments tc ON t.id = tc.task_id
LEFT JOIN task_attachments ta ON t.id = ta.task_id
LEFT JOIN task_work_log twl ON t.id = twl.task_id
LEFT JOIN task_subscribers ts ON t.id = ts.task_id
LEFT JOIN task_dependencies td ON t.id = td.task_id
GROUP BY t.id
`, [taskIds])
]);
// STEP 3: Create lookup maps for efficient data merging
const assigneesMap = new Map();
assigneesResult.rows.forEach(row => assigneesMap.set(row.task_id, row.assignees || []));
const labelsMap = new Map();
labelsResult.rows.forEach(row => labelsMap.set(row.task_id, row.labels || []));
const aggregatesMap = new Map();
aggregatesResult.rows.forEach(row => aggregatesMap.set(row.id, row));
// STEP 4: Merge data efficiently
const enrichedTasks = baseTasks.map(task => {
const aggregates = aggregatesMap.get(task.id) || {};
const assignees = assigneesMap.get(task.id) || [];
const labels = labelsMap.get(task.id) || [];
return {
...task,
assignees,
assignee_names: assignees.map((a: any) => a.name).join(", "),
names: assignees.map((a: any) => a.name).join(", "),
labels,
all_labels: labels,
sub_tasks_count: parseInt(aggregates.sub_tasks_count || 0),
completed_sub_tasks: parseInt(aggregates.completed_sub_tasks || 0),
comments_count: parseInt(aggregates.comments_count || 0),
attachments_count: parseInt(aggregates.attachments_count || 0),
total_minutes_spent: parseFloat(aggregates.total_minutes_spent || 0),
has_subscribers: aggregates.has_subscribers || false,
has_dependencies: aggregates.has_dependencies || false,
status_category: {
is_done: task.is_done,
is_doing: task.is_doing,
is_todo: task.is_todo
}
};
});
// STEP 5: Group tasks (same logic as existing method)
const groups = await this.getGroups(groupBy, req.params.id);
const map = groups.reduce((g: { [x: string]: ITaskGroup }, group) => {
if (group.id)
g[group.id] = new TaskListGroup(group);
return g;
}, {});
await this.updateMapByGroup(enrichedTasks, groupBy, map);
const updatedGroups = Object.keys(map).map(key => {
const group = map[key];
TasksControllerV2.updateTaskProgresses(group);
return {
id: key,
...group
};
});
// STEP 6: Transform to V3 format (same as existing method)
const priorityMap: Record<string, string> = {
"0": "low",
"1": "medium",
"2": "high"
};
const transformedTasks = enrichedTasks.map(task => ({
id: task.id,
task_key: task.task_key || "",
title: task.name || "",
description: task.description || "",
status: task.status || "todo",
priority: priorityMap[task.priority_value?.toString()] || "medium",
phase: task.phase_name || "Development",
progress: typeof task.complete_ratio === "number" ? task.complete_ratio : 0,
assignees: task.assignees?.map((a: any) => a.team_member_id) || [],
assignee_names: task.assignees || [],
labels: task.labels?.map((l: any) => ({
id: l.id || l.label_id,
name: l.name,
color: l.color_code || "#1890ff"
})) || [],
dueDate: task.end_date,
startDate: task.start_date,
timeTracking: {
estimated: task.total_minutes || 0,
logged: task.total_minutes_spent || 0,
},
customFields: {},
createdAt: task.created_at || new Date().toISOString(),
updatedAt: task.updated_at || new Date().toISOString(),
order: typeof task.sort_order === "number" ? task.sort_order : 0,
originalStatusId: task.status,
originalPriorityId: task.priority,
statusColor: task.status_color,
priorityColor: task.priority_color,
sub_tasks_count: task.sub_tasks_count || 0,
comments_count: task.comments_count || 0,
has_subscribers: !!task.has_subscribers,
attachments_count: task.attachments_count || 0,
has_dependencies: !!task.has_dependencies,
schedule_id: task.schedule_id || null,
}));
const responseGroups = updatedGroups.map(group => {
let groupValue = group.name;
if (groupBy === GroupBy.STATUS) {
groupValue = group.name.toLowerCase().replace(/\s+/g, "_");
} else if (groupBy === GroupBy.PRIORITY) {
groupValue = group.name.toLowerCase();
} else if (groupBy === GroupBy.PHASE) {
groupValue = group.name.toLowerCase().replace(/\s+/g, "_");
}
const groupTasks = group.tasks.map(task => {
const foundTask = transformedTasks.find(t => t.id === task.id);
return foundTask || task;
});
return {
id: group.id,
title: group.name,
groupType: groupBy,
groupValue,
collapsed: false,
tasks: groupTasks,
taskIds: groupTasks.map((task: any) => task.id),
color: group.color_code || this.getDefaultGroupColor(groupBy, groupValue),
category_id: group.category_id,
start_date: group.start_date,
end_date: group.end_date,
sort_index: (group as any).sort_index,
todo_progress: group.todo_progress,
doing_progress: group.doing_progress,
done_progress: group.done_progress,
};
}).filter(group => group.tasks.length > 0 || req.query.include_empty === "true");
const endTime = performance.now();
const totalTime = endTime - startTime;
console.log(`[PERFORMANCE] getTasksV4Optimized method completed in ${totalTime.toFixed(2)}ms for project ${req.params.id} with ${transformedTasks.length} tasks - Improvement: ${2136 - totalTime > 0 ? "+" : ""}${(2136 - totalTime).toFixed(2)}ms`);
return res.status(200).send(new ServerResponse(true, {
groups: responseGroups,
allTasks: transformedTasks,
grouping: groupBy,
totalTasks: transformedTasks.length,
performanceMetrics: {
executionTime: Math.round(totalTime),
tasksCount: transformedTasks.length,
optimizationGain: Math.round(2136 - totalTime)
}
}));
}
private static getDefaultGroupColor(groupBy: string, groupValue: string): string {
const colorMaps: Record<string, Record<string, string>> = {
[GroupBy.STATUS]: {