From 654446f06815d5c221c66e651b123dbf0656b5d8 Mon Sep 17 00:00:00 2001
From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com>
Date: Sun, 16 Apr 2023 19:56:19 +0200
Subject: [PATCH] Feature/improve handling of jobs (#1864)

* Improve handling of jobs
  * Remove jobs on complete
  * Refactor jobs removal

* Update changelog
---
 CHANGELOG.md                                  |  5 +++
 apps/api/src/app/admin/queue/queue.service.ts | 44 ++++++++++---------
 libs/common/src/lib/config.ts                 |  8 +---
 3 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 551628c1..d2ee88e6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Changed
+
+- Enabled the configuration to immediately remove queue jobs on complete
+- Refactored the implementation of removing queue jobs
+
 ### Fixed
 
 - Fixed the unique job ids of the gather asset profile process
diff --git a/apps/api/src/app/admin/queue/queue.service.ts b/apps/api/src/app/admin/queue/queue.service.ts
index ebaab6d9..81b32ddb 100644
--- a/apps/api/src/app/admin/queue/queue.service.ts
+++ b/apps/api/src/app/admin/queue/queue.service.ts
@@ -4,7 +4,7 @@ import {
 } from '@ghostfolio/common/config';
 import { AdminJobs } from '@ghostfolio/common/interfaces';
 import { InjectQueue } from '@nestjs/bull';
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
 import { JobStatus, Queue } from 'bull';
 
 @Injectable()
@@ -23,14 +23,11 @@ export class QueueService {
   }: {
     status?: JobStatus[];
   }) {
-    const jobs = await this.dataGatheringQueue.getJobs(status);
-
-    for (const job of jobs) {
-      try {
-        await job.remove();
-      } catch (error) {
-        Logger.warn(error, 'QueueService');
-      }
+    for (const statusItem of status) {
+      await this.dataGatheringQueue.clean(
+        300,
+        statusItem === 'waiting' ? 'wait' : statusItem
+      );
     }
   }
 
@@ -44,18 +41,23 @@ export class QueueService {
     const jobs = await this.dataGatheringQueue.getJobs(status);
 
     const jobsWithState = await Promise.all(
-      jobs.slice(0, limit).map(async (job) => {
-        return {
-          attemptsMade: job.attemptsMade + 1,
-          data: job.data,
-          finishedOn: job.finishedOn,
-          id: job.id,
-          name: job.name,
-          stacktrace: job.stacktrace,
-          state: await job.getState(),
-          timestamp: job.timestamp
-        };
-      })
+      jobs
+        .filter((job) => {
+          return job;
+        })
+        .slice(0, limit)
+        .map(async (job) => {
+          return {
+            attemptsMade: job.attemptsMade + 1,
+            data: job.data,
+            finishedOn: job.finishedOn,
+            id: job.id,
+            name: job.name,
+            stacktrace: job.stacktrace,
+            state: await job.getState(),
+            timestamp: job.timestamp
+          };
+        })
     );
 
     return {
diff --git a/libs/common/src/lib/config.ts b/libs/common/src/lib/config.ts
index 3319f9a3..171ce50d 100644
--- a/libs/common/src/lib/config.ts
+++ b/libs/common/src/lib/config.ts
@@ -49,9 +49,7 @@ export const GATHER_ASSET_PROFILE_PROCESS_OPTIONS: JobOptions = {
     type: 'exponential'
   },
   priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH,
-  removeOnComplete: {
-    age: ms('2 weeks') / 1000
-  }
+  removeOnComplete: true
 };
 export const GATHER_HISTORICAL_MARKET_DATA_PROCESS =
   'GATHER_HISTORICAL_MARKET_DATA';
@@ -62,9 +60,7 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS: JobOptions = {
     type: 'exponential'
   },
   priority: DATA_GATHERING_QUEUE_PRIORITY_LOW,
-  removeOnComplete: {
-    age: ms('2 weeks') / 1000
-  }
+  removeOnComplete: true
 };
 
 export const HEADER_KEY_IMPERSONATION = 'Impersonation-Id';