sudacode/ghostfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Feature/adapt priorities of data gathering jobs (#3262)

Browse Source 
* Adapt priorities of data gathering jobs

* Update changelog
This commit is contained in:
Thomas Kaul 2024-04-13 09:28:14 +02:00 committed by GitHub
parent 45340b581f
commit 4e7d93db13
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 136 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -10,9 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Added support to immediately execute a queue job from the admin control panel
- Added a priority column to the queue jobs view in the admin control panel
### Changed
- Adapted the priorities of queue jobs
- Upgraded `angular` from version `17.2.4` to `17.3.3`
- Upgraded `Nx` from version `18.1.2` to `18.2.3`
- Upgraded `prisma` from version `5.11.0` to `5.12.1`

16
apps/api/src/app/admin/admin.controller.ts
View File

@ -7,13 +7,12 @@ import { ManualService } from '@ghostfolio/api/services/data-provider/manual/man
import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
import { PropertyDto } from '@ghostfolio/api/services/property/property.dto';
import {
DATA_GATHERING_QUEUE_PRIORITY_HIGH,
DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_OPTIONS
} from '@ghostfolio/common/config';
import {
getAssetProfileIdentifier,
resetHours
} from '@ghostfolio/common/helper';
import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
import {
AdminData,
AdminMarketData,
@ -94,7 +93,8 @@ export class AdminController {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
jobId: getAssetProfileIdentifier({ dataSource, symbol })
jobId: getAssetProfileIdentifier({ dataSource, symbol }),
priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
}
};
})
@ -119,7 +119,8 @@ export class AdminController {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
jobId: getAssetProfileIdentifier({ dataSource, symbol })
jobId: getAssetProfileIdentifier({ dataSource, symbol }),
priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
}
};
})
@ -141,7 +142,8 @@ export class AdminController {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
jobId: getAssetProfileIdentifier({ dataSource, symbol })
jobId: getAssetProfileIdentifier({ dataSource, symbol }),
priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
}
});
}

1
apps/api/src/app/admin/queue/queue.service.ts
View File

@ -58,6 +58,7 @@ export class QueueService {
finishedOn: job.finishedOn,
id: job.id,
name: job.name,
opts: job.opts,
stacktrace: job.stacktrace,
state: await job.getState(),
timestamp: job.timestamp

13
apps/api/src/app/import/import.service.ts
View File

@ -13,6 +13,10 @@ import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/da
import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import {
DATA_GATHERING_QUEUE_PRIORITY_HIGH,
DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
} from '@ghostfolio/common/config';
import {
DATE_FORMAT,
getAssetProfileIdentifier,
@ -448,15 +452,16 @@ export class ImportService {
});
});
this.dataGatheringService.gatherSymbols(
uniqueActivities.map(({ date, SymbolProfile }) => {
this.dataGatheringService.gatherSymbols({
dataGatheringItems: uniqueActivities.map(({ date, SymbolProfile }) => {
return {
date,
dataSource: SymbolProfile.dataSource,
symbol: SymbolProfile.symbol
};
})
);
}),
priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
});
}
return activities;

22
apps/api/src/app/order/order.controller.ts
View File

@ -7,7 +7,10 @@ import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interc
import { ApiService } from '@ghostfolio/api/services/api/api.service';
import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';
import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config';
import {
DATA_GATHERING_QUEUE_PRIORITY_HIGH,
HEADER_KEY_IMPERSONATION
} from '@ghostfolio/common/config';
import { hasPermission, permissions } from '@ghostfolio/common/permissions';
import type { DateRange, RequestWithUser } from '@ghostfolio/common/types';
@ -160,13 +163,16 @@ export class OrderController {
if (data.dataSource && !order.isDraft) {
// Gather symbol data in the background, if data source is set
// (not MANUAL) and not draft
this.dataGatheringService.gatherSymbols([
{
dataSource: data.dataSource,
date: order.date,
symbol: data.symbol
}
]);
this.dataGatheringService.gatherSymbols({
dataGatheringItems: [
{
dataSource: data.dataSource,
date: order.date,
symbol: data.symbol
}
],
priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
});
}
return order;

22
apps/api/src/app/order/order.service.ts
View File

@ -4,6 +4,7 @@ import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-
import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import {
DATA_GATHERING_QUEUE_PRIORITY_HIGH,
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_OPTIONS
} from '@ghostfolio/common/config';
@ -101,7 +102,8 @@ export class OrderService {
jobId: getAssetProfileIdentifier({
dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
symbol: data.SymbolProfile.connectOrCreate.create.symbol
})
}),
priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
}
});
}
@ -427,13 +429,17 @@ export class OrderService {
if (!isDraft) {
// Gather symbol data of order in the background, if not draft
this.dataGatheringService.gatherSymbols([
{
dataSource: data.SymbolProfile.connect.dataSource_symbol.dataSource,
date: <Date>data.date,
symbol: data.SymbolProfile.connect.dataSource_symbol.symbol
}
]);
this.dataGatheringService.gatherSymbols({
dataGatheringItems: [
{
dataSource:
data.SymbolProfile.connect.dataSource_symbol.dataSource,
date: <Date>data.date,
symbol: data.SymbolProfile.connect.dataSource_symbol.symbol
}
],
priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
});
}
}

4
apps/api/src/services/cron.service.ts
View File

@ -1,4 +1,5 @@
import {
DATA_GATHERING_QUEUE_PRIORITY_LOW,
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
PROPERTY_IS_DATA_GATHERING_ENABLED
@ -56,7 +57,8 @@ export class CronService {
name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
jobId: getAssetProfileIdentifier({ dataSource, symbol })
jobId: getAssetProfileIdentifier({ dataSource, symbol }),
priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
}
};
})

40
apps/api/src/services/data-gathering/data-gathering.service.ts
View File

@ -8,6 +8,8 @@ import { PropertyService } from '@ghostfolio/api/services/property/property.serv
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import {
DATA_GATHERING_QUEUE,
DATA_GATHERING_QUEUE_PRIORITY_HIGH,
DATA_GATHERING_QUEUE_PRIORITY_LOW,
GATHER_HISTORICAL_MARKET_DATA_PROCESS,
GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
PROPERTY_BENCHMARKS
@ -61,24 +63,35 @@ export class DataGatheringService {
public async gather7Days() {
const dataGatheringItems = await this.getSymbols7D();
await this.gatherSymbols(dataGatheringItems);
await this.gatherSymbols({
dataGatheringItems,
priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
});
}
public async gatherMax() {
const dataGatheringItems = await this.getSymbolsMax();
await this.gatherSymbols(dataGatheringItems);
await this.gatherSymbols({
dataGatheringItems,
priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
});
}
public async gatherSymbol({ dataSource, symbol }: UniqueAsset) {
await this.marketDataService.deleteMany({ dataSource, symbol });
const symbols = (await this.getSymbolsMax()).filter((dataGatheringItem) => {
return (
dataGatheringItem.dataSource === dataSource &&
dataGatheringItem.symbol === symbol
);
const dataGatheringItems = (await this.getSymbolsMax()).filter(
(dataGatheringItem) => {
return (
dataGatheringItem.dataSource === dataSource &&
dataGatheringItem.symbol === symbol
);
}
);
await this.gatherSymbols({
dataGatheringItems,
priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
});
await this.gatherSymbols(symbols);
}
public async gatherSymbolForDate({
@ -230,9 +243,15 @@ export class DataGatheringService {
);
}
public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
public async gatherSymbols({
dataGatheringItems,
priority
}: {
dataGatheringItems: IDataGatheringItem[];
priority: number;
}) {
await this.addJobsToQueue(
aSymbolsWithStartDate.map(({ dataSource, date, symbol }) => {
dataGatheringItems.map(({ dataSource, date, symbol }) => {
return {
data: {
dataSource,
@ -242,6 +261,7 @@ export class DataGatheringService {
name: GATHER_HISTORICAL_MARKET_DATA_PROCESS,
opts: {
...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
priority,
jobId: `${getAssetProfileIdentifier({
dataSource,
symbol

13
apps/client/src/app/components/admin-jobs/admin-jobs.component.ts
View File

@ -1,6 +1,11 @@
import { AdminService } from '@ghostfolio/client/services/admin.service';
import { UserService } from '@ghostfolio/client/services/user/user.service';
import { QUEUE_JOB_STATUS_LIST } from '@ghostfolio/common/config';
import {
DATA_GATHERING_QUEUE_PRIORITY_HIGH,
DATA_GATHERING_QUEUE_PRIORITY_LOW,
DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,
QUEUE_JOB_STATUS_LIST
} from '@ghostfolio/common/config';
import { getDateWithTimeFormatString } from '@ghostfolio/common/helper';
import { AdminJobs, User } from '@ghostfolio/common/interfaces';
@ -24,6 +29,11 @@ import { takeUntil } from 'rxjs/operators';
templateUrl: './admin-jobs.html'
})
export class AdminJobsComponent implements OnDestroy, OnInit {
public DATA_GATHERING_QUEUE_PRIORITY_LOW = DATA_GATHERING_QUEUE_PRIORITY_LOW;
public DATA_GATHERING_QUEUE_PRIORITY_HIGH =
DATA_GATHERING_QUEUE_PRIORITY_HIGH;
public DATA_GATHERING_QUEUE_PRIORITY_MEDIUM =
DATA_GATHERING_QUEUE_PRIORITY_MEDIUM;
public defaultDateTimeFormat: string;
public filterForm: FormGroup;
public dataSource: MatTableDataSource<AdminJobs['jobs'][0]> =
@ -33,6 +43,7 @@ export class AdminJobsComponent implements OnDestroy, OnInit {
'type',
'symbol',
'dataSource',
'priority',
'attempts',
'created',
'finished',

42
apps/client/src/app/components/admin-jobs/admin-jobs.html
View File

@ -58,6 +58,25 @@
</td>
</ng-container>
<ng-container matColumnDef="priority">
<th *matHeaderCellDef class="px-1 py-2" mat-header-cell>
<ng-container i18n>Priority</ng-container>
</th>
<td *matCellDef="let element" class="px-1 py-2" mat-cell>
@if (element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_LOW) {
<ion-icon class="h6 mb-0" name="chevron-down-circle-outline" />
} @else if (
element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
) {
<ion-icon class="h6 mb-0" name="remove-circle-outline" />
} @else if (
element.opts.priority === DATA_GATHERING_QUEUE_PRIORITY_HIGH
) {
<ion-icon class="h6 mb-0" name="chevron-up-circle-outline" />
}
</td>
</ng-container>
<ng-container matColumnDef="attempts">
<th *matHeaderCellDef class="px-1 py-2 text-right" mat-header-cell>
<ng-container i18n>Attempts</ng-container>
@ -90,24 +109,37 @@
<ng-container i18n>Status</ng-container>
</th>
<td *matCellDef="let element" class="px-1 py-2" mat-cell>
<ion-icon *ngIf="element.state === 'active'" name="play-outline" />
<ion-icon
*ngIf="element.state === 'active'"
class="h6 mb-0"
name="play-outline"
/>
<ion-icon
*ngIf="element.state === 'completed'"
class="text-success"
class="h6 mb-0 text-success"
name="checkmark-circle-outline"
/>
<ion-icon
*ngIf="element.state === 'delayed'"
class="h6 mb-0"
name="time-outline"
[ngClass]="{ 'text-danger': element.stacktrace?.length > 0 }"
/>
<ion-icon
*ngIf="element.state === 'failed'"
class="text-danger"
class="h6 mb-0 text-danger"
name="alert-circle-outline"
/>
<ion-icon *ngIf="element.state === 'paused'" name="pause-outline" />
<ion-icon *ngIf="element.state === 'waiting'" name="cafe-outline" />
<ion-icon
*ngIf="element.state === 'paused'"
class="h6 mb-0"
name="pause-outline"
/>
<ion-icon
*ngIf="element.state === 'waiting'"
class="h6 mb-0"
name="cafe-outline"
/>
</td>
</ng-container>

7
libs/common/src/lib/config.ts
View File

@ -32,8 +32,11 @@ export const warnColorRgb = {
};
export const DATA_GATHERING_QUEUE = 'DATA_GATHERING_QUEUE';
export const DATA_GATHERING_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER;
export const DATA_GATHERING_QUEUE_PRIORITY_HIGH = 1;
export const DATA_GATHERING_QUEUE_PRIORITY_LOW = Number.MAX_SAFE_INTEGER;
export const DATA_GATHERING_QUEUE_PRIORITY_MEDIUM = Math.round(
DATA_GATHERING_QUEUE_PRIORITY_LOW / 2
);
export const DEFAULT_CURRENCY = 'USD';
export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy';
@ -69,7 +72,6 @@ export const GATHER_ASSET_PROFILE_PROCESS_OPTIONS: JobOptions = {
delay: ms('1 minute'),
type: 'exponential'
},
priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH,
removeOnComplete: true
};
export const GATHER_HISTORICAL_MARKET_DATA_PROCESS =
@ -80,7 +82,6 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS: JobOptions = {
delay: ms('1 minute'),
type: 'exponential'
},
priority: DATA_GATHERING_QUEUE_PRIORITY_LOW,
removeOnComplete: true
};

1
libs/common/src/lib/interfaces/admin-jobs.interface.ts
View File

@ -8,6 +8,7 @@ export interface AdminJobs {
| 'finishedOn'
| 'id'
| 'name'
| 'opts'
| 'stacktrace'
| 'timestamp'
> & {