sudacode/ghostfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Feature/harmonize processor concurrency environment variables (#3913)

Browse Source 
* Harmonize processor concurrency environment variables

* Update changelog
This commit is contained in:
Thomas Kaul 2024-10-12 19:35:43 +02:00 committed by GitHub
parent 3289bd5cc9
commit aee5e833a5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 31 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@ -15,12 +15,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Improved the labels of the chart of the holdings tab on the home page (experimental) - Improved the labels of the chart of the holdings tab on the home page (experimental)
- Exposed the timeout of the portfolio snapshot computation as an environment variable (`PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT`) - Exposed the timeout of the portfolio snapshot computation as an environment variable (`PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT`)
- Harmonized the processor concurrency environment variables
- Improved the portfolio unit tests to work with exported activity files - Improved the portfolio unit tests to work with exported activity files
### Fixed ### Fixed
- Considered the language of the user settings on login with _Security Token_ - Considered the language of the user settings on login with _Security Token_
### Todo
- Rename the environment variable from `PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE` to `PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY`
- Rename the environment variable from `PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA` to `PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY`
- Rename the environment variable from `PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT` to `PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY`
## 2.114.0 - 2024-10-10 ## 2.114.0 - 2024-10-10
### Added ### Added

18
apps/api/src/services/configuration/configuration.service.ts
View File

@ -1,9 +1,9 @@
import { Environment } from '@ghostfolio/api/services/interfaces/environment.interface'; import { Environment } from '@ghostfolio/api/services/interfaces/environment.interface';
import { import {
CACHE_TTL_NO_CACHE, CACHE_TTL_NO_CACHE,
DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE, DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY,
DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA, DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY,
DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT, DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY,
DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT, DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT,
DEFAULT_ROOT_URL DEFAULT_ROOT_URL
} from '@ghostfolio/common/config'; } from '@ghostfolio/common/config';
@ -51,14 +51,14 @@ export class ConfigurationService {
MAX_ACTIVITIES_TO_IMPORT: num({ default: Number.MAX_SAFE_INTEGER }), MAX_ACTIVITIES_TO_IMPORT: num({ default: Number.MAX_SAFE_INTEGER }),
MAX_CHART_ITEMS: num({ default: 365 }), MAX_CHART_ITEMS: num({ default: 365 }),
PORT: port({ default: 3333 }), PORT: port({ default: 3333 }),
PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE: num({ PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY: num({
default: DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE default: DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY
}), }),
PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA: num({ PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY: num({
default: DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA default: DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY
}), }),
PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT: num({ PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY: num({
default: DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT default: DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY
}), }),
PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT: num({ PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT: num({
default: DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT default: DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT

4
apps/api/src/services/interfaces/environment.interface.ts
View File

@ -30,6 +30,10 @@ export interface Environment extends CleanedEnvAccessors {
MAX_ACTIVITIES_TO_IMPORT: number; MAX_ACTIVITIES_TO_IMPORT: number;
MAX_CHART_ITEMS: number; MAX_CHART_ITEMS: number;
PORT: number; PORT: number;
PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY: number;
PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY: number;
PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY: number;
PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT: number;
REDIS_DB: number; REDIS_DB: number;
REDIS_HOST: string; REDIS_HOST: string;
REDIS_PASSWORD: string; REDIS_PASSWORD: string;

12
apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
View File

@ -3,8 +3,8 @@ import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfac
import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
import { import {
DATA_GATHERING_QUEUE, DATA_GATHERING_QUEUE,
DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE, DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY,
DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA, DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY,
GATHER_ASSET_PROFILE_PROCESS, GATHER_ASSET_PROFILE_PROCESS,
GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME
} from '@ghostfolio/common/config'; } from '@ghostfolio/common/config';
@ -38,8 +38,8 @@ export class DataGatheringProcessor {
@Process({ @Process({
concurrency: parseInt( concurrency: parseInt(
process.env.PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE ?? process.env.PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY ??
DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE.toString(), DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY.toString(),
10 10
), ),
name: GATHER_ASSET_PROFILE_PROCESS name: GATHER_ASSET_PROFILE_PROCESS
@ -69,8 +69,8 @@ export class DataGatheringProcessor {
@Process({ @Process({
concurrency: parseInt( concurrency: parseInt(
process.env.PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA ?? process.env.PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY ??
DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA.toString(), DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY.toString(),
10 10
), ),
name: GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME name: GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME

4
apps/api/src/services/queues/portfolio-snapshot/portfolio-snapshot.processor.ts
View File

@ -9,7 +9,7 @@ import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.s
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service'; import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
import { import {
CACHE_TTL_INFINITE, CACHE_TTL_INFINITE,
DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT, DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY,
PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME, PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME,
PORTFOLIO_SNAPSHOT_QUEUE PORTFOLIO_SNAPSHOT_QUEUE
} from '@ghostfolio/common/config'; } from '@ghostfolio/common/config';
@ -35,7 +35,7 @@ export class PortfolioSnapshotProcessor {
@Process({ @Process({
concurrency: parseInt( concurrency: parseInt(
process.env.PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT ?? process.env.PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT ??
DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT.toString(), DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY.toString(),
10 10
), ),
name: PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME name: PORTFOLIO_SNAPSHOT_PROCESS_JOB_NAME

6
libs/common/src/lib/config.ts
View File

@ -48,9 +48,9 @@ export const DEFAULT_CURRENCY = 'USD';
export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy'; export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy';
export const DEFAULT_LANGUAGE_CODE = 'en'; export const DEFAULT_LANGUAGE_CODE = 'en';
export const DEFAULT_PAGE_SIZE = 50; export const DEFAULT_PAGE_SIZE = 50;
export const DEFAULT_PROCESSOR_CONCURRENCY_GATHER_ASSET_PROFILE = 1; export const DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY = 1;
export const DEFAULT_PROCESSOR_CONCURRENCY_GATHER_HISTORICAL_MARKET_DATA = 1; export const DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY = 1;
export const DEFAULT_PROCESSOR_CONCURRENCY_PORTFOLIO_SNAPSHOT = 1; export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_CONCURRENCY = 1;
export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT = 30000; export const DEFAULT_PROCESSOR_PORTFOLIO_SNAPSHOT_COMPUTATION_TIMEOUT = 30000;
export const DEFAULT_ROOT_URL = 'https://localhost:4200'; export const DEFAULT_ROOT_URL = 'https://localhost:4200';