Release 1.270.0 (#1987)

* Check for duplicates in dividend import

* Update changelog

.env.example

@@ -12,5 +12,5 @@ POSTGRES_PASSWORD=<INSERT_POSTGRES_PASSWORD>
 
 ACCESS_TOKEN_SALT=<INSERT_RANDOM_STRING>
 ALPHA_VANTAGE_API_KEY=
-DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/${POSTGRES_DB}?sslmode=prefer
+DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/${POSTGRES_DB}?connect_timeout=300&sslmode=prefer
 JWT_SECRET_KEY=<INSERT_RANDOM_STRING>

.eslintrc.json

@@ -1,12 +1,12 @@
 {
   "root": true,
   "ignorePatterns": ["**/*"],
-  "plugins": ["@nrwl/nx"],
+  "plugins": ["@nx"],
   "overrides": [
     {
       "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
       "rules": {
-        "@nrwl/nx/enforce-module-boundaries": [
+        "@nx/enforce-module-boundaries": [
           "error",
           {
             "enforceBuildableLibDependency": true,
@@ -23,12 +23,12 @@
     },
     {
       "files": ["*.ts", "*.tsx"],
-      "extends": ["plugin:@nrwl/nx/typescript"],
+      "extends": ["plugin:@nx/typescript"],
       "rules": {}
     },
     {
       "files": ["*.js", "*.jsx"],
-      "extends": ["plugin:@nrwl/nx/javascript"],
+      "extends": ["plugin:@nx/javascript"],
       "rules": {}
     },
     {
@@ -113,5 +113,6 @@
         "radix": "error"
       }
     }
-  ]
+  ],
+  "extends": [null, "plugin:storybook/recommended"]
 }

.storybook/main.js

@@ -1,8 +0,0 @@
-module.exports = {
-  // uncomment the property below if you want to apply some webpack config globally
-  // webpackFinal: async (config, { configType }) => {
-  //   // Make whatever fine-grained changes you need that should apply to all storybook configs
-  //   // Return the altered config
-  //   return config;
-  // },
-};

.storybook/tsconfig.json

@@ -1,10 +0,0 @@
-{
-  "extends": "../tsconfig.base.json",
-  "exclude": [
-    "../**/*.spec.js",
-    "../**/*.spec.ts",
-    "../**/*.spec.tsx",
-    "../**/*.spec.jsx"
-  ],
-  "include": ["../**/*"]
-}

.vscode/launch.json

@@ -2,32 +2,33 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Debug Jest File",
-      "type": "node",
-      "request": "launch",
-      "program": "${workspaceFolder}/node_modules/@nrwl/cli/bin/nx",
       "args": [
         "test",
         "--codeCoverage=false",
         "--testFile=${workspaceFolder}/apps/api/src/app/portfolio/portfolio-calculator-novn-buy-and-sell.spec.ts"
       ],
+      "console": "internalConsole",
       "cwd": "${workspaceFolder}",
-      "console": "internalConsole"
+      "name": "Debug Jest",
+      "program": "${workspaceFolder}/node_modules/@nrwl/cli/bin/nx",
+      "request": "launch",
+      "type": "node"
     },
     {
-      "envFile": "${workspaceFolder}/.env",
-      "type": "node",
-      "request": "launch",
-      "name": "Launch Program",
-      "program": "${workspaceFolder}/apps/api/src/main.ts",
-      "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
-      "outFiles": ["${workspaceFolder}/dist/apps/api/**/*.js"],
       "autoAttachChildProcesses": true,
+      "console": "integratedTerminal",
+      "cwd": "${workspaceFolder}/apps/api",
+      "envFile": "${workspaceFolder}/.env",
+      "name": "Debug API",
+      "outFiles": ["${workspaceFolder}/dist/apps/api/**/*.js"],
+      "program": "${workspaceFolder}/apps/api/src/main.ts",
+      "request": "launch",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
       "skipFiles": [
         "${workspaceFolder}/node_modules/**/*.js",
         "<node_internals>/**/*.js"
       ],
-      "console": "integratedTerminal"
+      "type": "node"
     }
   ]
 }

CHANGELOG.md

@@ -5,6 +5,236 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 1.270.0 - 2023-05-19
+
+### Added
+
+- Added the cash balance and the value of equity to the account detail dialog
+- Added a check for duplicates to the preview step of the import dividends dialog
+- Added an error message for duplicates to the preview step of the activities import
+- Added a connection timeout to the environment variable `DATABASE_URL`
+- Introduced the _Open Startup_ (`/open`) page with aggregated key metrics including uptime
+
+### Changed
+
+- Improved the mobile layout of the portfolio summary tab on the home page
+- Improved the language localization for German (`de`)
+- Upgraded `prisma` from version `4.13.0` to `4.14.1`
+
+### Fixed
+
+- Improved the _Select all_ activities checkbox state after importing activities including a duplicate
+- Fixed an issue with the data source transformation in the import dividends dialog
+- Fixed the _Storybook_ setup
+
+## 1.269.0 - 2023-05-11
+
+### Added
+
+- Added `FINANCIAL_MODELING_PREP` as a new data source type
+
+### Changed
+
+- Improved the market price on the first buy date in the chart of the position detail dialog
+- Restructured the admin control panel with a new settings tab
+
+### Fixed
+
+- Fixed an error that occurred while editing an activity caused by the cash balance update
+
+## 1.268.0 - 2023-05-08
+
+### Added
+
+- Added `depends_on` and `healthcheck` for the _Postgres_ and _Redis_ services to the `docker-compose` files (`docker-compose.yml` and `docker-compose.build.yml`)
+
+### Changed
+
+- Improved the preview step of the activities import by unchecking duplicates
+- Upgraded `yahoo-finance2` from version `2.3.10` to `2.4.1`
+
+## 1.267.0 - 2023-05-07
+
+### Added
+
+- Added support for the _Stripe_ checkout to the pricing page
+
+### Changed
+
+- Improved the management of platforms in the admin control panel
+- Improved the style of the interstitial for the subscription
+- Improved the language localization for German (`de`)
+- Upgraded `Nx` from version `15.9.2` to `16.0.3`
+
+## 1.266.0 - 2023-05-06
+
+### Added
+
+- Introduced the option to update the cash balance of an account when adding an activity
+- Added support for the management of platforms in the admin control panel
+- Added _DEV Community_ to the _As seen in_ section on the landing page
+
+### Changed
+
+- Upgraded `class-transformer` from version `0.3.2` to `0.5.1`
+- Upgraded `class-validator` from version `0.13.1` to `0.14.0`
+- Upgraded `prisma` from version `4.12.0` to `4.13.0`
+
+### Fixed
+
+- Added a fallback to use `quoteSummary(symbol)` if `quote(symbols)` fails in the _Yahoo Finance_ service
+- Added the missing `dataSource` attribute to the activities import
+
+## 1.265.0 - 2023-05-01
+
+### Changed
+
+- Improved the tooltip of the portfolio proportion chart component
+
+### Fixed
+
+- Fixed the missing platform name in the allocations by platform chart on the allocations page
+
+## 1.264.0 - 2023-05-01
+
+### Added
+
+- Introduced the allocations by platform chart on the allocations page
+
+### Changed
+
+- Deprecated the use of the environment variable `BASE_CURRENCY`
+- Cleaned up initial values from the _X-ray_ section
+
+## 1.263.0 - 2023-04-30
+
+### Changed
+
+- Split the environment variable `DATA_SOURCE_PRIMARY` in `DATA_SOURCE_EXCHANGE_RATES` and `DATA_SOURCE_IMPORT`
+
+### Fixed
+
+- Fixed the exception on the accounts page
+
+## 1.262.0 - 2023-04-29
+
+### Added
+
+- Added the labels to the tabs to increase the usability
+- Extended the support of the impersonation mode for local development
+
+### Changed
+
+- Improved the queue jobs implementation by adding / updating historical market data in bulk
+- Improved the language localization for German (`de`)
+
+### Fixed
+
+- Improved the holdings table by showing the cash position also when the filter contains the accounts, so that we can see the total allocation for that account
+
+## 1.261.0 - 2023-04-25
+
+### Added
+
+- Introduced a new button to delete all activities from the portfolio activities page
+- Added `state` to the `MarketData` database schema to distinguish `CLOSE` and `INTRADAY` in the data gathering
+- Added the distance to now to the subscription expiration date in the users table of the admin control panel
+
+## 1.260.0 - 2023-04-23
+
+### Added
+
+- Added `dataSource` as a unique constraint to the `MarketData` database schema
+
+### Fixed
+
+- Removed the unnecessary sort header of the comment column in the historical market data table of the admin control panel
+
+## 1.259.0 - 2023-04-22
+
+### Added
+
+- Added a fallback to historical market data if a data provider does not provide live data
+- Added a general health check endpoint
+- Added health check endpoints for data providers
+
+### Changed
+
+- Persisted today's market data continuously
+
+### Fixed
+
+- Fixed the alignment of the performance column header in the holdings table
+- Removed the unnecessary sort header of the comment column in the activities table
+- Fixed the targets in `proxy.conf.json` from `http://localhost:3333` to `http://0.0.0.0:3333` for local development
+
+## 1.258.0 - 2023-04-20
+
+### Added
+
+- Introduced a data source mapping
+
+## 1.257.0 - 2023-04-18
+
+### Added
+
+- Introduced the allocations by ETF provider chart on the allocations page
+
+### Fixed
+
+- Fixed an issue in the global heat map component caused by manipulating an input property
+- Fixed an issue with the currency inconsistency in the _EOD Historical Data_ service (convert from `GBX` to `GBp`)
+
+## 1.256.0 - 2023-04-17
+
+### Added
+
+- Added the _Yahoo Finance_ data enhancer for countries, sectors and urls
+
+### Changed
+
+- Enabled the configuration to immediately remove queue jobs on complete
+- Refactored the implementation of removing queue jobs
+
+### Fixed
+
+- Fixed the unique job ids of the gather asset profile process
+- Fixed the style of the button to fetch the current market price
+
+## 1.255.0 - 2023-04-15
+
+### Added
+
+- Made the system message expandable
+
+### Changed
+
+- Skipped creating queue jobs for asset profiles with `MANUAL` data source not having a scraper configuration
+- Reduced the execution interval of the data gathering to every hour
+- Upgraded `prisma` from version `4.11.0` to `4.12.0`
+
+### Fixed
+
+- Improved the style of the system message
+
+## 1.254.0 - 2023-04-14
+
+### Changed
+
+- Improved the queue jobs implementation by adding in bulk
+- Improved the queue jobs implementation by introducing unique job ids
+- Reverted the execution interval of the data gathering from every 12 hours to every 4 hours
+
+## 1.253.0 - 2023-04-14
+
+### Changed
+
+- Reduced the execution interval of the data gathering to every 12 hours
+
+### Fixed
+
+- Fixed the background color of dialogs in dark mode
+
 ## 1.252.2 - 2023-04-11
 
 ### Changed

DEVELOPMENT.md

@@ -18,7 +18,13 @@
 
 ### Prisma
 
-#### Create schema migration (local)
+#### Synchronize schema with database for prototyping
+
+Run `yarn database:push`
+
+https://www.prisma.io/docs/concepts/components/prisma-migrate/db-push
+
+#### Create schema migration
 
 Run `yarn prisma migrate dev --name added_job_title`
 

README.md

@@ -232,6 +232,7 @@ Deprecated: `GET http://localhost:3333/api/v1/auth/anonymous/<INSERT_SECURITY_TO
 | Field      | Type                | Description                                        |
 | ---------- | ------------------- | -------------------------------------------------- |
 | accountId  | string (`optional`) | Id of the account                                  |
+| comment    | string (`optional`) | Comment of the activity                            |
 | currency   | string              | `CHF` \| `EUR` \| `USD` etc.                       |
 | dataSource | string              | `MANUAL` (for type `ITEM`) \| `YAHOO`              |
 | date       | string              | Date in the format `ISO-8601`                      |
@@ -274,6 +275,6 @@ If you like to support this project, get [**Ghostfolio Premium**](https://ghostf
 
 ## License
 
-© 2023 [Ghostfolio](https://ghostfol.io)
+© 2021 - 2023 [Ghostfolio](https://ghostfol.io)
 
 Licensed under the [AGPLv3 License](https://www.gnu.org/licenses/agpl-3.0.html).

apps/api/project.json

@@ -33,7 +33,7 @@
       "outputs": ["{options.outputPath}"]
     },
     "serve": {
-      "executor": "@nrwl/node:node",
+      "executor": "@nx/node:node",
       "options": {
         "buildTarget": "api:build"
       }
@@ -45,7 +45,7 @@
       }
     },
     "test": {
-      "executor": "@nrwl/jest:jest",
+      "executor": "@nx/jest:jest",
       "options": {
         "jestConfig": "apps/api/jest.config.ts",
         "passWithNoTests": true

apps/api/src/app/access/access.module.ts

@@ -1,4 +1,4 @@
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
 import { Module } from '@nestjs/common';
 
 import { AccessController } from './access.controller';

apps/api/src/app/access/access.service.ts

@@ -1,4 +1,4 @@
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { AccessWithGranteeUser } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
 import { Access, Prisma } from '@prisma/client';

apps/api/src/app/account/account.controller.ts

@@ -1,6 +1,6 @@
 import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service';
 import { RedactValuesInResponseInterceptor } from '@ghostfolio/api/interceptors/redact-values-in-response.interceptor';
-import { ImpersonationService } from '@ghostfolio/api/services/impersonation.service';
+import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';
 import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config';
 import { Accounts } from '@ghostfolio/common/interfaces';
 import { hasPermission, permissions } from '@ghostfolio/common/permissions';
@@ -87,10 +87,7 @@ export class AccountController {
     @Headers(HEADER_KEY_IMPERSONATION.toLowerCase()) impersonationId
   ): Promise<Accounts> {
     const impersonationUserId =
-      await this.impersonationService.validateImpersonationId(
+      await this.impersonationService.validateImpersonationId(impersonationId);
-        impersonationId,
-        this.request.user.id
-      );
 
     return this.portfolioService.getAccountsWithAggregations({
       userId: impersonationUserId || this.request.user.id,
@@ -106,10 +103,7 @@ export class AccountController {
     @Param('id') id: string
   ): Promise<AccountWithValue> {
     const impersonationUserId =
-      await this.impersonationService.validateImpersonationId(
+      await this.impersonationService.validateImpersonationId(impersonationId);
-        impersonationId,
-        this.request.user.id
-      );
 
     const accountsWithAggregations =
       await this.portfolioService.getAccountsWithAggregations({

apps/api/src/app/account/account.module.ts

@@ -1,11 +1,11 @@
 import { PortfolioModule } from '@ghostfolio/api/app/portfolio/portfolio.module';
 import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
 import { UserModule } from '@ghostfolio/api/app/user/user.module';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
+import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
-import { ImpersonationModule } from '@ghostfolio/api/services/impersonation.module';
+import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
 import { Module } from '@nestjs/common';
 
 import { AccountController } from './account.controller';

apps/api/src/app/account/account.service.ts

@@ -1,5 +1,5 @@
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { Filter } from '@ghostfolio/common/interfaces';
 import { Injectable } from '@nestjs/common';
 import { Account, Order, Platform, Prisma } from '@prisma/client';
@@ -172,4 +172,47 @@ export class AccountService {
       where
     });
   }
+
+  public async updateAccountBalance({
+    accountId,
+    amount,
+    currency,
+    date,
+    userId
+  }: {
+    accountId: string;
+    amount: number;
+    currency: string;
+    date: Date;
+    userId: string;
+  }) {
+    const { balance, currency: currencyOfAccount } = await this.account({
+      id_userId: {
+        userId,
+        id: accountId
+      }
+    });
+
+    const amountInCurrencyOfAccount =
+      await this.exchangeRateDataService.toCurrencyAtDate(
+        amount,
+        currency,
+        currencyOfAccount,
+        date
+      );
+
+    if (amountInCurrencyOfAccount) {
+      await this.prismaService.account.update({
+        data: {
+          balance: new Big(balance).plus(amountInCurrencyOfAccount).toNumber()
+        },
+        where: {
+          id_userId: {
+            userId,
+            id: accountId
+          }
+        }
+      });
+    }
+  }
 }

apps/api/src/app/admin/admin.controller.ts

@@ -1,5 +1,5 @@
-import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
-import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { PropertyDto } from '@ghostfolio/api/services/property/property.dto';
 import {
   GATHER_ASSET_PROFILE_PROCESS,
@@ -100,16 +100,21 @@ export class AdminController {
 
     const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
 
-    for (const { dataSource, symbol } of uniqueAssets) {
+    await this.dataGatheringService.addJobsToQueue(
-      await this.dataGatheringService.addJobToQueue(
+      uniqueAssets.map(({ dataSource, symbol }) => {
-        GATHER_ASSET_PROFILE_PROCESS,
+        return {
-        {
+          data: {
-          dataSource,
+            dataSource,
-          symbol
+            symbol
-        },
+          },
-        GATHER_ASSET_PROFILE_PROCESS_OPTIONS
+          name: GATHER_ASSET_PROFILE_PROCESS,
-      );
+          opts: {
-    }
+            ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+            jobId: `${dataSource}-${symbol}`
+          }
+        };
+      })
+    );
 
     this.dataGatheringService.gatherMax();
   }
@@ -131,16 +136,21 @@ export class AdminController {
 
     const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
 
-    for (const { dataSource, symbol } of uniqueAssets) {
+    await this.dataGatheringService.addJobsToQueue(
-      await this.dataGatheringService.addJobToQueue(
+      uniqueAssets.map(({ dataSource, symbol }) => {
-        GATHER_ASSET_PROFILE_PROCESS,
+        return {
-        {
+          data: {
-          dataSource,
+            dataSource,
-          symbol
+            symbol
-        },
+          },
-        GATHER_ASSET_PROFILE_PROCESS_OPTIONS
+          name: GATHER_ASSET_PROFILE_PROCESS,
-      );
+          opts: {
-    }
+            ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+            jobId: `${dataSource}-${symbol}`
+          }
+        };
+      })
+    );
   }
 
   @Post('gather/profile-data/:dataSource/:symbol')
@@ -161,14 +171,17 @@ export class AdminController {
       );
     }
 
-    await this.dataGatheringService.addJobToQueue(
+    await this.dataGatheringService.addJobToQueue({
-      GATHER_ASSET_PROFILE_PROCESS,
+      data: {
-      {
         dataSource,
         symbol
       },
-      GATHER_ASSET_PROFILE_PROCESS_OPTIONS
+      name: GATHER_ASSET_PROFILE_PROCESS,
-    );
+      opts: {
+        ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+        jobId: `${dataSource}-${symbol}`
+      }
+    });
   }
 
   @Post('gather/:dataSource/:symbol')
@@ -304,9 +317,10 @@ export class AdminController {
     const date = new Date(dateString);
 
     return this.marketDataService.updateMarketData({
-      data: { ...data, dataSource },
+      data: { marketPrice: data.marketPrice, state: 'CLOSE' },
       where: {
-        date_symbol: {
+        dataSource_date_symbol: {
+          dataSource,
           date,
           symbol
         }

apps/api/src/app/admin/admin.module.ts

@@ -1,12 +1,12 @@
 import { SubscriptionModule } from '@ghostfolio/api/app/subscription/subscription.module';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module';
+import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
+import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
-import { MarketDataModule } from '@ghostfolio/api/services/market-data.module';
+import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
 import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
-import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
+import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
 import { Module } from '@nestjs/common';
 
 import { AdminController } from './admin.controller';

apps/api/src/app/admin/admin.service.ts

@@ -1,10 +1,10 @@
 import { SubscriptionService } from '@ghostfolio/api/app/subscription/subscription.service';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
-import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
-import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
+import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
 import { PROPERTY_CURRENCIES } from '@ghostfolio/common/config';
 import {
   AdminData,

apps/api/src/app/admin/queue/queue.module.ts

@@ -1,4 +1,4 @@
-import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module';
+import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module';
 import { Module } from '@nestjs/common';
 
 import { QueueController } from './queue.controller';

apps/api/src/app/admin/queue/queue.service.ts

@@ -4,7 +4,7 @@ import {
 } from '@ghostfolio/common/config';
 import { AdminJobs } from '@ghostfolio/common/interfaces';
 import { InjectQueue } from '@nestjs/bull';
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
 import { JobStatus, Queue } from 'bull';
 
 @Injectable()
@@ -23,14 +23,11 @@ export class QueueService {
   }: {
     status?: JobStatus[];
   }) {
-    const jobs = await this.dataGatheringQueue.getJobs(status);
+    for (const statusItem of status) {
-
+      await this.dataGatheringQueue.clean(
-    for (const job of jobs) {
+        300,
-      try {
+        statusItem === 'waiting' ? 'wait' : statusItem
-        await job.remove();
+      );
-      } catch (error) {
-        Logger.warn(error, 'QueueService');
-      }
     }
   }
 
@@ -44,18 +41,23 @@ export class QueueService {
     const jobs = await this.dataGatheringQueue.getJobs(status);
 
     const jobsWithState = await Promise.all(
-      jobs.slice(0, limit).map(async (job) => {
+      jobs
-        return {
+        .filter((job) => {
-          attemptsMade: job.attemptsMade + 1,
+          return job;
-          data: job.data,
+        })
-          finishedOn: job.finishedOn,
+        .slice(0, limit)
-          id: job.id,
+        .map(async (job) => {
-          name: job.name,
+          return {
-          stacktrace: job.stacktrace,
+            attemptsMade: job.attemptsMade + 1,
-          state: await job.getState(),
+            data: job.data,
-          timestamp: job.timestamp
+            finishedOn: job.finishedOn,
-        };
+            id: job.id,
-      })
+            name: job.name,
+            stacktrace: job.stacktrace,
+            state: await job.getState(),
+            timestamp: job.timestamp
+          };
+        })
     );
 
     return {

apps/api/src/app/app.controller.ts

@@ -1,4 +1,4 @@
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { Controller } from '@nestjs/common';
 
 @Controller()

apps/api/src/app/app.module.ts

@@ -1,18 +1,18 @@
 import { join } from 'path';
 
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
+import { CronService } from '@ghostfolio/api/services/cron.service';
+import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module';
+import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
+import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
+import { TwitterBotModule } from '@ghostfolio/api/services/twitter-bot/twitter-bot.module';
 import { BullModule } from '@nestjs/bull';
 import { MiddlewareConsumer, Module, RequestMethod } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { ScheduleModule } from '@nestjs/schedule';
 import { ServeStaticModule } from '@nestjs/serve-static';
 
-import { ConfigurationModule } from '../services/configuration.module';
-import { CronService } from '../services/cron.service';
-import { DataGatheringModule } from '../services/data-gathering.module';
-import { DataProviderModule } from '../services/data-provider/data-provider.module';
-import { ExchangeRateDataModule } from '../services/exchange-rate-data.module';
-import { PrismaModule } from '../services/prisma.module';
-import { TwitterBotModule } from '../services/twitter-bot/twitter-bot.module';
 import { AccessModule } from './access/access.module';
 import { AccountModule } from './account/account.module';
 import { AdminModule } from './admin/admin.module';
@@ -24,10 +24,12 @@ import { CacheModule } from './cache/cache.module';
 import { ExchangeRateModule } from './exchange-rate/exchange-rate.module';
 import { ExportModule } from './export/export.module';
 import { FrontendMiddleware } from './frontend.middleware';
+import { HealthModule } from './health/health.module';
 import { ImportModule } from './import/import.module';
 import { InfoModule } from './info/info.module';
 import { LogoModule } from './logo/logo.module';
 import { OrderModule } from './order/order.module';
+import { PlatformModule } from './platform/platform.module';
 import { PortfolioModule } from './portfolio/portfolio.module';
 import { RedisCacheModule } from './redis-cache/redis-cache.module';
 import { SubscriptionModule } from './subscription/subscription.module';
@@ -57,10 +59,12 @@ import { UserModule } from './user/user.module';
     ExchangeRateModule,
     ExchangeRateDataModule,
     ExportModule,
+    HealthModule,
     ImportModule,
     InfoModule,
     LogoModule,
     OrderModule,
+    PlatformModule,
     PortfolioModule,
     PrismaModule,
     RedisCacheModule,

apps/api/src/app/auth-device/auth-device.module.ts

@@ -1,7 +1,7 @@
 import { AuthDeviceController } from '@ghostfolio/api/app/auth-device/auth-device.controller';
 import { AuthDeviceService } from '@ghostfolio/api/app/auth-device/auth-device.service';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
 import { Module } from '@nestjs/common';
 import { JwtModule } from '@nestjs/jwt';
 

apps/api/src/app/auth-device/auth-device.service.ts

@@ -1,5 +1,5 @@
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { Injectable } from '@nestjs/common';
 import { AuthDevice, Prisma } from '@prisma/client';
 

apps/api/src/app/auth/auth.controller.ts

@@ -1,5 +1,5 @@
 import { WebAuthService } from '@ghostfolio/api/app/auth/web-auth.service';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { DEFAULT_LANGUAGE_CODE } from '@ghostfolio/common/config';
 import { OAuthResponse } from '@ghostfolio/common/interfaces';
 import {

apps/api/src/app/auth/auth.module.ts

@@ -2,8 +2,8 @@ import { AuthDeviceService } from '@ghostfolio/api/app/auth-device/auth-device.s
 import { WebAuthService } from '@ghostfolio/api/app/auth/web-auth.service';
 import { SubscriptionModule } from '@ghostfolio/api/app/subscription/subscription.module';
 import { UserModule } from '@ghostfolio/api/app/user/user.module';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
 import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
 import { Module } from '@nestjs/common';
 import { JwtModule } from '@nestjs/jwt';

apps/api/src/app/auth/auth.service.ts

@@ -1,5 +1,5 @@
 import { UserService } from '@ghostfolio/api/app/user/user.service';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { Injectable, InternalServerErrorException } from '@nestjs/common';
 import { JwtService } from '@nestjs/jwt';

apps/api/src/app/auth/google.strategy.ts

@@ -1,4 +1,4 @@
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { Injectable, Logger } from '@nestjs/common';
 import { PassportStrategy } from '@nestjs/passport';
 import { Provider } from '@prisma/client';

apps/api/src/app/auth/jwt.strategy.ts

@@ -1,6 +1,6 @@
 import { UserService } from '@ghostfolio/api/app/user/user.service';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { HEADER_KEY_TIMEZONE } from '@ghostfolio/common/config';
 import { Injectable, UnauthorizedException } from '@nestjs/common';
 import { PassportStrategy } from '@nestjs/passport';

apps/api/src/app/auth/web-auth.service.ts

@@ -1,7 +1,7 @@
 import { AuthDeviceDto } from '@ghostfolio/api/app/auth-device/auth-device.dto';
 import { AuthDeviceService } from '@ghostfolio/api/app/auth-device/auth-device.service';
 import { UserService } from '@ghostfolio/api/app/user/user.service';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import type { RequestWithUser } from '@ghostfolio/common/types';
 import {
   Inject,

apps/api/src/app/benchmark/benchmark.module.ts

@@ -1,10 +1,10 @@
 import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
 import { SymbolModule } from '@ghostfolio/api/app/symbol/symbol.module';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { MarketDataModule } from '@ghostfolio/api/services/market-data.module';
+import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';
 import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
-import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
+import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
 import { Module } from '@nestjs/common';
 
 import { BenchmarkController } from './benchmark.controller';

apps/api/src/app/benchmark/benchmark.service.ts

@@ -1,9 +1,9 @@
 import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service';
 import { SymbolService } from '@ghostfolio/api/app/symbol/symbol.service';
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
-import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
-import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
+import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
 import {
   MAX_CHART_ITEMS,
   PROPERTY_BENCHMARKS

apps/api/src/app/cache/cache.module.ts

@@ -1,10 +1,10 @@
 import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module';
+import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
+import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
-import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
+import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
 import { Module } from '@nestjs/common';
 
 import { CacheController } from './cache.controller';

apps/api/src/app/exchange-rate/exchange-rate.module.ts

@@ -1,4 +1,4 @@
-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
+import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
 import { Module } from '@nestjs/common';
 
 import { ExchangeRateController } from './exchange-rate.controller';

apps/api/src/app/exchange-rate/exchange-rate.service.ts

@@ -1,4 +1,4 @@
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { Injectable } from '@nestjs/common';
 
 @Injectable()

apps/api/src/app/export/export.module.ts

@@ -1,8 +1,8 @@
 import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module';
+import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
 import { Module } from '@nestjs/common';
 
 import { ExportController } from './export.controller';

apps/api/src/app/export/export.service.ts

@@ -1,5 +1,5 @@
 import { environment } from '@ghostfolio/api/environments/environment';
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { Export } from '@ghostfolio/common/interfaces';
 import { Injectable } from '@nestjs/common';
 

apps/api/src/app/frontend.middleware.ts

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 
 import { environment } from '@ghostfolio/api/environments/environment';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { DEFAULT_LANGUAGE_CODE } from '@ghostfolio/common/config';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import { Injectable, NestMiddleware } from '@nestjs/common';

apps/api/src/app/health/health.controller.ts

@@ -0,0 +1,44 @@
+import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
+import {
+  Controller,
+  Get,
+  HttpException,
+  Param,
+  UseInterceptors
+} from '@nestjs/common';
+import { DataSource } from '@prisma/client';
+import { StatusCodes, getReasonPhrase } from 'http-status-codes';
+
+import { HealthService } from './health.service';
+
+@Controller('health')
+export class HealthController {
+  public constructor(private readonly healthService: HealthService) {}
+
+  @Get()
+  public async getHealth() {}
+
+  @Get('data-provider/:dataSource')
+  @UseInterceptors(TransformDataSourceInRequestInterceptor)
+  public async getHealthOfDataProvider(
+    @Param('dataSource') dataSource: DataSource
+  ) {
+    if (!DataSource[dataSource]) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.NOT_FOUND),
+        StatusCodes.NOT_FOUND
+      );
+    }
+
+    const hasResponse = await this.healthService.hasResponseFromDataProvider(
+      dataSource
+    );
+
+    if (hasResponse !== true) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.SERVICE_UNAVAILABLE),
+        StatusCodes.SERVICE_UNAVAILABLE
+      );
+    }
+  }
+}

apps/api/src/app/health/health.module.ts

@@ -0,0 +1,13 @@
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
+import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
+import { Module } from '@nestjs/common';
+
+import { HealthController } from './health.controller';
+import { HealthService } from './health.service';
+
+@Module({
+  controllers: [HealthController],
+  imports: [ConfigurationModule, DataProviderModule],
+  providers: [HealthService]
+})
+export class HealthModule {}

apps/api/src/app/health/health.service.ts

@@ -0,0 +1,14 @@
+import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
+import { Injectable } from '@nestjs/common';
+import { DataSource } from '@prisma/client';
+
+@Injectable()
+export class HealthService {
+  public constructor(
+    private readonly dataProviderService: DataProviderService
+  ) {}
+
+  public async hasResponseFromDataProvider(aDataSource: DataSource) {
+    return this.dataProviderService.checkQuote(aDataSource);
+  }
+}

apps/api/src/app/import/import.controller.ts

@@ -1,6 +1,6 @@
 import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
 import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { ImportResponse } from '@ghostfolio/common/interfaces';
 import { hasPermission, permissions } from '@ghostfolio/common/permissions';
 import type { RequestWithUser } from '@ghostfolio/common/types';
@@ -35,6 +35,8 @@ export class ImportController {
 
   @Post()
   @UseGuards(AuthGuard('jwt'))
+  @UseInterceptors(TransformDataSourceInRequestInterceptor)
+  @UseInterceptors(TransformDataSourceInResponseInterceptor)
   public async import(
     @Body() importData: ImportDataDto,
     @Query('dryRun') isDryRun?: boolean

apps/api/src/app/import/import.module.ts

@@ -1,19 +1,19 @@
 import { AccountModule } from '@ghostfolio/api/app/account/account.module';
 import { CacheModule } from '@ghostfolio/api/app/cache/cache.module';
 import { OrderModule } from '@ghostfolio/api/app/order/order.module';
+import { PlatformModule } from '@ghostfolio/api/app/platform/platform.module';
 import { PortfolioModule } from '@ghostfolio/api/app/portfolio/portfolio.module';
 import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module';
+import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
+import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
-import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
+import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
 import { Module } from '@nestjs/common';
 
 import { ImportController } from './import.controller';
 import { ImportService } from './import.service';
-import { PlatformModule } from '@ghostfolio/api/services/platform/platform.module';
 
 @Module({
   controllers: [ImportController],

apps/api/src/app/import/import.service.ts

@@ -1,13 +1,16 @@
 import { AccountService } from '@ghostfolio/api/app/account/account.service';
 import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';
 import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
-import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';
+import {
+  Activity,
+  ActivityError
+} from '@ghostfolio/api/app/order/interfaces/activities.interface';
 import { OrderService } from '@ghostfolio/api/app/order/order.service';
+import { PlatformService } from '@ghostfolio/api/app/platform/platform.service';
 import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service';
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
-import { PlatformService } from '@ghostfolio/api/services/platform/platform.service';
+import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
-import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
 import { parseDate } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import {
@@ -15,7 +18,7 @@ import {
   OrderWithAccount
 } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
-import { Prisma, SymbolProfile } from '@prisma/client';
+import { DataSource, Prisma, SymbolProfile } from '@prisma/client';
 import Big from 'big.js';
 import { endOfToday, isAfter, isSameDay, parseISO } from 'date-fns';
 import { v4 as uuidv4 } from 'uuid';
@@ -71,8 +74,25 @@ export class ImportService {
 
         const value = new Big(quantity).mul(marketPrice).toNumber();
 
+        const isDuplicate = orders.some((activity) => {
+          return (
+            activity.SymbolProfile.currency === assetProfile.currency &&
+            activity.SymbolProfile.dataSource === assetProfile.dataSource &&
+            isSameDay(activity.date, parseDate(dateString)) &&
+            activity.quantity === quantity &&
+            activity.SymbolProfile.symbol === assetProfile.symbol &&
+            activity.type === 'DIVIDEND' &&
+            activity.unitPrice === marketPrice
+          );
+        });
+
+        const error: ActivityError = isDuplicate
+          ? { code: 'IS_DUPLICATE' }
+          : undefined;
+
         return {
           Account,
+          error,
           quantity,
           value,
           accountId: Account?.id,
@@ -130,7 +150,7 @@ export class ImportService {
             }
           }
         }),
-        this.platformService.get()
+        this.platformService.getPlatforms()
       ]);
 
       for (const account of accountsDto) {
@@ -183,9 +203,10 @@ export class ImportService {
     for (const activity of activitiesDto) {
       if (!activity.dataSource) {
         if (activity.type === 'ITEM') {
-          activity.dataSource = 'MANUAL';
+          activity.dataSource = DataSource.MANUAL;
         } else {
-          activity.dataSource = this.dataProviderService.getPrimaryDataSource();
+          activity.dataSource =
+            this.dataProviderService.getDataSourceForImport();
         }
       }
 
@@ -203,9 +224,14 @@ export class ImportService {
       userId
     });
 
+    const activitiesExtendedWithErrors = await this.extendActivitiesWithErrors({
+      activitiesDto,
+      userId
+    });
+
     const accounts = (await this.accountService.getAccounts(userId)).map(
-      (account) => {
+      ({ id, name }) => {
-        return { id: account.id, name: account.name };
+        return { id, name };
       }
     );
 
@@ -220,16 +246,14 @@ export class ImportService {
     for (const {
       accountId,
       comment,
-      currency,
+      date,
-      dataSource,
+      error,
-      date: dateString,
       fee,
       quantity,
-      symbol,
+      SymbolProfile: assetProfile,
       type,
       unitPrice
-    } of activitiesDto) {
+    } of activitiesExtendedWithErrors) {
-      const date = parseISO(<string>(<unknown>dateString));
       const validatedAccount = accounts.find(({ id }) => {
         return id === accountId;
       });
@@ -254,6 +278,139 @@ export class ImportService {
           createdAt: new Date(),
           id: uuidv4(),
           isDraft: isAfter(date, endOfToday()),
+          SymbolProfile: {
+            assetClass: assetProfile.assetClass,
+            assetSubClass: assetProfile.assetSubClass,
+            comment: assetProfile.comment,
+            countries: assetProfile.countries,
+            createdAt: assetProfile.createdAt,
+            currency: assetProfile.currency,
+            dataSource: assetProfile.dataSource,
+            id: assetProfile.id,
+            isin: assetProfile.isin,
+            name: assetProfile.name,
+            scraperConfiguration: assetProfile.scraperConfiguration,
+            sectors: assetProfile.sectors,
+            symbol: assetProfile.currency,
+            symbolMapping: assetProfile.symbolMapping,
+            updatedAt: assetProfile.updatedAt,
+            url: assetProfile.url,
+            ...assetProfiles[assetProfile.symbol]
+          },
+          Account: validatedAccount,
+          symbolProfileId: undefined,
+          updatedAt: new Date()
+        };
+      } else {
+        if (error) {
+          continue;
+        }
+
+        order = await this.orderService.createOrder({
+          comment,
+          date,
+          fee,
+          quantity,
+          type,
+          unitPrice,
+          userId,
+          accountId: validatedAccount?.id,
+          SymbolProfile: {
+            connectOrCreate: {
+              create: {
+                currency: assetProfile.currency,
+                dataSource: assetProfile.dataSource,
+                symbol: assetProfile.symbol
+              },
+              where: {
+                dataSource_symbol: {
+                  dataSource: assetProfile.dataSource,
+                  symbol: assetProfile.symbol
+                }
+              }
+            }
+          },
+          updateAccountBalance: false,
+          User: { connect: { id: userId } }
+        });
+      }
+
+      const value = new Big(quantity).mul(unitPrice).toNumber();
+
+      //@ts-ignore
+      activities.push({
+        ...order,
+        error,
+        value,
+        feeInBaseCurrency: this.exchangeRateDataService.toCurrency(
+          fee,
+          assetProfile.currency,
+          userCurrency
+        ),
+        valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
+          value,
+          assetProfile.currency,
+          userCurrency
+        )
+      });
+    }
+
+    return activities;
+  }
+
+  private async extendActivitiesWithErrors({
+    activitiesDto,
+    userId
+  }: {
+    activitiesDto: Partial<CreateOrderDto>[];
+    userId: string;
+  }): Promise<Partial<Activity>[]> {
+    const existingActivities = await this.orderService.orders({
+      include: { SymbolProfile: true },
+      orderBy: { date: 'desc' },
+      where: { userId }
+    });
+
+    return activitiesDto.map(
+      ({
+        accountId,
+        comment,
+        currency,
+        dataSource,
+        date: dateString,
+        fee,
+        quantity,
+        symbol,
+        type,
+        unitPrice
+      }) => {
+        const date = parseISO(<string>(<unknown>dateString));
+        const isDuplicate = existingActivities.some((activity) => {
+          return (
+            activity.SymbolProfile.currency === currency &&
+            activity.SymbolProfile.dataSource === dataSource &&
+            isSameDay(activity.date, date) &&
+            activity.fee === fee &&
+            activity.quantity === quantity &&
+            activity.SymbolProfile.symbol === symbol &&
+            activity.type === type &&
+            activity.unitPrice === unitPrice
+          );
+        });
+
+        const error: ActivityError = isDuplicate
+          ? { code: 'IS_DUPLICATE' }
+          : undefined;
+
+        return {
+          accountId,
+          comment,
+          date,
+          error,
+          fee,
+          quantity,
+          type,
+          unitPrice,
           SymbolProfile: {
             currency,
             dataSource,
@@ -270,62 +427,11 @@ export class ImportService {
             sectors: null,
             symbolMapping: null,
             updatedAt: undefined,
-            url: null,
+            url: null
-            ...assetProfiles[symbol]
+          }
-          },
-          Account: validatedAccount,
-          symbolProfileId: undefined,
-          updatedAt: new Date()
         };
-      } else {
-        order = await this.orderService.createOrder({
-          comment,
-          date,
-          fee,
-          quantity,
-          type,
-          unitPrice,
-          userId,
-          accountId: validatedAccount?.id,
-          SymbolProfile: {
-            connectOrCreate: {
-              create: {
-                currency,
-                dataSource,
-                symbol
-              },
-              where: {
-                dataSource_symbol: {
-                  dataSource,
-                  symbol
-                }
-              }
-            }
-          },
-          User: { connect: { id: userId } }
-        });
       }
-
+    );
-      const value = new Big(quantity).mul(unitPrice).toNumber();
-
-      //@ts-ignore
-      activities.push({
-        ...order,
-        value,
-        feeInBaseCurrency: this.exchangeRateDataService.toCurrency(
-          fee,
-          currency,
-          userCurrency
-        ),
-        valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
-          value,
-          currency,
-          userCurrency
-        )
-      });
-    }
-
-    return activities;
   }
 
   private isUniqueAccount(accounts: AccountWithPlatform[]) {
@@ -354,33 +460,11 @@ export class ImportService {
     const assetProfiles: {
       [symbol: string]: Partial<SymbolProfile>;
     } = {};
-    const existingActivities = await this.orderService.orders({
-      include: { SymbolProfile: true },
-      orderBy: { date: 'desc' },
-      where: { userId }
-    });
 
     for (const [
       index,
-      { currency, dataSource, date, fee, quantity, symbol, type, unitPrice }
+      { currency, dataSource, symbol }
     ] of activitiesDto.entries()) {
-      const duplicateActivity = existingActivities.find((activity) => {
-        return (
-          activity.SymbolProfile.currency === currency &&
-          activity.SymbolProfile.dataSource === dataSource &&
-          isSameDay(activity.date, parseISO(<string>(<unknown>date))) &&
-          activity.fee === fee &&
-          activity.quantity === quantity &&
-          activity.SymbolProfile.symbol === symbol &&
-          activity.type === type &&
-          activity.unitPrice === unitPrice
-        );
-      });
-
-      if (duplicateActivity) {
-        throw new Error(`activities.${index} is a duplicate activity`);
-      }
-
       if (dataSource !== 'MANUAL') {
         const assetProfile = (
           await this.dataProviderService.getAssetProfiles([

apps/api/src/app/info/info.module.ts

@@ -1,12 +1,13 @@
 import { BenchmarkModule } from '@ghostfolio/api/app/benchmark/benchmark.module';
+import { PlatformModule } from '@ghostfolio/api/app/platform/platform.module';
 import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module';
+import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
+import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
 import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
-import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
+import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
 import { TagModule } from '@ghostfolio/api/services/tag/tag.module';
 import { Module } from '@nestjs/common';
 import { JwtModule } from '@nestjs/jwt';
@@ -26,6 +27,7 @@ import { InfoService } from './info.service';
       secret: process.env.JWT_SECRET_KEY,
       signOptions: { expiresIn: '30 days' }
     }),
+    PlatformModule,
     PrismaModule,
     PropertyModule,
     RedisCacheModule,

apps/api/src/app/info/info.service.ts

@@ -1,11 +1,13 @@
 import { BenchmarkService } from '@ghostfolio/api/app/benchmark/benchmark.service';
+import { PlatformService } from '@ghostfolio/api/app/platform/platform.service';
 import { RedisCacheService } from '@ghostfolio/api/app/redis-cache/redis-cache.service';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { TagService } from '@ghostfolio/api/services/tag/tag.service';
 import {
+  PROPERTY_BETTER_UPTIME_MONITOR_ID,
   PROPERTY_COUNTRIES_OF_SUBSCRIBERS,
   PROPERTY_DEMO_USER_ID,
   PROPERTY_IS_READ_ONLY_MODE,
@@ -38,6 +40,7 @@ export class InfoService {
     private readonly configurationService: ConfigurationService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
     private readonly jwtService: JwtService,
+    private readonly platformService: PlatformService,
     private readonly prismaService: PrismaService,
     private readonly propertyService: PropertyService,
     private readonly redisCacheService: RedisCacheService,
@@ -47,9 +50,12 @@ export class InfoService {
   public async get(): Promise<InfoItem> {
     const info: Partial<InfoItem> = {};
     let isReadOnlyMode: boolean;
-    const platforms = await this.prismaService.platform.findMany({
+    const platforms = (
-      orderBy: { name: 'asc' },
+      await this.platformService.getPlatforms({
-      select: { id: true, name: true }
+        orderBy: { name: 'asc' }
+      })
+    ).map(({ id, name }) => {
+      return { id, name };
     });
     let systemMessage: string;
 
@@ -110,19 +116,28 @@ export class InfoService {
       globalPermissions.push(permissions.createUserAccount);
     }
 
+    const [benchmarks, demoAuthToken, statistics, subscriptions, tags] =
+      await Promise.all([
+        this.benchmarkService.getBenchmarkAssetProfiles(),
+        this.getDemoAuthToken(),
+        this.getStatistics(),
+        this.getSubscriptions(),
+        this.tagService.get()
+      ]);
+
     return {
       ...info,
+      benchmarks,
+      demoAuthToken,
       globalPermissions,
       isReadOnlyMode,
       platforms,
+      statistics,
+      subscriptions,
       systemMessage,
+      tags,
       baseCurrency: this.configurationService.get('BASE_CURRENCY'),
-      benchmarks: await this.benchmarkService.getBenchmarkAssetProfiles(),
+      currencies: this.exchangeRateDataService.getCurrencies()
-      currencies: this.exchangeRateDataService.getCurrencies(),
-      demoAuthToken: await this.getDemoAuthToken(),
-      statistics: await this.getStatistics(),
-      subscriptions: await this.getSubscriptions(),
-      tags: await this.tagService.get()
     };
   }
 
@@ -286,6 +301,7 @@ export class InfoService {
     const gitHubContributors = await this.countGitHubContributors();
     const gitHubStargazers = await this.countGitHubStargazers();
     const slackCommunityUsers = await this.countSlackCommunityUsers();
+    const uptime = await this.getUptime();
 
     statistics = {
       activeUsers1d,
@@ -294,7 +310,8 @@ export class InfoService {
       gitHubContributors,
       gitHubStargazers,
       newUsers30d,
-      slackCommunityUsers
+      slackCommunityUsers,
+      uptime
     };
 
     await this.redisCacheService.set(
@@ -318,4 +335,33 @@ export class InfoService {
 
     return JSON.parse(stripeConfig.value);
   }
+
+  private async getUptime(): Promise<number> {
+    {
+      try {
+        const monitorId = (await this.propertyService.getByKey(
+          PROPERTY_BETTER_UPTIME_MONITOR_ID
+        )) as string;
+
+        const get = bent(
+          `https://betteruptime.com/api/v2/monitors/${monitorId}/sla`,
+          'GET',
+          'json',
+          200,
+          {
+            Authorization: `Bearer ${this.configurationService.get(
+              'BETTER_UPTIME_API_KEY'
+            )}`
+          }
+        );
+
+        const { data } = await get();
+        return data.attributes.availability / 100;
+      } catch (error) {
+        Logger.error(error, 'InfoService');
+
+        return undefined;
+      }
+    }
+  }
 }

apps/api/src/app/logo/logo.module.ts

@@ -1,5 +1,5 @@
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
+import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
 import { Module } from '@nestjs/common';
 
 import { LogoController } from './logo.controller';

apps/api/src/app/logo/logo.service.ts

@@ -1,4 +1,4 @@
-import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
+import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { HttpException, Injectable } from '@nestjs/common';
 import { DataSource } from '@prisma/client';

apps/api/src/app/order/create-order.dto.ts

@@ -8,6 +8,7 @@ import {
 import { Transform, TransformFnParams } from 'class-transformer';
 import {
   IsArray,
+  IsBoolean,
   IsEnum,
   IsISO8601,
   IsNumber,
@@ -64,4 +65,8 @@ export class CreateOrderDto {
 
   @IsNumber()
   unitPrice: number;
+
+  @IsBoolean()
+  @IsOptional()
+  updateAccountBalance?: boolean;
 }

apps/api/src/app/order/interfaces/activities.interface.ts

@@ -5,7 +5,14 @@ export interface Activities {
 }
 
 export interface Activity extends OrderWithAccount {
+  error?: ActivityError;
   feeInBaseCurrency: number;
+  updateAccountBalance?: boolean;
   value: number;
   valueInBaseCurrency: number;
 }
+
+export interface ActivityError {
+  code: 'IS_DUPLICATE';
+  message?: string;
+}

apps/api/src/app/order/order.controller.ts

@@ -2,7 +2,7 @@ import { RedactValuesInResponseInterceptor } from '@ghostfolio/api/interceptors/
 import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
 import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';
 import { ApiService } from '@ghostfolio/api/services/api/api.service';
-import { ImpersonationService } from '@ghostfolio/api/services/impersonation.service';
+import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';
 import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config';
 import { hasPermission, permissions } from '@ghostfolio/common/permissions';
 import type { RequestWithUser } from '@ghostfolio/common/types';
@@ -41,6 +41,23 @@ export class OrderController {
     @Inject(REQUEST) private readonly request: RequestWithUser
   ) {}
 
+  @Delete()
+  @UseGuards(AuthGuard('jwt'))
+  public async deleteOrders(): Promise<number> {
+    if (
+      !hasPermission(this.request.user.permissions, permissions.deleteOrder)
+    ) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.FORBIDDEN),
+        StatusCodes.FORBIDDEN
+      );
+    }
+
+    return this.orderService.deleteOrders({
+      userId: this.request.user.id
+    });
+  }
+
   @Delete(':id')
   @UseGuards(AuthGuard('jwt'))
   public async deleteOrder(@Param('id') id: string): Promise<OrderModel> {
@@ -79,10 +96,7 @@ export class OrderController {
     });
 
     const impersonationUserId =
-      await this.impersonationService.validateImpersonationId(
+      await this.impersonationService.validateImpersonationId(impersonationId);
-        impersonationId,
-        this.request.user.id
-      );
     const userCurrency = this.request.user.Settings.settings.baseCurrency;
 
     const activities = await this.orderService.getOrders({

apps/api/src/app/order/order.module.ts

@@ -3,13 +3,13 @@ import { CacheModule } from '@ghostfolio/api/app/cache/cache.module';
 import { RedisCacheModule } from '@ghostfolio/api/app/redis-cache/redis-cache.module';
 import { UserModule } from '@ghostfolio/api/app/user/user.module';
 import { ApiModule } from '@ghostfolio/api/services/api/api.module';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module';
+import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
+import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
-import { ImpersonationModule } from '@ghostfolio/api/services/impersonation.module';
+import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
-import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
+import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
 import { Module } from '@nestjs/common';
 
 import { OrderController } from './order.controller';

apps/api/src/app/order/order.service.ts

@@ -1,8 +1,8 @@
 import { AccountService } from '@ghostfolio/api/app/account/account.service';
-import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
-import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
+import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
 import {
   GATHER_ASSET_PROFILE_PROCESS,
   GATHER_ASSET_PROFILE_PROCESS_OPTIONS
@@ -73,6 +73,7 @@ export class OrderService {
       dataSource?: DataSource;
       symbol?: string;
       tags?: Tag[];
+      updateAccountBalance?: boolean;
       userId: string;
     }
   ): Promise<Order> {
@@ -89,12 +90,16 @@ export class OrderService {
       };
     }
 
+    const accountId = data.accountId;
+    let currency = data.currency;
     const tags = data.tags ?? [];
+    const updateAccountBalance = data.updateAccountBalance ?? false;
+    const userId = data.userId;
 
     if (data.type === 'ITEM') {
       const assetClass = data.assetClass;
       const assetSubClass = data.assetSubClass;
-      const currency = data.SymbolProfile.connectOrCreate.create.currency;
+      currency = data.SymbolProfile.connectOrCreate.create.currency;
       const dataSource: DataSource = 'MANUAL';
       const id = uuidv4();
       const name = data.SymbolProfile.connectOrCreate.create.symbol;
@@ -112,14 +117,17 @@ export class OrderService {
       };
     }
 
-    await this.dataGatheringService.addJobToQueue(
+    await this.dataGatheringService.addJobToQueue({
-      GATHER_ASSET_PROFILE_PROCESS,
+      data: {
-      {
         dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
         symbol: data.SymbolProfile.connectOrCreate.create.symbol
       },
-      GATHER_ASSET_PROFILE_PROCESS_OPTIONS
+      name: GATHER_ASSET_PROFILE_PROCESS,
-    );
+      opts: {
+        ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+        jobId: `${data.SymbolProfile.connectOrCreate.create.dataSource}-${data.SymbolProfile.connectOrCreate.create.symbol}`
+      }
+    });
 
     const isDraft = isAfter(data.date as Date, endOfToday());
 
@@ -146,11 +154,12 @@ export class OrderService {
     delete data.dataSource;
     delete data.symbol;
     delete data.tags;
+    delete data.updateAccountBalance;
     delete data.userId;
 
     const orderData: Prisma.OrderCreateInput = data;
 
-    return this.prismaService.order.create({
+    const order = await this.prismaService.order.create({
       data: {
         ...orderData,
         Account,
@@ -162,6 +171,27 @@ export class OrderService {
         }
       }
     });
+
+    if (updateAccountBalance === true) {
+      let amount = new Big(data.unitPrice)
+        .mul(data.quantity)
+        .plus(data.fee)
+        .toNumber();
+
+      if (data.type === 'BUY') {
+        amount = new Big(amount).mul(-1).toNumber();
+      }
+
+      await this.accountService.updateAccountBalance({
+        accountId,
+        amount,
+        currency,
+        userId,
+        date: data.date as Date
+      });
+    }
+
+    return order;
   }
 
   public async deleteOrder(
@@ -178,6 +208,14 @@ export class OrderService {
     return order;
   }
 
+  public async deleteOrders(where: Prisma.OrderWhereInput): Promise<number> {
+    const { count } = await this.prismaService.order.deleteMany({
+      where
+    });
+
+    return count;
+  }
+
   public async getOrders({
     filters,
     includeDrafts = false,

apps/api/src/app/order/update-order.dto.ts

@@ -8,6 +8,7 @@ import {
 import { Transform, TransformFnParams } from 'class-transformer';
 import {
   IsArray,
+  IsBoolean,
   IsEnum,
   IsISO8601,
   IsNumber,

apps/api/src/app/platform/create-platform.dto.ts

@@ -0,0 +1,9 @@
+import { IsString } from 'class-validator';
+
+export class CreatePlatformDto {
+  @IsString()
+  name: string;
+
+  @IsString()
+  url: string;
+}

apps/api/src/app/platform/platform.controller.ts

@@ -0,0 +1,114 @@
+import { hasPermission, permissions } from '@ghostfolio/common/permissions';
+import type { RequestWithUser } from '@ghostfolio/common/types';
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  HttpException,
+  Inject,
+  Param,
+  Post,
+  Put,
+  UseGuards
+} from '@nestjs/common';
+import { REQUEST } from '@nestjs/core';
+import { AuthGuard } from '@nestjs/passport';
+import { Platform } from '@prisma/client';
+import { StatusCodes, getReasonPhrase } from 'http-status-codes';
+
+import { CreatePlatformDto } from './create-platform.dto';
+import { PlatformService } from './platform.service';
+import { UpdatePlatformDto } from './update-platform.dto';
+
+@Controller('platform')
+export class PlatformController {
+  public constructor(
+    private readonly platformService: PlatformService,
+    @Inject(REQUEST) private readonly request: RequestWithUser
+  ) {}
+
+  @Get()
+  @UseGuards(AuthGuard('jwt'))
+  public async getPlatforms() {
+    return this.platformService.getPlatformsWithAccountCount();
+  }
+
+  @Post()
+  @UseGuards(AuthGuard('jwt'))
+  public async createPlatform(
+    @Body() data: CreatePlatformDto
+  ): Promise<Platform> {
+    if (
+      !hasPermission(this.request.user.permissions, permissions.createPlatform)
+    ) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.FORBIDDEN),
+        StatusCodes.FORBIDDEN
+      );
+    }
+    return this.platformService.createPlatform(data);
+  }
+
+  @Put(':id')
+  @UseGuards(AuthGuard('jwt'))
+  public async updatePlatform(
+    @Param('id') id: string,
+    @Body() data: UpdatePlatformDto
+  ) {
+    if (
+      !hasPermission(this.request.user.permissions, permissions.updatePlatform)
+    ) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.FORBIDDEN),
+        StatusCodes.FORBIDDEN
+      );
+    }
+
+    const originalPlatform = await this.platformService.getPlatform({
+      id
+    });
+
+    if (!originalPlatform) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.FORBIDDEN),
+        StatusCodes.FORBIDDEN
+      );
+    }
+
+    return this.platformService.updatePlatform({
+      data: {
+        ...data
+      },
+      where: {
+        id
+      }
+    });
+  }
+
+  @Delete(':id')
+  @UseGuards(AuthGuard('jwt'))
+  public async deletePlatform(@Param('id') id: string) {
+    if (
+      !hasPermission(this.request.user.permissions, permissions.deletePlatform)
+    ) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.FORBIDDEN),
+        StatusCodes.FORBIDDEN
+      );
+    }
+
+    const originalPlatform = await this.platformService.getPlatform({
+      id
+    });
+
+    if (!originalPlatform) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.FORBIDDEN),
+        StatusCodes.FORBIDDEN
+      );
+    }
+
+    return this.platformService.deletePlatform({ id });
+  }
+}

apps/api/src/app/platform/platform.module.ts

@@ -1,9 +1,11 @@
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
 import { Module } from '@nestjs/common';
 
+import { PlatformController } from './platform.controller';
 import { PlatformService } from './platform.service';
 
 @Module({
+  controllers: [PlatformController],
   exports: [PlatformService],
   imports: [PrismaModule],
   providers: [PlatformService]

apps/api/src/app/platform/platform.service.ts

@@ -0,0 +1,83 @@
+import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
+import { Injectable } from '@nestjs/common';
+import { Platform, Prisma } from '@prisma/client';
+
+@Injectable()
+export class PlatformService {
+  public constructor(private readonly prismaService: PrismaService) {}
+
+  public async getPlatform(
+    platformWhereUniqueInput: Prisma.PlatformWhereUniqueInput
+  ): Promise<Platform> {
+    return this.prismaService.platform.findUnique({
+      where: platformWhereUniqueInput
+    });
+  }
+
+  public async getPlatforms({
+    cursor,
+    orderBy,
+    skip,
+    take,
+    where
+  }: {
+    cursor?: Prisma.PlatformWhereUniqueInput;
+    orderBy?: Prisma.PlatformOrderByWithRelationInput;
+    skip?: number;
+    take?: number;
+    where?: Prisma.PlatformWhereInput;
+  } = {}) {
+    return this.prismaService.platform.findMany({
+      cursor,
+      orderBy,
+      skip,
+      take,
+      where
+    });
+  }
+
+  public async getPlatformsWithAccountCount() {
+    const platformsWithAccountCount =
+      await this.prismaService.platform.findMany({
+        include: {
+          _count: {
+            select: { Account: true }
+          }
+        }
+      });
+
+    return platformsWithAccountCount.map(({ _count, id, name, url }) => {
+      return {
+        id,
+        name,
+        url,
+        accountCount: _count.Account
+      };
+    });
+  }
+
+  public async createPlatform(data: Prisma.PlatformCreateInput) {
+    return this.prismaService.platform.create({
+      data
+    });
+  }
+
+  public async updatePlatform({
+    data,
+    where
+  }: {
+    data: Prisma.PlatformUpdateInput;
+    where: Prisma.PlatformWhereUniqueInput;
+  }): Promise<Platform> {
+    return this.prismaService.platform.update({
+      data,
+      where
+    });
+  }
+
+  public async deletePlatform(
+    where: Prisma.PlatformWhereUniqueInput
+  ): Promise<Platform> {
+    return this.prismaService.platform.delete({ where });
+  }
+}

apps/api/src/app/platform/update-platform.dto.ts

@@ -0,0 +1,12 @@
+import { IsString } from 'class-validator';
+
+export class UpdatePlatformDto {
+  @IsString()
+  id: string;
+
+  @IsString()
+  name: string;
+
+  @IsString()
+  url: string;
+}

apps/api/src/app/portfolio/current-rate.service.mock.ts

@@ -1,8 +1,8 @@
 import { parseDate, resetHours } from '@ghostfolio/common/helper';
-import { DataProviderInfo } from '@ghostfolio/common/interfaces';
 import { addDays, endOfDay, isBefore, isSameDay } from 'date-fns';
 
 import { GetValueObject } from './interfaces/get-value-object.interface';
+import { GetValuesObject } from './interfaces/get-values-object.interface';
 import { GetValuesParams } from './interfaces/get-values-params.interface';
 
 function mockGetValue(symbol: string, date: Date) {
@@ -49,11 +49,9 @@ export const CurrentRateServiceMock = {
   getValues: ({
     dataGatheringItems,
     dateQuery
-  }: GetValuesParams): Promise<{
+  }: GetValuesParams): Promise<GetValuesObject> => {
-    dataProviderInfos: DataProviderInfo[];
-    values: GetValueObject[];
-  }> => {
     const values: GetValueObject[] = [];
+
     if (dateQuery.lt) {
       for (
         let date = resetHours(dateQuery.gte);
@@ -85,6 +83,7 @@ export const CurrentRateServiceMock = {
         }
       }
     }
-    return Promise.resolve({ values, dataProviderInfos: [] });
+
+    return Promise.resolve({ values, dataProviderInfos: [], errors: [] });
   }
 };

apps/api/src/app/portfolio/current-rate.service.spec.ts

@@ -1,13 +1,13 @@
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
-import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
-import { DataProviderInfo } from '@ghostfolio/common/interfaces';
+import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { DataSource, MarketData } from '@prisma/client';
 
 import { CurrentRateService } from './current-rate.service';
-import { GetValueObject } from './interfaces/get-value-object.interface';
+import { GetValuesObject } from './interfaces/get-values-object.interface';
 
-jest.mock('@ghostfolio/api/services/market-data.service', () => {
+jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => {
   return {
     MarketDataService: jest.fn().mockImplementation(() => {
       return {
@@ -18,7 +18,8 @@ jest.mock('@ghostfolio/api/services/market-data.service', () => {
             createdAt: date,
             dataSource: DataSource.YAHOO,
             id: 'aefcbe3a-ee10-4c4f-9f2d-8ffad7b05584',
-            marketPrice: 1847.839966
+            marketPrice: 1847.839966,
+            state: 'CLOSE'
           });
         },
         getRange: ({
@@ -37,6 +38,7 @@ jest.mock('@ghostfolio/api/services/market-data.service', () => {
               date: dateRangeStart,
               id: '8fa48fde-f397-4b0d-adbc-fb940e830e6d',
               marketPrice: 1841.823902,
+              state: 'CLOSE',
               symbol: symbols[0]
             },
             {
@@ -45,6 +47,7 @@ jest.mock('@ghostfolio/api/services/market-data.service', () => {
               date: dateRangeEnd,
               id: '082d6893-df27-4c91-8a5d-092e84315b56',
               marketPrice: 1847.839966,
+              state: 'CLOSE',
               symbol: symbols[0]
             }
           ]);
@@ -54,14 +57,27 @@ jest.mock('@ghostfolio/api/services/market-data.service', () => {
   };
 });
 
-jest.mock('@ghostfolio/api/services/exchange-rate-data.service', () => {
+jest.mock(
+  '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service',
+  () => {
+    return {
+      ExchangeRateDataService: jest.fn().mockImplementation(() => {
+        return {
+          initialize: () => Promise.resolve(),
+          toCurrency: (value: number) => {
+            return 1 * value;
+          }
+        };
+      })
+    };
+  }
+);
+
+jest.mock('@ghostfolio/api/services/property/property.service', () => {
   return {
-    ExchangeRateDataService: jest.fn().mockImplementation(() => {
+    PropertyService: jest.fn().mockImplementation(() => {
       return {
-        initialize: () => Promise.resolve(),
+        getByKey: (key: string) => Promise.resolve({})
-        toCurrency: (value: number) => {
-          return 1 * value;
-        }
       };
     })
   };
@@ -72,9 +88,18 @@ describe('CurrentRateService', () => {
   let dataProviderService: DataProviderService;
   let exchangeRateDataService: ExchangeRateDataService;
   let marketDataService: MarketDataService;
+  let propertyService: PropertyService;
 
   beforeAll(async () => {
-    dataProviderService = new DataProviderService(null, [], null);
+    propertyService = new PropertyService(null);
+
+    dataProviderService = new DataProviderService(
+      null,
+      [],
+      null,
+      null,
+      propertyService
+    );
     exchangeRateDataService = new ExchangeRateDataService(
       null,
       null,
@@ -104,21 +129,14 @@ describe('CurrentRateService', () => {
         },
         userCurrency: 'CHF'
       })
-    ).toMatchObject<{
+    ).toMatchObject<GetValuesObject>({
-      dataProviderInfos: DataProviderInfo[];
-      values: GetValueObject[];
-    }>({
       dataProviderInfos: [],
+      errors: [],
       values: [
         {
           date: undefined,
           marketPriceInBaseCurrency: 1841.823902,
           symbol: 'AMZN'
-        },
-        {
-          date: undefined,
-          marketPriceInBaseCurrency: 1847.839966,
-          symbol: 'AMZN'
         }
       ]
     });

apps/api/src/app/portfolio/current-rate.service.ts

@@ -1,13 +1,14 @@
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
-import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { resetHours } from '@ghostfolio/common/helper';
-import { DataProviderInfo } from '@ghostfolio/common/interfaces';
+import { DataProviderInfo, ResponseError } from '@ghostfolio/common/interfaces';
 import { Injectable } from '@nestjs/common';
 import { isBefore, isToday } from 'date-fns';
-import { flatten } from 'lodash';
+import { flatten, isEmpty, uniqBy } from 'lodash';
 
 import { GetValueObject } from './interfaces/get-value-object.interface';
+import { GetValuesObject } from './interfaces/get-values-object.interface';
 import { GetValuesParams } from './interfaces/get-values-params.interface';
 
 @Injectable()
@@ -23,10 +24,7 @@ export class CurrentRateService {
     dataGatheringItems,
     dateQuery,
     userCurrency
-  }: GetValuesParams): Promise<{
+  }: GetValuesParams): Promise<GetValuesObject> {
-    dataProviderInfos: DataProviderInfo[];
-    values: GetValueObject[];
-  }> {
     const dataProviderInfos: DataProviderInfo[] = [];
     const includeToday =
       (!dateQuery.lt || isBefore(new Date(), dateQuery.lt)) &&
@@ -34,9 +32,10 @@ export class CurrentRateService {
       (!dateQuery.in || this.containsToday(dateQuery.in));
 
     const promises: Promise<GetValueObject[]>[] = [];
+    const quoteErrors: ResponseError['errors'] = [];
+    const today = resetHours(new Date());
 
     if (includeToday) {
-      const today = resetHours(new Date());
       promises.push(
         this.dataProviderService
           .getQuotes(dataGatheringItems)
@@ -51,18 +50,26 @@ export class CurrentRateService {
                 );
               }
 
-              result.push({
+              if (dataResultProvider?.[dataGatheringItem.symbol]?.marketPrice) {
-                date: today,
+                result.push({
-                marketPriceInBaseCurrency:
+                  date: today,
-                  this.exchangeRateDataService.toCurrency(
+                  marketPriceInBaseCurrency:
-                    dataResultProvider?.[dataGatheringItem.symbol]
+                    this.exchangeRateDataService.toCurrency(
-                      ?.marketPrice ?? 0,
+                      dataResultProvider?.[dataGatheringItem.symbol]
-                    dataResultProvider?.[dataGatheringItem.symbol]?.currency,
+                        ?.marketPrice,
-                    userCurrency
+                      dataResultProvider?.[dataGatheringItem.symbol]?.currency,
-                  ),
+                      userCurrency
-                symbol: dataGatheringItem.symbol
+                    ),
-              });
+                  symbol: dataGatheringItem.symbol
+                });
+              } else {
+                quoteErrors.push({
+                  dataSource: dataGatheringItem.dataSource,
+                  symbol: dataGatheringItem.symbol
+                });
+              }
             }
+
             return result;
           })
       );
@@ -94,10 +101,60 @@ export class CurrentRateService {
         })
     );
 
-    return {
+    const values = flatten(await Promise.all(promises));
+
+    const response: GetValuesObject = {
       dataProviderInfos,
-      values: flatten(await Promise.all(promises))
+      errors: quoteErrors.map(({ dataSource, symbol }) => {
+        return { dataSource, symbol };
+      }),
+      values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`)
     };
+
+    if (!isEmpty(quoteErrors)) {
+      for (const { symbol } of quoteErrors) {
+        try {
+          // If missing quote, fallback to the latest available historical market price
+          let value: GetValueObject = response.values.find((currentValue) => {
+            return currentValue.symbol === symbol && isToday(currentValue.date);
+          });
+
+          if (!value) {
+            value = {
+              symbol,
+              date: today,
+              marketPriceInBaseCurrency: 0
+            };
+
+            response.values.push(value);
+          }
+
+          const [latestValue] = response.values
+            .filter((currentValue) => {
+              return (
+                currentValue.symbol === symbol &&
+                currentValue.marketPriceInBaseCurrency
+              );
+            })
+            .sort((a, b) => {
+              if (a.date < b.date) {
+                return 1;
+              }
+
+              if (a.date > b.date) {
+                return -1;
+              }
+
+              return 0;
+            });
+
+          value.marketPriceInBaseCurrency =
+            latestValue.marketPriceInBaseCurrency;
+        } catch {}
+      }
+    }
+
+    return response;
   }
 
   private containsToday(dates: Date[]): boolean {

apps/api/src/app/portfolio/interfaces/get-values-object.interface.ts

@@ -0,0 +1,9 @@
+import { DataProviderInfo, ResponseError } from '@ghostfolio/common/interfaces';
+
+import { GetValueObject } from './get-value-object.interface';
+
+export interface GetValuesObject {
+  dataProviderInfos: DataProviderInfo[];
+  errors: ResponseError['errors'];
+  values: GetValueObject[];
+}

apps/api/src/app/portfolio/portfolio-calculator.ts

@@ -24,9 +24,10 @@ import {
   isSameYear,
   max,
   min,
-  set
+  set,
+  subDays
 } from 'date-fns';
-import { first, flatten, isNumber, last, sortBy } from 'lodash';
+import { first, flatten, isNumber, last, sortBy, uniq } from 'lodash';
 
 import { CurrentRateService } from './current-rate.service';
 import { CurrentPositions } from './interfaces/current-positions.interface';
@@ -360,7 +361,7 @@ export class PortfolioCalculator {
 
     let firstTransactionPoint: TransactionPoint = null;
     let firstIndex = transactionPointsBeforeEndDate.length;
-    const dates = [];
+    let dates = [];
     const dataGatheringItems: IDataGatheringItem[] = [];
     const currencies: { [symbol: string]: string } = {};
 
@@ -389,15 +390,37 @@ export class PortfolioCalculator {
 
     dates.push(resetHours(end));
 
-    const { dataProviderInfos, values: marketSymbols } =
+    // Add dates of last week for fallback
-      await this.currentRateService.getValues({
+    dates.push(subDays(resetHours(new Date()), 7));
-        currencies,
+    dates.push(subDays(resetHours(new Date()), 6));
-        dataGatheringItems,
+    dates.push(subDays(resetHours(new Date()), 5));
-        dateQuery: {
+    dates.push(subDays(resetHours(new Date()), 4));
-          in: dates
+    dates.push(subDays(resetHours(new Date()), 3));
-        },
+    dates.push(subDays(resetHours(new Date()), 2));
-        userCurrency: this.currency
+    dates.push(subDays(resetHours(new Date()), 1));
-      });
+    dates.push(resetHours(new Date()));
+
+    dates = uniq(
+      dates.map((date) => {
+        return date.getTime();
+      })
+    ).map((timestamp) => {
+      return new Date(timestamp);
+    });
+    dates.sort((a, b) => a.getTime() - b.getTime());
+
+    const {
+      dataProviderInfos,
+      errors: currentRateErrors,
+      values: marketSymbols
+    } = await this.currentRateService.getValues({
+      currencies,
+      dataGatheringItems,
+      dateQuery: {
+        in: dates
+      },
+      userCurrency: this.currency
+    });
 
     this.dataProviderInfos = dataProviderInfos;
 
@@ -472,7 +495,13 @@ export class PortfolioCalculator {
         transactionCount: item.transactionCount
       });
 
-      if (hasErrors && item.investment.gt(0)) {
+      if (
+        (hasErrors ||
+          currentRateErrors.find(({ dataSource, symbol }) => {
+            return dataSource === item.dataSource && symbol === item.symbol;
+          })) &&
+        item.investment.gt(0)
+      ) {
         errors.push({ dataSource: item.dataSource, symbol: item.symbol });
       }
     }
@@ -722,7 +751,7 @@ export class PortfolioCalculator {
         );
       } else if (!currentPosition.quantity.eq(0)) {
         Logger.warn(
-          `Missing initial value for symbol ${currentPosition.symbol} at ${currentPosition.firstBuyDate}`,
+          `Missing historical market data for symbol ${currentPosition.symbol}`,
           'PortfolioCalculator'
         );
         hasErrors = true;

apps/api/src/app/portfolio/portfolio.controller.ts

@@ -8,8 +8,8 @@ import { RedactValuesInResponseInterceptor } from '@ghostfolio/api/interceptors/
 import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
 import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';
 import { ApiService } from '@ghostfolio/api/services/api/api.service';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config';
 import {
   PortfolioDetails,
@@ -91,6 +91,7 @@ export class PortfolioController {
       filteredValueInPercentage,
       hasErrors,
       holdings,
+      platforms,
       summary,
       totalValueInBaseCurrency
     } = await this.portfolioService.getDetails({
@@ -136,9 +137,12 @@ export class PortfolioController {
           portfolioPosition.value / totalValue;
       }
 
-      for (const [name, { current, original }] of Object.entries(accounts)) {
+      for (const [name, { valueInBaseCurrency }] of Object.entries(accounts)) {
-        accounts[name].current = current / totalValue;
+        accounts[name].valueInPercentage = valueInBaseCurrency / totalValue;
-        accounts[name].original = original / totalInvestment;
+      }
+
+      for (const [name, { valueInBaseCurrency }] of Object.entries(platforms)) {
+        platforms[name].valueInPercentage = valueInBaseCurrency / totalValue;
       }
     }
 
@@ -182,6 +186,7 @@ export class PortfolioController {
       filteredValueInPercentage,
       hasError,
       holdings,
+      platforms,
       totalValueInBaseCurrency,
       summary: portfolioSummary
     };

apps/api/src/app/portfolio/portfolio.module.ts

@@ -3,14 +3,14 @@ import { AccountService } from '@ghostfolio/api/app/account/account.service';
 import { OrderModule } from '@ghostfolio/api/app/order/order.module';
 import { UserModule } from '@ghostfolio/api/app/user/user.module';
 import { ApiModule } from '@ghostfolio/api/services/api/api.module';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering.module';
+import { DataGatheringModule } from '@ghostfolio/api/services/data-gathering/data-gathering.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data.module';
+import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
-import { ImpersonationModule } from '@ghostfolio/api/services/impersonation.module';
+import { ImpersonationModule } from '@ghostfolio/api/services/impersonation/impersonation.module';
-import { MarketDataModule } from '@ghostfolio/api/services/market-data.module';
+import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
-import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
+import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
 import { Module } from '@nestjs/common';
 
 import { CurrentRateService } from './current-rate.service';

apps/api/src/app/portfolio/portfolio.service.ts

@@ -7,18 +7,15 @@ import { PortfolioOrder } from '@ghostfolio/api/app/portfolio/interfaces/portfol
 import { TransactionPoint } from '@ghostfolio/api/app/portfolio/interfaces/transaction-point.interface';
 import { UserService } from '@ghostfolio/api/app/user/user.service';
 import { AccountClusterRiskCurrentInvestment } from '@ghostfolio/api/models/rules/account-cluster-risk/current-investment';
-import { AccountClusterRiskInitialInvestment } from '@ghostfolio/api/models/rules/account-cluster-risk/initial-investment';
 import { AccountClusterRiskSingleAccount } from '@ghostfolio/api/models/rules/account-cluster-risk/single-account';
 import { CurrencyClusterRiskBaseCurrencyCurrentInvestment } from '@ghostfolio/api/models/rules/currency-cluster-risk/base-currency-current-investment';
-import { CurrencyClusterRiskBaseCurrencyInitialInvestment } from '@ghostfolio/api/models/rules/currency-cluster-risk/base-currency-initial-investment';
 import { CurrencyClusterRiskCurrentInvestment } from '@ghostfolio/api/models/rules/currency-cluster-risk/current-investment';
-import { CurrencyClusterRiskInitialInvestment } from '@ghostfolio/api/models/rules/currency-cluster-risk/initial-investment';
 import { FeeRatioInitialInvestment } from '@ghostfolio/api/models/rules/fees/fee-ratio-initial-investment';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
-import { ImpersonationService } from '@ghostfolio/api/services/impersonation.service';
+import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';
-import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
+import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
 import {
   EMERGENCY_FUND_TAG_ID,
   MAX_CHART_ITEMS,
@@ -149,7 +146,8 @@ export class PortfolioService {
         }
       }
 
-      const valueInBaseCurrency = details.accounts[account.id]?.current ?? 0;
+      const valueInBaseCurrency =
+        details.accounts[account.id]?.valueInBaseCurrency ?? 0;
 
       const result = {
         ...account,
@@ -462,10 +460,18 @@ export class PortfolioService {
     });
 
     const holdings: PortfolioDetails['holdings'] = {};
-    const totalInvestmentInBaseCurrency = currentPositions.totalInvestment.plus(
+    const totalValueInBaseCurrency = currentPositions.currentValue.plus(
       cashDetails.balanceInBaseCurrency
     );
-    let filteredValueInBaseCurrency = currentPositions.currentValue;
+
+    const isFilteredByAccount =
+      filters?.some((filter) => {
+        return filter.type === 'ACCOUNT';
+      }) ?? false;
+
+    let filteredValueInBaseCurrency = isFilteredByAccount
+      ? totalValueInBaseCurrency
+      : currentPositions.currentValue;
 
     if (
       filters?.length === 0 ||
@@ -484,13 +490,10 @@ export class PortfolioService {
         symbol: position.symbol
       };
     });
-    const symbols = currentPositions.positions.map(
-      (position) => position.symbol
-    );
 
     const [dataProviderResponses, symbolProfiles] = await Promise.all([
       this.dataProviderService.getQuotes(dataGatheringItems),
-      this.symbolProfileService.getSymbolProfilesBySymbols(symbols)
+      this.symbolProfileService.getSymbolProfiles(dataGatheringItems)
     ]);
 
     const symbolProfileMap: { [symbol: string]: EnhancedSymbolProfile } = {};
@@ -564,12 +567,11 @@ export class PortfolioService {
       };
     }
 
-    if (
+    const isFilteredByCash = filters?.some((filter) => {
-      filters?.length === 0 ||
+      return filter.type === 'ASSET_CLASS' && filter.id === 'CASH';
-      (filters?.length === 1 &&
+    });
-        filters[0].type === 'ASSET_CLASS' &&
+
-        filters[0].id === 'CASH')
+    if (filters?.length === 0 || isFilteredByAccount || isFilteredByCash) {
-    ) {
       const cashPositions = await this.getCashPositions({
         cashDetails,
         userCurrency,
@@ -581,7 +583,7 @@ export class PortfolioService {
       }
     }
 
-    const accounts = await this.getValueOfAccounts({
+    const { accounts, platforms } = await this.getValueOfAccountsAndPlatforms({
       filters,
       orders,
       portfolioItemsNow,
@@ -595,7 +597,7 @@ export class PortfolioService {
       filters[0].id === EMERGENCY_FUND_TAG_ID &&
       filters[0].type === 'TAG'
     ) {
-      const cashPositions = await this.getCashPositions({
+      const emergencyFundCashPositions = await this.getCashPositions({
         cashDetails,
         userCurrency,
         value: filteredValueInBaseCurrency
@@ -614,13 +616,12 @@ export class PortfolioService {
       accounts[UNKNOWN_KEY] = {
         balance: 0,
         currency: userCurrency,
-        current: emergencyFundInCash,
         name: UNKNOWN_KEY,
-        original: emergencyFundInCash
+        valueInBaseCurrency: emergencyFundInCash
       };
 
       holdings[userCurrency] = {
-        ...cashPositions[userCurrency],
+        ...emergencyFundCashPositions[userCurrency],
         investment: emergencyFundInCash,
         value: emergencyFundInCash
       };
@@ -640,6 +641,7 @@ export class PortfolioService {
     return {
       accounts,
       holdings,
+      platforms,
       summary,
       filteredValueInBaseCurrency: filteredValueInBaseCurrency.toNumber(),
       filteredValueInPercentage: summary.netWorth
@@ -792,16 +794,6 @@ export class PortfolioService {
       let maxPrice = Math.max(orders[0].unitPrice, marketPrice);
       let minPrice = Math.min(orders[0].unitPrice, marketPrice);
 
-      if (!historicalData?.[aSymbol]?.[firstBuyDate]) {
-        // Add historical entry for buy date, if no historical data available
-        historicalDataArray.push({
-          averagePrice: orders[0].unitPrice,
-          date: firstBuyDate,
-          marketPrice: orders[0].unitPrice,
-          quantity: orders[0].quantity
-        });
-      }
-
       if (historicalData[aSymbol]) {
         let j = -1;
         for (const [date, { marketPrice }] of Object.entries(
@@ -813,11 +805,16 @@ export class PortfolioService {
           ) {
             j++;
           }
+
           let currentAveragePrice = 0;
           let currentQuantity = 0;
+
           const currentSymbol = transactionPoints[j].items.find(
-            (item) => item.symbol === aSymbol
+            ({ symbol }) => {
+              return symbol === aSymbol;
+            }
           );
+
           if (currentSymbol) {
             currentAveragePrice = currentSymbol.quantity.eq(0)
               ? 0
@@ -827,14 +824,25 @@ export class PortfolioService {
 
           historicalDataArray.push({
             date,
-            marketPrice,
             averagePrice: currentAveragePrice,
+            marketPrice:
+              historicalDataArray.length > 0
+                ? marketPrice
+                : currentAveragePrice,
             quantity: currentQuantity
           });
 
           maxPrice = Math.max(marketPrice ?? 0, maxPrice);
           minPrice = Math.min(marketPrice ?? Number.MAX_SAFE_INTEGER, minPrice);
         }
+      } else {
+        // Add historical entry for buy date, if no historical data available
+        historicalDataArray.push({
+          averagePrice: orders[0].unitPrice,
+          date: firstBuyDate,
+          marketPrice: orders[0].unitPrice,
+          quantity: orders[0].quantity
+        });
       }
 
       return {
@@ -979,11 +987,13 @@ export class PortfolioService {
       };
     });
 
-    const symbols = positions.map((position) => position.symbol);
-
     const [dataProviderResponses, symbolProfiles] = await Promise.all([
       this.dataProviderService.getQuotes(dataGatheringItem),
-      this.symbolProfileService.getSymbolProfilesBySymbols(symbols)
+      this.symbolProfileService.getSymbolProfiles(
+        positions.map(({ dataSource, symbol }) => {
+          return { dataSource, symbol };
+        })
+      )
     ]);
 
     const symbolProfileMap: { [symbol: string]: EnhancedSymbolProfile } = {};
@@ -1168,7 +1178,7 @@ export class PortfolioService {
       portfolioItemsNow[position.symbol] = position;
     }
 
-    const accounts = await this.getValueOfAccounts({
+    const { accounts } = await this.getValueOfAccountsAndPlatforms({
       orders,
       portfolioItemsNow,
       userCurrency,
@@ -1179,10 +1189,6 @@ export class PortfolioService {
       rules: {
         accountClusterRisk: await this.rulesService.evaluate(
           [
-            new AccountClusterRiskInitialInvestment(
-              this.exchangeRateDataService,
-              accounts
-            ),
             new AccountClusterRiskCurrentInvestment(
               this.exchangeRateDataService,
               accounts
@@ -1196,18 +1202,10 @@ export class PortfolioService {
         ),
         currencyClusterRisk: await this.rulesService.evaluate(
           [
-            new CurrencyClusterRiskBaseCurrencyInitialInvestment(
-              this.exchangeRateDataService,
-              positions
-            ),
             new CurrencyClusterRiskBaseCurrencyCurrentInvestment(
               this.exchangeRateDataService,
               positions
             ),
-            new CurrencyClusterRiskInitialInvestment(
-              this.exchangeRateDataService,
-              positions
-            ),
             new CurrencyClusterRiskCurrentInvestment(
               this.exchangeRateDataService,
               positions
@@ -1700,7 +1698,7 @@ export class PortfolioService {
     };
   }
 
-  private async getValueOfAccounts({
+  private async getValueOfAccountsAndPlatforms({
     filters = [],
     orders,
     portfolioItemsNow,
@@ -1724,6 +1722,7 @@ export class PortfolioService {
     });
 
     const accounts: PortfolioDetails['accounts'] = {};
+    const platforms: PortfolioDetails['platforms'] = {};
 
     let currentAccounts: (Account & {
       Order?: Order[];
@@ -1734,6 +1733,7 @@ export class PortfolioService {
       currentAccounts = await this.accountService.getAccounts(userId);
     } else if (filters.length === 1 && filters[0].type === 'ACCOUNT') {
       currentAccounts = await this.accountService.accounts({
+        include: { Platform: true },
         where: { id: filters[0].id }
       });
     } else {
@@ -1744,6 +1744,7 @@ export class PortfolioService {
       );
 
       currentAccounts = await this.accountService.accounts({
+        include: { Platform: true },
         where: { id: { in: accountIds } }
       });
     }
@@ -1768,63 +1769,81 @@ export class PortfolioService {
       accounts[account.id] = {
         balance: account.balance,
         currency: account.currency,
-        current: this.exchangeRateDataService.toCurrency(
-          account.balance,
-          account.currency,
-          userCurrency
-        ),
         name: account.name,
-        original: this.exchangeRateDataService.toCurrency(
+        valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
           account.balance,
           account.currency,
           userCurrency
         )
       };
 
+      if (platforms[account.Platform?.id || UNKNOWN_KEY]?.valueInBaseCurrency) {
+        platforms[account.Platform?.id || UNKNOWN_KEY].valueInBaseCurrency +=
+          this.exchangeRateDataService.toCurrency(
+            account.balance,
+            account.currency,
+            userCurrency
+          );
+      } else {
+        platforms[account.Platform?.id || UNKNOWN_KEY] = {
+          balance: account.balance,
+          currency: account.currency,
+          name: account.Platform?.name,
+          valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
+            account.balance,
+            account.currency,
+            userCurrency
+          )
+        };
+      }
+
       for (const order of ordersByAccount) {
         let currentValueOfSymbolInBaseCurrency =
           order.quantity *
           (portfolioItemsNow[order.SymbolProfile.symbol]?.marketPrice ??
             order.unitPrice ??
             0);
-        let originalValueOfSymbolInBaseCurrency =
-          this.exchangeRateDataService.toCurrency(
-            order.quantity * order.unitPrice,
-            order.SymbolProfile.currency,
-            userCurrency
-          );
 
         if (order.type === 'SELL') {
           currentValueOfSymbolInBaseCurrency *= -1;
-          originalValueOfSymbolInBaseCurrency *= -1;
         }
 
-        if (accounts[order.Account?.id || UNKNOWN_KEY]?.current) {
+        if (accounts[order.Account?.id || UNKNOWN_KEY]?.valueInBaseCurrency) {
-          accounts[order.Account?.id || UNKNOWN_KEY].current +=
+          accounts[order.Account?.id || UNKNOWN_KEY].valueInBaseCurrency +=
             currentValueOfSymbolInBaseCurrency;
-          accounts[order.Account?.id || UNKNOWN_KEY].original +=
-            originalValueOfSymbolInBaseCurrency;
         } else {
           accounts[order.Account?.id || UNKNOWN_KEY] = {
             balance: 0,
             currency: order.Account?.currency,
-            current: currentValueOfSymbolInBaseCurrency,
             name: account.name,
-            original: originalValueOfSymbolInBaseCurrency
+            valueInBaseCurrency: currentValueOfSymbolInBaseCurrency
+          };
+        }
+
+        if (
+          platforms[order.Account?.Platform?.id || UNKNOWN_KEY]
+            ?.valueInBaseCurrency
+        ) {
+          platforms[
+            order.Account?.Platform?.id || UNKNOWN_KEY
+          ].valueInBaseCurrency += currentValueOfSymbolInBaseCurrency;
+        } else {
+          platforms[order.Account?.Platform?.id || UNKNOWN_KEY] = {
+            balance: 0,
+            currency: order.Account?.currency,
+            name: account.Platform?.name,
+            valueInBaseCurrency: currentValueOfSymbolInBaseCurrency
           };
         }
       }
     }
 
-    return accounts;
+    return { accounts, platforms };
   }
 
   private async getUserId(aImpersonationId: string, aUserId: string) {
     const impersonationUserId =
-      await this.impersonationService.validateImpersonationId(
+      await this.impersonationService.validateImpersonationId(aImpersonationId);
-        aImpersonationId,
-        aUserId
-      );
 
     return impersonationUserId || aUserId;
   }

apps/api/src/app/redis-cache/redis-cache.module.ts

@@ -1,5 +1,5 @@
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { CacheManagerOptions, CacheModule, Module } from '@nestjs/common';
 import * as redisStore from 'cache-manager-redis-store';
 

apps/api/src/app/redis-cache/redis-cache.service.ts

@@ -1,4 +1,4 @@
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common';
 import { Cache } from 'cache-manager';
 

apps/api/src/app/subscription/subscription.controller.ts

@@ -1,4 +1,4 @@
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import {
   DEFAULT_LANGUAGE_CODE,

apps/api/src/app/subscription/subscription.module.ts

@@ -1,5 +1,5 @@
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
 import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
 import { Module } from '@nestjs/common';
 

apps/api/src/app/subscription/subscription.service.ts

@@ -1,5 +1,5 @@
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import {
   DEFAULT_LANGUAGE_CODE,
   PROPERTY_STRIPE_CONFIG

apps/api/src/app/symbol/symbol.module.ts

@@ -1,7 +1,7 @@
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { MarketDataModule } from '@ghostfolio/api/services/market-data.module';
+import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
 import { Module } from '@nestjs/common';
 
 import { SymbolController } from './symbol.controller';

apps/api/src/app/symbol/symbol.service.ts

@@ -3,7 +3,7 @@ import {
   IDataGatheringItem,
   IDataProviderHistoricalResponse
 } from '@ghostfolio/api/services/interfaces/interfaces';
-import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import { HistoricalDataItem } from '@ghostfolio/common/interfaces';
 import { UserWithSettings } from '@ghostfolio/common/types';

apps/api/src/app/user/user.module.ts

@@ -1,6 +1,6 @@
 import { SubscriptionModule } from '@ghostfolio/api/app/subscription/subscription.module';
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
 import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
 import { TagModule } from '@ghostfolio/api/services/tag/tag.module';
 import { Module } from '@nestjs/common';

apps/api/src/app/user/user.service.ts

@@ -1,6 +1,7 @@
 import { SubscriptionService } from '@ghostfolio/api/app/subscription/subscription.service';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { environment } from '@ghostfolio/api/environments/environment';
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
+import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { TagService } from '@ghostfolio/api/services/tag/tag.service';
 import { PROPERTY_IS_READ_ONLY_MODE, locale } from '@ghostfolio/common/config';
@@ -165,7 +166,7 @@ export class UserService {
         this.subscriptionService.getSubscription(Subscription);
 
       if (
-        Analytics?.activityCount % 25 === 0 &&
+        Analytics?.activityCount % 20 === 0 &&
         user.subscription?.type === 'Basic'
       ) {
         currentPermissions.push(permissions.enableSubscriptionInterstitial);
@@ -196,6 +197,10 @@ export class UserService {
       }
     }
 
+    if (!environment.production && role === 'ADMIN') {
+      currentPermissions.push(permissions.impersonateAllUsers);
+    }
+
     user.Account = sortBy(user.Account, (account) => {
       return account.name;
     });

apps/api/src/interceptors/transform-data-source-in-request.interceptor.ts

@@ -1,3 +1,4 @@
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { decodeDataSource } from '@ghostfolio/common/helper';
 import {
   CallHandler,
@@ -5,10 +6,9 @@ import {
   Injectable,
   NestInterceptor
 } from '@nestjs/common';
+import { DataSource } from '@prisma/client';
 import { Observable } from 'rxjs';
 
-import { ConfigurationService } from '../services/configuration.service';
-
 @Injectable()
 export class TransformDataSourceInRequestInterceptor<T>
   implements NestInterceptor<T, any>
@@ -25,11 +25,24 @@ export class TransformDataSourceInRequestInterceptor<T>
     const request = http.getRequest();
 
     if (this.configurationService.get('ENABLE_FEATURE_SUBSCRIPTION')) {
-      if (request.body.dataSource) {
+      if (request.body.activities) {
+        request.body.activities = request.body.activities.map((activity) => {
+          if (DataSource[activity.dataSource]) {
+            return activity;
+          } else {
+            return {
+              ...activity,
+              dataSource: decodeDataSource(activity.dataSource)
+            };
+          }
+        });
+      }
+
+      if (request.body.dataSource && !DataSource[request.body.dataSource]) {
         request.body.dataSource = decodeDataSource(request.body.dataSource);
       }
 
-      if (request.params.dataSource) {
+      if (request.params.dataSource && !DataSource[request.params.dataSource]) {
         request.params.dataSource = decodeDataSource(request.params.dataSource);
       }
     }

apps/api/src/interceptors/transform-data-source-in-response.interceptor.ts

@@ -1,4 +1,5 @@
 import { redactAttributes } from '@ghostfolio/api/helper/object.helper';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { encodeDataSource } from '@ghostfolio/common/helper';
 import {
   CallHandler,
@@ -10,8 +11,6 @@ import { DataSource } from '@prisma/client';
 import { Observable } from 'rxjs';
 import { map } from 'rxjs/operators';
 
-import { ConfigurationService } from '../services/configuration.service';
-
 @Injectable()
 export class TransformDataSourceInResponseInterceptor<T>
   implements NestInterceptor<T, any>

apps/api/src/main.ts

@@ -32,12 +32,23 @@ async function bootstrap() {
   // Support 10mb csv/json files for importing activities
   app.use(bodyParser.json({ limit: '10mb' }));
 
+  const BASE_CURRENCY = configService.get<string>('BASE_CURRENCY');
   const HOST = configService.get<string>('HOST') || '0.0.0.0';
   const PORT = configService.get<number>('PORT') || 3333;
+
   await app.listen(PORT, HOST, () => {
     logLogo();
     Logger.log(`Listening at http://${HOST}:${PORT}`);
     Logger.log('');
+
+    if (BASE_CURRENCY) {
+      Logger.warn(
+        `The environment variable "BASE_CURRENCY" is deprecated and will be removed in Ghostfolio 2.0.`
+      );
+      Logger.warn(
+        'Please use the currency converter in the activity dialog instead.'
+      );
+    }
   });
 }
 

apps/api/src/models/order.ts

@@ -1,8 +1,7 @@
+import { IOrder } from '@ghostfolio/api/services/interfaces/interfaces';
 import { Account, SymbolProfile, Type as TypeOfOrder } from '@prisma/client';
 import { v4 as uuidv4 } from 'uuid';
 
-import { IOrder } from '../services/interfaces/interfaces';
-
 export class Order {
   private account: Account;
   private currency: string;

apps/api/src/models/rule.ts

@@ -1,5 +1,5 @@
 import { RuleSettings } from '@ghostfolio/api/models/interfaces/rule-settings.interface';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { groupBy } from '@ghostfolio/common/helper';
 import { TimelinePosition, UserSettings } from '@ghostfolio/common/interfaces';
 

apps/api/src/models/rules/account-cluster-risk/current-investment.ts

@@ -1,5 +1,5 @@
 import { RuleSettings } from '@ghostfolio/api/models/interfaces/rule-settings.interface';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import {
   PortfolioDetails,
   PortfolioPosition,
@@ -14,7 +14,7 @@ export class AccountClusterRiskCurrentInvestment extends Rule<Settings> {
     private accounts: PortfolioDetails['accounts']
   ) {
     super(exchangeRateDataService, {
-      name: 'Current Investment'
+      name: 'Investment'
     });
   }
 
@@ -28,7 +28,7 @@ export class AccountClusterRiskCurrentInvestment extends Rule<Settings> {
     for (const [accountId, account] of Object.entries(this.accounts)) {
       accounts[accountId] = {
         name: account.name,
-        investment: account.current
+        investment: account.valueInBaseCurrency
       };
     }
 

apps/api/src/models/rules/account-cluster-risk/initial-investment.ts

@@ -1,88 +0,0 @@
-import { RuleSettings } from '@ghostfolio/api/models/interfaces/rule-settings.interface';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
-import {
-  PortfolioDetails,
-  PortfolioPosition,
-  UserSettings
-} from '@ghostfolio/common/interfaces';
-
-import { Rule } from '../../rule';
-
-export class AccountClusterRiskInitialInvestment extends Rule<Settings> {
-  public constructor(
-    protected exchangeRateDataService: ExchangeRateDataService,
-    private accounts: PortfolioDetails['accounts']
-  ) {
-    super(exchangeRateDataService, {
-      name: 'Initial Investment'
-    });
-  }
-
-  public evaluate(ruleSettings?: Settings) {
-    const accounts: {
-      [symbol: string]: Pick<PortfolioPosition, 'name'> & {
-        investment: number;
-      };
-    } = {};
-
-    for (const [accountId, account] of Object.entries(this.accounts)) {
-      accounts[accountId] = {
-        name: account.name,
-        investment: account.original
-      };
-    }
-
-    let maxItem;
-    let totalInvestment = 0;
-
-    for (const account of Object.values(accounts)) {
-      if (!maxItem) {
-        maxItem = account;
-      }
-
-      // Calculate total investment
-      totalInvestment += account.investment;
-
-      // Find maximum
-      if (account.investment > maxItem?.investment) {
-        maxItem = account;
-      }
-    }
-
-    const maxInvestmentRatio = maxItem.investment / totalInvestment;
-
-    if (maxInvestmentRatio > ruleSettings.threshold) {
-      return {
-        evaluation: `Over ${
-          ruleSettings.threshold * 100
-        }% of your initial investment is at ${maxItem.name} (${(
-          maxInvestmentRatio * 100
-        ).toPrecision(3)}%)`,
-        value: false
-      };
-    }
-
-    return {
-      evaluation: `The major part of your initial investment is at ${
-        maxItem.name
-      } (${(maxInvestmentRatio * 100).toPrecision(3)}%) and does not exceed ${
-        ruleSettings.threshold * 100
-      }%`,
-      value: true
-    };
-  }
-
-  public getSettings(aUserSettings: UserSettings): Settings {
-    return {
-      baseCurrency: aUserSettings.baseCurrency,
-      isActive: true,
-      threshold: 0.5
-    };
-  }
-}
-
-interface Settings extends RuleSettings {
-  baseCurrency: string;
-  isActive: boolean;
-  threshold: number;
-}

apps/api/src/models/rules/account-cluster-risk/single-account.ts

@@ -1,5 +1,5 @@
 import { RuleSettings } from '@ghostfolio/api/models/interfaces/rule-settings.interface';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { PortfolioDetails, UserSettings } from '@ghostfolio/common/interfaces';
 
 import { Rule } from '../../rule';

apps/api/src/models/rules/currency-cluster-risk/base-currency-current-investment.ts

@@ -1,5 +1,5 @@
 import { RuleSettings } from '@ghostfolio/api/models/interfaces/rule-settings.interface';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { TimelinePosition, UserSettings } from '@ghostfolio/common/interfaces';
 
 import { Rule } from '../../rule';
@@ -10,7 +10,7 @@ export class CurrencyClusterRiskBaseCurrencyCurrentInvestment extends Rule<Setti
     private positions: TimelinePosition[]
   ) {
     super(exchangeRateDataService, {
-      name: 'Current Investment: Base Currency'
+      name: 'Investment: Base Currency'
     });
   }
 

apps/api/src/models/rules/currency-cluster-risk/base-currency-initial-investment.ts

@@ -1,71 +0,0 @@
-import { RuleSettings } from '@ghostfolio/api/models/interfaces/rule-settings.interface';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
-import { TimelinePosition, UserSettings } from '@ghostfolio/common/interfaces';
-
-import { Rule } from '../../rule';
-
-export class CurrencyClusterRiskBaseCurrencyInitialInvestment extends Rule<Settings> {
-  public constructor(
-    protected exchangeRateDataService: ExchangeRateDataService,
-    private positions: TimelinePosition[]
-  ) {
-    super(exchangeRateDataService, {
-      name: 'Initial Investment: Base Currency'
-    });
-  }
-
-  public evaluate(ruleSettings: Settings) {
-    const positionsGroupedByCurrency = this.groupCurrentPositionsByAttribute(
-      this.positions,
-      'currency',
-      ruleSettings.baseCurrency
-    );
-
-    let maxItem = positionsGroupedByCurrency[0];
-    let totalInvestment = 0;
-
-    positionsGroupedByCurrency.forEach((groupItem) => {
-      // Calculate total investment
-      totalInvestment += groupItem.investment;
-
-      // Find maximum
-      if (groupItem.investment > maxItem.investment) {
-        maxItem = groupItem;
-      }
-    });
-
-    const baseCurrencyItem = positionsGroupedByCurrency.find((item) => {
-      return item.groupKey === ruleSettings.baseCurrency;
-    });
-
-    const baseCurrencyInvestmentRatio =
-      baseCurrencyItem?.investment / totalInvestment || 0;
-
-    if (maxItem.groupKey !== ruleSettings.baseCurrency) {
-      return {
-        evaluation: `The major part of your initial investment is not in your base currency (${(
-          baseCurrencyInvestmentRatio * 100
-        ).toPrecision(3)}% in ${ruleSettings.baseCurrency})`,
-        value: false
-      };
-    }
-
-    return {
-      evaluation: `The major part of your initial investment is in your base currency (${(
-        baseCurrencyInvestmentRatio * 100
-      ).toPrecision(3)}% in ${ruleSettings.baseCurrency})`,
-      value: true
-    };
-  }
-
-  public getSettings(aUserSettings: UserSettings): Settings {
-    return {
-      baseCurrency: aUserSettings.baseCurrency,
-      isActive: true
-    };
-  }
-}
-
-interface Settings extends RuleSettings {
-  baseCurrency: string;
-}

apps/api/src/models/rules/currency-cluster-risk/current-investment.ts

@@ -1,5 +1,5 @@
 import { RuleSettings } from '@ghostfolio/api/models/interfaces/rule-settings.interface';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { TimelinePosition, UserSettings } from '@ghostfolio/common/interfaces';
 
 import { Rule } from '../../rule';
@@ -10,7 +10,7 @@ export class CurrencyClusterRiskCurrentInvestment extends Rule<Settings> {
     private positions: TimelinePosition[]
   ) {
     super(exchangeRateDataService, {
-      name: 'Current Investment'
+      name: 'Investment'
     });
   }
 

apps/api/src/models/rules/currency-cluster-risk/initial-investment.ts

@@ -1,72 +0,0 @@
-import { RuleSettings } from '@ghostfolio/api/models/interfaces/rule-settings.interface';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
-import { TimelinePosition, UserSettings } from '@ghostfolio/common/interfaces';
-
-import { Rule } from '../../rule';
-
-export class CurrencyClusterRiskInitialInvestment extends Rule<Settings> {
-  public constructor(
-    protected exchangeRateDataService: ExchangeRateDataService,
-    private positions: TimelinePosition[]
-  ) {
-    super(exchangeRateDataService, {
-      name: 'Initial Investment'
-    });
-  }
-
-  public evaluate(ruleSettings: Settings) {
-    const positionsGroupedByCurrency = this.groupCurrentPositionsByAttribute(
-      this.positions,
-      'currency',
-      ruleSettings.baseCurrency
-    );
-
-    let maxItem = positionsGroupedByCurrency[0];
-    let totalInvestment = 0;
-
-    positionsGroupedByCurrency.forEach((groupItem) => {
-      // Calculate total investment
-      totalInvestment += groupItem.investment;
-
-      // Find maximum
-      if (groupItem.investment > maxItem.investment) {
-        maxItem = groupItem;
-      }
-    });
-
-    const maxInvestmentRatio = maxItem.investment / totalInvestment;
-
-    if (maxInvestmentRatio > ruleSettings.threshold) {
-      return {
-        evaluation: `Over ${
-          ruleSettings.threshold * 100
-        }% of your initial investment is in ${maxItem.groupKey} (${(
-          maxInvestmentRatio * 100
-        ).toPrecision(3)}%)`,
-        value: false
-      };
-    }
-
-    return {
-      evaluation: `The major part of your initial investment is in ${
-        maxItem.groupKey
-      } (${(maxInvestmentRatio * 100).toPrecision(3)}%) and does not exceed ${
-        ruleSettings.threshold * 100
-      }%`,
-      value: true
-    };
-  }
-
-  public getSettings(aUserSettings: UserSettings): Settings {
-    return {
-      baseCurrency: aUserSettings.baseCurrency,
-      isActive: true,
-      threshold: 0.5
-    };
-  }
-}
-
-interface Settings extends RuleSettings {
-  baseCurrency: string;
-  threshold: number;
-}

apps/api/src/models/rules/fees/fee-ratio-initial-investment.ts

@@ -1,5 +1,5 @@
 import { RuleSettings } from '@ghostfolio/api/models/interfaces/rule-settings.interface';
-import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { UserSettings } from '@ghostfolio/common/interfaces';
 
 import { Rule } from '../../rule';
@@ -11,7 +11,7 @@ export class FeeRatioInitialInvestment extends Rule<Settings> {
     private fees: number
   ) {
     super(exchangeRateDataService, {
-      name: 'Initial Investment'
+      name: 'Investment'
     });
   }
 

apps/api/src/services/configuration/configuration.module.ts

@@ -1,4 +1,4 @@
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { Module } from '@nestjs/common';
 
 @Module({

apps/api/src/services/configuration/configuration.service.ts

@@ -1,9 +1,8 @@
+import { Environment } from '@ghostfolio/api/services/interfaces/environment.interface';
 import { Injectable } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
 import { bool, cleanEnv, host, json, num, port, str } from 'envalid';
 
-import { Environment } from './interfaces/environment.interface';
-
 @Injectable()
 export class ConfigurationService {
   private readonly environmentConfiguration: Environment;
@@ -16,8 +15,10 @@ export class ConfigurationService {
         choices: ['AUD', 'CAD', 'CNY', 'EUR', 'GBP', 'JPY', 'RUB', 'USD'],
         default: 'USD'
       }),
+      BETTER_UPTIME_API_KEY: str({ default: '' }),
       CACHE_TTL: num({ default: 1 }),
-      DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }),
+      DATA_SOURCE_EXCHANGE_RATES: str({ default: DataSource.YAHOO }),
+      DATA_SOURCE_IMPORT: str({ default: DataSource.YAHOO }),
       DATA_SOURCES: json({
         default: [DataSource.COINGECKO, DataSource.MANUAL, DataSource.YAHOO]
       }),
@@ -29,6 +30,7 @@ export class ConfigurationService {
       ENABLE_FEATURE_SUBSCRIPTION: bool({ default: false }),
       ENABLE_FEATURE_SYSTEM_MESSAGE: bool({ default: false }),
       EOD_HISTORICAL_DATA_API_KEY: str({ default: '' }),
+      FINANCIAL_MODELING_PREP_API_KEY: str({ default: '' }),
       GOOGLE_CLIENT_ID: str({ default: 'dummyClientId' }),
       GOOGLE_SECRET: str({ default: 'dummySecret' }),
       GOOGLE_SHEETS_ACCOUNT: str({ default: '' }),

apps/api/src/services/cron.service.ts

@@ -5,8 +5,8 @@ import {
 import { Injectable } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
 
-import { DataGatheringService } from './data-gathering.service';
+import { DataGatheringService } from './data-gathering/data-gathering.service';
-import { ExchangeRateDataService } from './exchange-rate-data.service';
+import { ExchangeRateDataService } from './exchange-rate-data/exchange-rate-data.service';
 import { TwitterBotService } from './twitter-bot/twitter-bot.service';
 
 @Injectable()
@@ -19,8 +19,8 @@ export class CronService {
     private readonly twitterBotService: TwitterBotService
   ) {}
 
-  @Cron(CronExpression.EVERY_4_HOURS)
+  @Cron(CronExpression.EVERY_HOUR)
-  public async runEveryFourHours() {
+  public async runEveryHour() {
     await this.dataGatheringService.gather7Days();
   }
 
@@ -38,15 +38,20 @@ export class CronService {
   public async runEverySundayAtTwelvePm() {
     const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
 
-    for (const { dataSource, symbol } of uniqueAssets) {
+    await this.dataGatheringService.addJobsToQueue(
-      await this.dataGatheringService.addJobToQueue(
+      uniqueAssets.map(({ dataSource, symbol }) => {
-        GATHER_ASSET_PROFILE_PROCESS,
+        return {
-        {
+          data: {
-          dataSource,
+            dataSource,
-          symbol
+            symbol
-        },
+          },
-        GATHER_ASSET_PROFILE_PROCESS_OPTIONS
+          name: GATHER_ASSET_PROFILE_PROCESS,
-      );
+          opts: {
-    }
+            ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+            jobId: `${dataSource}-${symbol}`
+          }
+        };
+      })
+    );
   }
 }

apps/api/src/services/data-gathering/data-gathering.module.ts

@@ -1,17 +1,17 @@
-import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
-import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
 import { DataEnhancerModule } from '@ghostfolio/api/services/data-provider/data-enhancer/data-enhancer.module';
 import { DataProviderModule } from '@ghostfolio/api/services/data-provider/data-provider.module';
-import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
+import { ExchangeRateDataModule } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.module';
+import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module';
+import { PrismaModule } from '@ghostfolio/api/services/prisma/prisma.module';
+import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile/symbol-profile.module';
 import { DATA_GATHERING_QUEUE } from '@ghostfolio/common/config';
 import { BullModule } from '@nestjs/bull';
 import { Module } from '@nestjs/common';
 import ms from 'ms';
 
 import { DataGatheringProcessor } from './data-gathering.processor';
-import { ExchangeRateDataModule } from './exchange-rate-data.module';
-import { MarketDataModule } from './market-data.module';
-import { SymbolProfileModule } from './symbol-profile.module';
 
 @Module({
   imports: [

apps/api/src/services/data-gathering/data-gathering.processor.ts

@@ -1,12 +1,16 @@
+import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
+import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import {
   DATA_GATHERING_QUEUE,
   GATHER_ASSET_PROFILE_PROCESS,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS
 } from '@ghostfolio/common/config';
-import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { Process, Processor } from '@nestjs/bull';
 import { Injectable, Logger } from '@nestjs/common';
+import { Prisma } from '@prisma/client';
 import { Job } from 'bull';
 import {
   format,
@@ -18,9 +22,6 @@ import {
 } from 'date-fns';
 
 import { DataGatheringService } from './data-gathering.service';
-import { DataProviderService } from './data-provider/data-provider.service';
-import { IDataGatheringItem } from './interfaces/interfaces';
-import { PrismaService } from './prisma.service';
 
 @Injectable()
 @Processor(DATA_GATHERING_QUEUE)
@@ -28,10 +29,10 @@ export class DataGatheringProcessor {
   public constructor(
     private readonly dataGatheringService: DataGatheringService,
     private readonly dataProviderService: DataProviderService,
-    private readonly prismaService: PrismaService
+    private readonly marketDataService: MarketDataService
   ) {}
 
-  @Process(GATHER_ASSET_PROFILE_PROCESS)
+  @Process({ concurrency: 1, name: GATHER_ASSET_PROFILE_PROCESS })
   public async gatherAssetProfile(job: Job<UniqueAsset>) {
     try {
       await this.dataGatheringService.gatherAssetProfiles([job.data]);
@@ -45,18 +46,27 @@ export class DataGatheringProcessor {
     }
   }
 
-  @Process(GATHER_HISTORICAL_MARKET_DATA_PROCESS)
+  @Process({ concurrency: 1, name: GATHER_HISTORICAL_MARKET_DATA_PROCESS })
   public async gatherHistoricalMarketData(job: Job<IDataGatheringItem>) {
     try {
       const { dataSource, date, symbol } = job.data;
+      let currentDate = parseISO(<string>(<unknown>date));
+
+      Logger.log(
+        `Historical market data gathering has been started for ${symbol} (${dataSource}) at ${format(
+          currentDate,
+          DATE_FORMAT
+        )}`,
+        `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS})`
+      );
 
       const historicalData = await this.dataProviderService.getHistoricalRaw(
         [{ dataSource, symbol }],
-        parseISO(<string>(<unknown>date)),
+        currentDate,
         new Date()
       );
 
-      let currentDate = parseISO(<string>(<unknown>date));
+      const data: Prisma.MarketDataUpdateInput[] = [];
       let lastMarketPrice: number;
 
       while (
@@ -82,23 +92,13 @@ export class DataGatheringProcessor {
         }
 
         if (lastMarketPrice) {
-          try {
+          data.push({
-            await this.prismaService.marketData.create({
+            dataSource,
-              data: {
+            symbol,
-                dataSource,
+            date: getStartOfUtcDate(currentDate),
-                symbol,
+            marketPrice: lastMarketPrice,
-                date: new Date(
+            state: 'CLOSE'
-                  Date.UTC(
+          });
-                    getYear(currentDate),
-                    getMonth(currentDate),
-                    getDate(currentDate),
-                    0
-                  )
-                ),
-                marketPrice: lastMarketPrice
-              }
-            });
-          } catch {}
         }
 
         // Count month one up for iteration
@@ -112,8 +112,13 @@ export class DataGatheringProcessor {
         );
       }
 
+      await this.marketDataService.updateMany({ data });
+
       Logger.log(
-        `Historical market data gathering has been completed for ${symbol} (${dataSource}).`,
+        `Historical market data gathering has been completed for ${symbol} (${dataSource}) at ${format(
+          currentDate,
+          DATE_FORMAT
+        )}`,
         `DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS})`
       );
     } catch (error) {

apps/api/src/services/data-gathering/data-gathering.service.ts

@@ -1,9 +1,14 @@
-import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
+import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
+import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
+import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
+import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
+import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
+import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
+import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
 import {
   DATA_GATHERING_QUEUE,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS,
-  GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
+  GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS
-  QUEUE_JOB_STATUS_LIST
 } from '@ghostfolio/common/config';
 import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
@@ -12,13 +17,7 @@ import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
 import { JobOptions, Queue } from 'bull';
 import { format, min, subDays, subYears } from 'date-fns';
-
+import { isEmpty } from 'lodash';
-import { DataProviderService } from './data-provider/data-provider.service';
-import { DataEnhancerInterface } from './data-provider/interfaces/data-enhancer.interface';
-import { ExchangeRateDataService } from './exchange-rate-data.service';
-import { IDataGatheringItem } from './interfaces/interfaces';
-import { MarketDataService } from './market-data.service';
-import { PrismaService } from './prisma.service';
 
 @Injectable()
 export class DataGatheringService {
@@ -34,17 +33,22 @@ export class DataGatheringService {
     private readonly symbolProfileService: SymbolProfileService
   ) {}
 
-  public async addJobToQueue(name: string, data: any, options?: JobOptions) {
+  public async addJobToQueue({
-    const hasJob = await this.hasJob(name, data);
+    data,
+    name,
+    opts
+  }: {
+    data: any;
+    name: string;
+    opts?: JobOptions;
+  }) {
+    return this.dataGatheringQueue.add(name, data, opts);
+  }
 
-    if (hasJob) {
+  public async addJobsToQueue(
-      Logger.log(
+    jobs: { data: any; name: string; opts?: JobOptions }[]
-        `Job ${name} with data ${JSON.stringify(data)} already exists.`,
+  ) {
-        'DataGatheringService'
+    return this.dataGatheringQueue.addBulk(jobs);
-      );
-    } else {
-      return this.dataGatheringQueue.add(name, data, options);
-    }
   }
 
   public async gather7Days() {
@@ -97,7 +101,7 @@ export class DataGatheringService {
             symbol
           },
           update: { marketPrice },
-          where: { date_symbol: { date, symbol } }
+          where: { dataSource_date_symbol: { dataSource, date, symbol } }
         });
       }
     } catch (error) {
@@ -119,12 +123,9 @@ export class DataGatheringService {
     const assetProfiles = await this.dataProviderService.getAssetProfiles(
       uniqueAssets
     );
-    const symbolProfiles =
+    const symbolProfiles = await this.symbolProfileService.getSymbolProfiles(
-      await this.symbolProfileService.getSymbolProfilesBySymbols(
+      uniqueAssets
-        uniqueAssets.map(({ symbol }) => {
+    );
-          return symbol;
-        })
-      );
 
     for (const [symbol, assetProfile] of Object.entries(assetProfiles)) {
       const symbolMapping = symbolProfiles.find((symbolProfile) => {
@@ -209,59 +210,22 @@ export class DataGatheringService {
   }
 
   public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
-    for (const { dataSource, date, symbol } of aSymbolsWithStartDate) {
+    await this.addJobsToQueue(
-      await this.addJobToQueue(
+      aSymbolsWithStartDate.map(({ dataSource, date, symbol }) => {
-        GATHER_HISTORICAL_MARKET_DATA_PROCESS,
-        {
-          dataSource,
-          date,
-          symbol
-        },
-        GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS
-      );
-    }
-  }
-
-  public async getSymbolsMax(): Promise<IDataGatheringItem[]> {
-    const startDate =
-      (
-        await this.prismaService.order.findFirst({
-          orderBy: [{ date: 'asc' }]
-        })
-      )?.date ?? new Date();
-
-    const currencyPairsToGather = this.exchangeRateDataService
-      .getCurrencyPairs()
-      .map(({ dataSource, symbol }) => {
         return {
-          dataSource,
+          data: {
-          symbol,
+            dataSource,
-          date: min([startDate, subYears(new Date(), 10)])
+            date,
-        };
+            symbol
-      });
-
-    const symbolProfilesToGather = (
-      await this.prismaService.symbolProfile.findMany({
-        orderBy: [{ symbol: 'asc' }],
-        select: {
-          dataSource: true,
-          Order: {
-            orderBy: [{ date: 'asc' }],
-            select: { date: true },
-            take: 1
           },
-          scraperConfiguration: true,
+          name: GATHER_HISTORICAL_MARKET_DATA_PROCESS,
-          symbol: true
+          opts: {
-        }
+            ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
+            jobId: `${dataSource}-${symbol}-${format(date, DATE_FORMAT)}`
+          }
+        };
       })
-    ).map((symbolProfile) => {
+    );
-      return {
-        ...symbolProfile,
-        date: symbolProfile.Order?.[0]?.date ?? startDate
-      };
-    });
-
-    return [...currencyPairsToGather, ...symbolProfilesToGather];
   }
 
   public async getUniqueAssets(): Promise<UniqueAsset[]> {
@@ -298,13 +262,14 @@ export class DataGatheringService {
 
     // Only consider symbols with incomplete market data for the last
     // 7 days
-    const symbolsNotToGather = (
+    const symbolsWithCompleteMarketData = (
       await this.prismaService.marketData.groupBy({
         _count: true,
         by: ['symbol'],
         orderBy: [{ symbol: 'asc' }],
         where: {
-          date: { gt: startDate }
+          date: { gt: startDate },
+          state: 'CLOSE'
         }
       })
     )
@@ -316,8 +281,14 @@ export class DataGatheringService {
       });
 
     const symbolProfilesToGather = symbolProfiles
-      .filter(({ symbol }) => {
+      .filter(({ dataSource, scraperConfiguration, symbol }) => {
-        return !symbolsNotToGather.includes(symbol);
+        const manualDataSourceWithScraperConfiguration =
+          dataSource === 'MANUAL' && !isEmpty(scraperConfiguration);
+
+        return (
+          !symbolsWithCompleteMarketData.includes(symbol) &&
+          (dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration)
+        );
       })
       .map((symbolProfile) => {
         return {
@@ -329,7 +300,7 @@ export class DataGatheringService {
     const currencyPairsToGather = this.exchangeRateDataService
       .getCurrencyPairs()
       .filter(({ symbol }) => {
-        return !symbolsNotToGather.includes(symbol);
+        return !symbolsWithCompleteMarketData.includes(symbol);
       })
       .map(({ dataSource, symbol }) => {
         return {
@@ -342,17 +313,56 @@ export class DataGatheringService {
     return [...currencyPairsToGather, ...symbolProfilesToGather];
   }
 
-  private async hasJob(name: string, data: any) {
+  private async getSymbolsMax(): Promise<IDataGatheringItem[]> {
-    const jobs = await this.dataGatheringQueue.getJobs(
+    const startDate =
-      QUEUE_JOB_STATUS_LIST.filter((status) => {
+      (
-        return status !== 'completed';
+        await this.prismaService.order.findFirst({
-      })
+          orderBy: [{ date: 'asc' }]
-    );
+        })
+      )?.date ?? new Date();
 
-    return jobs.some((job) => {
+    const currencyPairsToGather = this.exchangeRateDataService
-      return (
+      .getCurrencyPairs()
-        job.name === name && JSON.stringify(job.data) === JSON.stringify(data)
+      .map(({ dataSource, symbol }) => {
-      );
+        return {
-    });
+          dataSource,
+          symbol,
+          date: min([startDate, subYears(new Date(), 10)])
+        };
+      });
+
+    const symbolProfilesToGather = (
+      await this.prismaService.symbolProfile.findMany({
+        orderBy: [{ symbol: 'asc' }],
+        select: {
+          dataSource: true,
+          Order: {
+            orderBy: [{ date: 'asc' }],
+            select: { date: true },
+            take: 1
+          },
+          scraperConfiguration: true,
+          symbol: true
+        }
+      })
+    )
+      .filter((symbolProfile) => {
+        const manualDataSourceWithScraperConfiguration =
+          symbolProfile.dataSource === 'MANUAL' &&
+          !isEmpty(symbolProfile.scraperConfiguration);
+
+        return (
+          symbolProfile.dataSource !== 'MANUAL' ||
+          manualDataSourceWithScraperConfiguration
+        );
+      })
+      .map((symbolProfile) => {
+        return {
+          ...symbolProfile,
+          date: symbolProfile.Order?.[0]?.date ?? startDate
+        };
+      });
+
+    return [...currencyPairsToGather, ...symbolProfilesToGather];
   }
 }

apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts

@@ -1,5 +1,5 @@
 import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
 import {
   IDataProviderHistoricalResponse,
@@ -7,7 +7,7 @@ import {
 } from '@ghostfolio/api/services/interfaces/interfaces';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
 import { DataSource, SymbolProfile } from '@prisma/client';
 import { format, isAfter, isBefore, parse } from 'date-fns';
 
@@ -33,7 +33,8 @@ export class AlphaVantageService implements DataProviderInterface {
     aSymbol: string
   ): Promise<Partial<SymbolProfile>> {
     return {
-      dataSource: this.getName()
+      dataSource: this.getName(),
+      symbol: aSymbol
     };
   }
 
@@ -109,6 +110,10 @@ export class AlphaVantageService implements DataProviderInterface {
     return {};
   }
 
+  public getTestSymbol() {
+    return undefined;
+  }
+
   public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
     const result = await this.alphaVantage.data.search(aQuery);
 

apps/api/src/services/data-provider/coingecko/coingecko.service.ts

@@ -1,5 +1,5 @@
 import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
 import {
   IDataProviderHistoricalResponse,
@@ -160,6 +160,10 @@ export class CoinGeckoService implements DataProviderInterface {
     return results;
   }
 
+  public getTestSymbol() {
+    return 'bitcoin';
+  }
+
   public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
     let items: LookupItem[] = [];
 

apps/api/src/services/data-provider/data-enhancer/data-enhancer.module.ts

@@ -1,15 +1,27 @@
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
+import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';
 import { TrackinsightDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/trackinsight/trackinsight.service';
+import { YahooFinanceDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service';
 import { Module } from '@nestjs/common';
 
 @Module({
-  exports: ['DataEnhancers', TrackinsightDataEnhancerService],
+  exports: [
+    'DataEnhancers',
+    TrackinsightDataEnhancerService,
+    YahooFinanceDataEnhancerService
+  ],
+  imports: [ConfigurationModule, CryptocurrencyModule],
   providers: [
+    TrackinsightDataEnhancerService,
+    YahooFinanceDataEnhancerService,
     {
-      inject: [TrackinsightDataEnhancerService],
+      inject: [
+        TrackinsightDataEnhancerService,
+        YahooFinanceDataEnhancerService
+      ],
       provide: 'DataEnhancers',
-      useFactory: (trackinsight) => [trackinsight]
+      useFactory: (trackinsight, yahooFinance) => [trackinsight, yahooFinance]
-    },
+    }
-    TrackinsightDataEnhancerService
   ]
 })
 export class DataEnhancerModule {}

apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts

@@ -1,11 +1,13 @@
 import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
 import { Country } from '@ghostfolio/common/interfaces/country.interface';
 import { Sector } from '@ghostfolio/common/interfaces/sector.interface';
+import { Injectable } from '@nestjs/common';
 import { SymbolProfile } from '@prisma/client';
 import bent from 'bent';
 
 const getJSON = bent('json');
 
+@Injectable()
 export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
   private static baseUrl = 'https://data.trackinsight.com';
   private static countries = require('countries-list/dist/countries.json');

apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.spec.ts

@@ -1,7 +1,7 @@
-import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service';
 
-import { YahooFinanceService } from './yahoo-finance.service';
+import { YahooFinanceDataEnhancerService } from './yahoo-finance.service';
 
 jest.mock(
   '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service',
@@ -25,16 +25,16 @@ jest.mock(
   }
 );
 
-describe('YahooFinanceService', () => {
+describe('YahooFinanceDataEnhancerService', () => {
   let configurationService: ConfigurationService;
   let cryptocurrencyService: CryptocurrencyService;
-  let yahooFinanceService: YahooFinanceService;
+  let yahooFinanceDataEnhancerService: YahooFinanceDataEnhancerService;
 
   beforeAll(async () => {
     configurationService = new ConfigurationService();
     cryptocurrencyService = new CryptocurrencyService();
 
-    yahooFinanceService = new YahooFinanceService(
+    yahooFinanceDataEnhancerService = new YahooFinanceDataEnhancerService(
       configurationService,
       cryptocurrencyService
     );
@@ -42,25 +42,37 @@ describe('YahooFinanceService', () => {
 
   it('convertFromYahooFinanceSymbol', async () => {
     expect(
-      await yahooFinanceService.convertFromYahooFinanceSymbol('BRK-B')
+      await yahooFinanceDataEnhancerService.convertFromYahooFinanceSymbol(
+        'BRK-B'
+      )
     ).toEqual('BRK-B');
     expect(
-      await yahooFinanceService.convertFromYahooFinanceSymbol('BTC-USD')
+      await yahooFinanceDataEnhancerService.convertFromYahooFinanceSymbol(
+        'BTC-USD'
+      )
     ).toEqual('BTCUSD');
     expect(
-      await yahooFinanceService.convertFromYahooFinanceSymbol('EURUSD=X')
+      await yahooFinanceDataEnhancerService.convertFromYahooFinanceSymbol(
+        'EURUSD=X'
+      )
     ).toEqual('EURUSD');
   });
 
   it('convertToYahooFinanceSymbol', async () => {
     expect(
-      await yahooFinanceService.convertToYahooFinanceSymbol('BTCUSD')
+      await yahooFinanceDataEnhancerService.convertToYahooFinanceSymbol(
+        'BTCUSD'
+      )
     ).toEqual('BTC-USD');
     expect(
-      await yahooFinanceService.convertToYahooFinanceSymbol('DOGEUSD')
+      await yahooFinanceDataEnhancerService.convertToYahooFinanceSymbol(
+        'DOGEUSD'
+      )
     ).toEqual('DOGE-USD');
     expect(
-      await yahooFinanceService.convertToYahooFinanceSymbol('USDCHF')
+      await yahooFinanceDataEnhancerService.convertToYahooFinanceSymbol(
+        'USDCHF'
+      )
     ).toEqual('USDCHF=X');
   });
 });

apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts

@@ -0,0 +1,325 @@
+import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
+import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service';
+import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
+import { UNKNOWN_KEY } from '@ghostfolio/common/config';
+import { isCurrency } from '@ghostfolio/common/helper';
+import { Injectable, Logger } from '@nestjs/common';
+import {
+  AssetClass,
+  AssetSubClass,
+  DataSource,
+  SymbolProfile
+} from '@prisma/client';
+import { countries } from 'countries-list';
+import yahooFinance from 'yahoo-finance2';
+import type { Price } from 'yahoo-finance2/dist/esm/src/modules/quoteSummary-iface';
+
+@Injectable()
+export class YahooFinanceDataEnhancerService implements DataEnhancerInterface {
+  private baseCurrency: string;
+
+  public constructor(
+    private readonly configurationService: ConfigurationService,
+    private readonly cryptocurrencyService: CryptocurrencyService
+  ) {
+    this.baseCurrency = this.configurationService.get('BASE_CURRENCY');
+  }
+
+  public convertFromYahooFinanceSymbol(aYahooFinanceSymbol: string) {
+    let symbol = aYahooFinanceSymbol.replace(
+      new RegExp(`-${this.baseCurrency}$`),
+      this.baseCurrency
+    );
+
+    if (symbol.includes('=X') && !symbol.includes(this.baseCurrency)) {
+      symbol = `${this.baseCurrency}${symbol}`;
+    }
+
+    return symbol.replace('=X', '');
+  }
+
+  /**
+   * Converts a symbol to a Yahoo Finance symbol
+   *
+   * Currency:        USDCHF  -> USDCHF=X
+   * Cryptocurrency:  BTCUSD  -> BTC-USD
+   *                  DOGEUSD -> DOGE-USD
+   */
+  public convertToYahooFinanceSymbol(aSymbol: string) {
+    if (
+      aSymbol.includes(this.baseCurrency) &&
+      aSymbol.length > this.baseCurrency.length
+    ) {
+      if (
+        isCurrency(
+          aSymbol.substring(0, aSymbol.length - this.baseCurrency.length)
+        )
+      ) {
+        return `${aSymbol}=X`;
+      } else if (
+        this.cryptocurrencyService.isCryptocurrency(
+          aSymbol.replace(
+            new RegExp(`-${this.baseCurrency}$`),
+            this.baseCurrency
+          )
+        )
+      ) {
+        // Add a dash before the last three characters
+        // BTCUSD  -> BTC-USD
+        // DOGEUSD -> DOGE-USD
+        // SOL1USD -> SOL1-USD
+        return aSymbol.replace(
+          new RegExp(`-?${this.baseCurrency}$`),
+          `-${this.baseCurrency}`
+        );
+      }
+    }
+
+    return aSymbol;
+  }
+
+  public async enhance({
+    response,
+    symbol
+  }: {
+    response: Partial<SymbolProfile>;
+    symbol: string;
+  }): Promise<Partial<SymbolProfile>> {
+    if (response.dataSource !== 'YAHOO' && !response.isin) {
+      return response;
+    }
+
+    try {
+      let yahooSymbol: string;
+
+      if (response.dataSource === 'YAHOO') {
+        yahooSymbol = symbol;
+      } else {
+        const { quotes } = await yahooFinance.search(response.isin);
+        yahooSymbol = quotes[0].symbol;
+      }
+
+      const { countries, sectors, url } = await this.getAssetProfile(
+        yahooSymbol
+      );
+
+      if (countries) {
+        response.countries = countries;
+      }
+
+      if (sectors) {
+        response.sectors = sectors;
+      }
+
+      if (url) {
+        response.url = url;
+      }
+    } catch (error) {
+      Logger.error(error, 'YahooFinanceDataEnhancerService');
+    }
+
+    return response;
+  }
+
+  public formatName({
+    longName,
+    quoteType,
+    shortName,
+    symbol
+  }: {
+    longName: Price['longName'];
+    quoteType: Price['quoteType'];
+    shortName: Price['shortName'];
+    symbol: Price['symbol'];
+  }) {
+    let name = longName;
+
+    if (name) {
+      name = name.replace('Amundi Index Solutions - ', '');
+      name = name.replace('iShares ETF (CH) - ', '');
+      name = name.replace('iShares III Public Limited Company - ', '');
+      name = name.replace('iShares V PLC - ', '');
+      name = name.replace('iShares VI Public Limited Company - ', '');
+      name = name.replace('iShares VII PLC - ', '');
+      name = name.replace('Multi Units Luxembourg - ', '');
+      name = name.replace('VanEck ETFs N.V. - ', '');
+      name = name.replace('Vaneck Vectors Ucits Etfs Plc - ', '');
+      name = name.replace('Vanguard Funds Public Limited Company - ', '');
+      name = name.replace('Vanguard Index Funds - ', '');
+      name = name.replace('Xtrackers (IE) Plc - ', '');
+    }
+
+    if (quoteType === 'FUTURE') {
+      // "Gold Jun 22" -> "Gold"
+      name = shortName?.slice(0, -7);
+    }
+
+    return name || shortName || symbol;
+  }
+
+  public async getAssetProfile(
+    aSymbol: string
+  ): Promise<Partial<SymbolProfile>> {
+    const response: Partial<SymbolProfile> = {};
+
+    try {
+      const symbol = this.convertToYahooFinanceSymbol(aSymbol);
+      const assetProfile = await yahooFinance.quoteSummary(symbol, {
+        modules: ['price', 'summaryProfile', 'topHoldings']
+      });
+
+      const { assetClass, assetSubClass } = this.parseAssetClass({
+        quoteType: assetProfile.price.quoteType,
+        shortName: assetProfile.price.shortName
+      });
+
+      response.assetClass = assetClass;
+      response.assetSubClass = assetSubClass;
+      response.currency = assetProfile.price.currency;
+      response.dataSource = this.getName();
+      response.name = this.formatName({
+        longName: assetProfile.price.longName,
+        quoteType: assetProfile.price.quoteType,
+        shortName: assetProfile.price.shortName,
+        symbol: assetProfile.price.symbol
+      });
+      response.symbol = aSymbol;
+
+      if (assetSubClass === AssetSubClass.MUTUALFUND) {
+        response.sectors = [];
+
+        for (const sectorWeighting of assetProfile.topHoldings
+          ?.sectorWeightings ?? []) {
+          for (const [sector, weight] of Object.entries(sectorWeighting)) {
+            response.sectors.push({ weight, name: this.parseSector(sector) });
+          }
+        }
+      } else if (
+        assetSubClass === AssetSubClass.STOCK &&
+        assetProfile.summaryProfile?.country
+      ) {
+        // Add country if asset is stock and country available
+
+        try {
+          const [code] = Object.entries(countries).find(([, country]) => {
+            return country.name === assetProfile.summaryProfile?.country;
+          });
+
+          if (code) {
+            response.countries = [{ code, weight: 1 }];
+          }
+        } catch {}
+
+        if (assetProfile.summaryProfile?.sector) {
+          response.sectors = [
+            { name: assetProfile.summaryProfile?.sector, weight: 1 }
+          ];
+        }
+      }
+
+      const url = assetProfile.summaryProfile?.website;
+      if (url) {
+        response.url = url;
+      }
+    } catch (error) {
+      Logger.error(error, 'YahooFinanceService');
+    }
+
+    return response;
+  }
+
+  public getName() {
+    return DataSource.YAHOO;
+  }
+
+  public parseAssetClass({
+    quoteType,
+    shortName
+  }: {
+    quoteType: string;
+    shortName: string;
+  }): {
+    assetClass: AssetClass;
+    assetSubClass: AssetSubClass;
+  } {
+    let assetClass: AssetClass;
+    let assetSubClass: AssetSubClass;
+
+    switch (quoteType?.toLowerCase()) {
+      case 'cryptocurrency':
+        assetClass = AssetClass.CASH;
+        assetSubClass = AssetSubClass.CRYPTOCURRENCY;
+        break;
+      case 'equity':
+        assetClass = AssetClass.EQUITY;
+        assetSubClass = AssetSubClass.STOCK;
+        break;
+      case 'etf':
+        assetClass = AssetClass.EQUITY;
+        assetSubClass = AssetSubClass.ETF;
+        break;
+      case 'future':
+        assetClass = AssetClass.COMMODITY;
+        assetSubClass = AssetSubClass.COMMODITY;
+
+        if (
+          shortName?.toLowerCase()?.startsWith('gold') ||
+          shortName?.toLowerCase()?.startsWith('palladium') ||
+          shortName?.toLowerCase()?.startsWith('platinum') ||
+          shortName?.toLowerCase()?.startsWith('silver')
+        ) {
+          assetSubClass = AssetSubClass.PRECIOUS_METAL;
+        }
+
+        break;
+      case 'mutualfund':
+        assetClass = AssetClass.EQUITY;
+        assetSubClass = AssetSubClass.MUTUALFUND;
+        break;
+    }
+
+    return { assetClass, assetSubClass };
+  }
+
+  private parseSector(aString: string): string {
+    let sector = UNKNOWN_KEY;
+
+    switch (aString) {
+      case 'basic_materials':
+        sector = 'Basic Materials';
+        break;
+      case 'communication_services':
+        sector = 'Communication Services';
+        break;
+      case 'consumer_cyclical':
+        sector = 'Consumer Cyclical';
+        break;
+      case 'consumer_defensive':
+        sector = 'Consumer Staples';
+        break;
+      case 'energy':
+        sector = 'Energy';
+        break;
+      case 'financial_services':
+        sector = 'Financial Services';
+        break;
+      case 'healthcare':
+        sector = 'Healthcare';
+        break;
+      case 'industrials':
+        sector = 'Industrials';
+        break;
+      case 'realestate':
+        sector = 'Real Estate';
+        break;
+      case 'technology':
+        sector = 'Technology';
+        break;
+      case 'utilities':
+        sector = 'Utilities';
+        break;
+    }
+
+    return sector;
+  }
+}