Release 1.257.0 (#1872)

* Clone countries before manipulation

* Update changelog

CHANGELOG.md

@@ -5,6 +5,67 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 1.257.0 - 2023-04-18
+
+### Added
+
+- Introduced the allocations by ETF provider chart on the allocations page
+
+### Fixed
+
+- Fixed an issue in the global heat map component caused by manipulating an input property
+- Fixed an issue with the currency inconsistency in the _EOD Historical Data_ service (convert from `GBX` to `GBp`)
+
+## 1.256.0 - 2023-04-17
+
+### Added
+
+- Added the _Yahoo Finance_ data enhancer for countries, sectors and urls
+
+### Changed
+
+- Enabled the configuration to immediately remove queue jobs on complete
+- Refactored the implementation of removing queue jobs
+
+### Fixed
+
+- Fixed the unique job ids of the gather asset profile process
+- Fixed the style of the button to fetch the current market price
+
+## 1.255.0 - 2023-04-15
+
+### Added
+
+- Made the system message expandable
+
+### Changed
+
+- Skipped creating queue jobs for asset profiles with `MANUAL` data source not having a scraper configuration
+- Reduced the execution interval of the data gathering to every hour
+- Upgraded `prisma` from version `4.11.0` to `4.12.0`
+
+### Fixed
+
+- Improved the style of the system message
+
+## 1.254.0 - 2023-04-14
+
+### Changed
+
+- Improved the queue jobs implementation by adding in bulk
+- Improved the queue jobs implementation by introducing unique job ids
+- Reverted the execution interval of the data gathering from every 12 hours to every 4 hours
+
+## 1.253.0 - 2023-04-14
+
+### Changed
+
+- Reduced the execution interval of the data gathering to every 12 hours
+
+### Fixed
+
+- Fixed the background color of dialogs in dark mode
+
 ## 1.252.2 - 2023-04-11
 
 ### Changed

apps/api/src/app/admin/admin.controller.ts

@@ -100,16 +100,21 @@ export class AdminController {
 
     const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
 
-    for (const { dataSource, symbol } of uniqueAssets) {
-      await this.dataGatheringService.addJobToQueue(
-        GATHER_ASSET_PROFILE_PROCESS,
-        {
-          dataSource,
-          symbol
-        },
-        GATHER_ASSET_PROFILE_PROCESS_OPTIONS
-      );
-    }
+    await this.dataGatheringService.addJobsToQueue(
+      uniqueAssets.map(({ dataSource, symbol }) => {
+        return {
+          data: {
+            dataSource,
+            symbol
+          },
+          name: GATHER_ASSET_PROFILE_PROCESS,
+          opts: {
+            ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+            jobId: `${dataSource}-${symbol}`
+          }
+        };
+      })
+    );
 
     this.dataGatheringService.gatherMax();
   }
@@ -131,16 +136,21 @@ export class AdminController {
 
     const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
 
-    for (const { dataSource, symbol } of uniqueAssets) {
-      await this.dataGatheringService.addJobToQueue(
-        GATHER_ASSET_PROFILE_PROCESS,
-        {
-          dataSource,
-          symbol
-        },
-        GATHER_ASSET_PROFILE_PROCESS_OPTIONS
-      );
-    }
+    await this.dataGatheringService.addJobsToQueue(
+      uniqueAssets.map(({ dataSource, symbol }) => {
+        return {
+          data: {
+            dataSource,
+            symbol
+          },
+          name: GATHER_ASSET_PROFILE_PROCESS,
+          opts: {
+            ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+            jobId: `${dataSource}-${symbol}`
+          }
+        };
+      })
+    );
   }
 
   @Post('gather/profile-data/:dataSource/:symbol')
@@ -161,14 +171,17 @@ export class AdminController {
       );
     }
 
-    await this.dataGatheringService.addJobToQueue(
-      GATHER_ASSET_PROFILE_PROCESS,
-      {
+    await this.dataGatheringService.addJobToQueue({
+      data: {
         dataSource,
         symbol
       },
-      GATHER_ASSET_PROFILE_PROCESS_OPTIONS
-    );
+      name: GATHER_ASSET_PROFILE_PROCESS,
+      opts: {
+        ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+        jobId: `${dataSource}-${symbol}`
+      }
+    });
   }
 
   @Post('gather/:dataSource/:symbol')

apps/api/src/app/admin/queue/queue.service.ts

@@ -4,7 +4,7 @@ import {
 } from '@ghostfolio/common/config';
 import { AdminJobs } from '@ghostfolio/common/interfaces';
 import { InjectQueue } from '@nestjs/bull';
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
 import { JobStatus, Queue } from 'bull';
 
 @Injectable()
@@ -23,14 +23,11 @@ export class QueueService {
   }: {
     status?: JobStatus[];
   }) {
-    const jobs = await this.dataGatheringQueue.getJobs(status);
-
-    for (const job of jobs) {
-      try {
-        await job.remove();
-      } catch (error) {
-        Logger.warn(error, 'QueueService');
-      }
+    for (const statusItem of status) {
+      await this.dataGatheringQueue.clean(
+        300,
+        statusItem === 'waiting' ? 'wait' : statusItem
+      );
     }
   }
 
@@ -44,18 +41,23 @@ export class QueueService {
     const jobs = await this.dataGatheringQueue.getJobs(status);
 
     const jobsWithState = await Promise.all(
-      jobs.slice(0, limit).map(async (job) => {
-        return {
-          attemptsMade: job.attemptsMade + 1,
-          data: job.data,
-          finishedOn: job.finishedOn,
-          id: job.id,
-          name: job.name,
-          stacktrace: job.stacktrace,
-          state: await job.getState(),
-          timestamp: job.timestamp
-        };
-      })
+      jobs
+        .filter((job) => {
+          return job;
+        })
+        .slice(0, limit)
+        .map(async (job) => {
+          return {
+            attemptsMade: job.attemptsMade + 1,
+            data: job.data,
+            finishedOn: job.finishedOn,
+            id: job.id,
+            name: job.name,
+            stacktrace: job.stacktrace,
+            state: await job.getState(),
+            timestamp: job.timestamp
+          };
+        })
     );
 
     return {

apps/api/src/app/order/order.service.ts

@@ -112,14 +112,17 @@ export class OrderService {
       };
     }
 
-    await this.dataGatheringService.addJobToQueue(
-      GATHER_ASSET_PROFILE_PROCESS,
-      {
+    await this.dataGatheringService.addJobToQueue({
+      data: {
         dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
         symbol: data.SymbolProfile.connectOrCreate.create.symbol
       },
-      GATHER_ASSET_PROFILE_PROCESS_OPTIONS
-    );
+      name: GATHER_ASSET_PROFILE_PROCESS,
+      opts: {
+        ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+        jobId: `${data.SymbolProfile.connectOrCreate.create.dataSource}-${data.SymbolProfile.connectOrCreate.create.symbol}`
+      }
+    });
 
     const isDraft = isAfter(data.date as Date, endOfToday());
 

apps/api/src/app/portfolio/portfolio-calculator.ts

@@ -722,7 +722,7 @@ export class PortfolioCalculator {
         );
       } else if (!currentPosition.quantity.eq(0)) {
         Logger.warn(
-          `Missing initial value for symbol ${currentPosition.symbol} at ${currentPosition.firstBuyDate}`,
+          `Missing historical market data for symbol ${currentPosition.symbol}`,
           'PortfolioCalculator'
         );
         hasErrors = true;

apps/api/src/services/cron.service.ts

@@ -19,8 +19,8 @@ export class CronService {
     private readonly twitterBotService: TwitterBotService
   ) {}
 
-  @Cron(CronExpression.EVERY_4_HOURS)
-  public async runEveryFourHours() {
+  @Cron(CronExpression.EVERY_HOUR)
+  public async runEveryHour() {
     await this.dataGatheringService.gather7Days();
   }
 
@@ -38,15 +38,20 @@ export class CronService {
   public async runEverySundayAtTwelvePm() {
     const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
 
-    for (const { dataSource, symbol } of uniqueAssets) {
-      await this.dataGatheringService.addJobToQueue(
-        GATHER_ASSET_PROFILE_PROCESS,
-        {
-          dataSource,
-          symbol
-        },
-        GATHER_ASSET_PROFILE_PROCESS_OPTIONS
-      );
-    }
+    await this.dataGatheringService.addJobsToQueue(
+      uniqueAssets.map(({ dataSource, symbol }) => {
+        return {
+          data: {
+            dataSource,
+            symbol
+          },
+          name: GATHER_ASSET_PROFILE_PROCESS,
+          opts: {
+            ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
+            jobId: `${dataSource}-${symbol}`
+          }
+        };
+      })
+    );
   }
 }

apps/api/src/services/data-gathering.service.ts

@@ -2,8 +2,7 @@ import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.se
 import {
   DATA_GATHERING_QUEUE,
   GATHER_HISTORICAL_MARKET_DATA_PROCESS,
-  GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
-  QUEUE_JOB_STATUS_LIST
+  GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
 import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
@@ -12,6 +11,7 @@ import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
 import { JobOptions, Queue } from 'bull';
 import { format, min, subDays, subYears } from 'date-fns';
+import { isEmpty } from 'lodash';
 
 import { DataProviderService } from './data-provider/data-provider.service';
 import { DataEnhancerInterface } from './data-provider/interfaces/data-enhancer.interface';
@@ -34,17 +34,22 @@ export class DataGatheringService {
     private readonly symbolProfileService: SymbolProfileService
   ) {}
 
-  public async addJobToQueue(name: string, data: any, options?: JobOptions) {
-    const hasJob = await this.hasJob(name, data);
+  public async addJobToQueue({
+    data,
+    name,
+    opts
+  }: {
+    data: any;
+    name: string;
+    opts?: JobOptions;
+  }) {
+    return this.dataGatheringQueue.add(name, data, opts);
+  }
 
-    if (hasJob) {
-      Logger.log(
-        `Job ${name} with data ${JSON.stringify(data)} already exists.`,
-        'DataGatheringService'
-      );
-    } else {
-      return this.dataGatheringQueue.add(name, data, options);
-    }
+  public async addJobsToQueue(
+    jobs: { data: any; name: string; opts?: JobOptions }[]
+  ) {
+    return this.dataGatheringQueue.addBulk(jobs);
   }
 
   public async gather7Days() {
@@ -209,59 +214,22 @@ export class DataGatheringService {
   }
 
   public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
-    for (const { dataSource, date, symbol } of aSymbolsWithStartDate) {
-      await this.addJobToQueue(
-        GATHER_HISTORICAL_MARKET_DATA_PROCESS,
-        {
-          dataSource,
-          date,
-          symbol
-        },
-        GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS
-      );
-    }
-  }
-
-  public async getSymbolsMax(): Promise<IDataGatheringItem[]> {
-    const startDate =
-      (
-        await this.prismaService.order.findFirst({
-          orderBy: [{ date: 'asc' }]
-        })
-      )?.date ?? new Date();
-
-    const currencyPairsToGather = this.exchangeRateDataService
-      .getCurrencyPairs()
-      .map(({ dataSource, symbol }) => {
+    await this.addJobsToQueue(
+      aSymbolsWithStartDate.map(({ dataSource, date, symbol }) => {
         return {
-          dataSource,
-          symbol,
-          date: min([startDate, subYears(new Date(), 10)])
-        };
-      });
-
-    const symbolProfilesToGather = (
-      await this.prismaService.symbolProfile.findMany({
-        orderBy: [{ symbol: 'asc' }],
-        select: {
-          dataSource: true,
-          Order: {
-            orderBy: [{ date: 'asc' }],
-            select: { date: true },
-            take: 1
+          data: {
+            dataSource,
+            date,
+            symbol
           },
-          scraperConfiguration: true,
-          symbol: true
-        }
+          name: GATHER_HISTORICAL_MARKET_DATA_PROCESS,
+          opts: {
+            ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
+            jobId: `${dataSource}-${symbol}-${format(date, DATE_FORMAT)}`
+          }
+        };
       })
-    ).map((symbolProfile) => {
-      return {
-        ...symbolProfile,
-        date: symbolProfile.Order?.[0]?.date ?? startDate
-      };
-    });
-
-    return [...currencyPairsToGather, ...symbolProfilesToGather];
+    );
   }
 
   public async getUniqueAssets(): Promise<UniqueAsset[]> {
@@ -298,7 +266,7 @@ export class DataGatheringService {
 
     // Only consider symbols with incomplete market data for the last
     // 7 days
-    const symbolsNotToGather = (
+    const symbolsWithCompleteMarketData = (
       await this.prismaService.marketData.groupBy({
         _count: true,
         by: ['symbol'],
@@ -316,8 +284,14 @@ export class DataGatheringService {
       });
 
     const symbolProfilesToGather = symbolProfiles
-      .filter(({ symbol }) => {
-        return !symbolsNotToGather.includes(symbol);
+      .filter(({ dataSource, scraperConfiguration, symbol }) => {
+        const manualDataSourceWithScraperConfiguration =
+          dataSource === 'MANUAL' && !isEmpty(scraperConfiguration);
+
+        return (
+          !symbolsWithCompleteMarketData.includes(symbol) &&
+          (dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration)
+        );
       })
       .map((symbolProfile) => {
         return {
@@ -329,7 +303,7 @@ export class DataGatheringService {
     const currencyPairsToGather = this.exchangeRateDataService
       .getCurrencyPairs()
       .filter(({ symbol }) => {
-        return !symbolsNotToGather.includes(symbol);
+        return !symbolsWithCompleteMarketData.includes(symbol);
       })
       .map(({ dataSource, symbol }) => {
         return {
@@ -342,17 +316,56 @@ export class DataGatheringService {
     return [...currencyPairsToGather, ...symbolProfilesToGather];
   }
 
-  private async hasJob(name: string, data: any) {
-    const jobs = await this.dataGatheringQueue.getJobs(
-      QUEUE_JOB_STATUS_LIST.filter((status) => {
-        return status !== 'completed';
-      })
-    );
+  private async getSymbolsMax(): Promise<IDataGatheringItem[]> {
+    const startDate =
+      (
+        await this.prismaService.order.findFirst({
+          orderBy: [{ date: 'asc' }]
+        })
+      )?.date ?? new Date();
 
-    return jobs.some((job) => {
-      return (
-        job.name === name && JSON.stringify(job.data) === JSON.stringify(data)
-      );
-    });
+    const currencyPairsToGather = this.exchangeRateDataService
+      .getCurrencyPairs()
+      .map(({ dataSource, symbol }) => {
+        return {
+          dataSource,
+          symbol,
+          date: min([startDate, subYears(new Date(), 10)])
+        };
+      });
+
+    const symbolProfilesToGather = (
+      await this.prismaService.symbolProfile.findMany({
+        orderBy: [{ symbol: 'asc' }],
+        select: {
+          dataSource: true,
+          Order: {
+            orderBy: [{ date: 'asc' }],
+            select: { date: true },
+            take: 1
+          },
+          scraperConfiguration: true,
+          symbol: true
+        }
+      })
+    )
+      .filter((symbolProfile) => {
+        const manualDataSourceWithScraperConfiguration =
+          symbolProfile.dataSource === 'MANUAL' &&
+          !isEmpty(symbolProfile.scraperConfiguration);
+
+        return (
+          symbolProfile.dataSource !== 'MANUAL' ||
+          manualDataSourceWithScraperConfiguration
+        );
+      })
+      .map((symbolProfile) => {
+        return {
+          ...symbolProfile,
+          date: symbolProfile.Order?.[0]?.date ?? startDate
+        };
+      });
+
+    return [...currencyPairsToGather, ...symbolProfilesToGather];
   }
 }

apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts

@@ -33,7 +33,8 @@ export class AlphaVantageService implements DataProviderInterface {
     aSymbol: string
   ): Promise<Partial<SymbolProfile>> {
     return {
-      dataSource: this.getName()
+      dataSource: this.getName(),
+      symbol: aSymbol
     };
   }
 

apps/api/src/services/data-provider/data-enhancer/data-enhancer.module.ts

@@ -1,15 +1,27 @@
+import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module';
+import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';
 import { TrackinsightDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/trackinsight/trackinsight.service';
+import { YahooFinanceDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service';
 import { Module } from '@nestjs/common';
 
 @Module({
-  exports: ['DataEnhancers', TrackinsightDataEnhancerService],
+  exports: [
+    'DataEnhancers',
+    TrackinsightDataEnhancerService,
+    YahooFinanceDataEnhancerService
+  ],
+  imports: [ConfigurationModule, CryptocurrencyModule],
   providers: [
+    TrackinsightDataEnhancerService,
+    YahooFinanceDataEnhancerService,
     {
-      inject: [TrackinsightDataEnhancerService],
+      inject: [
+        TrackinsightDataEnhancerService,
+        YahooFinanceDataEnhancerService
+      ],
       provide: 'DataEnhancers',
-      useFactory: (trackinsight) => [trackinsight]
-    },
-    TrackinsightDataEnhancerService
+      useFactory: (trackinsight, yahooFinance) => [trackinsight, yahooFinance]
+    }
   ]
 })
 export class DataEnhancerModule {}

apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts

@@ -1,11 +1,13 @@
 import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
 import { Country } from '@ghostfolio/common/interfaces/country.interface';
 import { Sector } from '@ghostfolio/common/interfaces/sector.interface';
+import { Injectable } from '@nestjs/common';
 import { SymbolProfile } from '@prisma/client';
 import bent from 'bent';
 
 const getJSON = bent('json');
 
+@Injectable()
 export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
   private static baseUrl = 'https://data.trackinsight.com';
   private static countries = require('countries-list/dist/countries.json');

apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.spec.ts

@@ -1,7 +1,7 @@
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
 import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service';
 
-import { YahooFinanceService } from './yahoo-finance.service';
+import { YahooFinanceDataEnhancerService } from './yahoo-finance.service';
 
 jest.mock(
   '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service',
@@ -25,16 +25,16 @@ jest.mock(
   }
 );
 
-describe('YahooFinanceService', () => {
+describe('YahooFinanceDataEnhancerService', () => {
   let configurationService: ConfigurationService;
   let cryptocurrencyService: CryptocurrencyService;
-  let yahooFinanceService: YahooFinanceService;
+  let yahooFinanceDataEnhancerService: YahooFinanceDataEnhancerService;
 
   beforeAll(async () => {
     configurationService = new ConfigurationService();
     cryptocurrencyService = new CryptocurrencyService();
 
-    yahooFinanceService = new YahooFinanceService(
+    yahooFinanceDataEnhancerService = new YahooFinanceDataEnhancerService(
       configurationService,
       cryptocurrencyService
     );
@@ -42,25 +42,37 @@ describe('YahooFinanceService', () => {
 
   it('convertFromYahooFinanceSymbol', async () => {
     expect(
-      await yahooFinanceService.convertFromYahooFinanceSymbol('BRK-B')
+      await yahooFinanceDataEnhancerService.convertFromYahooFinanceSymbol(
+        'BRK-B'
+      )
     ).toEqual('BRK-B');
     expect(
-      await yahooFinanceService.convertFromYahooFinanceSymbol('BTC-USD')
+      await yahooFinanceDataEnhancerService.convertFromYahooFinanceSymbol(
+        'BTC-USD'
+      )
     ).toEqual('BTCUSD');
     expect(
-      await yahooFinanceService.convertFromYahooFinanceSymbol('EURUSD=X')
+      await yahooFinanceDataEnhancerService.convertFromYahooFinanceSymbol(
+        'EURUSD=X'
+      )
     ).toEqual('EURUSD');
   });
 
   it('convertToYahooFinanceSymbol', async () => {
     expect(
-      await yahooFinanceService.convertToYahooFinanceSymbol('BTCUSD')
+      await yahooFinanceDataEnhancerService.convertToYahooFinanceSymbol(
+        'BTCUSD'
+      )
     ).toEqual('BTC-USD');
     expect(
-      await yahooFinanceService.convertToYahooFinanceSymbol('DOGEUSD')
+      await yahooFinanceDataEnhancerService.convertToYahooFinanceSymbol(
+        'DOGEUSD'
+      )
     ).toEqual('DOGE-USD');
     expect(
-      await yahooFinanceService.convertToYahooFinanceSymbol('USDCHF')
+      await yahooFinanceDataEnhancerService.convertToYahooFinanceSymbol(
+        'USDCHF'
+      )
     ).toEqual('USDCHF=X');
   });
 });

apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts

@@ -0,0 +1,325 @@
+import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service';
+import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
+import { UNKNOWN_KEY } from '@ghostfolio/common/config';
+import { isCurrency } from '@ghostfolio/common/helper';
+import { Injectable, Logger } from '@nestjs/common';
+import {
+  AssetClass,
+  AssetSubClass,
+  DataSource,
+  SymbolProfile
+} from '@prisma/client';
+import { countries } from 'countries-list';
+import yahooFinance from 'yahoo-finance2';
+import type { Price } from 'yahoo-finance2/dist/esm/src/modules/quoteSummary-iface';
+
+@Injectable()
+export class YahooFinanceDataEnhancerService implements DataEnhancerInterface {
+  private baseCurrency: string;
+
+  public constructor(
+    private readonly configurationService: ConfigurationService,
+    private readonly cryptocurrencyService: CryptocurrencyService
+  ) {
+    this.baseCurrency = this.configurationService.get('BASE_CURRENCY');
+  }
+
+  public convertFromYahooFinanceSymbol(aYahooFinanceSymbol: string) {
+    let symbol = aYahooFinanceSymbol.replace(
+      new RegExp(`-${this.baseCurrency}$`),
+      this.baseCurrency
+    );
+
+    if (symbol.includes('=X') && !symbol.includes(this.baseCurrency)) {
+      symbol = `${this.baseCurrency}${symbol}`;
+    }
+
+    return symbol.replace('=X', '');
+  }
+
+  /**
+   * Converts a symbol to a Yahoo Finance symbol
+   *
+   * Currency:        USDCHF  -> USDCHF=X
+   * Cryptocurrency:  BTCUSD  -> BTC-USD
+   *                  DOGEUSD -> DOGE-USD
+   */
+  public convertToYahooFinanceSymbol(aSymbol: string) {
+    if (
+      aSymbol.includes(this.baseCurrency) &&
+      aSymbol.length > this.baseCurrency.length
+    ) {
+      if (
+        isCurrency(
+          aSymbol.substring(0, aSymbol.length - this.baseCurrency.length)
+        )
+      ) {
+        return `${aSymbol}=X`;
+      } else if (
+        this.cryptocurrencyService.isCryptocurrency(
+          aSymbol.replace(
+            new RegExp(`-${this.baseCurrency}$`),
+            this.baseCurrency
+          )
+        )
+      ) {
+        // Add a dash before the last three characters
+        // BTCUSD  -> BTC-USD
+        // DOGEUSD -> DOGE-USD
+        // SOL1USD -> SOL1-USD
+        return aSymbol.replace(
+          new RegExp(`-?${this.baseCurrency}$`),
+          `-${this.baseCurrency}`
+        );
+      }
+    }
+
+    return aSymbol;
+  }
+
+  public async enhance({
+    response,
+    symbol
+  }: {
+    response: Partial<SymbolProfile>;
+    symbol: string;
+  }): Promise<Partial<SymbolProfile>> {
+    if (response.dataSource !== 'YAHOO' && !response.isin) {
+      return response;
+    }
+
+    try {
+      let yahooSymbol: string;
+
+      if (response.dataSource === 'YAHOO') {
+        yahooSymbol = symbol;
+      } else {
+        const { quotes } = await yahooFinance.search(response.isin);
+        yahooSymbol = quotes[0].symbol;
+      }
+
+      const { countries, sectors, url } = await this.getAssetProfile(
+        yahooSymbol
+      );
+
+      if (countries) {
+        response.countries = countries;
+      }
+
+      if (sectors) {
+        response.sectors = sectors;
+      }
+
+      if (url) {
+        response.url = url;
+      }
+    } catch (error) {
+      Logger.error(error, 'YahooFinanceDataEnhancerService');
+    }
+
+    return response;
+  }
+
+  public formatName({
+    longName,
+    quoteType,
+    shortName,
+    symbol
+  }: {
+    longName: Price['longName'];
+    quoteType: Price['quoteType'];
+    shortName: Price['shortName'];
+    symbol: Price['symbol'];
+  }) {
+    let name = longName;
+
+    if (name) {
+      name = name.replace('Amundi Index Solutions - ', '');
+      name = name.replace('iShares ETF (CH) - ', '');
+      name = name.replace('iShares III Public Limited Company - ', '');
+      name = name.replace('iShares V PLC - ', '');
+      name = name.replace('iShares VI Public Limited Company - ', '');
+      name = name.replace('iShares VII PLC - ', '');
+      name = name.replace('Multi Units Luxembourg - ', '');
+      name = name.replace('VanEck ETFs N.V. - ', '');
+      name = name.replace('Vaneck Vectors Ucits Etfs Plc - ', '');
+      name = name.replace('Vanguard Funds Public Limited Company - ', '');
+      name = name.replace('Vanguard Index Funds - ', '');
+      name = name.replace('Xtrackers (IE) Plc - ', '');
+    }
+
+    if (quoteType === 'FUTURE') {
+      // "Gold Jun 22" -> "Gold"
+      name = shortName?.slice(0, -7);
+    }
+
+    return name || shortName || symbol;
+  }
+
+  public async getAssetProfile(
+    aSymbol: string
+  ): Promise<Partial<SymbolProfile>> {
+    const response: Partial<SymbolProfile> = {};
+
+    try {
+      const symbol = this.convertToYahooFinanceSymbol(aSymbol);
+      const assetProfile = await yahooFinance.quoteSummary(symbol, {
+        modules: ['price', 'summaryProfile', 'topHoldings']
+      });
+
+      const { assetClass, assetSubClass } = this.parseAssetClass({
+        quoteType: assetProfile.price.quoteType,
+        shortName: assetProfile.price.shortName
+      });
+
+      response.assetClass = assetClass;
+      response.assetSubClass = assetSubClass;
+      response.currency = assetProfile.price.currency;
+      response.dataSource = this.getName();
+      response.name = this.formatName({
+        longName: assetProfile.price.longName,
+        quoteType: assetProfile.price.quoteType,
+        shortName: assetProfile.price.shortName,
+        symbol: assetProfile.price.symbol
+      });
+      response.symbol = aSymbol;
+
+      if (assetSubClass === AssetSubClass.MUTUALFUND) {
+        response.sectors = [];
+
+        for (const sectorWeighting of assetProfile.topHoldings
+          ?.sectorWeightings ?? []) {
+          for (const [sector, weight] of Object.entries(sectorWeighting)) {
+            response.sectors.push({ weight, name: this.parseSector(sector) });
+          }
+        }
+      } else if (
+        assetSubClass === AssetSubClass.STOCK &&
+        assetProfile.summaryProfile?.country
+      ) {
+        // Add country if asset is stock and country available
+
+        try {
+          const [code] = Object.entries(countries).find(([, country]) => {
+            return country.name === assetProfile.summaryProfile?.country;
+          });
+
+          if (code) {
+            response.countries = [{ code, weight: 1 }];
+          }
+        } catch {}
+
+        if (assetProfile.summaryProfile?.sector) {
+          response.sectors = [
+            { name: assetProfile.summaryProfile?.sector, weight: 1 }
+          ];
+        }
+      }
+
+      const url = assetProfile.summaryProfile?.website;
+      if (url) {
+        response.url = url;
+      }
+    } catch (error) {
+      Logger.error(error, 'YahooFinanceService');
+    }
+
+    return response;
+  }
+
+  public getName() {
+    return DataSource.YAHOO;
+  }
+
+  public parseAssetClass({
+    quoteType,
+    shortName
+  }: {
+    quoteType: string;
+    shortName: string;
+  }): {
+    assetClass: AssetClass;
+    assetSubClass: AssetSubClass;
+  } {
+    let assetClass: AssetClass;
+    let assetSubClass: AssetSubClass;
+
+    switch (quoteType?.toLowerCase()) {
+      case 'cryptocurrency':
+        assetClass = AssetClass.CASH;
+        assetSubClass = AssetSubClass.CRYPTOCURRENCY;
+        break;
+      case 'equity':
+        assetClass = AssetClass.EQUITY;
+        assetSubClass = AssetSubClass.STOCK;
+        break;
+      case 'etf':
+        assetClass = AssetClass.EQUITY;
+        assetSubClass = AssetSubClass.ETF;
+        break;
+      case 'future':
+        assetClass = AssetClass.COMMODITY;
+        assetSubClass = AssetSubClass.COMMODITY;
+
+        if (
+          shortName?.toLowerCase()?.startsWith('gold') ||
+          shortName?.toLowerCase()?.startsWith('palladium') ||
+          shortName?.toLowerCase()?.startsWith('platinum') ||
+          shortName?.toLowerCase()?.startsWith('silver')
+        ) {
+          assetSubClass = AssetSubClass.PRECIOUS_METAL;
+        }
+
+        break;
+      case 'mutualfund':
+        assetClass = AssetClass.EQUITY;
+        assetSubClass = AssetSubClass.MUTUALFUND;
+        break;
+    }
+
+    return { assetClass, assetSubClass };
+  }
+
+  private parseSector(aString: string): string {
+    let sector = UNKNOWN_KEY;
+
+    switch (aString) {
+      case 'basic_materials':
+        sector = 'Basic Materials';
+        break;
+      case 'communication_services':
+        sector = 'Communication Services';
+        break;
+      case 'consumer_cyclical':
+        sector = 'Consumer Cyclical';
+        break;
+      case 'consumer_defensive':
+        sector = 'Consumer Staples';
+        break;
+      case 'energy':
+        sector = 'Energy';
+        break;
+      case 'financial_services':
+        sector = 'Financial Services';
+        break;
+      case 'healthcare':
+        sector = 'Healthcare';
+        break;
+      case 'industrials':
+        sector = 'Industrials';
+        break;
+      case 'realestate':
+        sector = 'Real Estate';
+        break;
+      case 'technology':
+        sector = 'Technology';
+        break;
+      case 'utilities':
+        sector = 'Utilities';
+        break;
+    }
+
+    return sector;
+  }
+}

apps/api/src/services/data-provider/data-provider.module.ts

@@ -11,12 +11,15 @@ import { PrismaModule } from '@ghostfolio/api/services/prisma.module';
 import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module';
 import { Module } from '@nestjs/common';
 
+import { DataEnhancerModule } from './data-enhancer/data-enhancer.module';
+import { YahooFinanceDataEnhancerService } from './data-enhancer/yahoo-finance/yahoo-finance.service';
 import { DataProviderService } from './data-provider.service';
 
 @Module({
   imports: [
     ConfigurationModule,
     CryptocurrencyModule,
+    DataEnhancerModule,
     PrismaModule,
     SymbolProfileModule
   ],
@@ -57,7 +60,8 @@ import { DataProviderService } from './data-provider.service';
         rapidApiService,
         yahooFinanceService
       ]
-    }
+    },
+    YahooFinanceDataEnhancerService
   ],
   exports: [DataProviderService, YahooFinanceService]
 })

apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts

@@ -40,10 +40,11 @@ export class EodHistoricalDataService implements DataProviderInterface {
     return {
       assetClass: searchResult?.assetClass,
       assetSubClass: searchResult?.assetSubClass,
-      currency: searchResult?.currency,
+      currency: this.convertCurrency(searchResult?.currency),
       dataSource: this.getName(),
       isin: searchResult?.isin,
-      name: searchResult?.name
+      name: searchResult?.name,
+      symbol: aSymbol
     };
   }
 
@@ -146,7 +147,7 @@ export class EodHistoricalDataService implements DataProviderInterface {
           { close, code, timestamp }
         ) => {
           result[code] = {
-            currency: searchResponse?.items[0]?.currency,
+            currency: this.convertCurrency(searchResponse?.items[0]?.currency),
             dataSource: DataSource.EOD_HISTORICAL_DATA,
             marketPrice: close,
             marketState: isToday(new Date(timestamp * 1000)) ? 'open' : 'closed'
@@ -183,16 +184,26 @@ export class EodHistoricalDataService implements DataProviderInterface {
             return {
               assetClass,
               assetSubClass,
-              currency,
               dataSource,
               name,
-              symbol
+              symbol,
+              currency: this.convertCurrency(currency)
             };
           }
         )
     };
   }
 
+  private convertCurrency(aCurrency: string) {
+    let currency = aCurrency;
+
+    if (currency === 'GBX') {
+      currency = 'GBp';
+    }
+
+    return currency;
+  }
+
   private async getSearchResult(aQuery: string): Promise<
     (LookupItem & {
       assetClass: AssetClass;
@@ -212,14 +223,7 @@ export class EodHistoricalDataService implements DataProviderInterface {
       const response = await get();
 
       searchResult = response.map(
-        ({
-          Code,
-          Currency: currency,
-          Exchange,
-          ISIN: isin,
-          Name: name,
-          Type
-        }) => {
+        ({ Code, Currency, Exchange, ISIN: isin, Name: name, Type }) => {
           const { assetClass, assetSubClass } = this.parseAssetClass({
             Exchange,
             Type
@@ -228,9 +232,9 @@ export class EodHistoricalDataService implements DataProviderInterface {
           return {
             assetClass,
             assetSubClass,
-            currency,
             isin,
             name,
+            currency: this.convertCurrency(Currency),
             dataSource: this.getName(),
             symbol: `${Code}.${Exchange}`
           };

apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts

@@ -30,7 +30,8 @@ export class GoogleSheetsService implements DataProviderInterface {
     aSymbol: string
   ): Promise<Partial<SymbolProfile>> {
     return {
-      dataSource: this.getName()
+      dataSource: this.getName(),
+      symbol: aSymbol
     };
   }
 

apps/api/src/services/data-provider/manual/manual.service.ts

@@ -34,7 +34,8 @@ export class ManualService implements DataProviderInterface {
     aSymbol: string
   ): Promise<Partial<SymbolProfile>> {
     return {
-      dataSource: this.getName()
+      dataSource: this.getName(),
+      symbol: aSymbol
     };
   }
 

apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts

@@ -27,7 +27,8 @@ export class RapidApiService implements DataProviderInterface {
     aSymbol: string
   ): Promise<Partial<SymbolProfile>> {
     return {
-      dataSource: this.getName()
+      dataSource: this.getName(),
+      symbol: aSymbol
     };
   }
 

apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts

@@ -1,26 +1,19 @@
 import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
 import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service';
+import { YahooFinanceDataEnhancerService } from '@ghostfolio/api/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service';
 import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
 import {
   IDataProviderHistoricalResponse,
   IDataProviderResponse
 } from '@ghostfolio/api/services/interfaces/interfaces';
-import { UNKNOWN_KEY } from '@ghostfolio/common/config';
-import { DATE_FORMAT, isCurrency } from '@ghostfolio/common/helper';
+import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable, Logger } from '@nestjs/common';
-import {
-  AssetClass,
-  AssetSubClass,
-  DataSource,
-  SymbolProfile
-} from '@prisma/client';
+import { DataSource, SymbolProfile } from '@prisma/client';
 import Big from 'big.js';
-import { countries } from 'countries-list';
 import { addDays, format, isSameDay } from 'date-fns';
 import yahooFinance from 'yahoo-finance2';
-import type { Price } from 'yahoo-finance2/dist/esm/src/modules/quoteSummary-iface';
 
 @Injectable()
 export class YahooFinanceService implements DataProviderInterface {
@@ -28,7 +21,8 @@ export class YahooFinanceService implements DataProviderInterface {
 
   public constructor(
     private readonly configurationService: ConfigurationService,
-    private readonly cryptocurrencyService: CryptocurrencyService
+    private readonly cryptocurrencyService: CryptocurrencyService,
+    private readonly yahooFinanceDataEnhancerService: YahooFinanceDataEnhancerService
   ) {
     this.baseCurrency = this.configurationService.get('BASE_CURRENCY');
   }
@@ -37,128 +31,20 @@ export class YahooFinanceService implements DataProviderInterface {
     return true;
   }
 
-  public convertFromYahooFinanceSymbol(aYahooFinanceSymbol: string) {
-    let symbol = aYahooFinanceSymbol.replace(
-      new RegExp(`-${this.baseCurrency}$`),
-      this.baseCurrency
-    );
-
-    if (symbol.includes('=X') && !symbol.includes(this.baseCurrency)) {
-      symbol = `${this.baseCurrency}${symbol}`;
-    }
-
-    return symbol.replace('=X', '');
-  }
-
-  /**
-   * Converts a symbol to a Yahoo Finance symbol
-   *
-   * Currency:        USDCHF  -> USDCHF=X
-   * Cryptocurrency:  BTCUSD  -> BTC-USD
-   *                  DOGEUSD -> DOGE-USD
-   */
-  public convertToYahooFinanceSymbol(aSymbol: string) {
-    if (
-      aSymbol.includes(this.baseCurrency) &&
-      aSymbol.length > this.baseCurrency.length
-    ) {
-      if (
-        isCurrency(
-          aSymbol.substring(0, aSymbol.length - this.baseCurrency.length)
-        )
-      ) {
-        return `${aSymbol}=X`;
-      } else if (
-        this.cryptocurrencyService.isCryptocurrency(
-          aSymbol.replace(
-            new RegExp(`-${this.baseCurrency}$`),
-            this.baseCurrency
-          )
-        )
-      ) {
-        // Add a dash before the last three characters
-        // BTCUSD  -> BTC-USD
-        // DOGEUSD -> DOGE-USD
-        // SOL1USD -> SOL1-USD
-        return aSymbol.replace(
-          new RegExp(`-?${this.baseCurrency}$`),
-          `-${this.baseCurrency}`
-        );
-      }
-    }
-
-    return aSymbol;
-  }
-
   public async getAssetProfile(
     aSymbol: string
   ): Promise<Partial<SymbolProfile>> {
-    const response: Partial<SymbolProfile> = {};
+    const { assetClass, assetSubClass, currency, name } =
+      await this.yahooFinanceDataEnhancerService.getAssetProfile(aSymbol);
 
-    try {
-      const symbol = this.convertToYahooFinanceSymbol(aSymbol);
-      const assetProfile = await yahooFinance.quoteSummary(symbol, {
-        modules: ['price', 'summaryProfile', 'topHoldings']
-      });
-
-      const { assetClass, assetSubClass } = this.parseAssetClass({
-        quoteType: assetProfile.price.quoteType,
-        shortName: assetProfile.price.shortName
-      });
-
-      response.assetClass = assetClass;
-      response.assetSubClass = assetSubClass;
-      response.currency = assetProfile.price.currency;
-      response.dataSource = this.getName();
-      response.name = this.formatName({
-        longName: assetProfile.price.longName,
-        quoteType: assetProfile.price.quoteType,
-        shortName: assetProfile.price.shortName,
-        symbol: assetProfile.price.symbol
-      });
-      response.symbol = aSymbol;
-
-      if (assetSubClass === AssetSubClass.MUTUALFUND) {
-        response.sectors = [];
-
-        for (const sectorWeighting of assetProfile.topHoldings
-          ?.sectorWeightings ?? []) {
-          for (const [sector, weight] of Object.entries(sectorWeighting)) {
-            response.sectors.push({ weight, name: this.parseSector(sector) });
-          }
-        }
-      } else if (
-        assetSubClass === AssetSubClass.STOCK &&
-        assetProfile.summaryProfile?.country
-      ) {
-        // Add country if asset is stock and country available
-
-        try {
-          const [code] = Object.entries(countries).find(([, country]) => {
-            return country.name === assetProfile.summaryProfile?.country;
-          });
-
-          if (code) {
-            response.countries = [{ code, weight: 1 }];
-          }
-        } catch {}
-
-        if (assetProfile.summaryProfile?.sector) {
-          response.sectors = [
-            { name: assetProfile.summaryProfile?.sector, weight: 1 }
-          ];
-        }
-      }
-
-      const url = assetProfile.summaryProfile?.website;
-      if (url) {
-        response.url = url;
-      }
-    } catch (error) {
-      Logger.error(error, 'YahooFinanceService');
-    }
-
-    return response;
+    return {
+      assetClass,
+      assetSubClass,
+      currency,
+      name,
+      dataSource: this.getName(),
+      symbol: aSymbol
+    };
   }
 
   public async getDividends({
@@ -178,7 +64,9 @@ export class YahooFinanceService implements DataProviderInterface {
 
     try {
       const historicalResult = await yahooFinance.historical(
-        this.convertToYahooFinanceSymbol(symbol),
+        this.yahooFinanceDataEnhancerService.convertToYahooFinanceSymbol(
+          symbol
+        ),
         {
           events: 'dividends',
           interval: granularity === 'month' ? '1mo' : '1d',
@@ -228,7 +116,9 @@ export class YahooFinanceService implements DataProviderInterface {
 
     try {
       const historicalResult = await yahooFinance.historical(
-        this.convertToYahooFinanceSymbol(aSymbol),
+        this.yahooFinanceDataEnhancerService.convertToYahooFinanceSymbol(
+          aSymbol
+        ),
         {
           interval: '1d',
           period1: format(from, DATE_FORMAT),
@@ -278,7 +168,7 @@ export class YahooFinanceService implements DataProviderInterface {
       return {};
     }
     const yahooFinanceSymbols = aSymbols.map((symbol) =>
-      this.convertToYahooFinanceSymbol(symbol)
+      this.yahooFinanceDataEnhancerService.convertToYahooFinanceSymbol(symbol)
     );
 
     try {
@@ -288,7 +178,10 @@ export class YahooFinanceService implements DataProviderInterface {
 
       for (const quote of quotes) {
         // Convert symbols back
-        const symbol = this.convertFromYahooFinanceSymbol(quote.symbol);
+        const symbol =
+          this.yahooFinanceDataEnhancerService.convertFromYahooFinanceSymbol(
+            quote.symbol
+          );
 
         response[symbol] = {
           currency: quote.currency,
@@ -405,14 +298,16 @@ export class YahooFinanceService implements DataProviderInterface {
           return currentQuote.symbol === marketDataItem.symbol;
         });
 
-        const symbol = this.convertFromYahooFinanceSymbol(
-          marketDataItem.symbol
-        );
+        const symbol =
+          this.yahooFinanceDataEnhancerService.convertFromYahooFinanceSymbol(
+            marketDataItem.symbol
+          );
 
-        const { assetClass, assetSubClass } = this.parseAssetClass({
-          quoteType: quote.quoteType,
-          shortName: quote.shortname
-        });
+        const { assetClass, assetSubClass } =
+          this.yahooFinanceDataEnhancerService.parseAssetClass({
+            quoteType: quote.quoteType,
+            shortName: quote.shortname
+          });
 
         items.push({
           assetClass,
@@ -420,7 +315,7 @@ export class YahooFinanceService implements DataProviderInterface {
           symbol,
           currency: marketDataItem.currency,
           dataSource: this.getName(),
-          name: this.formatName({
+          name: this.yahooFinanceDataEnhancerService.formatName({
             longName: quote.longname,
             quoteType: quote.quoteType,
             shortName: quote.shortname,
@@ -435,42 +330,6 @@ export class YahooFinanceService implements DataProviderInterface {
     return { items };
   }
 
-  private formatName({
-    longName,
-    quoteType,
-    shortName,
-    symbol
-  }: {
-    longName: Price['longName'];
-    quoteType: Price['quoteType'];
-    shortName: Price['shortName'];
-    symbol: Price['symbol'];
-  }) {
-    let name = longName;
-
-    if (name) {
-      name = name.replace('Amundi Index Solutions - ', '');
-      name = name.replace('iShares ETF (CH) - ', '');
-      name = name.replace('iShares III Public Limited Company - ', '');
-      name = name.replace('iShares V PLC - ', '');
-      name = name.replace('iShares VI Public Limited Company - ', '');
-      name = name.replace('iShares VII PLC - ', '');
-      name = name.replace('Multi Units Luxembourg - ', '');
-      name = name.replace('VanEck ETFs N.V. - ', '');
-      name = name.replace('Vaneck Vectors Ucits Etfs Plc - ', '');
-      name = name.replace('Vanguard Funds Public Limited Company - ', '');
-      name = name.replace('Vanguard Index Funds - ', '');
-      name = name.replace('Xtrackers (IE) Plc - ', '');
-    }
-
-    if (quoteType === 'FUTURE') {
-      // "Gold Jun 22" -> "Gold"
-      name = shortName?.slice(0, -6);
-    }
-
-    return name || shortName || symbol;
-  }
-
   private getConvertedValue({
     symbol,
     value
@@ -491,95 +350,4 @@ export class YahooFinanceService implements DataProviderInterface {
 
     return value;
   }
-
-  private parseAssetClass({
-    quoteType,
-    shortName
-  }: {
-    quoteType: string;
-    shortName: string;
-  }): {
-    assetClass: AssetClass;
-    assetSubClass: AssetSubClass;
-  } {
-    let assetClass: AssetClass;
-    let assetSubClass: AssetSubClass;
-
-    switch (quoteType?.toLowerCase()) {
-      case 'cryptocurrency':
-        assetClass = AssetClass.CASH;
-        assetSubClass = AssetSubClass.CRYPTOCURRENCY;
-        break;
-      case 'equity':
-        assetClass = AssetClass.EQUITY;
-        assetSubClass = AssetSubClass.STOCK;
-        break;
-      case 'etf':
-        assetClass = AssetClass.EQUITY;
-        assetSubClass = AssetSubClass.ETF;
-        break;
-      case 'future':
-        assetClass = AssetClass.COMMODITY;
-        assetSubClass = AssetSubClass.COMMODITY;
-
-        if (
-          shortName?.toLowerCase()?.startsWith('gold') ||
-          shortName?.toLowerCase()?.startsWith('palladium') ||
-          shortName?.toLowerCase()?.startsWith('platinum') ||
-          shortName?.toLowerCase()?.startsWith('silver')
-        ) {
-          assetSubClass = AssetSubClass.PRECIOUS_METAL;
-        }
-
-        break;
-      case 'mutualfund':
-        assetClass = AssetClass.EQUITY;
-        assetSubClass = AssetSubClass.MUTUALFUND;
-        break;
-    }
-
-    return { assetClass, assetSubClass };
-  }
-
-  private parseSector(aString: string): string {
-    let sector = UNKNOWN_KEY;
-
-    switch (aString) {
-      case 'basic_materials':
-        sector = 'Basic Materials';
-        break;
-      case 'communication_services':
-        sector = 'Communication Services';
-        break;
-      case 'consumer_cyclical':
-        sector = 'Consumer Cyclical';
-        break;
-      case 'consumer_defensive':
-        sector = 'Consumer Staples';
-        break;
-      case 'energy':
-        sector = 'Energy';
-        break;
-      case 'financial_services':
-        sector = 'Financial Services';
-        break;
-      case 'healthcare':
-        sector = 'Healthcare';
-        break;
-      case 'industrials':
-        sector = 'Industrials';
-        break;
-      case 'realestate':
-        sector = 'Real Estate';
-        break;
-      case 'technology':
-        sector = 'Technology';
-        break;
-      case 'utilities':
-        sector = 'Utilities';
-        break;
-    }
-
-    return sector;
-  }
 }

apps/client/src/app/app.component.html

@@ -31,7 +31,8 @@
         >
         <div
           *ngIf="!canCreateAccount && info?.systemMessage && user"
-          class="d-inline-block info-message px-3 py-2"
+          class="cursor-pointer d-inline-block info-message px-3 py-2 text-truncate"
+          (click)="onShowSystemMessage()"
         >
           {{ info.systemMessage }}
         </div>

apps/client/src/app/app.component.scss

@@ -13,9 +13,10 @@
       margin-top: -0.5rem;
 
       .info-message {
-        background-color: rgba(0, 0, 0, $alpha-hover);
+        background-color: rgba(var(--palette-foreground-text), 0.05);
         border-radius: 2rem;
         font-size: 80%;
+        max-width: 100%;
 
         .a {
           color: rgba(var(--palette-primary-500), 1);
@@ -30,3 +31,13 @@
     line-height: 1;
   }
 }
+
+:host-context(.is-dark-theme) {
+  main {
+    .info-message-container {
+      .info-message {
+        background-color: rgba(var(--palette-foreground-text-dark), 0.05);
+      }
+    }
+  }
+}

apps/client/src/app/app.component.ts

@@ -100,6 +100,10 @@ export class AppComponent implements OnDestroy, OnInit {
     this.tokenStorageService.signOut();
   }
 
+  public onShowSystemMessage() {
+    alert(this.info.systemMessage);
+  }
+
   public onSignOut() {
     this.tokenStorageService.signOut();
     this.userService.remove();

apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.html

@@ -33,7 +33,7 @@
         <span class="ml-2" matTextSuffix>{{ data.currency }}</span>
       </mat-form-field>
       <button
-        class="apply-current-market-price ml-2 no-min-width"
+        class="ml-2 mt-1 no-min-width"
         mat-button
         title="Fetch market price"
         (click)="onFetchSymbolForDate()"

apps/client/src/app/components/admin-market-data-detail/market-data-detail-dialog/market-data-detail-dialog.scss

@@ -3,11 +3,5 @@
 
   .mat-mdc-dialog-content {
     max-height: unset;
-
-    .mat-mdc-button {
-      &.apply-current-market-price {
-        height: 56px;
-      }
-    }
   }
 }

apps/client/src/app/components/world-map-chart/world-map-chart.component.ts

@@ -30,6 +30,9 @@ export class WorldMapChartComponent implements OnChanges, OnDestroy, OnInit {
   public ngOnInit() {}
 
   public ngOnChanges() {
+    // Create a copy before manipulating countries object
+    this.countries = structuredClone(this.countries);
+
     if (this.countries) {
       this.isLoading = true;
 

apps/client/src/app/pages/portfolio/activities/create-or-update-activity-dialog/create-or-update-activity-dialog.html

@@ -153,7 +153,7 @@
       </mat-form-field>
       <button
         *ngIf="currentMarketPrice && (data.activity.type === 'BUY' || data.activity.type === 'SELL')"
-        class="apply-current-market-price ml-2 no-min-width"
+        class="ml-2 mt-1 no-min-width"
         mat-button
         title="Apply current market price"
         type="button"

apps/client/src/app/pages/portfolio/activities/create-or-update-activity-dialog/create-or-update-activity-dialog.scss

@@ -15,12 +15,6 @@
         color: var(--dark-primary-text);
       }
     }
-
-    .mat-mdc-button {
-      &.apply-current-market-price {
-        height: 56px;
-      }
-    }
   }
 }
 

apps/client/src/app/pages/portfolio/allocations/allocations-page.component.ts

@@ -65,7 +65,7 @@ export class AllocationsPageComponent implements OnDestroy, OnInit {
       | 'exchange'
       | 'name'
       | 'value'
-    >;
+    > & { etfProvider: string };
   };
   public sectors: {
     [name: string]: { name: string; value: number };
@@ -249,7 +249,7 @@ export class AllocationsPageComponent implements OnDestroy, OnInit {
   public initializeAnalysisData() {
     this.initialize();
 
-    for (const [id, { current, name, original }] of Object.entries(
+    for (const [id, { current, name }] of Object.entries(
       this.portfolioDetails.accounts
     )) {
       this.accounts[id] = {
@@ -275,6 +275,10 @@ export class AllocationsPageComponent implements OnDestroy, OnInit {
         assetClass: position.assetClass,
         assetSubClass: position.assetSubClass,
         currency: position.currency,
+        etfProvider: this.extractEtfProvider({
+          assetSubClass: position.assetSubClass,
+          name: position.name
+        }),
         exchange: position.exchange,
         name: position.name
       };
@@ -452,4 +456,19 @@ export class AllocationsPageComponent implements OnDestroy, OnInit {
           });
       });
   }
+
+  private extractEtfProvider({
+    assetSubClass,
+    name
+  }: {
+    assetSubClass: PortfolioPosition['assetSubClass'];
+    name: string;
+  }) {
+    if (assetSubClass === 'ETF') {
+      const [firstWord] = name.split(' ');
+      return firstWord;
+    }
+
+    return UNKNOWN_KEY;
+  }
 }

apps/client/src/app/pages/portfolio/allocations/allocations-page.html

@@ -249,4 +249,29 @@
       </mat-card>
     </div>
   </div>
+  <div class="row">
+    <div class="col-md-4">
+      <mat-card appearance="outlined" class="mb-3">
+        <mat-card-header class="overflow-hidden w-100">
+          <mat-card-title class="align-items-center d-flex text-truncate"
+            ><span i18n>By ETF Provider</span
+            ><gf-premium-indicator
+              *ngIf="user?.subscription?.type === 'Basic'"
+              class="ml-1"
+            ></gf-premium-indicator
+          ></mat-card-title>
+        </mat-card-header>
+        <mat-card-content>
+          <gf-portfolio-proportion-chart
+            [baseCurrency]="user?.settings?.baseCurrency"
+            [colorScheme]="user?.settings?.colorScheme"
+            [isInPercent]="hasImpersonationId || user.settings.isRestrictedView"
+            [keys]="['etfProvider']"
+            [locale]="user?.settings?.locale"
+            [positions]="positions"
+          ></gf-portfolio-proportion-chart>
+        </mat-card-content>
+      </mat-card>
+    </div>
+  </div>
 </div>

apps/client/src/styles.scss

@@ -237,7 +237,7 @@ body {
     }
 
     .mat-mdc-dialog-container {
-      background: var(--dark-background);
+      --mdc-dialog-container-color: var(--dark-background);
 
       .mdc-dialog__content {
         --mdc-dialog-supporting-text-color: rgba(var(--light-primary-text));
@@ -464,7 +464,7 @@ ngx-skeleton-loader {
 }
 
 .with-info-message {
-  height: calc(100vh - 5rem - 3.5rem) !important;
+  height: calc(100vh - 5rem - 3.5rem + 0.5rem) !important;
 }
 
 .with-placeholder-as-option {

libs/common/src/lib/config.ts

@@ -49,9 +49,7 @@ export const GATHER_ASSET_PROFILE_PROCESS_OPTIONS: JobOptions = {
     type: 'exponential'
   },
   priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH,
-  removeOnComplete: {
-    age: ms('2 weeks') / 1000
-  }
+  removeOnComplete: true
 };
 export const GATHER_HISTORICAL_MARKET_DATA_PROCESS =
   'GATHER_HISTORICAL_MARKET_DATA';
@@ -62,9 +60,7 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS: JobOptions = {
     type: 'exponential'
   },
   priority: DATA_GATHERING_QUEUE_PRIORITY_LOW,
-  removeOnComplete: {
-    age: ms('2 weeks') / 1000
-  }
+  removeOnComplete: true
 };
 
 export const HEADER_KEY_IMPERSONATION = 'Impersonation-Id';

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ghostfolio",
-  "version": "1.252.2",
+  "version": "1.257.0",
   "homepage": "https://ghostfol.io",
   "license": "AGPL-3.0",
   "scripts": {
@@ -80,7 +80,7 @@
     "@nestjs/schedule": "2.1.0",
     "@nestjs/serve-static": "3.0.0",
     "@nrwl/angular": "15.9.2",
-    "@prisma/client": "4.11.0",
+    "@prisma/client": "4.12.0",
     "@simplewebauthn/browser": "5.2.1",
     "@simplewebauthn/server": "5.2.1",
     "@stripe/stripe-js": "1.47.0",
@@ -120,7 +120,7 @@
     "passport": "0.6.0",
     "passport-google-oauth20": "2.0.0",
     "passport-jwt": "4.0.0",
-    "prisma": "4.11.0",
+    "prisma": "4.12.0",
     "reflect-metadata": "0.1.13",
     "rxjs": "7.5.6",
     "stripe": "11.12.0",

yarn.lock

@@ -4887,22 +4887,22 @@
   dependencies:
     esquery "^1.0.1"
 
-"@prisma/client@4.11.0":
-  version "4.11.0"
-  resolved "https://registry.yarnpkg.com/@prisma/client/-/client-4.11.0.tgz#41d5664dea4172c954190a432f70b86d3e2e629b"
-  integrity sha512-0INHYkQIqgAjrt7NzhYpeDQi8x3Nvylc2uDngKyFDDj1tTRQ4uV1HnVmd1sQEraeVAN63SOK0dgCKQHlvjL0KA==
+"@prisma/client@4.12.0":
+  version "4.12.0"
+  resolved "https://registry.yarnpkg.com/@prisma/client/-/client-4.12.0.tgz#119b692888b1fe0fd3305c7d0e0ac48520aa6839"
+  integrity sha512-j9/ighfWwux97J2dS15nqhl60tYoH8V0IuSsgZDb6bCFcQD3fXbXmxjYC8GHhIgOk3lB7Pq+8CwElz2MiDpsSg==
   dependencies:
-    "@prisma/engines-version" "4.11.0-57.8fde8fef4033376662cad983758335009d522acb"
+    "@prisma/engines-version" "4.12.0-67.659ef412370fa3b41cd7bf6e94587c1dfb7f67e7"
 
-"@prisma/engines-version@4.11.0-57.8fde8fef4033376662cad983758335009d522acb":
-  version "4.11.0-57.8fde8fef4033376662cad983758335009d522acb"
-  resolved "https://registry.yarnpkg.com/@prisma/engines-version/-/engines-version-4.11.0-57.8fde8fef4033376662cad983758335009d522acb.tgz#74af5ff56170c78e93ce46c56510160f58cd3c01"
-  integrity sha512-3Vd8Qq06d5xD8Ch5WauWcUUrsVPdMC6Ge8ILji8RFfyhUpqon6qSyGM0apvr1O8n8qH8cKkEFqRPsYjuz5r83g==
+"@prisma/engines-version@4.12.0-67.659ef412370fa3b41cd7bf6e94587c1dfb7f67e7":
+  version "4.12.0-67.659ef412370fa3b41cd7bf6e94587c1dfb7f67e7"
+  resolved "https://registry.yarnpkg.com/@prisma/engines-version/-/engines-version-4.12.0-67.659ef412370fa3b41cd7bf6e94587c1dfb7f67e7.tgz#51a1cc5c886564b542acde64a873645d0dee2566"
+  integrity sha512-JIHNj5jlXb9mcaJwakM0vpgRYJIAurxTUqM0iX0tfEQA5XLZ9ONkIckkhuAKdAzocZ+80GYg7QSsfpjg7OxbOA==
 
-"@prisma/engines@4.11.0":
-  version "4.11.0"
-  resolved "https://registry.yarnpkg.com/@prisma/engines/-/engines-4.11.0.tgz#c99749bfe20f58e8f4d2b5e04fee0785eba440e1"
-  integrity sha512-0AEBi2HXGV02cf6ASsBPhfsVIbVSDC9nbQed4iiY5eHttW9ZtMxHThuKZE1pnESbr8HRdgmFSa/Kn4OSNYuibg==
+"@prisma/engines@4.12.0":
+  version "4.12.0"
+  resolved "https://registry.yarnpkg.com/@prisma/engines/-/engines-4.12.0.tgz#68d99078b70b2d9c339d0e8cbf2e99f00b72aa8c"
+  integrity sha512-0alKtnxhNB5hYU+ymESBlGI4b9XrGGSdv7Ud+8TE/fBNOEhIud0XQsAR+TrvUZgS4na5czubiMsODw0TUrgkIA==
 
 "@samverschueren/stream-to-observable@^0.3.0":
   version "0.3.1"
@@ -18083,12 +18083,12 @@ pretty-hrtime@^1.0.3:
   resolved "https://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz"
   integrity sha512-66hKPCr+72mlfiSjlEB1+45IjXSqvVAIy6mocupoww4tBFE9R9IhwwUGoI4G++Tc9Aq+2rxOt0RFU6gPcrte0A==
 
-prisma@4.11.0:
-  version "4.11.0"
-  resolved "https://registry.yarnpkg.com/prisma/-/prisma-4.11.0.tgz#9695ba4129a43eab3e76b5f7a033c6c020377725"
-  integrity sha512-4zZmBXssPUEiX+GeL0MUq/Yyie4ltiKmGu7jCJFnYMamNrrulTBc+D+QwAQSJ01tyzeGHlD13kOnqPwRipnlNw==
+prisma@4.12.0:
+  version "4.12.0"
+  resolved "https://registry.yarnpkg.com/prisma/-/prisma-4.12.0.tgz#1080eda951928cb3b0274ad29da9ae4f93143d68"
+  integrity sha512-xqVper4mbwl32BWzLpdznHAYvYDWQQWK2tBfXjdUD397XaveRyAP7SkBZ6kFlIg8kKayF4hvuaVtYwXd9BodAg==
   dependencies:
-    "@prisma/engines" "4.11.0"
+    "@prisma/engines" "4.12.0"
 
 prismjs@^1.28.0:
   version "1.28.0"