Feature/disable caching in health check endpoints for data providers (#2147)

* Disable caching in health check endpoint

* Update changelog

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Disabled the caching in the health check endpoints for data providers
+
 ## 1.289.0 - 2023-07-14
 
 ### Changed

apps/api/src/app/benchmark/benchmark.service.ts

@@ -66,11 +66,11 @@ export class BenchmarkService {
 
     const promises: Promise<number>[] = [];
 
-    const quotes = await this.dataProviderService.getQuotes(
+    const quotes = await this.dataProviderService.getQuotes({
-      benchmarkAssetProfiles.map(({ dataSource, symbol }) => {
+      items: benchmarkAssetProfiles.map(({ dataSource, symbol }) => {
         return { dataSource, symbol };
       })
-    );
+    });
 
     for (const { dataSource, symbol } of benchmarkAssetProfiles) {
       promises.push(this.marketDataService.getMax({ dataSource, symbol }));

apps/api/src/app/portfolio/current-rate.service.ts

@@ -38,7 +38,7 @@ export class CurrentRateService {
     if (includeToday) {
       promises.push(
         this.dataProviderService
-          .getQuotes(dataGatheringItems)
+          .getQuotes({ items: dataGatheringItems })
           .then((dataResultProvider) => {
             const result: GetValueObject[] = [];
             for (const dataGatheringItem of dataGatheringItems) {

apps/api/src/app/portfolio/portfolio.service.ts

@@ -504,15 +504,17 @@ export class PortfolioService {
       );
     }
 
-    const dataGatheringItems = currentPositions.positions.map((position) => {
+    const dataGatheringItems = currentPositions.positions.map(
-      return {
+      ({ dataSource, symbol }) => {
-        dataSource: position.dataSource,
+        return {
-        symbol: position.symbol
+          dataSource,
-      };
+          symbol
-    });
+        };
+      }
+    );
 
     const [dataProviderResponses, symbolProfiles] = await Promise.all([
-      this.dataProviderService.getQuotes(dataGatheringItems),
+      this.dataProviderService.getQuotes({ items: dataGatheringItems }),
       this.symbolProfileService.getSymbolProfiles(dataGatheringItems)
     ]);
 
@@ -897,9 +899,9 @@ export class PortfolioService {
         )
       };
     } else {
-      const currentData = await this.dataProviderService.getQuotes([
+      const currentData = await this.dataProviderService.getQuotes({
-        { dataSource: DataSource.YAHOO, symbol: aSymbol }
+        items: [{ dataSource: DataSource.YAHOO, symbol: aSymbol }]
-      ]);
+      });
       const marketPrice = currentData[aSymbol]?.marketPrice;
 
       let historicalData = await this.dataProviderService.getHistorical(
@@ -1000,15 +1002,15 @@ export class PortfolioService {
       (item) => !item.quantity.eq(0)
     );
 
-    const dataGatheringItem = positions.map((position) => {
+    const dataGatheringItems = positions.map(({ dataSource, symbol }) => {
       return {
-        dataSource: position.dataSource,
+        dataSource,
-        symbol: position.symbol
+        symbol
       };
     });
 
     const [dataProviderResponses, symbolProfiles] = await Promise.all([
-      this.dataProviderService.getQuotes(dataGatheringItem),
+      this.dataProviderService.getQuotes({ items: dataGatheringItems }),
       this.symbolProfileService.getSymbolProfiles(
         positions.map(({ dataSource, symbol }) => {
           return { dataSource, symbol };

apps/api/src/app/symbol/symbol.service.ts

@@ -27,9 +27,9 @@ export class SymbolService {
     dataGatheringItem: IDataGatheringItem;
     includeHistoricalData?: number;
   }): Promise<SymbolItem> {
-    const quotes = await this.dataProviderService.getQuotes([
+    const quotes = await this.dataProviderService.getQuotes({
-      dataGatheringItem
+      items: [dataGatheringItem]
-    ]);
+    });
     const { currency, marketPrice } = quotes[dataGatheringItem.symbol] ?? {};
 
     if (dataGatheringItem.dataSource && marketPrice >= 0) {

apps/api/src/services/data-provider/data-provider.service.ts

@@ -3,7 +3,6 @@ import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.in
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
 import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
 import {
-  IDataGatheringItem,
   IDataProviderHistoricalResponse,
   IDataProviderResponse
 } from '@ghostfolio/api/services/interfaces/interfaces';
@@ -12,6 +11,7 @@ import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config';
 import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
+import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import type { Granularity, UserWithSettings } from '@ghostfolio/common/types';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
@@ -45,12 +45,15 @@ export class DataProviderService {
     const dataProvider = this.getDataProvider(dataSource);
     const symbol = dataProvider.getTestSymbol();
 
-    const quotes = await this.getQuotes([
+    const quotes = await this.getQuotes({
-      {
+      items: [
-        dataSource,
+        {
-        symbol
+          dataSource,
-      }
+          symbol
-    ]);
+        }
+      ],
+      useCache: false
+    });
 
     if (quotes[symbol]?.marketPrice > 0) {
       return true;
@@ -59,14 +62,16 @@ export class DataProviderService {
     return false;
   }
 
-  public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{
+  public async getAssetProfiles(items: UniqueAsset[]): Promise<{
     [symbol: string]: Partial<SymbolProfile>;
   }> {
     const response: {
       [symbol: string]: Partial<SymbolProfile>;
     } = {};
 
-    const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
+    const itemsGroupedByDataSource = groupBy(items, ({ dataSource }) => {
+      return dataSource;
+    });
 
     const promises = [];
 
@@ -127,7 +132,7 @@ export class DataProviderService {
   }
 
   public async getHistorical(
-    aItems: IDataGatheringItem[],
+    aItems: UniqueAsset[],
     aGranularity: Granularity = 'month',
     from: Date,
     to: Date
@@ -155,11 +160,11 @@ export class DataProviderService {
           )}'`
         : '';
 
-    const dataSources = aItems.map((item) => {
+    const dataSources = aItems.map(({ dataSource }) => {
-      return item.dataSource;
+      return dataSource;
     });
-    const symbols = aItems.map((item) => {
+    const symbols = aItems.map(({ symbol }) => {
-      return item.symbol;
+      return symbol;
     });
 
     try {
@@ -192,7 +197,7 @@ export class DataProviderService {
   }
 
   public async getHistoricalRaw(
-    aDataGatheringItems: IDataGatheringItem[],
+    aDataGatheringItems: UniqueAsset[],
     from: Date,
     to: Date
   ): Promise<{
@@ -229,7 +234,13 @@ export class DataProviderService {
     return result;
   }
 
-  public async getQuotes(items: IDataGatheringItem[]): Promise<{
+  public async getQuotes({
+    items,
+    useCache = true
+  }: {
+    items: UniqueAsset[];
+    useCache?: boolean;
+  }): Promise<{
     [symbol: string]: IDataProviderResponse;
   }> {
     const response: {
@@ -238,23 +249,24 @@ export class DataProviderService {
     const startTimeTotal = performance.now();
 
     // Get items from cache
-    const itemsToFetch: IDataGatheringItem[] = [];
+    const itemsToFetch: UniqueAsset[] = [];
 
     for (const { dataSource, symbol } of items) {
-      const quoteString = await this.redisCacheService.get(
+      if (useCache) {
-        this.redisCacheService.getQuoteKey({ dataSource, symbol })
+        const quoteString = await this.redisCacheService.get(
-      );
+          this.redisCacheService.getQuoteKey({ dataSource, symbol })
+        );
 
-      if (quoteString) {
+        if (quoteString) {
-        try {
+          try {
-          const cachedDataProviderResponse = JSON.parse(quoteString);
+            const cachedDataProviderResponse = JSON.parse(quoteString);
-          response[symbol] = cachedDataProviderResponse;
+            response[symbol] = cachedDataProviderResponse;
-        } catch {}
+            continue;
+          } catch {}
+        }
       }
 
-      if (!quoteString) {
+      itemsToFetch.push({ dataSource, symbol });
-        itemsToFetch.push({ dataSource, symbol });
-      }
     }
 
     const numberOfItemsInCache = Object.keys(response)?.length;

apps/api/src/services/exchange-rate-data/exchange-rate-data.service.ts

@@ -64,11 +64,11 @@ export class ExchangeRateDataService {
     if (Object.keys(result).length !== this.currencyPairs.length) {
       // Load currencies directly from data provider as a fallback
       // if historical data is not fully available
-      const quotes = await this.dataProviderService.getQuotes(
+      const quotes = await this.dataProviderService.getQuotes({
-        this.currencyPairs.map(({ dataSource, symbol }) => {
+        items: this.currencyPairs.map(({ dataSource, symbol }) => {
           return { dataSource, symbol };
         })
-      );
+      });
 
       for (const symbol of Object.keys(quotes)) {
         if (isNumber(quotes[symbol].marketPrice)) {