Feature/increase robustness if live data is missing (#1884)

* Continuously persist today's market data

* Add fallback to historical market data if data provider does not provide live data

* Update changelog

CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Added a fallback to historical market data if a data provider does not provide live data
+
+### Changed
+
+- Persisted today's market data continuously
+
 ### Fixed
 
 - Fixed the alignment of the performance column header in the holdings table

apps/api/src/app/portfolio/current-rate.service.mock.ts

@@ -1,9 +1,9 @@
 import { parseDate, resetHours } from '@ghostfolio/common/helper';
-import { DataProviderInfo } from '@ghostfolio/common/interfaces';
 import { addDays, endOfDay, isBefore, isSameDay } from 'date-fns';
 
 import { GetValueObject } from './interfaces/get-value-object.interface';
 import { GetValuesParams } from './interfaces/get-values-params.interface';
+import { GetValuesObject } from './interfaces/get-values-object.interface';
 
 function mockGetValue(symbol: string, date: Date) {
   switch (symbol) {
@@ -49,11 +49,9 @@ export const CurrentRateServiceMock = {
   getValues: ({
     dataGatheringItems,
     dateQuery
-  }: GetValuesParams): Promise<{
+  }: GetValuesParams): Promise<GetValuesObject> => {
-    dataProviderInfos: DataProviderInfo[];
-    values: GetValueObject[];
-  }> => {
     const values: GetValueObject[] = [];
+
     if (dateQuery.lt) {
       for (
         let date = resetHours(dateQuery.gte);
@@ -85,6 +83,7 @@ export const CurrentRateServiceMock = {
         }
       }
     }
-    return Promise.resolve({ values, dataProviderInfos: [] });
+
+    return Promise.resolve({ values, dataProviderInfos: [], errors: [] });
   }
 };

apps/api/src/app/portfolio/current-rate.service.spec.ts

@@ -1,12 +1,11 @@
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
 import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
-import { DataProviderInfo } from '@ghostfolio/common/interfaces';
+import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { DataSource, MarketData } from '@prisma/client';
 
 import { CurrentRateService } from './current-rate.service';
-import { GetValueObject } from './interfaces/get-value-object.interface';
+import { GetValuesObject } from './interfaces/get-values-object.interface';
-import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 
 jest.mock('@ghostfolio/api/services/market-data.service', () => {
   return {
@@ -123,21 +122,14 @@ describe('CurrentRateService', () => {
         },
         userCurrency: 'CHF'
       })
-    ).toMatchObject<{
+    ).toMatchObject<GetValuesObject>({
-      dataProviderInfos: DataProviderInfo[];
-      values: GetValueObject[];
-    }>({
       dataProviderInfos: [],
+      errors: [],
       values: [
         {
           date: undefined,
           marketPriceInBaseCurrency: 1841.823902,
           symbol: 'AMZN'
-        },
-        {
-          date: undefined,
-          marketPriceInBaseCurrency: 1847.839966,
-          symbol: 'AMZN'
         }
       ]
     });

apps/api/src/app/portfolio/current-rate.service.ts

@@ -2,13 +2,14 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
 import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
 import { resetHours } from '@ghostfolio/common/helper';
-import { DataProviderInfo } from '@ghostfolio/common/interfaces';
+import { DataProviderInfo, ResponseError } from '@ghostfolio/common/interfaces';
 import { Injectable } from '@nestjs/common';
 import { isBefore, isToday } from 'date-fns';
-import { flatten } from 'lodash';
+import { flatten, isEmpty, uniqBy } from 'lodash';
 
 import { GetValueObject } from './interfaces/get-value-object.interface';
 import { GetValuesParams } from './interfaces/get-values-params.interface';
+import { GetValuesObject } from './interfaces/get-values-object.interface';
 
 @Injectable()
 export class CurrentRateService {
@@ -23,10 +24,7 @@ export class CurrentRateService {
     dataGatheringItems,
     dateQuery,
     userCurrency
-  }: GetValuesParams): Promise<{
+  }: GetValuesParams): Promise<GetValuesObject> {
-    dataProviderInfos: DataProviderInfo[];
-    values: GetValueObject[];
-  }> {
     const dataProviderInfos: DataProviderInfo[] = [];
     const includeToday =
       (!dateQuery.lt || isBefore(new Date(), dateQuery.lt)) &&
@@ -34,9 +32,10 @@ export class CurrentRateService {
       (!dateQuery.in || this.containsToday(dateQuery.in));
 
     const promises: Promise<GetValueObject[]>[] = [];
+    const quoteErrors: ResponseError['errors'] = [];
+    const today = resetHours(new Date());
 
     if (includeToday) {
-      const today = resetHours(new Date());
       promises.push(
         this.dataProviderService
           .getQuotes(dataGatheringItems)
@@ -51,18 +50,26 @@ export class CurrentRateService {
                 );
               }
 
-              result.push({
+              if (dataResultProvider?.[dataGatheringItem.symbol]?.marketPrice) {
-                date: today,
+                result.push({
-                marketPriceInBaseCurrency:
+                  date: today,
-                  this.exchangeRateDataService.toCurrency(
+                  marketPriceInBaseCurrency:
-                    dataResultProvider?.[dataGatheringItem.symbol]
+                    this.exchangeRateDataService.toCurrency(
-                      ?.marketPrice ?? 0,
+                      dataResultProvider?.[dataGatheringItem.symbol]
-                    dataResultProvider?.[dataGatheringItem.symbol]?.currency,
+                        ?.marketPrice,
-                    userCurrency
+                      dataResultProvider?.[dataGatheringItem.symbol]?.currency,
-                  ),
+                      userCurrency
-                symbol: dataGatheringItem.symbol
+                    ),
-              });
+                  symbol: dataGatheringItem.symbol
+                });
+              } else {
+                quoteErrors.push({
+                  dataSource: dataGatheringItem.dataSource,
+                  symbol: dataGatheringItem.symbol
+                });
+              }
             }
+
             return result;
           })
       );
@@ -94,10 +101,60 @@ export class CurrentRateService {
         })
     );
 
-    return {
+    const values = flatten(await Promise.all(promises));
+
+    const response: GetValuesObject = {
       dataProviderInfos,
-      values: flatten(await Promise.all(promises))
+      errors: quoteErrors.map(({ dataSource, symbol }) => {
+        return { dataSource, symbol };
+      }),
+      values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`)
     };
+
+    if (!isEmpty(quoteErrors)) {
+      for (const { symbol } of quoteErrors) {
+        try {
+          // If missing quote, fallback to the latest available historical market price
+          let value: GetValueObject = response.values.find((currentValue) => {
+            return currentValue.symbol === symbol && isToday(currentValue.date);
+          });
+
+          if (!value) {
+            value = {
+              symbol,
+              date: today,
+              marketPriceInBaseCurrency: 0
+            };
+
+            response.values.push(value);
+          }
+
+          const [latestValue] = response.values
+            .filter((currentValue) => {
+              return (
+                currentValue.symbol === symbol &&
+                currentValue.marketPriceInBaseCurrency
+              );
+            })
+            .sort((a, b) => {
+              if (a.date < b.date) {
+                return 1;
+              }
+
+              if (a.date > b.date) {
+                return -1;
+              }
+
+              return 0;
+            });
+
+          value.marketPriceInBaseCurrency =
+            latestValue.marketPriceInBaseCurrency;
+        } catch {}
+      }
+    }
+
+    return response;
   }
 
   private containsToday(dates: Date[]): boolean {

apps/api/src/app/portfolio/interfaces/get-values-object.interface.ts

@@ -0,0 +1,9 @@
+import { DataProviderInfo, ResponseError } from '@ghostfolio/common/interfaces';
+
+import { GetValueObject } from './get-value-object.interface';
+
+export interface GetValuesObject {
+  dataProviderInfos: DataProviderInfo[];
+  errors: ResponseError['errors'];
+  values: GetValueObject[];
+}

apps/api/src/app/portfolio/portfolio-calculator.ts

@@ -24,9 +24,10 @@ import {
   isSameYear,
   max,
   min,
-  set
+  set,
+  subDays
 } from 'date-fns';
-import { first, flatten, isNumber, last, sortBy } from 'lodash';
+import { first, flatten, isNumber, last, sortBy, uniq } from 'lodash';
 
 import { CurrentRateService } from './current-rate.service';
 import { CurrentPositions } from './interfaces/current-positions.interface';
@@ -360,7 +361,7 @@ export class PortfolioCalculator {
 
     let firstTransactionPoint: TransactionPoint = null;
     let firstIndex = transactionPointsBeforeEndDate.length;
-    const dates = [];
+    let dates = [];
     const dataGatheringItems: IDataGatheringItem[] = [];
     const currencies: { [symbol: string]: string } = {};
 
@@ -389,15 +390,37 @@ export class PortfolioCalculator {
 
     dates.push(resetHours(end));
 
-    const { dataProviderInfos, values: marketSymbols } =
+    // Add dates of last week for fallback
-      await this.currentRateService.getValues({
+    dates.push(subDays(resetHours(new Date()), 7));
-        currencies,
+    dates.push(subDays(resetHours(new Date()), 6));
-        dataGatheringItems,
+    dates.push(subDays(resetHours(new Date()), 5));
-        dateQuery: {
+    dates.push(subDays(resetHours(new Date()), 4));
-          in: dates
+    dates.push(subDays(resetHours(new Date()), 3));
-        },
+    dates.push(subDays(resetHours(new Date()), 2));
-        userCurrency: this.currency
+    dates.push(subDays(resetHours(new Date()), 1));
-      });
+    dates.push(resetHours(new Date()));
+
+    dates = uniq(
+      dates.map((date) => {
+        return date.getTime();
+      })
+    ).map((timestamp) => {
+      return new Date(timestamp);
+    });
+    dates.sort((a, b) => a.getTime() - b.getTime());
+
+    const {
+      dataProviderInfos,
+      errors: currentRateErrors,
+      values: marketSymbols
+    } = await this.currentRateService.getValues({
+      currencies,
+      dataGatheringItems,
+      dateQuery: {
+        in: dates
+      },
+      userCurrency: this.currency
+    });
 
     this.dataProviderInfos = dataProviderInfos;
 
@@ -472,7 +495,13 @@ export class PortfolioCalculator {
         transactionCount: item.transactionCount
       });
 
-      if (hasErrors && item.investment.gt(0)) {
+      if (
+        (hasErrors ||
+          currentRateErrors.find(({ dataSource, symbol }) => {
+            return dataSource === item.dataSource && symbol === item.symbol;
+          })) &&
+        item.investment.gt(0)
+      ) {
         errors.push({ dataSource: item.dataSource, symbol: item.symbol });
       }
     }

apps/api/src/services/data-provider/data-provider.service.ts

@@ -7,13 +7,13 @@ import {
   IDataProviderResponse
 } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
-import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
 import { UserWithSettings } from '@ghostfolio/common/types';
 import { Granularity } from '@ghostfolio/common/types';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
 import { format, isValid } from 'date-fns';
-import { groupBy, isEmpty } from 'lodash';
+import { groupBy, isEmpty, isNumber } from 'lodash';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config';
 
@@ -241,7 +241,7 @@ export class DataProviderService {
         const promise = Promise.resolve(dataProvider.getQuotes(symbolsChunk));
 
         promises.push(
-          promise.then((result) => {
+          promise.then(async (result) => {
             for (const [symbol, dataProviderResponse] of Object.entries(
               result
             )) {
@@ -256,6 +256,38 @@ export class DataProviderService {
                 1000
               ).toFixed(3)} seconds`
             );
+
+            try {
+              const date = getStartOfUtcDate(new Date());
+
+              // Upsert quotes by imitating missing upsertMany functionality
+              // with $transaction
+              const upsertPromises = Object.keys(response)
+                .filter((symbol) => {
+                  return (
+                    isNumber(response[symbol].marketPrice) &&
+                    response[symbol].marketPrice > 0
+                  );
+                })
+                .map((symbol) =>
+                  this.prismaService.marketData.upsert({
+                    create: {
+                      date,
+                      symbol,
+                      dataSource: response[symbol].dataSource,
+                      marketPrice: response[symbol].marketPrice
+                    },
+                    update: {
+                      marketPrice: response[symbol].marketPrice
+                    },
+                    where: {
+                      date_symbol: { date, symbol }
+                    }
+                  })
+                );
+
+              await this.prismaService.$transaction(upsertPromises);
+            } catch {}
           })
         );
       }

libs/common/src/lib/helper.ts

@@ -152,6 +152,13 @@ export function getNumberFormatGroup(aLocale?: string) {
   }).value;
 }
 
+export function getStartOfUtcDate(aDate: Date) {
+  const date = new Date(aDate);
+  date.setUTCHours(0, 0, 0, 0);
+
+  return date;
+}
+
 export function getSum(aArray: Big[]) {
   if (aArray?.length > 0) {
     return aArray.reduce((a, b) => a.plus(b), new Big(0));