Feature/increase robustness if live data is missing (#1884)

* Continuously persist today's market data * Add fallback to historical market data if data provider does not provide live data * Update changelog
2023-04-22 14:43:57 +02:00 · 2023-04-22 14:43:57 +02:00 · aafedd5f75
commit aafedd5f75
parent 32956ae04c
8 changed files with 187 additions and 54 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ## Unreleased

+### Added
+
+- Added a fallback to historical market data if a data provider does not provide live data
+
+### Changed
+
+- Persisted today's market data continuously
+
 ### Fixed

 - Fixed the alignment of the performance column header in the holdings table
--- a/apps/api/src/app/portfolio/current-rate.service.mock.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.mock.ts
@ -1,9 +1,9 @@
 import { parseDate, resetHours } from '@ghostfolio/common/helper';
-import { DataProviderInfo } from '@ghostfolio/common/interfaces';
 import { addDays, endOfDay, isBefore, isSameDay } from 'date-fns';

 import { GetValueObject } from './interfaces/get-value-object.interface';
 import { GetValuesParams } from './interfaces/get-values-params.interface';
+import { GetValuesObject } from './interfaces/get-values-object.interface';

 function mockGetValue(symbol: string, date: Date) {
  switch (symbol) {
@ -49,11 +49,9 @@ export const CurrentRateServiceMock = {
  getValues: ({
    dataGatheringItems,
    dateQuery
-  }: GetValuesParams): Promise<{
-    dataProviderInfos: DataProviderInfo[];
-    values: GetValueObject[];
-  }> => {
+  }: GetValuesParams): Promise<GetValuesObject> => {
    const values: GetValueObject[] = [];
+
    if (dateQuery.lt) {
      for (
        let date = resetHours(dateQuery.gte);
@ -85,6 +83,7 @@ export const CurrentRateServiceMock = {
        }
      }
    }
-    return Promise.resolve({ values, dataProviderInfos: [] });
+
+    return Promise.resolve({ values, dataProviderInfos: [], errors: [] });
  }
 };
--- a/apps/api/src/app/portfolio/current-rate.service.spec.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts
@ -1,12 +1,11 @@
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
 import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
-import { DataProviderInfo } from '@ghostfolio/common/interfaces';
+import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { DataSource, MarketData } from '@prisma/client';

 import { CurrentRateService } from './current-rate.service';
-import { GetValueObject } from './interfaces/get-value-object.interface';
-import { PropertyService } from '@ghostfolio/api/services/property/property.service';
+import { GetValuesObject } from './interfaces/get-values-object.interface';

 jest.mock('@ghostfolio/api/services/market-data.service', () => {
  return {
@ -123,21 +122,14 @@ describe('CurrentRateService', () => {
        },
        userCurrency: 'CHF'
      })
-    ).toMatchObject<{
-      dataProviderInfos: DataProviderInfo[];
-      values: GetValueObject[];
-    }>({
+    ).toMatchObject<GetValuesObject>({
      dataProviderInfos: [],
+      errors: [],
      values: [
        {
          date: undefined,
          marketPriceInBaseCurrency: 1841.823902,
          symbol: 'AMZN'
-        },
-        {
-          date: undefined,
-          marketPriceInBaseCurrency: 1847.839966,
-          symbol: 'AMZN'
        }
      ]
    });
--- a/apps/api/src/app/portfolio/current-rate.service.ts
+++ b/apps/api/src/app/portfolio/current-rate.service.ts
@ -2,13 +2,14 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
 import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
 import { resetHours } from '@ghostfolio/common/helper';
-import { DataProviderInfo } from '@ghostfolio/common/interfaces';
+import { DataProviderInfo, ResponseError } from '@ghostfolio/common/interfaces';
 import { Injectable } from '@nestjs/common';
 import { isBefore, isToday } from 'date-fns';
-import { flatten } from 'lodash';
+import { flatten, isEmpty, uniqBy } from 'lodash';

 import { GetValueObject } from './interfaces/get-value-object.interface';
 import { GetValuesParams } from './interfaces/get-values-params.interface';
+import { GetValuesObject } from './interfaces/get-values-object.interface';

@Injectable()
 export class CurrentRateService {
@ -23,10 +24,7 @@ export class CurrentRateService {
    dataGatheringItems,
    dateQuery,
    userCurrency
-  }: GetValuesParams): Promise<{
-    dataProviderInfos: DataProviderInfo[];
-    values: GetValueObject[];
-  }> {
+  }: GetValuesParams): Promise<GetValuesObject> {
    const dataProviderInfos: DataProviderInfo[] = [];
    const includeToday =
      (!dateQuery.lt || isBefore(new Date(), dateQuery.lt)) &&
@ -34,9 +32,10 @@ export class CurrentRateService {
      (!dateQuery.in || this.containsToday(dateQuery.in));

    const promises: Promise<GetValueObject[]>[] = [];
+    const quoteErrors: ResponseError['errors'] = [];
+    const today = resetHours(new Date());

    if (includeToday) {
-      const today = resetHours(new Date());
      promises.push(
        this.dataProviderService
          .getQuotes(dataGatheringItems)
@ -51,18 +50,26 @@ export class CurrentRateService {
                );
              }

-              result.push({
-                date: today,
-                marketPriceInBaseCurrency:
-                  this.exchangeRateDataService.toCurrency(
-                    dataResultProvider?.[dataGatheringItem.symbol]
-                      ?.marketPrice ?? 0,
-                    dataResultProvider?.[dataGatheringItem.symbol]?.currency,
-                    userCurrency
-                  ),
-                symbol: dataGatheringItem.symbol
-              });
+              if (dataResultProvider?.[dataGatheringItem.symbol]?.marketPrice) {
+                result.push({
+                  date: today,
+                  marketPriceInBaseCurrency:
+                    this.exchangeRateDataService.toCurrency(
+                      dataResultProvider?.[dataGatheringItem.symbol]
+                        ?.marketPrice,
+                      dataResultProvider?.[dataGatheringItem.symbol]?.currency,
+                      userCurrency
+                    ),
+                  symbol: dataGatheringItem.symbol
+                });
+              } else {
+                quoteErrors.push({
+                  dataSource: dataGatheringItem.dataSource,
+                  symbol: dataGatheringItem.symbol
+                });
+              }
            }
+
            return result;
          })
      );
@ -94,10 +101,60 @@ export class CurrentRateService {
        })
    );

-    return {
+    const values = flatten(await Promise.all(promises));
+
+    const response: GetValuesObject = {
      dataProviderInfos,
-      values: flatten(await Promise.all(promises))
+      errors: quoteErrors.map(({ dataSource, symbol }) => {
+        return { dataSource, symbol };
+      }),
+      values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`)
    };
+
+    if (!isEmpty(quoteErrors)) {
+      for (const { symbol } of quoteErrors) {
+        try {
+          // If missing quote, fallback to the latest available historical market price
+          let value: GetValueObject = response.values.find((currentValue) => {
+            return currentValue.symbol === symbol && isToday(currentValue.date);
+          });
+
+          if (!value) {
+            value = {
+              symbol,
+              date: today,
+              marketPriceInBaseCurrency: 0
+            };
+
+            response.values.push(value);
+          }
+
+          const [latestValue] = response.values
+            .filter((currentValue) => {
+              return (
+                currentValue.symbol === symbol &&
+                currentValue.marketPriceInBaseCurrency
+              );
+            })
+            .sort((a, b) => {
+              if (a.date < b.date) {
+                return 1;
+              }
+
+              if (a.date > b.date) {
+                return -1;
+              }
+
+              return 0;
+            });
+
+          value.marketPriceInBaseCurrency =
+            latestValue.marketPriceInBaseCurrency;
+        } catch {}
+      }
+    }
+
+    return response;
  }

  private containsToday(dates: Date[]): boolean {
--- a/apps/api/src/app/portfolio/interfaces/get-values-object.interface.ts
+++ b/apps/api/src/app/portfolio/interfaces/get-values-object.interface.ts
@ -0,0 +1,9 @@
+import { DataProviderInfo, ResponseError } from '@ghostfolio/common/interfaces';
+
+import { GetValueObject } from './get-value-object.interface';
+
+export interface GetValuesObject {
+  dataProviderInfos: DataProviderInfo[];
+  errors: ResponseError['errors'];
+  values: GetValueObject[];
+}
--- a/apps/api/src/app/portfolio/portfolio-calculator.ts
+++ b/apps/api/src/app/portfolio/portfolio-calculator.ts
@ -24,9 +24,10 @@ import {
  isSameYear,
  max,
  min,
-  set
+  set,
+  subDays
 } from 'date-fns';
-import { first, flatten, isNumber, last, sortBy } from 'lodash';
+import { first, flatten, isNumber, last, sortBy, uniq } from 'lodash';

 import { CurrentRateService } from './current-rate.service';
 import { CurrentPositions } from './interfaces/current-positions.interface';
@ -360,7 +361,7 @@ export class PortfolioCalculator {

    let firstTransactionPoint: TransactionPoint = null;
    let firstIndex = transactionPointsBeforeEndDate.length;
-    const dates = [];
+    let dates = [];
    const dataGatheringItems: IDataGatheringItem[] = [];
    const currencies: { [symbol: string]: string } = {};

@ -389,15 +390,37 @@ export class PortfolioCalculator {

    dates.push(resetHours(end));

-    const { dataProviderInfos, values: marketSymbols } =
-      await this.currentRateService.getValues({
-        currencies,
-        dataGatheringItems,
-        dateQuery: {
-          in: dates
-        },
-        userCurrency: this.currency
-      });
+    // Add dates of last week for fallback
+    dates.push(subDays(resetHours(new Date()), 7));
+    dates.push(subDays(resetHours(new Date()), 6));
+    dates.push(subDays(resetHours(new Date()), 5));
+    dates.push(subDays(resetHours(new Date()), 4));
+    dates.push(subDays(resetHours(new Date()), 3));
+    dates.push(subDays(resetHours(new Date()), 2));
+    dates.push(subDays(resetHours(new Date()), 1));
+    dates.push(resetHours(new Date()));
+
+    dates = uniq(
+      dates.map((date) => {
+        return date.getTime();
+      })
+    ).map((timestamp) => {
+      return new Date(timestamp);
+    });
+    dates.sort((a, b) => a.getTime() - b.getTime());
+
+    const {
+      dataProviderInfos,
+      errors: currentRateErrors,
+      values: marketSymbols
+    } = await this.currentRateService.getValues({
+      currencies,
+      dataGatheringItems,
+      dateQuery: {
+        in: dates
+      },
+      userCurrency: this.currency
+    });

    this.dataProviderInfos = dataProviderInfos;

@ -472,7 +495,13 @@ export class PortfolioCalculator {
        transactionCount: item.transactionCount
      });

-      if (hasErrors && item.investment.gt(0)) {
+      if (
+        (hasErrors ||
+          currentRateErrors.find(({ dataSource, symbol }) => {
+            return dataSource === item.dataSource && symbol === item.symbol;
+          })) &&
+        item.investment.gt(0)
+      ) {
        errors.push({ dataSource: item.dataSource, symbol: item.symbol });
      }
    }
--- a/apps/api/src/services/data-provider/data-provider.service.ts
+++ b/apps/api/src/services/data-provider/data-provider.service.ts
@ -7,13 +7,13 @@ import {
  IDataProviderResponse
 } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
-import { DATE_FORMAT } from '@ghostfolio/common/helper';
+import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
 import { UserWithSettings } from '@ghostfolio/common/types';
 import { Granularity } from '@ghostfolio/common/types';
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
 import { format, isValid } from 'date-fns';
-import { groupBy, isEmpty } from 'lodash';
+import { groupBy, isEmpty, isNumber } from 'lodash';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { PROPERTY_DATA_SOURCE_MAPPING } from '@ghostfolio/common/config';

@ -241,7 +241,7 @@ export class DataProviderService {
        const promise = Promise.resolve(dataProvider.getQuotes(symbolsChunk));

        promises.push(
-          promise.then((result) => {
+          promise.then(async (result) => {
            for (const [symbol, dataProviderResponse] of Object.entries(
              result
            )) {
@ -256,6 +256,38 @@ export class DataProviderService {
                1000
              ).toFixed(3)} seconds`
            );
+
+            try {
+              const date = getStartOfUtcDate(new Date());
+
+              // Upsert quotes by imitating missing upsertMany functionality
+              // with $transaction
+              const upsertPromises = Object.keys(response)
+                .filter((symbol) => {
+                  return (
+                    isNumber(response[symbol].marketPrice) &&
+                    response[symbol].marketPrice > 0
+                  );
+                })
+                .map((symbol) =>
+                  this.prismaService.marketData.upsert({
+                    create: {
+                      date,
+                      symbol,
+                      dataSource: response[symbol].dataSource,
+                      marketPrice: response[symbol].marketPrice
+                    },
+                    update: {
+                      marketPrice: response[symbol].marketPrice
+                    },
+                    where: {
+                      date_symbol: { date, symbol }
+                    }
+                  })
+                );
+
+              await this.prismaService.$transaction(upsertPromises);
+            } catch {}
          })
        );
      }
--- a/libs/common/src/lib/helper.ts
+++ b/libs/common/src/lib/helper.ts
@ -152,6 +152,13 @@ export function getNumberFormatGroup(aLocale?: string) {
  }).value;
 }

+export function getStartOfUtcDate(aDate: Date) {
+  const date = new Date(aDate);
+  date.setUTCHours(0, 0, 0, 0);
+
+  return date;
+}
+
 export function getSum(aArray: Big[]) {
  if (aArray?.length > 0) {
    return aArray.reduce((a, b) => a.plus(b), new Big(0));