Feature/improve performance of chart calculation (#1271)

* Improve performance chart calculation

Co-Authored-By: gizmodus <11334553+gizmodus@users.noreply.github.com>

* Update changelog

Co-Authored-By: gizmodus <11334553+gizmodus@users.noreply.github.com>

* Improve chart tooltip of benchmark comparator

* Update changelog

Co-authored-by: gizmodus <11334553+gizmodus@users.noreply.github.com>

CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Improved the algorithm of the performance chart calculation
+
+### Fixed
+
+- Improved the loading indicator of the benchmark comparator
+
 ## 1.194.0 - 17.09.2022
 
 ### Added

apps/api/src/app/benchmark/benchmark.service.ts

@@ -4,7 +4,10 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data
 import { MarketDataService } from '@ghostfolio/api/services/market-data.service';
 import { PropertyService } from '@ghostfolio/api/services/property/property.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
-import { PROPERTY_BENCHMARKS } from '@ghostfolio/common/config';
+import {
+  MAX_CHART_ITEMS,
+  PROPERTY_BENCHMARKS
+} from '@ghostfolio/common/config';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import {
   BenchmarkMarketDataDetails,
@@ -16,7 +19,6 @@ import { SymbolProfile } from '@prisma/client';
 import Big from 'big.js';
 import { format } from 'date-fns';
 import ms from 'ms';
-import { v4 as uuidv4 } from 'uuid';
 
 @Injectable()
 export class BenchmarkService {
@@ -157,27 +159,38 @@ export class BenchmarkService {
       })
     ]);
 
-    marketDataItems.push({
+    const step = Math.round(
-      ...currentSymbolItem,
+      marketDataItems.length / Math.min(marketDataItems.length, MAX_CHART_ITEMS)
-      createdAt: new Date(),
+    );
-      date: new Date(),
-      id: uuidv4()
-    });
 
     const marketPriceAtStartDate = marketDataItems?.[0]?.marketPrice ?? 0;
     return {
-      marketData: marketDataItems.map((marketDataItem) => {
+      marketData: [
-        return {
+        ...marketDataItems
-          date: format(marketDataItem.date, DATE_FORMAT),
+          .filter((marketDataItem, index) => {
+            return index % step === 0;
+          })
+          .map((marketDataItem) => {
+            return {
+              date: format(marketDataItem.date, DATE_FORMAT),
+              value:
+                marketPriceAtStartDate === 0
+                  ? 0
+                  : this.calculateChangeInPercentage(
+                      marketPriceAtStartDate,
+                      marketDataItem.marketPrice
+                    ) * 100
+            };
+          }),
+        {
+          date: format(new Date(), DATE_FORMAT),
           value:
-            marketPriceAtStartDate === 0
+            this.calculateChangeInPercentage(
-              ? 0
+              marketPriceAtStartDate,
-              : this.calculateChangeInPercentage(
+              currentSymbolItem.marketPrice
-                  marketPriceAtStartDate,
+            ) * 100
-                  marketDataItem.marketPrice
+        }
-                ) * 100
+      ]
-        };
-      })
     };
   }
 

apps/api/src/app/portfolio/portfolio-calculator.ts

@@ -16,12 +16,11 @@ import {
   isBefore,
   isSameMonth,
   isSameYear,
-  isWithinInterval,
   max,
   min,
   set
 } from 'date-fns';
-import { first, flatten, isNumber, sortBy } from 'lodash';
+import { first, flatten, isNumber, last, sortBy } from 'lodash';
 
 import { CurrentRateService } from './current-rate.service';
 import { CurrentPositions } from './interfaces/current-positions.interface';
@@ -168,6 +167,131 @@ export class PortfolioCalculator {
     this.transactionPoints = transactionPoints;
   }
 
+  public async getChartData(start: Date, end = new Date(Date.now()), step = 1) {
+    const symbols: { [symbol: string]: boolean } = {};
+
+    const transactionPointsBeforeEndDate =
+      this.transactionPoints?.filter((transactionPoint) => {
+        return isBefore(parseDate(transactionPoint.date), end);
+      }) ?? [];
+
+    const firstIndex = transactionPointsBeforeEndDate.length;
+    const dates: Date[] = [];
+    const dataGatheringItems: IDataGatheringItem[] = [];
+    const currencies: { [symbol: string]: string } = {};
+
+    let day = start;
+
+    while (isBefore(day, end)) {
+      dates.push(resetHours(day));
+      day = addDays(day, step);
+    }
+
+    dates.push(resetHours(end));
+
+    for (const item of transactionPointsBeforeEndDate[firstIndex - 1].items) {
+      dataGatheringItems.push({
+        dataSource: item.dataSource,
+        symbol: item.symbol
+      });
+      currencies[item.symbol] = item.currency;
+      symbols[item.symbol] = true;
+    }
+
+    const marketSymbols = await this.currentRateService.getValues({
+      currencies,
+      dataGatheringItems,
+      dateQuery: {
+        in: dates
+      },
+      userCurrency: this.currency
+    });
+
+    const marketSymbolMap: {
+      [date: string]: { [symbol: string]: Big };
+    } = {};
+
+    for (const marketSymbol of marketSymbols) {
+      const dateString = format(marketSymbol.date, DATE_FORMAT);
+      if (!marketSymbolMap[dateString]) {
+        marketSymbolMap[dateString] = {};
+      }
+      if (marketSymbol.marketPriceInBaseCurrency) {
+        marketSymbolMap[dateString][marketSymbol.symbol] = new Big(
+          marketSymbol.marketPriceInBaseCurrency
+        );
+      }
+    }
+
+    const netPerformanceValuesBySymbol: {
+      [symbol: string]: { [date: string]: Big };
+    } = {};
+
+    const investmentValuesBySymbol: {
+      [symbol: string]: { [date: string]: Big };
+    } = {};
+
+    const totalNetPerformanceValues: { [date: string]: Big } = {};
+    const totalInvestmentValues: { [date: string]: Big } = {};
+
+    for (const symbol of Object.keys(symbols)) {
+      const { netPerformanceValues, investmentValues } = this.getSymbolMetrics({
+        end,
+        marketSymbolMap,
+        start,
+        step,
+        symbol,
+        isChartMode: true
+      });
+
+      netPerformanceValuesBySymbol[symbol] = netPerformanceValues;
+      investmentValuesBySymbol[symbol] = investmentValues;
+    }
+
+    for (const currentDate of dates) {
+      const dateString = format(currentDate, DATE_FORMAT);
+
+      for (const symbol of Object.keys(netPerformanceValuesBySymbol)) {
+        totalNetPerformanceValues[dateString] =
+          totalNetPerformanceValues[dateString] ?? new Big(0);
+
+        if (netPerformanceValuesBySymbol[symbol]?.[dateString]) {
+          totalNetPerformanceValues[dateString] = totalNetPerformanceValues[
+            dateString
+          ].add(netPerformanceValuesBySymbol[symbol][dateString]);
+        }
+
+        totalInvestmentValues[dateString] =
+          totalInvestmentValues[dateString] ?? new Big(0);
+
+        if (investmentValuesBySymbol[symbol]?.[dateString]) {
+          totalInvestmentValues[dateString] = totalInvestmentValues[
+            dateString
+          ].add(investmentValuesBySymbol[symbol][dateString]);
+        }
+      }
+    }
+
+    const isInPercentage = true;
+
+    return Object.keys(totalNetPerformanceValues).map((date) => {
+      return isInPercentage
+        ? {
+            date,
+            value: totalInvestmentValues[date].eq(0)
+              ? 0
+              : totalNetPerformanceValues[date]
+                  .div(totalInvestmentValues[date])
+                  .mul(100)
+                  .toNumber()
+          }
+        : {
+            date,
+            value: totalNetPerformanceValues[date].toNumber()
+          };
+    });
+  }
+
   public async getCurrentPositions(
     start: Date,
     end = new Date(Date.now())
@@ -710,15 +834,19 @@ export class PortfolioCalculator {
 
   private getSymbolMetrics({
     end,
+    isChartMode = false,
     marketSymbolMap,
     start,
+    step = 1,
     symbol
   }: {
     end: Date;
+    isChartMode?: boolean;
     marketSymbolMap: {
       [date: string]: { [symbol: string]: Big };
     };
     start: Date;
+    step?: number;
     symbol: string;
   }) {
     let orders: PortfolioOrderItem[] = this.orders.filter((order) => {
@@ -767,10 +895,12 @@ export class PortfolioCalculator {
     let grossPerformanceFromSells = new Big(0);
     let initialValue: Big;
     let investmentAtStartDate: Big;
+    const investmentValues: { [date: string]: Big } = {};
     let lastAveragePrice = new Big(0);
     let lastTransactionInvestment = new Big(0);
     let lastValueOfInvestmentBeforeTransaction = new Big(0);
     let maxTotalInvestment = new Big(0);
+    const netPerformanceValues: { [date: string]: Big } = {};
     let timeWeightedGrossPerformancePercentage = new Big(1);
     let timeWeightedNetPerformancePercentage = new Big(1);
     let totalInvestment = new Big(0);
@@ -805,6 +935,41 @@ export class PortfolioCalculator {
       unitPrice: unitPriceAtEndDate
     });
 
+    let day = start;
+    let lastUnitPrice: Big;
+
+    if (isChartMode) {
+      const datesWithOrders = {};
+
+      for (const order of orders) {
+        datesWithOrders[order.date] = true;
+      }
+
+      while (isBefore(day, end)) {
+        const hasDate = datesWithOrders[format(day, DATE_FORMAT)];
+
+        if (!hasDate) {
+          orders.push({
+            symbol,
+            currency: null,
+            date: format(day, DATE_FORMAT),
+            dataSource: null,
+            fee: new Big(0),
+            name: '',
+            quantity: new Big(0),
+            type: TypeOfOrder.BUY,
+            unitPrice:
+              marketSymbolMap[format(day, DATE_FORMAT)]?.[symbol] ??
+              lastUnitPrice
+          });
+        }
+
+        lastUnitPrice = last(orders).unitPrice;
+
+        day = addDays(day, step);
+      }
+    }
+
     // Sort orders so that the start and end placeholder order are at the right
     // position
     orders = sortBy(orders, (order) => {
@@ -968,6 +1133,14 @@ export class PortfolioCalculator {
         grossPerformanceAtStartDate = grossPerformance;
       }
 
+      if (isChartMode && i > indexOfStartOrder) {
+        netPerformanceValues[order.date] = grossPerformance
+          .minus(grossPerformanceAtStartDate)
+          .minus(fees.minus(feesAtStartDate));
+
+        investmentValues[order.date] = totalInvestment;
+      }
+
       if (i === indexOfEndOrder) {
         break;
       }
@@ -1056,7 +1229,9 @@ export class PortfolioCalculator {
     return {
       initialValue,
       grossPerformancePercentage,
+      investmentValues,
       netPerformancePercentage,
+      netPerformanceValues,
       hasErrors: totalUnits.gt(0) && (!initialValue || !unitPriceAtEndDate),
       netPerformance: totalNetPerformance,
       grossPerformance: totalGrossPerformance

apps/api/src/app/portfolio/portfolio.service.ts

@@ -21,6 +21,7 @@ import { ImpersonationService } from '@ghostfolio/api/services/impersonation.ser
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
 import {
   ASSET_SUB_CLASS_EMERGENCY_FUND,
+  MAX_CHART_ITEMS,
   UNKNOWN_KEY
 } from '@ghostfolio/common/config';
 import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper';
@@ -57,7 +58,6 @@ import {
 } from '@prisma/client';
 import Big from 'big.js';
 import {
-  addDays,
   differenceInDays,
   endOfToday,
   format,
@@ -72,7 +72,7 @@ import {
   subDays,
   subYears
 } from 'date-fns';
-import { isEmpty, last, sortBy, uniq, uniqBy } from 'lodash';
+import { isEmpty, sortBy, uniq, uniqBy } from 'lodash';
 
 import {
   HistoricalDataContainer,
@@ -86,7 +86,6 @@ const emergingMarkets = require('../../assets/countries/emerging-markets.json');
 
 @Injectable()
 export class PortfolioService {
-  private static readonly MAX_CHART_ITEMS = 250;
   private baseCurrency: string;
 
   public constructor(
@@ -388,43 +387,19 @@ export class PortfolioService {
 
     const daysInMarket = differenceInDays(new Date(), startDate);
     const step = Math.round(
-      daysInMarket / Math.min(daysInMarket, PortfolioService.MAX_CHART_ITEMS)
+      daysInMarket / Math.min(daysInMarket, MAX_CHART_ITEMS)
     );
 
-    const items: HistoricalDataItem[] = [];
+    const items = await portfolioCalculator.getChartData(
-
+      startDate,
-    let currentEndDate = startDate;
+      endDate,
-
+      step
-    while (isBefore(currentEndDate, endDate)) {
+    );
-      const currentPositions = await portfolioCalculator.getCurrentPositions(
-        startDate,
-        currentEndDate
-      );
-
-      items.push({
-        date: format(currentEndDate, DATE_FORMAT),
-        value: currentPositions.netPerformancePercentage.toNumber() * 100
-      });
-
-      currentEndDate = addDays(currentEndDate, step);
-    }
-
-    const today = new Date();
-
-    if (last(items)?.date !== format(today, DATE_FORMAT)) {
-      // Add today
-      const { netPerformancePercentage } =
-        await portfolioCalculator.getCurrentPositions(startDate, today);
-      items.push({
-        date: format(today, DATE_FORMAT),
-        value: netPerformancePercentage.toNumber() * 100
-      });
-    }
 
     return {
+      items,
       isAllTimeHigh: false,
-      isAllTimeLow: false,
+      isAllTimeLow: false
-      items: items
     };
   }
 

apps/client/src/app/components/benchmark-comparator/benchmark-comparator.component.ts

@@ -23,11 +23,7 @@ import {
   getTextColor,
   parseDate
 } from '@ghostfolio/common/helper';
-import {
+import { LineChartItem, User } from '@ghostfolio/common/interfaces';
-  LineChartItem,
-  UniqueAsset,
-  User
-} from '@ghostfolio/common/interfaces';
 import { DateRange } from '@ghostfolio/common/types';
 import {
   Chart,
@@ -215,7 +211,7 @@ export class BenchmarkComparatorComponent implements OnChanges, OnDestroy {
         locale: this.locale,
         unit: '%'
       }),
-      mode: 'index',
+      mode: 'x',
       position: <unknown>'top',
       xAlign: 'center',
       yAlign: 'bottom'

libs/common/src/lib/config.ts

@@ -67,6 +67,8 @@ export const GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS: JobOptions = {
   }
 };
 
+export const MAX_CHART_ITEMS = 365;
+
 export const PROPERTY_BENCHMARKS = 'BENCHMARKS';
 export const PROPERTY_COUPONS = 'COUPONS';
 export const PROPERTY_CURRENCIES = 'CURRENCIES';