Feature/optimize import validation by reducing to unique asset profiles (#2198)

* Optimize activities validation * Optimize data gathering in import * Update changelog
2023-08-01 09:10:13 +02:00 · 2023-08-01 09:10:13 +02:00 · 286e41eb21
commit 286e41eb21
parent 4973d0261d
9 changed files with 153 additions and 62 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

+## Unreleased
+
+### Changed
+
+- Optimized the validation in the activities import by reducing the list to unique asset profiles
+- Optimized the data gathering in the activities import
+
 ## 1.295.0 - 2023-07-30

 ### Added
--- a/apps/api/src/app/admin/admin.controller.ts
+++ b/apps/api/src/app/admin/admin.controller.ts
@ -7,6 +7,7 @@ import {
  GATHER_ASSET_PROFILE_PROCESS,
  GATHER_ASSET_PROFILE_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import {
  AdminData,
  AdminMarketData,
@ -116,7 +117,7 @@ export class AdminController {
          name: GATHER_ASSET_PROFILE_PROCESS,
          opts: {
            ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-            jobId: `${dataSource}-${symbol}`
+            jobId: getAssetProfileIdentifier({ dataSource, symbol })
          }
        };
      })
@ -152,7 +153,7 @@ export class AdminController {
          name: GATHER_ASSET_PROFILE_PROCESS,
          opts: {
            ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-            jobId: `${dataSource}-${symbol}`
+            jobId: getAssetProfileIdentifier({ dataSource, symbol })
          }
        };
      })
@ -185,7 +186,7 @@ export class AdminController {
      name: GATHER_ASSET_PROFILE_PROCESS,
      opts: {
        ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-        jobId: `${dataSource}-${symbol}`
+        jobId: getAssetProfileIdentifier({ dataSource, symbol })
      }
    });
  }
--- a/apps/api/src/app/import/import.service.ts
+++ b/apps/api/src/app/import/import.service.ts
@ -8,10 +8,14 @@ import {
 import { OrderService } from '@ghostfolio/api/app/order/order.service';
 import { PlatformService } from '@ghostfolio/api/app/platform/platform.service';
 import { PortfolioService } from '@ghostfolio/api/app/portfolio/portfolio.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
 import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data/exchange-rate-data.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
-import { parseDate } from '@ghostfolio/common/helper';
+import {
+  getAssetProfileIdentifier,
+  parseDate
+} from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import {
  AccountWithPlatform,
@ -21,12 +25,14 @@ import { Injectable } from '@nestjs/common';
 import { DataSource, Prisma, SymbolProfile } from '@prisma/client';
 import Big from 'big.js';
 import { endOfToday, isAfter, isSameDay, parseISO } from 'date-fns';
+import { uniqBy } from 'lodash';
 import { v4 as uuidv4 } from 'uuid';

@Injectable()
 export class ImportService {
  public constructor(
    private readonly accountService: AccountService,
+    private readonly dataGatheringService: DataGatheringService,
    private readonly dataProviderService: DataProviderService,
    private readonly exchangeRateDataService: ExchangeRateDataService,
    private readonly orderService: OrderService,
@ -220,8 +226,7 @@ export class ImportService {

    const assetProfiles = await this.validateActivities({
      activitiesDto,
-      maxActivitiesToImport,
-      userId
+      maxActivitiesToImport
    });

    const activitiesExtendedWithErrors = await this.extendActivitiesWithErrors({
@ -250,10 +255,37 @@ export class ImportService {
      error,
      fee,
      quantity,
-      SymbolProfile: assetProfile,
+      SymbolProfile,
      type,
      unitPrice
    } of activitiesExtendedWithErrors) {
+      const assetProfile = assetProfiles[
+        getAssetProfileIdentifier({
+          dataSource: SymbolProfile.dataSource,
+          symbol: SymbolProfile.symbol
+        })
+      ] ?? {
+        currency: SymbolProfile.currency,
+        dataSource: SymbolProfile.dataSource,
+        symbol: SymbolProfile.symbol
+      };
+      const {
+        assetClass,
+        assetSubClass,
+        countries,
+        createdAt,
+        currency,
+        dataSource,
+        id,
+        isin,
+        name,
+        scraperConfiguration,
+        sectors,
+        symbol,
+        symbolMapping,
+        url,
+        updatedAt
+      } = assetProfile;
      const validatedAccount = accounts.find(({ id }) => {
        return id === accountId;
      });
@ -279,23 +311,22 @@ export class ImportService {
          id: uuidv4(),
          isDraft: isAfter(date, endOfToday()),
          SymbolProfile: {
-            assetClass: assetProfile.assetClass,
-            assetSubClass: assetProfile.assetSubClass,
-            comment: assetProfile.comment,
-            countries: assetProfile.countries,
-            createdAt: assetProfile.createdAt,
-            currency: assetProfile.currency,
-            dataSource: assetProfile.dataSource,
-            id: assetProfile.id,
-            isin: assetProfile.isin,
-            name: assetProfile.name,
-            scraperConfiguration: assetProfile.scraperConfiguration,
-            sectors: assetProfile.sectors,
-            symbol: assetProfile.currency,
-            symbolMapping: assetProfile.symbolMapping,
-            updatedAt: assetProfile.updatedAt,
-            url: assetProfile.url,
-            ...assetProfiles[assetProfile.symbol]
+            assetClass,
+            assetSubClass,
+            countries,
+            createdAt,
+            currency,
+            dataSource,
+            id,
+            isin,
+            name,
+            scraperConfiguration,
+            sectors,
+            symbol,
+            symbolMapping,
+            updatedAt,
+            url,
+            comment: assetProfile.comment
          },
          Account: validatedAccount,
          symbolProfileId: undefined,
@ -318,14 +349,14 @@ export class ImportService {
          SymbolProfile: {
            connectOrCreate: {
              create: {
-                currency: assetProfile.currency,
-                dataSource: assetProfile.dataSource,
-                symbol: assetProfile.symbol
+                currency,
+                dataSource,
+                symbol
              },
              where: {
                dataSource_symbol: {
-                  dataSource: assetProfile.dataSource,
-                  symbol: assetProfile.symbol
+                  dataSource,
+                  symbol
                }
              }
            }
@ -337,24 +368,49 @@ export class ImportService {

      const value = new Big(quantity).mul(unitPrice).toNumber();

-      //@ts-ignore
      activities.push({
        ...order,
        error,
        value,
        feeInBaseCurrency: this.exchangeRateDataService.toCurrency(
          fee,
-          assetProfile.currency,
+          currency,
          userCurrency
        ),
+        //@ts-ignore
+        SymbolProfile: assetProfile,
        valueInBaseCurrency: this.exchangeRateDataService.toCurrency(
          value,
-          assetProfile.currency,
+          currency,
          userCurrency
        )
      });
    }

+    activities.sort((activity1, activity2) => {
+      return Number(activity1.date) - Number(activity2.date);
+    });
+
+    if (!isDryRun) {
+      // Gather symbol data in the background, if not dry run
+      const uniqueActivities = uniqBy(activities, ({ SymbolProfile }) => {
+        return getAssetProfileIdentifier({
+          dataSource: SymbolProfile.dataSource,
+          symbol: SymbolProfile.symbol
+        });
+      });
+
+      this.dataGatheringService.gatherSymbols(
+        uniqueActivities.map(({ date, SymbolProfile }) => {
+          return {
+            date,
+            dataSource: SymbolProfile.dataSource,
+            symbol: SymbolProfile.symbol
+          };
+        })
+      );
+    }
+
    return activities;
  }

@ -446,25 +502,30 @@ export class ImportService {

  private async validateActivities({
    activitiesDto,
-    maxActivitiesToImport,
-    userId
+    maxActivitiesToImport
  }: {
    activitiesDto: Partial<CreateOrderDto>[];
    maxActivitiesToImport: number;
-    userId: string;
  }) {
    if (activitiesDto?.length > maxActivitiesToImport) {
      throw new Error(`Too many activities (${maxActivitiesToImport} at most)`);
    }

    const assetProfiles: {
-      [symbol: string]: Partial<SymbolProfile>;
+      [assetProfileIdentifier: string]: Partial<SymbolProfile>;
    } = {};

+    const uniqueActivitiesDto = uniqBy(
+      activitiesDto,
+      ({ dataSource, symbol }) => {
+        return getAssetProfileIdentifier({ dataSource, symbol });
+      }
+    );
+
    for (const [
      index,
      { currency, dataSource, symbol }
-    ] of activitiesDto.entries()) {
+    ] of uniqueActivitiesDto.entries()) {
      if (dataSource !== 'MANUAL') {
        const assetProfile = (
          await this.dataProviderService.getAssetProfiles([
@ -484,7 +545,8 @@ export class ImportService {
          );
        }

-        assetProfiles[symbol] = assetProfile;
+        assetProfiles[getAssetProfileIdentifier({ dataSource, symbol })] =
+          assetProfile;
      }
    }

--- a/apps/api/src/app/order/order.controller.ts
+++ b/apps/api/src/app/order/order.controller.ts
@ -2,6 +2,7 @@ import { RedactValuesInResponseInterceptor } from '@ghostfolio/api/interceptors/
 import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
 import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';
 import { ApiService } from '@ghostfolio/api/services/api/api.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
 import { ImpersonationService } from '@ghostfolio/api/services/impersonation/impersonation.service';
 import { HEADER_KEY_IMPERSONATION } from '@ghostfolio/common/config';
 import { hasPermission, permissions } from '@ghostfolio/common/permissions';
@ -36,6 +37,7 @@ import { UpdateOrderDto } from './update-order.dto';
 export class OrderController {
  public constructor(
    private readonly apiService: ApiService,
+    private readonly dataGatheringService: DataGatheringService,
    private readonly impersonationService: ImpersonationService,
    private readonly orderService: OrderService,
    @Inject(REQUEST) private readonly request: RequestWithUser
@ -123,7 +125,7 @@ export class OrderController {
      );
    }

-    return this.orderService.createOrder({
+    const order = await this.orderService.createOrder({
      ...data,
      date: parseISO(data.date),
      SymbolProfile: {
@ -144,6 +146,19 @@ export class OrderController {
      User: { connect: { id: this.request.user.id } },
      userId: this.request.user.id
    });
+
+    if (!order.isDraft) {
+      // Gather symbol data in the background, if not draft
+      this.dataGatheringService.gatherSymbols([
+        {
+          dataSource: data.dataSource,
+          date: order.date,
+          symbol: data.symbol
+        }
+      ]);
+    }
+
+    return order;
  }

  @Put(':id')
--- a/apps/api/src/app/order/order.service.ts
+++ b/apps/api/src/app/order/order.service.ts
@ -7,6 +7,7 @@ import {
  GATHER_ASSET_PROFILE_PROCESS,
  GATHER_ASSET_PROFILE_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import { Filter } from '@ghostfolio/common/interfaces';
 import { OrderWithAccount } from '@ghostfolio/common/types';
 import { Injectable } from '@nestjs/common';
@ -117,7 +118,7 @@ export class OrderService {
      };
    }

-    await this.dataGatheringService.addJobToQueue({
+    this.dataGatheringService.addJobToQueue({
      data: {
        dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
        symbol: data.SymbolProfile.connectOrCreate.create.symbol
@ -125,26 +126,13 @@ export class OrderService {
      name: GATHER_ASSET_PROFILE_PROCESS,
      opts: {
        ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-        jobId: `${data.SymbolProfile.connectOrCreate.create.dataSource}-${data.SymbolProfile.connectOrCreate.create.symbol}`
+        jobId: getAssetProfileIdentifier({
+          dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
+          symbol: data.SymbolProfile.connectOrCreate.create.symbol
+        })
      }
    });

-    const isDraft =
-      data.type === 'LIABILITY'
-        ? false
-        : isAfter(data.date as Date, endOfToday());
-
-    if (!isDraft) {
-      // Gather symbol data of order in the background, if not draft
-      this.dataGatheringService.gatherSymbols([
-        {
-          dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
-          date: <Date>data.date,
-          symbol: data.SymbolProfile.connectOrCreate.create.symbol
-        }
-      ]);
-    }
-
    delete data.accountId;
    delete data.assetClass;
    delete data.assetSubClass;
@ -162,6 +150,11 @@ export class OrderService {

    const orderData: Prisma.OrderCreateInput = data;

+    const isDraft =
+      data.type === 'LIABILITY'
+        ? false
+        : isAfter(data.date as Date, endOfToday());
+
    const order = await this.prismaService.order.create({
      data: {
        ...orderData,
--- a/apps/api/src/app/redis-cache/redis-cache.service.ts
+++ b/apps/api/src/app/redis-cache/redis-cache.service.ts
@ -1,4 +1,5 @@
 import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { CACHE_MANAGER, Inject, Injectable, Logger } from '@nestjs/common';

@ -22,7 +23,7 @@ export class RedisCacheService {
  }

  public getQuoteKey({ dataSource, symbol }: UniqueAsset) {
-    return `quote-${dataSource}-${symbol}`;
+    return `quote-${getAssetProfileIdentifier({ dataSource, symbol })}`;
  }

  public async remove(key: string) {
--- a/apps/api/src/services/cron.service.ts
+++ b/apps/api/src/services/cron.service.ts
@ -2,6 +2,7 @@ import {
  GATHER_ASSET_PROFILE_PROCESS,
  GATHER_ASSET_PROFILE_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
+import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
 import { Injectable } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';

@ -48,7 +49,7 @@ export class CronService {
          name: GATHER_ASSET_PROFILE_PROCESS,
          opts: {
            ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
-            jobId: `${dataSource}-${symbol}`
+            jobId: getAssetProfileIdentifier({ dataSource, symbol })
          }
        };
      })
--- a/apps/api/src/services/data-gathering/data-gathering.service.ts
+++ b/apps/api/src/services/data-gathering/data-gathering.service.ts
@ -10,7 +10,11 @@ import {
  GATHER_HISTORICAL_MARKET_DATA_PROCESS,
  GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS
 } from '@ghostfolio/common/config';
-import { DATE_FORMAT, resetHours } from '@ghostfolio/common/helper';
+import {
+  DATE_FORMAT,
+  getAssetProfileIdentifier,
+  resetHours
+} from '@ghostfolio/common/helper';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { InjectQueue } from '@nestjs/bull';
 import { Inject, Injectable, Logger } from '@nestjs/common';
@ -221,7 +225,10 @@ export class DataGatheringService {
          name: GATHER_HISTORICAL_MARKET_DATA_PROCESS,
          opts: {
            ...GATHER_HISTORICAL_MARKET_DATA_PROCESS_OPTIONS,
-            jobId: `${dataSource}-${symbol}-${format(date, DATE_FORMAT)}`
+            jobId: `${getAssetProfileIdentifier({
+              dataSource,
+              symbol
+            })}-${format(date, DATE_FORMAT)}`
          }
        };
      })
--- a/libs/common/src/lib/helper.ts
+++ b/libs/common/src/lib/helper.ts
@ -5,7 +5,7 @@ import { getDate, getMonth, getYear, parse, subDays } from 'date-fns';
 import { de, es, fr, it, nl, pt } from 'date-fns/locale';

 import { ghostfolioScraperApiSymbolPrefix, locale } from './config';
-import { Benchmark } from './interfaces';
+import { Benchmark, UniqueAsset } from './interfaces';
 import { ColorScheme } from './types';

 const NUMERIC_REGEXP = /[-]{0,1}[\d]*[.,]{0,1}[\d]+/g;
@ -64,6 +64,10 @@ export function extractNumberFromString(aString: string): number {
  }
 }

+export function getAssetProfileIdentifier({ dataSource, symbol }: UniqueAsset) {
+  return `${dataSource}-${symbol}`;
+}
+
 export function getBackgroundColor(aColorScheme: ColorScheme) {
  return getCssVariable(
    aColorScheme === 'DARK' ||