Feature/add accounts import export (#1635)

* Add accounts to activities export * Add logic for importing accounts * Update changelog
2023-02-07 02:29:59 +05:30 · 2023-02-07 02:29:59 +05:30 · a79f31b006
commit a79f31b006
parent 45cfd61dbb
11 changed files with 242 additions and 34 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ## Unreleased

+### Added
+
+- Added support to export accounts
+- Added suport to import accounts
+
 ### Changed

 - Improved the styling in the admin control panel
--- a/apps/api/src/app/account/create-account.dto.ts
+++ b/apps/api/src/app/account/create-account.dto.ts
@ -17,6 +17,10 @@ export class CreateAccountDto {
  @IsString()
  currency: string;

+  @IsOptional()
+  @IsString()
+  id?: string;
+
  @IsBoolean()
  @IsOptional()
  isExcluded?: boolean;
--- a/apps/api/src/app/export/export.service.ts
+++ b/apps/api/src/app/export/export.service.ts
@ -14,6 +14,22 @@ export class ExportService {
    activityIds?: string[];
    userId: string;
  }): Promise<Export> {
+    const accounts = await this.prismaService.account.findMany({
+      orderBy: {
+        name: 'asc'
+      },
+      select: {
+        accountType: true,
+        balance: true,
+        currency: true,
+        id: true,
+        isExcluded: true,
+        name: true,
+        platformId: true
+      },
+      where: { userId }
+    });
+
    let activities = await this.prismaService.order.findMany({
      orderBy: { date: 'desc' },
      select: {
@ -38,6 +54,7 @@ export class ExportService {

    return {
      meta: { date: new Date().toISOString(), version: environment.version },
+      accounts,
      activities: activities.map(
        ({
          accountId,
--- a/apps/api/src/app/import/import-data.dto.ts
+++ b/apps/api/src/app/import/import-data.dto.ts
@ -1,8 +1,15 @@
+import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';
 import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
 import { Type } from 'class-transformer';
-import { IsArray, ValidateNested } from 'class-validator';
+import { IsArray, IsOptional, ValidateNested } from 'class-validator';

 export class ImportDataDto {
+  @IsOptional()
+  @IsArray()
+  @Type(() => CreateAccountDto)
+  @ValidateNested({ each: true })
+  accounts: CreateAccountDto[];
+
  @IsArray()
  @Type(() => CreateOrderDto)
  @ValidateNested({ each: true })
--- a/apps/api/src/app/import/import.controller.ts
+++ b/apps/api/src/app/import/import.controller.ts
@ -2,6 +2,7 @@ import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interce
 import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
 import { ImportResponse } from '@ghostfolio/common/interfaces';
+import { hasPermission, permissions } from '@ghostfolio/common/permissions';
 import type { RequestWithUser } from '@ghostfolio/common/types';
 import {
  Body,
@ -38,7 +39,10 @@ export class ImportController {
    @Body() importData: ImportDataDto,
    @Query('dryRun') isDryRun?: boolean
  ): Promise<ImportResponse> {
-    if (!this.configurationService.get('ENABLE_FEATURE_IMPORT')) {
+    if (
+      !this.configurationService.get('ENABLE_FEATURE_IMPORT') ||
+      !hasPermission(this.request.user.permissions, permissions.createAccount)
+    ) {
      throw new HttpException(
        getReasonPhrase(StatusCodes.FORBIDDEN),
        StatusCodes.FORBIDDEN
@ -60,9 +64,10 @@ export class ImportController {

    try {
      const activities = await this.importService.import({
-        maxActivitiesToImport,
        isDryRun,
+        maxActivitiesToImport,
        userCurrency,
+        accountsDto: importData.accounts ?? [],
        activitiesDto: importData.activities,
        userId: this.request.user.id
      });
--- a/apps/api/src/app/import/import.service.ts
+++ b/apps/api/src/app/import/import.service.ts
@ -1,4 +1,5 @@
 import { AccountService } from '@ghostfolio/api/app/account/account.service';
+import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';
 import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
 import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';
 import { OrderService } from '@ghostfolio/api/app/order/order.service';
@ -100,18 +101,75 @@ export class ImportService {
  }

  public async import({
+    accountsDto,
    activitiesDto,
    isDryRun = false,
    maxActivitiesToImport,
    userCurrency,
    userId
  }: {
+    accountsDto: Partial<CreateAccountDto>[];
    activitiesDto: Partial<CreateOrderDto>[];
    isDryRun?: boolean;
    maxActivitiesToImport: number;
    userCurrency: string;
    userId: string;
  }): Promise<Activity[]> {
+    const accountIdMapping: { [oldAccountId: string]: string } = {};
+
+    if (!isDryRun && accountsDto?.length) {
+      const existingAccounts = await this.accountService.accounts({
+        where: {
+          id: {
+            in: accountsDto.map(({ id }) => {
+              return id;
+            })
+          }
+        }
+      });
+
+      for (const account of accountsDto) {
+        // Check if there is any existing account with the same ID
+        const accountWithSameId = existingAccounts.find(
+          (existingAccount) => existingAccount.id === account.id
+        );
+
+        // If there is no account or if the account belongs to a different user then create a new account
+        if (!accountWithSameId || accountWithSameId.userId !== userId) {
+          let oldAccountId: string;
+          const platformId = account.platformId;
+
+          delete account.platformId;
+
+          if (accountWithSameId) {
+            oldAccountId = account.id;
+            delete account.id;
+          }
+
+          const newAccountObject = {
+            ...account,
+            User: { connect: { id: userId } }
+          };
+
+          if (platformId) {
+            Object.assign(newAccountObject, {
+              Platform: { connect: { id: platformId } }
+            });
+          }
+
+          const newAccount = await this.accountService.createAccount(
+            newAccountObject,
+            userId
+          );
+
+          // Store the new to old account ID mappings for updating activities
+          if (accountWithSameId && oldAccountId) {
+            accountIdMapping[oldAccountId] = newAccount.id;
+          }
+        }
+      }
+    }
+
    for (const activity of activitiesDto) {
      if (!activity.dataSource) {
        if (activity.type === 'ITEM') {
@ -120,6 +178,13 @@ export class ImportService {
          activity.dataSource = this.dataProviderService.getPrimaryDataSource();
        }
      }
+
+      // If a new account is created, then update the accountId in all activities
+      if (!isDryRun) {
+        if (Object.keys(accountIdMapping).includes(activity.accountId)) {
+          activity.accountId = accountIdMapping[activity.accountId];
+        }
+      }
    }

    const assetProfiles = await this.validateActivities({
@ -128,12 +193,18 @@ export class ImportService {
      userId
    });

-    const accountIds = (await this.accountService.getAccounts(userId)).map(
+    const accounts = (await this.accountService.getAccounts(userId)).map(
      (account) => {
-        return account.id;
+        return { id: account.id, name: account.name };
      }
    );

+    if (isDryRun) {
+      accountsDto.forEach(({ id, name }) => {
+        accounts.push({ id, name });
+      });
+    }
+
    const activities: Activity[] = [];

    for (const {
@ -149,11 +220,15 @@ export class ImportService {
      unitPrice
    } of activitiesDto) {
      const date = parseISO(<string>(<unknown>dateString));
-      const validatedAccountId = accountIds.includes(accountId)
-        ? accountId
-        : undefined;
+      const validatedAccount = accounts.find(({ id }) => {
+        return id === accountId;
+      });

-      let order: OrderWithAccount;
+      let order:
+        | OrderWithAccount
+        | (Omit<OrderWithAccount, 'Account'> & {
+            Account?: { id: string; name: string };
+          });

      if (isDryRun) {
        order = {
@ -164,7 +239,7 @@ export class ImportService {
          type,
          unitPrice,
          userId,
-          accountId: validatedAccountId,
+          accountId: validatedAccount?.id,
          accountUserId: undefined,
          createdAt: new Date(),
          id: uuidv4(),
@ -187,6 +262,7 @@ export class ImportService {
            url: null,
            ...assetProfiles[symbol]
          },
+          Account: validatedAccount,
          symbolProfileId: undefined,
          updatedAt: new Date()
        };
@ -199,7 +275,7 @@ export class ImportService {
          type,
          unitPrice,
          userId,
-          accountId: validatedAccountId,
+          accountId: validatedAccount?.id,
          SymbolProfile: {
            connectOrCreate: {
              create: {
@ -221,6 +297,7 @@ export class ImportService {

      const value = new Big(quantity).mul(unitPrice).toNumber();

+      //@ts-ignore
      activities.push({
        ...order,
        value,
--- a/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts
+++ b/apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts
@ -11,6 +11,7 @@ import {
  MatLegacyDialogRef as MatDialogRef
 } from '@angular/material/legacy-dialog';
 import { MatLegacySnackBar as MatSnackBar } from '@angular/material/legacy-snack-bar';
+import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';
 import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';
 import { DataService } from '@ghostfolio/client/services/data.service';
 import { ImportActivitiesService } from '@ghostfolio/client/services/import-activities.service';
@ -28,6 +29,7 @@ import { ImportActivitiesDialogParams } from './interfaces/interfaces';
  templateUrl: 'import-activities-dialog.html'
 })
 export class ImportActivitiesDialog implements OnDestroy {
+  public accounts: CreateAccountDto[] = [];
  public activities: Activity[] = [];
  public details: any[] = [];
  public errorMessages: string[] = [];
@ -91,9 +93,10 @@ export class ImportActivitiesDialog implements OnDestroy {
    try {
      this.snackBar.open('⏳ ' + $localize`Importing data...`);

-      await this.importActivitiesService.importSelectedActivities(
-        this.selectedActivities
-      );
+      await this.importActivitiesService.importSelectedActivities({
+        accounts: this.accounts,
+        activities: this.selectedActivities
+      });

      this.snackBar.open(
        '✅ ' + $localize`Import has been completed`,
@ -163,6 +166,8 @@ export class ImportActivitiesDialog implements OnDestroy {
          if (file.name.endsWith('.json')) {
            const content = JSON.parse(fileContent);

+            this.accounts = content.accounts;
+
            if (!isArray(content.activities)) {
              if (isArray(content.orders)) {
                this.handleImportError({
@ -180,10 +185,13 @@ export class ImportActivitiesDialog implements OnDestroy {
            }

            try {
-              this.activities = await this.importActivitiesService.importJson({
-                content: content.activities,
-                isDryRun: true
-              });
+              const { activities } =
+                await this.importActivitiesService.importJson({
+                  accounts: content.accounts,
+                  activities: content.activities,
+                  isDryRun: true
+                });
+              this.activities = activities;
            } catch (error) {
              console.error(error);
              this.handleImportError({ error, activities: content.activities });
@ -192,11 +200,12 @@ export class ImportActivitiesDialog implements OnDestroy {
            return;
          } else if (file.name.endsWith('.csv')) {
            try {
-              this.activities = await this.importActivitiesService.importCsv({
+              const data = await this.importActivitiesService.importCsv({
                fileContent,
                isDryRun: true,
                userAccounts: this.data.user.accounts
              });
+              this.activities = data.activities;
            } catch (error) {
              console.error(error);
              this.handleImportError({
--- a/apps/client/src/app/services/import-activities.service.ts
+++ b/apps/client/src/app/services/import-activities.service.ts
@ -1,5 +1,6 @@
 import { HttpClient } from '@angular/common/http';
 import { Injectable } from '@angular/core';
+import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';
 import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
 import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';
 import { Account, DataSource, Type } from '@prisma/client';
@ -33,7 +34,9 @@ export class ImportActivitiesService {
    fileContent: string;
    isDryRun?: boolean;
    userAccounts: Account[];
-  }): Promise<Activity[]> {
+  }): Promise<{
+    activities: Activity[];
+  }> {
    const content = csvToJson(fileContent, {
      dynamicTyping: true,
      header: true,
@ -55,20 +58,26 @@ export class ImportActivitiesService {
      });
    }

-    return await this.importJson({ isDryRun, content: activities });
+    return await this.importJson({ activities, isDryRun });
  }

  public importJson({
-    content,
+    accounts,
+    activities,
    isDryRun = false
  }: {
-    content: CreateOrderDto[];
+    activities: CreateOrderDto[];
+    accounts?: CreateAccountDto[];
    isDryRun?: boolean;
-  }): Promise<Activity[]> {
+  }): Promise<{
+    activities: Activity[];
+    accounts?: CreateAccountDto[];
+  }> {
    return new Promise((resolve, reject) => {
      this.postImport(
        {
-          activities: content
+          accounts,
+          activities
        },
        isDryRun
      )
@ -80,22 +89,29 @@ export class ImportActivitiesService {
        )
        .subscribe({
          next: (data) => {
-            resolve(data.activities);
+            resolve(data);
          }
        });
    });
  }

-  public importSelectedActivities(
-    selectedActivities: Activity[]
-  ): Promise<Activity[]> {
+  public importSelectedActivities({
+    accounts,
+    activities
+  }: {
+    accounts: CreateAccountDto[];
+    activities: Activity[];
+  }): Promise<{
+    activities: Activity[];
+    accounts?: CreateAccountDto[];
+  }> {
    const importData: CreateOrderDto[] = [];

-    for (const activity of selectedActivities) {
+    for (const activity of activities) {
      importData.push(this.convertToCreateOrderDto(activity));
    }

-    return this.importJson({ content: importData });
+    return this.importJson({ accounts, activities: importData });
  }

  private convertToCreateOrderDto({
@ -347,7 +363,7 @@ export class ImportActivitiesService {
  }

  private postImport(
-    aImportData: { activities: CreateOrderDto[] },
+    aImportData: { accounts: CreateAccountDto[]; activities: CreateOrderDto[] },
    aIsDryRun = false
  ) {
    return this.http.post<{ activities: Activity[] }>(
--- a/libs/common/src/lib/interfaces/export.interface.ts
+++ b/libs/common/src/lib/interfaces/export.interface.ts
@ -1,10 +1,11 @@
-import { Order } from '@prisma/client';
+import { Account, Order } from '@prisma/client';

 export interface Export {
  meta: {
    date: string;
    version: string;
  };
+  accounts: Omit<Account, 'createdAt' | 'isDefault' | 'updatedAt' | 'userId'>[];
  activities: (Omit<
    Order,
    | 'accountUserId'
--- a/test/import/ok-without-accounts.json
+++ b/test/import/ok-without-accounts.json
@ -0,0 +1,48 @@
+{
+  "meta": {
+    "date": "2022-04-01T00:00:00.000Z",
+    "version": "dev"
+  },
+  "activities": [
+    {
+      "fee": 0,
+      "quantity": 0,
+      "type": "BUY",
+      "unitPrice": 0,
+      "currency": "USD",
+      "dataSource": "YAHOO",
+      "date": "2050-06-05T22:00:00.000Z",
+      "symbol": "MSFT"
+    },
+    {
+      "fee": 0,
+      "quantity": 1,
+      "type": "ITEM",
+      "unitPrice": 500000,
+      "currency": "USD",
+      "dataSource": "MANUAL",
+      "date": "2021-12-31T22:00:00.000Z",
+      "symbol": "Penthouse Apartment"
+    },
+    {
+      "fee": 0,
+      "quantity": 5,
+      "type": "DIVIDEND",
+      "unitPrice": 0.62,
+      "currency": "USD",
+      "dataSource": "YAHOO",
+      "date": "2021-11-16T22:00:00.000Z",
+      "symbol": "MSFT"
+    },
+    {
+      "fee": 19,
+      "quantity": 5,
+      "type": "BUY",
+      "unitPrice": 298.58,
+      "currency": "USD",
+      "dataSource": "YAHOO",
+      "date": "2021-09-15T22:00:00.000Z",
+      "symbol": "MSFT"
+    }
+  ]
+}
--- a/test/import/ok.json
+++ b/test/import/ok.json
@ -1,10 +1,23 @@
 {
  "meta": {
-    "date": "2022-04-01T00:00:00.000Z",
+    "date": "2023-02-05T00:00:00.000Z",
    "version": "dev"
  },
+  "accounts": [
+    {
+      "accountType": "SECURITIES",
+      "balance": 2000,
+      "currency": "USD",
+      "id": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
+      "isExcluded": false,
+      "name": "My Online Trading Account",
+      "platformId": null
+    }
+  ],
  "activities": [
    {
+      "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
+      "comment": null,
      "fee": 0,
      "quantity": 0,
      "type": "BUY",
@ -15,6 +28,8 @@
      "symbol": "MSFT"
    },
    {
+      "accountId": null,
+      "comment": null,
      "fee": 0,
      "quantity": 1,
      "type": "ITEM",
@ -25,6 +40,8 @@
      "symbol": "Penthouse Apartment"
    },
    {
+      "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
+      "comment": null,
      "fee": 0,
      "quantity": 5,
      "type": "DIVIDEND",
@ -35,6 +52,8 @@
      "symbol": "MSFT"
    },
    {
+      "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
+      "comment": "My first order",
      "fee": 19,
      "quantity": 5,
      "type": "BUY",