Feature/add accounts import export (#1635)

* Add accounts to activities export

* Add logic for importing accounts

* Update changelog

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Added support to export accounts
+- Added suport to import accounts
+
 ### Changed
 
 - Improved the styling in the admin control panel

apps/api/src/app/account/create-account.dto.ts

@@ -17,6 +17,10 @@ export class CreateAccountDto {
   @IsString()
   currency: string;
 
+  @IsOptional()
+  @IsString()
+  id?: string;
+
   @IsBoolean()
   @IsOptional()
   isExcluded?: boolean;

apps/api/src/app/export/export.service.ts

@@ -14,6 +14,22 @@ export class ExportService {
     activityIds?: string[];
     userId: string;
   }): Promise<Export> {
+    const accounts = await this.prismaService.account.findMany({
+      orderBy: {
+        name: 'asc'
+      },
+      select: {
+        accountType: true,
+        balance: true,
+        currency: true,
+        id: true,
+        isExcluded: true,
+        name: true,
+        platformId: true
+      },
+      where: { userId }
+    });
+
     let activities = await this.prismaService.order.findMany({
       orderBy: { date: 'desc' },
       select: {
@@ -38,6 +54,7 @@ export class ExportService {
 
     return {
       meta: { date: new Date().toISOString(), version: environment.version },
+      accounts,
       activities: activities.map(
         ({
           accountId,

apps/api/src/app/import/import-data.dto.ts

@@ -1,8 +1,15 @@
+import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';
 import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
 import { Type } from 'class-transformer';
-import { IsArray, ValidateNested } from 'class-validator';
+import { IsArray, IsOptional, ValidateNested } from 'class-validator';
 
 export class ImportDataDto {
+  @IsOptional()
+  @IsArray()
+  @Type(() => CreateAccountDto)
+  @ValidateNested({ each: true })
+  accounts: CreateAccountDto[];
+
   @IsArray()
   @Type(() => CreateOrderDto)
   @ValidateNested({ each: true })

apps/api/src/app/import/import.controller.ts

@@ -2,6 +2,7 @@ import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interce
 import { TransformDataSourceInResponseInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-response.interceptor';
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
 import { ImportResponse } from '@ghostfolio/common/interfaces';
+import { hasPermission, permissions } from '@ghostfolio/common/permissions';
 import type { RequestWithUser } from '@ghostfolio/common/types';
 import {
   Body,
@@ -38,7 +39,10 @@ export class ImportController {
     @Body() importData: ImportDataDto,
     @Query('dryRun') isDryRun?: boolean
   ): Promise<ImportResponse> {
-    if (!this.configurationService.get('ENABLE_FEATURE_IMPORT')) {
+    if (
+      !this.configurationService.get('ENABLE_FEATURE_IMPORT') ||
+      !hasPermission(this.request.user.permissions, permissions.createAccount)
+    ) {
       throw new HttpException(
         getReasonPhrase(StatusCodes.FORBIDDEN),
         StatusCodes.FORBIDDEN
@@ -60,9 +64,10 @@ export class ImportController {
 
     try {
       const activities = await this.importService.import({
-        maxActivitiesToImport,
         isDryRun,
+        maxActivitiesToImport,
         userCurrency,
+        accountsDto: importData.accounts ?? [],
         activitiesDto: importData.activities,
         userId: this.request.user.id
       });

apps/api/src/app/import/import.service.ts

@@ -1,4 +1,5 @@
 import { AccountService } from '@ghostfolio/api/app/account/account.service';
+import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';
 import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
 import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';
 import { OrderService } from '@ghostfolio/api/app/order/order.service';
@@ -100,18 +101,75 @@ export class ImportService {
   }
 
   public async import({
+    accountsDto,
     activitiesDto,
     isDryRun = false,
     maxActivitiesToImport,
     userCurrency,
     userId
   }: {
+    accountsDto: Partial<CreateAccountDto>[];
     activitiesDto: Partial<CreateOrderDto>[];
     isDryRun?: boolean;
     maxActivitiesToImport: number;
     userCurrency: string;
     userId: string;
   }): Promise<Activity[]> {
+    const accountIdMapping: { [oldAccountId: string]: string } = {};
+
+    if (!isDryRun && accountsDto?.length) {
+      const existingAccounts = await this.accountService.accounts({
+        where: {
+          id: {
+            in: accountsDto.map(({ id }) => {
+              return id;
+            })
+          }
+        }
+      });
+
+      for (const account of accountsDto) {
+        // Check if there is any existing account with the same ID
+        const accountWithSameId = existingAccounts.find(
+          (existingAccount) => existingAccount.id === account.id
+        );
+
+        // If there is no account or if the account belongs to a different user then create a new account
+        if (!accountWithSameId || accountWithSameId.userId !== userId) {
+          let oldAccountId: string;
+          const platformId = account.platformId;
+
+          delete account.platformId;
+
+          if (accountWithSameId) {
+            oldAccountId = account.id;
+            delete account.id;
+          }
+
+          const newAccountObject = {
+            ...account,
+            User: { connect: { id: userId } }
+          };
+
+          if (platformId) {
+            Object.assign(newAccountObject, {
+              Platform: { connect: { id: platformId } }
+            });
+          }
+
+          const newAccount = await this.accountService.createAccount(
+            newAccountObject,
+            userId
+          );
+
+          // Store the new to old account ID mappings for updating activities
+          if (accountWithSameId && oldAccountId) {
+            accountIdMapping[oldAccountId] = newAccount.id;
+          }
+        }
+      }
+    }
+
     for (const activity of activitiesDto) {
       if (!activity.dataSource) {
         if (activity.type === 'ITEM') {
@@ -120,6 +178,13 @@ export class ImportService {
           activity.dataSource = this.dataProviderService.getPrimaryDataSource();
         }
       }
+
+      // If a new account is created, then update the accountId in all activities
+      if (!isDryRun) {
+        if (Object.keys(accountIdMapping).includes(activity.accountId)) {
+          activity.accountId = accountIdMapping[activity.accountId];
+        }
+      }
     }
 
     const assetProfiles = await this.validateActivities({
@@ -128,12 +193,18 @@ export class ImportService {
       userId
     });
 
-    const accountIds = (await this.accountService.getAccounts(userId)).map(
+    const accounts = (await this.accountService.getAccounts(userId)).map(
       (account) => {
-        return account.id;
+        return { id: account.id, name: account.name };
       }
     );
 
+    if (isDryRun) {
+      accountsDto.forEach(({ id, name }) => {
+        accounts.push({ id, name });
+      });
+    }
+
     const activities: Activity[] = [];
 
     for (const {
@@ -149,11 +220,15 @@ export class ImportService {
       unitPrice
     } of activitiesDto) {
       const date = parseISO(<string>(<unknown>dateString));
-      const validatedAccountId = accountIds.includes(accountId)
+      const validatedAccount = accounts.find(({ id }) => {
-        ? accountId
+        return id === accountId;
-        : undefined;
+      });
 
-      let order: OrderWithAccount;
+      let order:
+        | OrderWithAccount
+        | (Omit<OrderWithAccount, 'Account'> & {
+            Account?: { id: string; name: string };
+          });
 
       if (isDryRun) {
         order = {
@@ -164,7 +239,7 @@ export class ImportService {
           type,
           unitPrice,
           userId,
-          accountId: validatedAccountId,
+          accountId: validatedAccount?.id,
           accountUserId: undefined,
           createdAt: new Date(),
           id: uuidv4(),
@@ -187,6 +262,7 @@ export class ImportService {
             url: null,
             ...assetProfiles[symbol]
           },
+          Account: validatedAccount,
           symbolProfileId: undefined,
           updatedAt: new Date()
         };
@@ -199,7 +275,7 @@ export class ImportService {
           type,
           unitPrice,
           userId,
-          accountId: validatedAccountId,
+          accountId: validatedAccount?.id,
           SymbolProfile: {
             connectOrCreate: {
               create: {
@@ -221,6 +297,7 @@ export class ImportService {
 
       const value = new Big(quantity).mul(unitPrice).toNumber();
 
+      //@ts-ignore
       activities.push({
         ...order,
         value,

apps/client/src/app/pages/portfolio/activities/import-activities-dialog/import-activities-dialog.component.ts

@@ -11,6 +11,7 @@ import {
   MatLegacyDialogRef as MatDialogRef
 } from '@angular/material/legacy-dialog';
 import { MatLegacySnackBar as MatSnackBar } from '@angular/material/legacy-snack-bar';
+import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';
 import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';
 import { DataService } from '@ghostfolio/client/services/data.service';
 import { ImportActivitiesService } from '@ghostfolio/client/services/import-activities.service';
@@ -28,6 +29,7 @@ import { ImportActivitiesDialogParams } from './interfaces/interfaces';
   templateUrl: 'import-activities-dialog.html'
 })
 export class ImportActivitiesDialog implements OnDestroy {
+  public accounts: CreateAccountDto[] = [];
   public activities: Activity[] = [];
   public details: any[] = [];
   public errorMessages: string[] = [];
@@ -91,9 +93,10 @@ export class ImportActivitiesDialog implements OnDestroy {
     try {
       this.snackBar.open('⏳ ' + $localize`Importing data...`);
 
-      await this.importActivitiesService.importSelectedActivities(
+      await this.importActivitiesService.importSelectedActivities({
-        this.selectedActivities
+        accounts: this.accounts,
-      );
+        activities: this.selectedActivities
+      });
 
       this.snackBar.open(
         '✅ ' + $localize`Import has been completed`,
@@ -163,6 +166,8 @@ export class ImportActivitiesDialog implements OnDestroy {
           if (file.name.endsWith('.json')) {
             const content = JSON.parse(fileContent);
 
+            this.accounts = content.accounts;
+
             if (!isArray(content.activities)) {
               if (isArray(content.orders)) {
                 this.handleImportError({
@@ -180,10 +185,13 @@ export class ImportActivitiesDialog implements OnDestroy {
             }
 
             try {
-              this.activities = await this.importActivitiesService.importJson({
+              const { activities } =
-                content: content.activities,
+                await this.importActivitiesService.importJson({
-                isDryRun: true
+                  accounts: content.accounts,
-              });
+                  activities: content.activities,
+                  isDryRun: true
+                });
+              this.activities = activities;
             } catch (error) {
               console.error(error);
               this.handleImportError({ error, activities: content.activities });
@@ -192,11 +200,12 @@ export class ImportActivitiesDialog implements OnDestroy {
             return;
           } else if (file.name.endsWith('.csv')) {
             try {
-              this.activities = await this.importActivitiesService.importCsv({
+              const data = await this.importActivitiesService.importCsv({
                 fileContent,
                 isDryRun: true,
                 userAccounts: this.data.user.accounts
               });
+              this.activities = data.activities;
             } catch (error) {
               console.error(error);
               this.handleImportError({

apps/client/src/app/services/import-activities.service.ts

@@ -1,5 +1,6 @@
 import { HttpClient } from '@angular/common/http';
 import { Injectable } from '@angular/core';
+import { CreateAccountDto } from '@ghostfolio/api/app/account/create-account.dto';
 import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
 import { Activity } from '@ghostfolio/api/app/order/interfaces/activities.interface';
 import { Account, DataSource, Type } from '@prisma/client';
@@ -33,7 +34,9 @@ export class ImportActivitiesService {
     fileContent: string;
     isDryRun?: boolean;
     userAccounts: Account[];
-  }): Promise<Activity[]> {
+  }): Promise<{
+    activities: Activity[];
+  }> {
     const content = csvToJson(fileContent, {
       dynamicTyping: true,
       header: true,
@@ -55,20 +58,26 @@ export class ImportActivitiesService {
       });
     }
 
-    return await this.importJson({ isDryRun, content: activities });
+    return await this.importJson({ activities, isDryRun });
   }
 
   public importJson({
-    content,
+    accounts,
+    activities,
     isDryRun = false
   }: {
-    content: CreateOrderDto[];
+    activities: CreateOrderDto[];
+    accounts?: CreateAccountDto[];
     isDryRun?: boolean;
-  }): Promise<Activity[]> {
+  }): Promise<{
+    activities: Activity[];
+    accounts?: CreateAccountDto[];
+  }> {
     return new Promise((resolve, reject) => {
       this.postImport(
         {
-          activities: content
+          accounts,
+          activities
         },
         isDryRun
       )
@@ -80,22 +89,29 @@ export class ImportActivitiesService {
         )
         .subscribe({
           next: (data) => {
-            resolve(data.activities);
+            resolve(data);
           }
         });
     });
   }
 
-  public importSelectedActivities(
+  public importSelectedActivities({
-    selectedActivities: Activity[]
+    accounts,
-  ): Promise<Activity[]> {
+    activities
+  }: {
+    accounts: CreateAccountDto[];
+    activities: Activity[];
+  }): Promise<{
+    activities: Activity[];
+    accounts?: CreateAccountDto[];
+  }> {
     const importData: CreateOrderDto[] = [];
 
-    for (const activity of selectedActivities) {
+    for (const activity of activities) {
       importData.push(this.convertToCreateOrderDto(activity));
     }
 
-    return this.importJson({ content: importData });
+    return this.importJson({ accounts, activities: importData });
   }
 
   private convertToCreateOrderDto({
@@ -347,7 +363,7 @@ export class ImportActivitiesService {
   }
 
   private postImport(
-    aImportData: { activities: CreateOrderDto[] },
+    aImportData: { accounts: CreateAccountDto[]; activities: CreateOrderDto[] },
     aIsDryRun = false
   ) {
     return this.http.post<{ activities: Activity[] }>(

libs/common/src/lib/interfaces/export.interface.ts

@@ -1,10 +1,11 @@
-import { Order } from '@prisma/client';
+import { Account, Order } from '@prisma/client';
 
 export interface Export {
   meta: {
     date: string;
     version: string;
   };
+  accounts: Omit<Account, 'createdAt' | 'isDefault' | 'updatedAt' | 'userId'>[];
   activities: (Omit<
     Order,
     | 'accountUserId'

test/import/ok-without-accounts.json

@@ -0,0 +1,48 @@
+{
+  "meta": {
+    "date": "2022-04-01T00:00:00.000Z",
+    "version": "dev"
+  },
+  "activities": [
+    {
+      "fee": 0,
+      "quantity": 0,
+      "type": "BUY",
+      "unitPrice": 0,
+      "currency": "USD",
+      "dataSource": "YAHOO",
+      "date": "2050-06-05T22:00:00.000Z",
+      "symbol": "MSFT"
+    },
+    {
+      "fee": 0,
+      "quantity": 1,
+      "type": "ITEM",
+      "unitPrice": 500000,
+      "currency": "USD",
+      "dataSource": "MANUAL",
+      "date": "2021-12-31T22:00:00.000Z",
+      "symbol": "Penthouse Apartment"
+    },
+    {
+      "fee": 0,
+      "quantity": 5,
+      "type": "DIVIDEND",
+      "unitPrice": 0.62,
+      "currency": "USD",
+      "dataSource": "YAHOO",
+      "date": "2021-11-16T22:00:00.000Z",
+      "symbol": "MSFT"
+    },
+    {
+      "fee": 19,
+      "quantity": 5,
+      "type": "BUY",
+      "unitPrice": 298.58,
+      "currency": "USD",
+      "dataSource": "YAHOO",
+      "date": "2021-09-15T22:00:00.000Z",
+      "symbol": "MSFT"
+    }
+  ]
+}

test/import/ok.json

@@ -1,10 +1,23 @@
 {
   "meta": {
-    "date": "2022-04-01T00:00:00.000Z",
+    "date": "2023-02-05T00:00:00.000Z",
     "version": "dev"
   },
+  "accounts": [
+    {
+      "accountType": "SECURITIES",
+      "balance": 2000,
+      "currency": "USD",
+      "id": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
+      "isExcluded": false,
+      "name": "My Online Trading Account",
+      "platformId": null
+    }
+  ],
   "activities": [
     {
+      "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
+      "comment": null,
       "fee": 0,
       "quantity": 0,
       "type": "BUY",
@@ -15,6 +28,8 @@
       "symbol": "MSFT"
     },
     {
+      "accountId": null,
+      "comment": null,
       "fee": 0,
       "quantity": 1,
       "type": "ITEM",
@@ -25,6 +40,8 @@
       "symbol": "Penthouse Apartment"
     },
     {
+      "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
+      "comment": null,
       "fee": 0,
       "quantity": 5,
       "type": "DIVIDEND",
@@ -35,6 +52,8 @@
       "symbol": "MSFT"
     },
     {
+      "accountId": "b2d3fe1d-d6a8-41a3-be39-07ef5e9480f0",
+      "comment": "My first order",
       "fee": 19,
       "quantity": 5,
       "type": "BUY",