Feature/extend import by csv files (#419)

* Support import of csv files

* Update changelog

apps/client/src/app/pages/portfolio/transactions/transactions-page.component.ts

@@ -6,14 +6,15 @@ import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
 import { UpdateOrderDto } from '@ghostfolio/api/app/order/update-order.dto';
 import { DataService } from '@ghostfolio/client/services/data.service';
 import { ImpersonationStorageService } from '@ghostfolio/client/services/impersonation-storage.service';
+import { ImportTransactionsService } from '@ghostfolio/client/services/import-transactions.service';
 import { UserService } from '@ghostfolio/client/services/user/user.service';
 import { User } from '@ghostfolio/common/interfaces';
 import { hasPermission, permissions } from '@ghostfolio/common/permissions';
-import { Order as OrderModel } from '@prisma/client';
+import { DataSource, Order as OrderModel } from '@prisma/client';
 import { format, parseISO } from 'date-fns';
 import { DeviceDetectorService } from 'ngx-device-detector';
-import { EMPTY, Subject, Subscription } from 'rxjs';
-import { catchError, takeUntil } from 'rxjs/operators';
+import { Subject, Subscription } from 'rxjs';
+import { takeUntil } from 'rxjs/operators';
 
 import { CreateOrUpdateTransactionDialog } from './create-or-update-transaction-dialog/create-or-update-transaction-dialog.component';
 import { ImportTransactionDialog } from './import-transaction-dialog/import-transaction-dialog.component';
@@ -35,6 +36,7 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
   public transactions: OrderModel[];
   public user: User;
 
+  private primaryDataSource: DataSource;
   private unsubscribeSubject = new Subject<void>();
 
   /**
@@ -46,11 +48,15 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
     private deviceService: DeviceDetectorService,
     private dialog: MatDialog,
     private impersonationStorageService: ImpersonationStorageService,
+    private importTransactionsService: ImportTransactionsService,
     private route: ActivatedRoute,
     private router: Router,
     private snackBar: MatSnackBar,
     private userService: UserService
   ) {
+    const { primaryDataSource } = this.dataService.fetchInfo();
+    this.primaryDataSource = primaryDataSource;
+
     this.routeQueryParams = route.queryParams
       .pipe(takeUntil(this.unsubscribeSubject))
       .subscribe((params) => {
@@ -58,8 +64,8 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
           this.openCreateTransactionDialog();
         } else if (params['editDialog']) {
           if (this.transactions) {
-            const transaction = this.transactions.find((transaction) => {
-              return transaction.id === params['transactionId'];
+            const transaction = this.transactions.find(({ id }) => {
+              return id === params['transactionId'];
             });
 
             this.openUpdateTransactionDialog(transaction);
@@ -164,9 +170,12 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
 
   public onImport() {
     const input = document.createElement('input');
+    input.accept = 'application/JSON, .csv';
     input.type = 'file';
 
     input.onchange = (event) => {
+      this.snackBar.open('⏳ Importing data...');
+
       // Getting the file reference
       const file = (event.target as HTMLInputElement).files[0];
 
@@ -174,35 +183,43 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
       const reader = new FileReader();
       reader.readAsText(file, 'UTF-8');
 
-      reader.onload = (readerEvent) => {
+      reader.onload = async (readerEvent) => {
+        const fileContent = readerEvent.target.result as string;
+
         try {
-          const content = JSON.parse(readerEvent.target.result as string);
+          if (file.type === 'application/json') {
+            const content = JSON.parse(fileContent);
+            try {
+              await this.importTransactionsService.importJson({
+                content: content.orders,
+                defaultAccountId: this.defaultAccountId
+              });
 
-          this.snackBar.open('⏳ Importing data...');
+              this.handleImportSuccess();
+            } catch (error) {
+              this.handleImportError(error);
+            }
 
-          this.dataService
-            .postImport({
-              orders: content.orders.map((order) => {
-                return { ...order, accountId: this.defaultAccountId };
-              })
-            })
-            .pipe(
-              catchError((error) => {
-                this.handleImportError(error);
+            return;
+          } else if (file.type === 'text/csv') {
+            try {
+              await this.importTransactionsService.importCsv({
+                fileContent,
+                defaultAccountId: this.defaultAccountId,
+                primaryDataSource: this.primaryDataSource
+              });
 
-                return EMPTY;
-              }),
-              takeUntil(this.unsubscribeSubject)
-            )
-            .subscribe({
-              next: () => {
-                this.fetchOrders();
+              this.handleImportSuccess();
+            } catch (error) {
+              this.handleImportError({
+                error: { message: error?.error?.message ?? [error?.message] }
+              });
+            }
 
-                this.snackBar.open('✅ Import has been completed', undefined, {
-                  duration: 3000
-                });
-              }
-            });
+            return;
+          }
+
+          throw new Error();
         } catch (error) {
           this.handleImportError({ error: { message: ['Unexpected format'] } });
         }
@@ -302,6 +319,14 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
     });
   }
 
+  private handleImportSuccess() {
+    this.fetchOrders();
+
+    this.snackBar.open('✅ Import has been completed', undefined, {
+      duration: 3000
+    });
+  }
+
   private openCreateTransactionDialog(aTransaction?: OrderModel): void {
     const dialogRef = this.dialog.open(CreateOrUpdateTransactionDialog, {
       data: {

apps/client/src/app/pages/portfolio/transactions/transactions-page.module.ts

@@ -4,6 +4,7 @@ import { MatButtonModule } from '@angular/material/button';
 import { MatSnackBarModule } from '@angular/material/snack-bar';
 import { RouterModule } from '@angular/router';
 import { GfTransactionsTableModule } from '@ghostfolio/client/components/transactions-table/transactions-table.module';
+import { ImportTransactionsService } from '@ghostfolio/client/services/import-transactions.service';
 
 import { GfCreateOrUpdateTransactionDialogModule } from './create-or-update-transaction-dialog/create-or-update-transaction-dialog.module';
 import { GfImportTransactionDialogModule } from './import-transaction-dialog/import-transaction-dialog.module';
@@ -23,7 +24,7 @@ import { TransactionsPageComponent } from './transactions-page.component';
     RouterModule,
     TransactionsPageRoutingModule
   ],
-  providers: [],
+  providers: [ImportTransactionsService],
   schemas: [CUSTOM_ELEMENTS_SCHEMA]
 })
 export class TransactionsPageModule {}

apps/client/src/app/services/data.service.ts

@@ -194,10 +194,6 @@ export class DataService {
     return this.http.post<OrderModel>(`/api/account`, aAccount);
   }
 
-  public postImport(aImportData: ImportDataDto) {
-    return this.http.post<void>('/api/import', aImportData);
-  }
-
   public postOrder(aOrder: CreateOrderDto) {
     return this.http.post<OrderModel>(`/api/order`, aOrder);
   }

apps/client/src/app/services/import-transactions.service.ts

@@ -0,0 +1,195 @@
+import { HttpClient } from '@angular/common/http';
+import { Injectable } from '@angular/core';
+import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
+import { DataSource, Type } from '@prisma/client';
+import { parse } from 'date-fns';
+import { isNumber } from 'lodash';
+import { parse as csvToJson } from 'papaparse';
+import { EMPTY } from 'rxjs';
+import { catchError } from 'rxjs/operators';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class ImportTransactionsService {
+  private static CURRENCY_KEYS = ['ccy', 'currency'];
+  private static DATE_KEYS = ['date'];
+  private static FEE_KEYS = ['commission', 'fee'];
+  private static QUANTITY_KEYS = ['qty', 'quantity', 'shares'];
+  private static SYMBOL_KEYS = ['code', 'symbol'];
+  private static TYPE_KEYS = ['action', 'type'];
+  private static UNIT_PRICE_KEYS = ['price', 'unitprice', 'value'];
+
+  public constructor(private http: HttpClient) {}
+
+  public async importCsv({
+    defaultAccountId,
+    fileContent,
+    primaryDataSource
+  }: {
+    defaultAccountId: string;
+    fileContent: string;
+    primaryDataSource: DataSource;
+  }) {
+    const content = csvToJson(fileContent, {
+      dynamicTyping: true,
+      header: true,
+      skipEmptyLines: true
+    }).data;
+
+    const orders: CreateOrderDto[] = [];
+
+    for (const item of content) {
+      orders.push({
+        accountId: defaultAccountId,
+        currency: this.parseCurrency(item),
+        dataSource: primaryDataSource,
+        date: this.parseDate(item),
+        fee: this.parseFee(item),
+        quantity: this.parseQuantity(item),
+        symbol: this.parseSymbol(item),
+        type: this.parseType(item),
+        unitPrice: this.parseUnitPrice(item)
+      });
+    }
+
+    await this.importJson({ defaultAccountId, content: orders });
+  }
+
+  public importJson({
+    content,
+    defaultAccountId
+  }: {
+    content: CreateOrderDto[];
+    defaultAccountId: string;
+  }): Promise<void> {
+    return new Promise((resolve, reject) => {
+      this.postImport({
+        orders: content.map((order) => {
+          return { ...order, accountId: defaultAccountId };
+        })
+      })
+        .pipe(
+          catchError((error) => {
+            reject(error);
+            return EMPTY;
+          })
+        )
+        .subscribe({
+          next: () => {
+            resolve();
+          }
+        });
+    });
+  }
+
+  private lowercaseKeys(aObject: any) {
+    return Object.keys(aObject).reduce((acc, key) => {
+      acc[key.toLowerCase()] = aObject[key];
+      return acc;
+    }, {});
+  }
+
+  private parseCurrency(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+
+    for (const key of ImportTransactionsService.CURRENCY_KEYS) {
+      if (item[key]) {
+        return item[key];
+      }
+    }
+
+    throw new Error('Could not parse currency');
+  }
+
+  private parseDate(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+    let date: string;
+
+    for (const key of ImportTransactionsService.DATE_KEYS) {
+      if (item[key]) {
+        try {
+          date = parse(item[key], 'dd-MM-yyyy', new Date()).toISOString();
+        } catch {}
+
+        try {
+          date = parse(item[key], 'dd/MM/yyyy', new Date()).toISOString();
+        } catch {}
+
+        if (date) {
+          return date;
+        }
+      }
+    }
+
+    throw new Error('Could not parse date');
+  }
+
+  private parseFee(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+
+    for (const key of ImportTransactionsService.FEE_KEYS) {
+      if ((item[key] || item[key] === 0) && isNumber(item[key])) {
+        return item[key];
+      }
+    }
+
+    throw new Error('Could not parse fee');
+  }
+
+  private parseQuantity(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+
+    for (const key of ImportTransactionsService.QUANTITY_KEYS) {
+      if (item[key] && isNumber(item[key])) {
+        return item[key];
+      }
+    }
+
+    throw new Error('Could not parse quantity');
+  }
+
+  private parseSymbol(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+
+    for (const key of ImportTransactionsService.SYMBOL_KEYS) {
+      if (item[key]) {
+        return item[key];
+      }
+    }
+
+    throw new Error('Could not parse symbol');
+  }
+
+  private parseType(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+
+    for (const key of ImportTransactionsService.TYPE_KEYS) {
+      if (item[key]) {
+        if (item[key].toLowerCase() === 'buy') {
+          return Type.BUY;
+        } else if (item[key].toLowerCase() === 'sell') {
+          return Type.SELL;
+        }
+      }
+    }
+
+    throw new Error('Could not parse type');
+  }
+
+  private parseUnitPrice(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+
+    for (const key of ImportTransactionsService.UNIT_PRICE_KEYS) {
+      if (item[key] && isNumber(item[key])) {
+        return item[key];
+      }
+    }
+
+    throw new Error('Could not parse unit price (unitPrice)');
+  }
+
+  private postImport(aImportData: { orders: CreateOrderDto[] }) {
+    return this.http.post<void>('/api/import', aImportData);
+  }
+}