Feature/extend import by csv files (#419)

* Support import of csv files

* Update changelog

CHANGELOG.md

@@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Added
+
+- Extended the import functionality for transactions by `csv` files
+- Introduced the primary data source
+
+### Changed
+
+- Restricted the file selector of the import functionality for transactions to `csv` and `json`
+
 ## 1.60.0 - 13.10.2021
 
 ### Added

apps/api/src/app/info/info.service.ts

@@ -1,5 +1,6 @@
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
 import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
+import { DataProviderService } from '@ghostfolio/api/services/data-provider/data-provider.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { InfoItem } from '@ghostfolio/common/interfaces';
@@ -16,6 +17,7 @@ export class InfoService {
 
   public constructor(
     private readonly configurationService: ConfigurationService,
+    private readonly dataProviderService: DataProviderService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
     private readonly dataGatheringService: DataGatheringService,
     private readonly jwtService: JwtService,
@@ -60,6 +62,7 @@ export class InfoService {
       currencies: this.exchangeRateDataService.getCurrencies(),
       demoAuthToken: this.getDemoAuthToken(),
       lastDataGathering: await this.getLastDataGathering(),
+      primaryDataSource: this.dataProviderService.getPrimaryDataSource(),
       statistics: await this.getStatistics(),
       subscriptions: await this.getSubscriptions()
     };

apps/api/src/app/portfolio/current-rate.service.ts

@@ -2,7 +2,6 @@ import { DataProviderService } from '@ghostfolio/api/services/data-provider/data
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
 import { resetHours } from '@ghostfolio/common/helper';
 import { Injectable } from '@nestjs/common';
-import { DataSource } from '@prisma/client';
 import { isBefore, isToday } from 'date-fns';
 import { flatten } from 'lodash';
 
@@ -27,7 +26,10 @@ export class CurrentRateService {
   }: GetValueParams): Promise<GetValueObject> {
     if (isToday(date)) {
       const dataProviderResult = await this.dataProviderService.get([
-        { symbol, dataSource: DataSource.YAHOO }
+        {
+          symbol,
+          dataSource: this.dataProviderService.getPrimaryDataSource()
+        }
       ]);
       return {
         symbol,

apps/api/src/services/data-provider/data-provider.service.ts

@@ -199,6 +199,10 @@ export class DataProviderService {
     };
   }
 
+  public getPrimaryDataSource(): DataSource {
+    return DataSource[this.configurationService.get('DATA_SOURCES')[0]];
+  }
+
   private getDataProvider(providerName: DataSource) {
     switch (providerName) {
       case DataSource.ALPHA_VANTAGE:

apps/api/src/services/exchange-rate-data.service.ts

@@ -210,7 +210,7 @@ export class ExchangeRateDataService {
         return {
           currency1: baseCurrency,
           currency2: currency,
-          dataSource: DataSource.YAHOO,
+          dataSource: this.dataProviderService.getPrimaryDataSource(),
           symbol: `${baseCurrency}${currency}`
         };
       });

apps/client/src/app/pages/portfolio/transactions/transactions-page.component.ts

@@ -6,14 +6,15 @@ import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
 import { UpdateOrderDto } from '@ghostfolio/api/app/order/update-order.dto';
 import { DataService } from '@ghostfolio/client/services/data.service';
 import { ImpersonationStorageService } from '@ghostfolio/client/services/impersonation-storage.service';
+import { ImportTransactionsService } from '@ghostfolio/client/services/import-transactions.service';
 import { UserService } from '@ghostfolio/client/services/user/user.service';
 import { User } from '@ghostfolio/common/interfaces';
 import { hasPermission, permissions } from '@ghostfolio/common/permissions';
-import { Order as OrderModel } from '@prisma/client';
+import { DataSource, Order as OrderModel } from '@prisma/client';
 import { format, parseISO } from 'date-fns';
 import { DeviceDetectorService } from 'ngx-device-detector';
-import { EMPTY, Subject, Subscription } from 'rxjs';
-import { catchError, takeUntil } from 'rxjs/operators';
+import { Subject, Subscription } from 'rxjs';
+import { takeUntil } from 'rxjs/operators';
 
 import { CreateOrUpdateTransactionDialog } from './create-or-update-transaction-dialog/create-or-update-transaction-dialog.component';
 import { ImportTransactionDialog } from './import-transaction-dialog/import-transaction-dialog.component';
@@ -35,6 +36,7 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
   public transactions: OrderModel[];
   public user: User;
 
+  private primaryDataSource: DataSource;
   private unsubscribeSubject = new Subject<void>();
 
   /**
@@ -46,11 +48,15 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
     private deviceService: DeviceDetectorService,
     private dialog: MatDialog,
     private impersonationStorageService: ImpersonationStorageService,
+    private importTransactionsService: ImportTransactionsService,
     private route: ActivatedRoute,
     private router: Router,
     private snackBar: MatSnackBar,
     private userService: UserService
   ) {
+    const { primaryDataSource } = this.dataService.fetchInfo();
+    this.primaryDataSource = primaryDataSource;
+
     this.routeQueryParams = route.queryParams
       .pipe(takeUntil(this.unsubscribeSubject))
       .subscribe((params) => {
@@ -58,8 +64,8 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
           this.openCreateTransactionDialog();
         } else if (params['editDialog']) {
           if (this.transactions) {
-            const transaction = this.transactions.find((transaction) => {
-              return transaction.id === params['transactionId'];
+            const transaction = this.transactions.find(({ id }) => {
+              return id === params['transactionId'];
             });
 
             this.openUpdateTransactionDialog(transaction);
@@ -164,9 +170,12 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
 
   public onImport() {
     const input = document.createElement('input');
+    input.accept = 'application/JSON, .csv';
     input.type = 'file';
 
     input.onchange = (event) => {
+      this.snackBar.open('⏳ Importing data...');
+
       // Getting the file reference
       const file = (event.target as HTMLInputElement).files[0];
 
@@ -174,35 +183,43 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
       const reader = new FileReader();
       reader.readAsText(file, 'UTF-8');
 
-      reader.onload = (readerEvent) => {
+      reader.onload = async (readerEvent) => {
+        const fileContent = readerEvent.target.result as string;
+
         try {
-          const content = JSON.parse(readerEvent.target.result as string);
+          if (file.type === 'application/json') {
+            const content = JSON.parse(fileContent);
+            try {
+              await this.importTransactionsService.importJson({
+                content: content.orders,
+                defaultAccountId: this.defaultAccountId
+              });
 
-          this.snackBar.open('⏳ Importing data...');
-
-          this.dataService
-            .postImport({
-              orders: content.orders.map((order) => {
-                return { ...order, accountId: this.defaultAccountId };
-              })
-            })
-            .pipe(
-              catchError((error) => {
+              this.handleImportSuccess();
+            } catch (error) {
               this.handleImportError(error);
+            }
 
-                return EMPTY;
-              }),
-              takeUntil(this.unsubscribeSubject)
-            )
-            .subscribe({
-              next: () => {
-                this.fetchOrders();
+            return;
+          } else if (file.type === 'text/csv') {
+            try {
+              await this.importTransactionsService.importCsv({
+                fileContent,
+                defaultAccountId: this.defaultAccountId,
+                primaryDataSource: this.primaryDataSource
+              });
 
-                this.snackBar.open('✅ Import has been completed', undefined, {
-                  duration: 3000
+              this.handleImportSuccess();
+            } catch (error) {
+              this.handleImportError({
+                error: { message: error?.error?.message ?? [error?.message] }
               });
             }
-            });
+
+            return;
+          }
+
+          throw new Error();
         } catch (error) {
           this.handleImportError({ error: { message: ['Unexpected format'] } });
         }
@@ -302,6 +319,14 @@ export class TransactionsPageComponent implements OnDestroy, OnInit {
     });
   }
 
+  private handleImportSuccess() {
+    this.fetchOrders();
+
+    this.snackBar.open('✅ Import has been completed', undefined, {
+      duration: 3000
+    });
+  }
+
   private openCreateTransactionDialog(aTransaction?: OrderModel): void {
     const dialogRef = this.dialog.open(CreateOrUpdateTransactionDialog, {
       data: {

apps/client/src/app/pages/portfolio/transactions/transactions-page.module.ts

@@ -4,6 +4,7 @@ import { MatButtonModule } from '@angular/material/button';
 import { MatSnackBarModule } from '@angular/material/snack-bar';
 import { RouterModule } from '@angular/router';
 import { GfTransactionsTableModule } from '@ghostfolio/client/components/transactions-table/transactions-table.module';
+import { ImportTransactionsService } from '@ghostfolio/client/services/import-transactions.service';
 
 import { GfCreateOrUpdateTransactionDialogModule } from './create-or-update-transaction-dialog/create-or-update-transaction-dialog.module';
 import { GfImportTransactionDialogModule } from './import-transaction-dialog/import-transaction-dialog.module';
@@ -23,7 +24,7 @@ import { TransactionsPageComponent } from './transactions-page.component';
     RouterModule,
     TransactionsPageRoutingModule
   ],
-  providers: [],
+  providers: [ImportTransactionsService],
   schemas: [CUSTOM_ELEMENTS_SCHEMA]
 })
 export class TransactionsPageModule {}

apps/client/src/app/services/data.service.ts

@@ -194,10 +194,6 @@ export class DataService {
     return this.http.post<OrderModel>(`/api/account`, aAccount);
   }
 
-  public postImport(aImportData: ImportDataDto) {
-    return this.http.post<void>('/api/import', aImportData);
-  }
-
   public postOrder(aOrder: CreateOrderDto) {
     return this.http.post<OrderModel>(`/api/order`, aOrder);
   }

apps/client/src/app/services/import-transactions.service.ts

@@ -0,0 +1,195 @@
+import { HttpClient } from '@angular/common/http';
+import { Injectable } from '@angular/core';
+import { CreateOrderDto } from '@ghostfolio/api/app/order/create-order.dto';
+import { DataSource, Type } from '@prisma/client';
+import { parse } from 'date-fns';
+import { isNumber } from 'lodash';
+import { parse as csvToJson } from 'papaparse';
+import { EMPTY } from 'rxjs';
+import { catchError } from 'rxjs/operators';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class ImportTransactionsService {
+  private static CURRENCY_KEYS = ['ccy', 'currency'];
+  private static DATE_KEYS = ['date'];
+  private static FEE_KEYS = ['commission', 'fee'];
+  private static QUANTITY_KEYS = ['qty', 'quantity', 'shares'];
+  private static SYMBOL_KEYS = ['code', 'symbol'];
+  private static TYPE_KEYS = ['action', 'type'];
+  private static UNIT_PRICE_KEYS = ['price', 'unitprice', 'value'];
+
+  public constructor(private http: HttpClient) {}
+
+  public async importCsv({
+    defaultAccountId,
+    fileContent,
+    primaryDataSource
+  }: {
+    defaultAccountId: string;
+    fileContent: string;
+    primaryDataSource: DataSource;
+  }) {
+    const content = csvToJson(fileContent, {
+      dynamicTyping: true,
+      header: true,
+      skipEmptyLines: true
+    }).data;
+
+    const orders: CreateOrderDto[] = [];
+
+    for (const item of content) {
+      orders.push({
+        accountId: defaultAccountId,
+        currency: this.parseCurrency(item),
+        dataSource: primaryDataSource,
+        date: this.parseDate(item),
+        fee: this.parseFee(item),
+        quantity: this.parseQuantity(item),
+        symbol: this.parseSymbol(item),
+        type: this.parseType(item),
+        unitPrice: this.parseUnitPrice(item)
+      });
+    }
+
+    await this.importJson({ defaultAccountId, content: orders });
+  }
+
+  public importJson({
+    content,
+    defaultAccountId
+  }: {
+    content: CreateOrderDto[];
+    defaultAccountId: string;
+  }): Promise<void> {
+    return new Promise((resolve, reject) => {
+      this.postImport({
+        orders: content.map((order) => {
+          return { ...order, accountId: defaultAccountId };
+        })
+      })
+        .pipe(
+          catchError((error) => {
+            reject(error);
+            return EMPTY;
+          })
+        )
+        .subscribe({
+          next: () => {
+            resolve();
+          }
+        });
+    });
+  }
+
+  private lowercaseKeys(aObject: any) {
+    return Object.keys(aObject).reduce((acc, key) => {
+      acc[key.toLowerCase()] = aObject[key];
+      return acc;
+    }, {});
+  }
+
+  private parseCurrency(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+
+    for (const key of ImportTransactionsService.CURRENCY_KEYS) {
+      if (item[key]) {
+        return item[key];
+      }
+    }
+
+    throw new Error('Could not parse currency');
+  }
+
+  private parseDate(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+    let date: string;
+
+    for (const key of ImportTransactionsService.DATE_KEYS) {
+      if (item[key]) {
+        try {
+          date = parse(item[key], 'dd-MM-yyyy', new Date()).toISOString();
+        } catch {}
+
+        try {
+          date = parse(item[key], 'dd/MM/yyyy', new Date()).toISOString();
+        } catch {}
+
+        if (date) {
+          return date;
+        }
+      }
+    }
+
+    throw new Error('Could not parse date');
+  }
+
+  private parseFee(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+
+    for (const key of ImportTransactionsService.FEE_KEYS) {
+      if ((item[key] || item[key] === 0) && isNumber(item[key])) {
+        return item[key];
+      }
+    }
+
+    throw new Error('Could not parse fee');
+  }
+
+  private parseQuantity(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+
+    for (const key of ImportTransactionsService.QUANTITY_KEYS) {
+      if (item[key] && isNumber(item[key])) {
+        return item[key];
+      }
+    }
+
+    throw new Error('Could not parse quantity');
+  }
+
+  private parseSymbol(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+
+    for (const key of ImportTransactionsService.SYMBOL_KEYS) {
+      if (item[key]) {
+        return item[key];
+      }
+    }
+
+    throw new Error('Could not parse symbol');
+  }
+
+  private parseType(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+
+    for (const key of ImportTransactionsService.TYPE_KEYS) {
+      if (item[key]) {
+        if (item[key].toLowerCase() === 'buy') {
+          return Type.BUY;
+        } else if (item[key].toLowerCase() === 'sell') {
+          return Type.SELL;
+        }
+      }
+    }
+
+    throw new Error('Could not parse type');
+  }
+
+  private parseUnitPrice(aItem: any) {
+    const item = this.lowercaseKeys(aItem);
+
+    for (const key of ImportTransactionsService.UNIT_PRICE_KEYS) {
+      if (item[key] && isNumber(item[key])) {
+        return item[key];
+      }
+    }
+
+    throw new Error('Could not parse unit price (unitPrice)');
+  }
+
+  private postImport(aImportData: { orders: CreateOrderDto[] }) {
+    return this.http.post<void>('/api/import', aImportData);
+  }
+}

libs/common/src/lib/interfaces/info-item.interface.ts

@@ -1,3 +1,4 @@
+import { DataSource } from '@prisma/client';
 import { Statistics } from './statistics.interface';
 import { Subscription } from './subscription.interface';
 
@@ -11,6 +12,7 @@ export interface InfoItem {
     type: string;
   };
   platforms: { id: string; name: string }[];
+  primaryDataSource: DataSource;
   statistics: Statistics;
   stripePublicKey?: string;
   subscriptions: Subscription[];

package.json

@@ -73,6 +73,7 @@
     "@simplewebauthn/server": "4.1.0",
     "@simplewebauthn/typescript-types": "4.0.0",
     "@stripe/stripe-js": "1.15.0",
+    "@types/papaparse": "5.2.6",
     "alphavantage": "2.2.0",
     "angular-material-css-vars": "2.1.2",
     "bent": "7.3.12",
@@ -99,6 +100,7 @@
     "ngx-markdown": "12.0.1",
     "ngx-skeleton-loader": "2.9.1",
     "ngx-stripe": "12.0.2",
+    "papaparse": "5.3.1",
     "passport": "0.4.1",
     "passport-google-oauth20": "2.0.0",
     "passport-jwt": "4.0.0",

yarn.lock

@@ -3920,6 +3920,13 @@
   resolved "https://registry.yarnpkg.com/@types/overlayscrollbars/-/overlayscrollbars-1.12.1.tgz#fb637071b545834fb12aea94ee309a2ff4cdc0a8"
   integrity sha512-V25YHbSoKQN35UasHf0EKD9U2vcmexRSp78qa8UglxFH8H3D+adEa9zGZwrqpH4TdvqeMrgMqVqsLB4woAryrQ==
 
+"@types/papaparse@5.2.6":
+  version "5.2.6"
+  resolved "https://registry.yarnpkg.com/@types/papaparse/-/papaparse-5.2.6.tgz#0bba18de4d15eff65883bc7c0794e0134de9e7c7"
+  integrity sha512-xGKSd0UTn58N1h0+zf8mW863Rv8BvXcGibEgKFtBIXZlcDXAmX/T4RdDO2mwmrmOypUDt5vRgo2v32a78JdqUA==
+  dependencies:
+    "@types/node" "*"
+
 "@types/parse-json@^4.0.0":
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0"
@@ -13146,6 +13153,11 @@ pako@^1.0.3, pako@~1.0.5:
   resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
   integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
 
+papaparse@5.3.1:
+  version "5.3.1"
+  resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-5.3.1.tgz#770b7a9124d821d4b2132132b7bd7dce7194b5b1"
+  integrity sha512-Dbt2yjLJrCwH2sRqKFFJaN5XgIASO9YOFeFP8rIBRG2Ain8mqk5r1M6DkfvqEVozVcz3r3HaUGw253hA1nLIcA==
+
 parallel-transform@^1.1.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/parallel-transform/-/parallel-transform-1.2.0.tgz#9049ca37d6cb2182c3b1d2c720be94d14a5814fc"