Feature/add data gathering for symbol profile data (#228)

* Implement profile data gathering

* Update changelog

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Extended the data management by symbol profile data
+- Added a currency attribute to the symbol profile model
+
 ### Changed
 
 - Improved the style of the active page in the navigation on desktop

apps/api/src/app/admin/admin.controller.ts

@@ -61,8 +61,29 @@ export class AdminController {
       );
     }
 
+    await this.dataGatheringService.gatherProfileData();
     this.dataGatheringService.gatherMax();
 
     return;
   }
+
+  @Post('gather/profile-data')
+  @UseGuards(AuthGuard('jwt'))
+  public async gatherProfileData(): Promise<void> {
+    if (
+      !hasPermission(
+        getPermissions(this.request.user.role),
+        permissions.accessAdminControl
+      )
+    ) {
+      throw new HttpException(
+        getReasonPhrase(StatusCodes.FORBIDDEN),
+        StatusCodes.FORBIDDEN
+      );
+    }
+
+    this.dataGatheringService.gatherProfileData();
+
+    return;
+  }
 }

apps/api/src/app/order/order.service.ts

@@ -62,6 +62,8 @@ export class OrderService {
       ]);
     }
 
+    this.dataGatheringService.gatherProfileData([data.symbol]);
+
     await this.cacheService.flush(aUserId);
 
     return this.prisma.order.create({

apps/api/src/services/cron.service.ts

@@ -18,6 +18,7 @@ export class CronService {
 
   @Cron(CronExpression.EVERY_12_HOURS)
   public async runEveryTwelveHours() {
+    await this.dataGatheringService.gatherProfileData();
     await this.exchangeRateDataService.loadCurrencies();
   }
 }

apps/api/src/services/data-gathering.service.ts

@@ -2,6 +2,7 @@ import { benchmarks, currencyPairs } from '@ghostfolio/common/config';
 import {
   getUtc,
   isGhostfolioScraperApiSymbol,
+  isRakutenRapidApiSymbol,
   resetHours
 } from '@ghostfolio/common/helper';
 import { Injectable } from '@nestjs/common';
@@ -37,7 +38,7 @@ export class DataGatheringService {
 
     if (isDataGatheringNeeded) {
       console.log('7d data gathering has been started.');
-      console.time('data-gathering');
+      console.time('7d-data-gathering');
 
       await this.prisma.property.create({
         data: {
@@ -70,7 +71,7 @@ export class DataGatheringService {
       });
 
       console.log('7d data gathering has been completed.');
-      console.timeEnd('data-gathering');
+      console.timeEnd('7d-data-gathering');
     }
   }
 
@@ -81,7 +82,7 @@ export class DataGatheringService {
 
     if (!isDataGatheringLocked) {
       console.log('Max data gathering has been started.');
-      console.time('data-gathering');
+      console.time('max-data-gathering');
 
       await this.prisma.property.create({
         data: {
@@ -114,10 +115,56 @@ export class DataGatheringService {
       });
 
       console.log('Max data gathering has been completed.');
-      console.timeEnd('data-gathering');
+      console.timeEnd('max-data-gathering');
     }
   }
 
+  public async gatherProfileData(aSymbols?: string[]) {
+    console.log('Profile data gathering has been started.');
+    console.time('profile-data-gathering');
+
+    let symbols = aSymbols;
+
+    if (!symbols) {
+      const dataGatheringItems = await this.getSymbolsProfileData();
+      symbols = dataGatheringItems.map((dataGatheringItem) => {
+        return dataGatheringItem.symbol;
+      });
+    }
+
+    const currentData = await this.dataProviderService.get(symbols);
+
+    for (const [symbol, { currency, dataSource, name }] of Object.entries(
+      currentData
+    )) {
+      try {
+        await this.prisma.symbolProfile.upsert({
+          create: {
+            currency,
+            dataSource,
+            name,
+            symbol
+          },
+          update: {
+            currency,
+            name
+          },
+          where: {
+            dataSource_symbol: {
+              dataSource,
+              symbol
+            }
+          }
+        });
+      } catch (error) {
+        console.error(`${symbol}: ${error?.meta?.cause}`);
+      }
+    }
+
+    console.log('Profile data gathering has been completed.');
+    console.timeEnd('profile-data-gathering');
+  }
+
   public async gatherSymbols(aSymbolsWithStartDate: IDataGatheringItem[]) {
     let hasError = false;
 
@@ -303,6 +350,25 @@ export class DataGatheringService {
     ];
   }
 
+  private async getSymbolsProfileData(): Promise<IDataGatheringItem[]> {
+    const startDate = subDays(resetHours(new Date()), 7);
+
+    const distinctOrders = await this.prisma.order.findMany({
+      distinct: ['symbol'],
+      orderBy: [{ symbol: 'asc' }],
+      select: { dataSource: true, symbol: true }
+    });
+
+    return [...this.getBenchmarksToGather(startDate), ...distinctOrders].filter(
+      (distinctOrder) => {
+        return (
+          distinctOrder.dataSource !== DataSource.GHOSTFOLIO &&
+          distinctOrder.dataSource !== DataSource.RAKUTEN
+        );
+      }
+    );
+  }
+
   private async isDataGatheringNeeded() {
     const lastDataGathering = await this.prisma.property.findUnique({
       where: { key: 'LAST_DATA_GATHERING' }

apps/api/src/services/data-provider.service.ts

@@ -46,7 +46,10 @@ export class DataProviderService {
     }
 
     const yahooFinanceSymbols = aSymbols.filter((symbol) => {
-      return !isGhostfolioScraperApiSymbol(symbol);
+      return (
+        !isGhostfolioScraperApiSymbol(symbol) &&
+        !isRakutenRapidApiSymbol(symbol)
+      );
     });
 
     const response = await this.yahooFinanceService.get(yahooFinanceSymbols);
@@ -57,13 +60,24 @@ export class DataProviderService {
 
     for (const symbol of ghostfolioScraperApiSymbols) {
       if (symbol) {
-        const ghostfolioScraperApiResult = await this.ghostfolioScraperApiService.get(
+        const ghostfolioScraperApiResult =
-          [symbol]
+          await this.ghostfolioScraperApiService.get([symbol]);
-        );
         response[symbol] = ghostfolioScraperApiResult[symbol];
       }
     }
 
+    const rakutenRapidApiSymbols = aSymbols.filter((symbol) => {
+      return isRakutenRapidApiSymbol(symbol);
+    });
+
+    for (const symbol of rakutenRapidApiSymbols) {
+      if (symbol) {
+        const rakutenRapidApiResult =
+          await this.ghostfolioScraperApiService.get([symbol]);
+        response[symbol] = rakutenRapidApiResult[symbol];
+      }
+    }
+
     return response;
   }
 

apps/client/src/app/pages/admin/admin-page.component.ts

@@ -85,6 +85,13 @@ export class AdminPageComponent implements OnDestroy, OnInit {
     }
   }
 
+  public onGatherProfileData() {
+    this.adminService
+      .gatherProfileData()
+      .pipe(takeUntil(this.unsubscribeSubject))
+      .subscribe(() => {});
+  }
+
   public formatDistanceToNow(aDateString: string) {
     if (aDateString) {
       const distanceString = formatDistanceToNowStrict(parseISO(aDateString), {

apps/client/src/app/pages/admin/admin-page.html

@@ -27,25 +27,46 @@
                 >
               </div>
               <div class="mt-2 overflow-hidden">
-                <button
+                <div class="mb-2">
-                  class="mb-2 mr-2 mw-100"
+                  <button
-                  color="accent"
+                    class="mw-100"
-                  mat-flat-button
+                    color="accent"
-                  (click)="onFlushCache()"
+                    mat-flat-button
-                >
+                    (click)="onFlushCache()"
-                  <ion-icon class="mr-1" name="close-circle-outline"></ion-icon>
+                  >
-                  <span i18n>Reset Data Gathering</span>
+                    <ion-icon
-                </button>
+                      class="mr-1"
-                <button
+                      name="close-circle-outline"
-                  class="mw-100"
+                    ></ion-icon>
-                  color="warn"
+                    <span i18n>Reset Data Gathering</span>
-                  mat-flat-button
+                  </button>
-                  [disabled]="dataGatheringInProgress"
+                </div>
-                  (click)="onGatherMax()"
+                <div class="mb-2">
-                >
+                  <button
-                  <ion-icon class="mr-1" name="warning-outline"></ion-icon>
+                    class="mw-100"
-                  <span i18n>Gather All Data</span>
+                    color="warn"
-                </button>
+                    mat-flat-button
+                    [disabled]="dataGatheringInProgress"
+                    (click)="onGatherMax()"
+                  >
+                    <ion-icon class="mr-1" name="warning-outline"></ion-icon>
+                    <span i18n>Gather All Data</span>
+                  </button>
+                </div>
+                <div>
+                  <button
+                    class="mb-2 mr-2 mw-100"
+                    color="accent"
+                    mat-flat-button
+                    (click)="onGatherProfileData()"
+                  >
+                    <ion-icon
+                      class="mr-1"
+                      name="cloud-download-outline"
+                    ></ion-icon>
+                    <span i18n>Gather Profile Data</span>
+                  </button>
+                </div>
               </div>
             </div>
           </div>

apps/client/src/app/services/admin.service.ts

@@ -10,4 +10,8 @@ export class AdminService {
   public gatherMax() {
     return this.http.post<void>(`/api/admin/gather/max`, {});
   }
+
+  public gatherProfileData() {
+    return this.http.post<void>(`/api/admin/gather/profile-data`, {});
+  }
 }

prisma/migrations/20210724160404_added_currency_to_symbol_profile/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "SymbolProfile" ADD COLUMN     "currency" "Currency";

prisma/schema.prisma

@@ -117,6 +117,7 @@ model Settings {
 model SymbolProfile {
   countries  Json?
   createdAt  DateTime   @default(now())
+  currency   Currency?
   dataSource DataSource
   id         String     @id @default(uuid())
   name       String?