Release 1.36.0 (#281)

* Eliminate name from scraper config

* Update changelog

CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 1.36.0 - 09.08.2021
+
+### Changed
+
+- Improved the data gathering handling on server restart
+- Respected the cash balance on the allocations page
+- Eliminated the name from the scraper configuration
+
+### Fixed
+
+- Fixed hidden cryptocurrency holdings
+
 ## 1.35.0 - 08.08.2021
 
 ### Changed

apps/api/src/app/admin/admin.service.ts

@@ -1,3 +1,4 @@
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { AdminData } from '@ghostfolio/common/interfaces';
@@ -7,6 +8,7 @@ import { Currency } from '@prisma/client';
 @Injectable()
 export class AdminService {
   public constructor(
+    private readonly dataGatheringService: DataGatheringService,
     private readonly exchangeRateDataService: ExchangeRateDataService,
     private readonly prismaService: PrismaService
   ) {}
@@ -68,18 +70,15 @@ export class AdminService {
   }
 
   private async getLastDataGathering() {
-    const lastDataGathering = await this.prismaService.property.findUnique({
-      where: { key: 'LAST_DATA_GATHERING' }
-    });
+    const lastDataGathering =
+      await this.dataGatheringService.getLastDataGathering();
 
-    if (lastDataGathering?.value) {
-      return new Date(lastDataGathering.value);
+    if (lastDataGathering) {
+      return lastDataGathering;
     }
 
     const dataGatheringInProgress =
-      await this.prismaService.property.findUnique({
-        where: { key: 'LOCKED_DATA_GATHERING' }
-      });
+      await this.dataGatheringService.getIsInProgress();
 
     if (dataGatheringInProgress) {
       return 'IN_PROGRESS';

apps/api/src/app/app.controller.ts

@@ -1,12 +1,12 @@
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { Controller } from '@nestjs/common';
 
-import { PrismaService } from '../services/prisma.service';
 import { RedisCacheService } from './redis-cache/redis-cache.service';
 
 @Controller()
 export class AppController {
   public constructor(
-    private readonly prismaService: PrismaService,
+    private readonly dataGatheringService: DataGatheringService,
     private readonly redisCacheService: RedisCacheService
   ) {
     this.initialize();
@@ -15,17 +15,12 @@ export class AppController {
   private async initialize() {
     this.redisCacheService.reset();
 
-    const isDataGatheringLocked = await this.prismaService.property.findUnique({
-      where: { key: 'LOCKED_DATA_GATHERING' }
-    });
+    const isDataGatheringInProgress =
+      await this.dataGatheringService.getIsInProgress();
 
-    if (!isDataGatheringLocked) {
-      // Prepare for automatical data gather if not locked
-      await this.prismaService.property.deleteMany({
-        where: {
-          OR: [{ key: 'LAST_DATA_GATHERING' }, { key: 'LOCKED_DATA_GATHERING' }]
-        }
-      });
+    if (isDataGatheringInProgress) {
+      // Prepare for automatical data gathering, if hung up in progress state
+      await this.dataGatheringService.reset();
     }
   }
 }

apps/api/src/app/cache/cache.module.ts

@@ -1,3 +1,10 @@
+import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
+import { DataProviderService } from '@ghostfolio/api/services/data-provider.service';
+import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
+import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
+import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
+import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { Module } from '@nestjs/common';
 
@@ -8,6 +15,16 @@ import { CacheService } from './cache.service';
 @Module({
   imports: [RedisCacheModule],
   controllers: [CacheController],
-  providers: [CacheService, PrismaService]
+  providers: [
+    AlphaVantageService,
+    CacheService,
+    ConfigurationService,
+    DataGatheringService,
+    DataProviderService,
+    GhostfolioScraperApiService,
+    PrismaService,
+    RakutenRapidApiService,
+    YahooFinanceService
+  ]
 })
 export class CacheModule {}

apps/api/src/app/cache/cache.service.ts

@@ -1,16 +1,14 @@
-import { PrismaService } from '@ghostfolio/api/services/prisma.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { Injectable } from '@nestjs/common';
 
 @Injectable()
 export class CacheService {
-  public constructor(private readonly prismaService: PrismaService) {}
+  public constructor(
+    private readonly dataGaterhingService: DataGatheringService
+  ) {}
 
   public async flush(): Promise<void> {
-    await this.prismaService.property.deleteMany({
-      where: {
-        OR: [{ key: 'LAST_DATA_GATHERING' }, { key: 'LOCKED_DATA_GATHERING' }]
-      }
-    });
+    await this.dataGaterhingService.reset();
 
     return;
   }

apps/api/src/app/info/info.module.ts

@@ -1,4 +1,10 @@
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
+import { DataProviderService } from '@ghostfolio/api/services/data-provider.service';
+import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
+import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
+import { RakutenRapidApiService } from '@ghostfolio/api/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
+import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { Module } from '@nestjs/common';
 import { JwtModule } from '@nestjs/jwt';
@@ -14,6 +20,16 @@ import { InfoService } from './info.service';
     })
   ],
   controllers: [InfoController],
-  providers: [ConfigurationService, InfoService, PrismaService]
+  providers: [
+    AlphaVantageService,
+    ConfigurationService,
+    DataGatheringService,
+    DataProviderService,
+    GhostfolioScraperApiService,
+    InfoService,
+    PrismaService,
+    RakutenRapidApiService,
+    YahooFinanceService
+  ]
 })
 export class InfoModule {}

apps/api/src/app/info/info.service.ts

@@ -1,4 +1,5 @@
 import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
+import { DataGatheringService } from '@ghostfolio/api/services/data-gathering.service';
 import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { InfoItem } from '@ghostfolio/common/interfaces';
 import { Subscription } from '@ghostfolio/common/interfaces/subscription.interface';
@@ -15,6 +16,7 @@ export class InfoService {
 
   public constructor(
     private readonly configurationService: ConfigurationService,
+    private readonly dataGatheringService: DataGatheringService,
     private readonly jwtService: JwtService,
     private readonly prismaService: PrismaService
   ) {}
@@ -116,11 +118,10 @@ export class InfoService {
   }
 
   private async getLastDataGathering() {
-    const lastDataGathering = await this.prismaService.property.findUnique({
-      where: { key: 'LAST_DATA_GATHERING' }
-    });
+    const lastDataGathering =
+      await this.dataGatheringService.getLastDataGathering();
 
-    return lastDataGathering?.value ? new Date(lastDataGathering.value) : null;
+    return lastDataGathering ?? null;
   }
 
   private async getStatistics() {

apps/api/src/app/portfolio/portfolio.service.ts

@@ -1,4 +1,5 @@
 import { AccountService } from '@ghostfolio/api/app/account/account.service';
+import { CashDetails } from '@ghostfolio/api/app/account/interfaces/cash-details.interface';
 import { CurrentRateService } from '@ghostfolio/api/app/core/current-rate.service';
 import { PortfolioOrder } from '@ghostfolio/api/app/core/interfaces/portfolio-order.interface';
 import { TimelineSpecification } from '@ghostfolio/api/app/core/interfaces/timeline-specification.interface';
@@ -17,10 +18,11 @@ import { FeeRatioInitialInvestment } from '@ghostfolio/api/models/rules/fees/fee
 import { DataProviderService } from '@ghostfolio/api/services/data-provider.service';
 import { ExchangeRateDataService } from '@ghostfolio/api/services/exchange-rate-data.service';
 import { ImpersonationService } from '@ghostfolio/api/services/impersonation.service';
+import { MarketState } from '@ghostfolio/api/services/interfaces/interfaces';
 import { EnhancedSymbolProfile } from '@ghostfolio/api/services/interfaces/symbol-profile.interface';
 import { RulesService } from '@ghostfolio/api/services/rules.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
-import { UNKNOWN_KEY } from '@ghostfolio/common/config';
+import { UNKNOWN_KEY, ghostfolioCashSymbol } from '@ghostfolio/common/config';
 import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper';
 import {
   PortfolioOverview,
@@ -38,7 +40,12 @@ import {
 } from '@ghostfolio/common/types';
 import { Inject, Injectable } from '@nestjs/common';
 import { REQUEST } from '@nestjs/core';
-import { Currency, DataSource, Type as TypeOfOrder } from '@prisma/client';
+import {
+  AssetClass,
+  Currency,
+  DataSource,
+  Type as TypeOfOrder
+} from '@prisma/client';
 import Big from 'big.js';
 import {
   endOfToday,
@@ -201,8 +208,16 @@ export class PortfolioService {
       throw new Error('Missing information');
     }
 
+    const cashDetails = await this.accountService.getCashDetails(
+      userId,
+      userCurrency
+    );
+
     const result: { [symbol: string]: PortfolioPosition } = {};
-    const totalValue = currentPositions.currentValue;
+    const totalInvestment = currentPositions.totalInvestment.plus(
+      cashDetails.balance
+    );
+    const totalValue = currentPositions.currentValue.plus(cashDetails.balance);
 
     const symbols = currentPositions.positions.map(
       (position) => position.symbol
@@ -231,9 +246,7 @@ export class PortfolioService {
       result[item.symbol] = {
         accounts,
         allocationCurrent: value.div(totalValue).toNumber(),
-        allocationInvestment: item.investment
-          .div(currentPositions.totalInvestment)
-          .toNumber(),
+        allocationInvestment: item.investment.div(totalInvestment).toNumber(),
         assetClass: symbolProfile.assetClass,
         countries: symbolProfile.countries,
         currency: item.currency,
@@ -252,6 +265,13 @@ export class PortfolioService {
       };
     }
 
+    // TODO: Add a cash position for each currency
+    result[ghostfolioCashSymbol] = await this.getCashPosition({
+      cashDetails,
+      investment: totalInvestment,
+      value: totalValue
+    });
+
     return result;
   }
 
@@ -660,6 +680,46 @@ export class PortfolioService {
     };
   }
 
+  private async getCashPosition({
+    cashDetails,
+    investment,
+    value
+  }: {
+    cashDetails: CashDetails;
+    investment: Big;
+    value: Big;
+  }) {
+    const accounts = {};
+    const cashValue = new Big(cashDetails.balance);
+
+    cashDetails.accounts.forEach((account) => {
+      accounts[account.name] = {
+        current: account.balance,
+        original: account.balance
+      };
+    });
+
+    return {
+      accounts,
+      allocationCurrent: cashValue.div(value).toNumber(),
+      allocationInvestment: cashValue.div(investment).toNumber(),
+      assetClass: AssetClass.CASH,
+      countries: [],
+      currency: Currency.CHF,
+      grossPerformance: 0,
+      grossPerformancePercent: 0,
+      investment: cashValue.toNumber(),
+      marketPrice: 0,
+      marketState: MarketState.open,
+      name: 'Cash',
+      quantity: 0,
+      sectors: [],
+      symbol: ghostfolioCashSymbol,
+      transactionCount: 0,
+      value: cashValue.toNumber()
+    };
+  }
+
   private getStartDate(aDateRange: DateRange, portfolioStart: Date) {
     switch (aDateRange) {
       case '1d':

apps/api/src/app/symbol/symbol.service.ts

@@ -1,6 +1,5 @@
 import { DataProviderService } from '@ghostfolio/api/services/data-provider.service';
-import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
-import { convertFromYahooSymbol } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service';
+import { PrismaService } from '@ghostfolio/api/services/prisma.service';
 import { Injectable } from '@nestjs/common';
 import { Currency, DataSource } from '@prisma/client';
 
@@ -11,7 +10,7 @@ import { SymbolItem } from './interfaces/symbol-item.interface';
 export class SymbolService {
   public constructor(
     private readonly dataProviderService: DataProviderService,
-    private readonly ghostfolioScraperApiService: GhostfolioScraperApiService
+    private readonly prismaService: PrismaService
   ) {}
 
   public async get(aSymbol: string): Promise<SymbolItem> {
@@ -37,16 +36,28 @@ export class SymbolService {
       results.items = items;
 
       // Add custom symbols
-      const scraperConfigurations = await this.ghostfolioScraperApiService.getScraperConfigurations();
-      scraperConfigurations.forEach((scraperConfiguration) => {
-        if (scraperConfiguration.name.toLowerCase().startsWith(aQuery)) {
-          results.items.push({
-            dataSource: DataSource.GHOSTFOLIO,
-            name: scraperConfiguration.name,
-            symbol: scraperConfiguration.symbol
-          });
-        }
-      });
+      const ghostfolioSymbolProfiles =
+        await this.prismaService.symbolProfile.findMany({
+          select: {
+            dataSource: true,
+            name: true,
+            symbol: true
+          },
+          where: {
+            AND: [
+              {
+                dataSource: DataSource.GHOSTFOLIO,
+                name: {
+                  startsWith: aQuery
+                }
+              }
+            ]
+          }
+        });
+
+      for (const ghostfolioSymbolProfile of ghostfolioSymbolProfiles) {
+        results.items.push(ghostfolioSymbolProfile);
+      }
 
       return results;
     } catch (error) {

apps/api/src/services/data-gathering.service.ts

@@ -250,6 +250,34 @@ export class DataGatheringService {
     });
   }
 
+  public async getIsInProgress() {
+    return await this.prismaService.property.findUnique({
+      where: { key: 'LOCKED_DATA_GATHERING' }
+    });
+  }
+
+  public async getLastDataGathering() {
+    const lastDataGathering = await this.prismaService.property.findUnique({
+      where: { key: 'LAST_DATA_GATHERING' }
+    });
+
+    if (lastDataGathering?.value) {
+      return new Date(lastDataGathering.value);
+    }
+
+    return undefined;
+  }
+
+  public async reset() {
+    console.log('Data gathering has been reset.');
+
+    await this.prismaService.property.deleteMany({
+      where: {
+        OR: [{ key: 'LAST_DATA_GATHERING' }, { key: 'LOCKED_DATA_GATHERING' }]
+      }
+    });
+  }
+
   private getBenchmarksToGather(startDate: Date): IDataGatheringItem[] {
     const benchmarksToGather = benchmarks.map(({ dataSource, symbol }) => {
       return {
@@ -305,9 +333,8 @@ export class DataGatheringService {
       }
     );
 
-    const customSymbolsToGather = await this.getCustomSymbolsToGather(
-      startDate
-    );
+    const customSymbolsToGather =
+      await this.ghostfolioScraperApi.getCustomSymbolsToGather(startDate);
 
     return [
       ...this.getBenchmarksToGather(startDate),
@@ -320,9 +347,8 @@ export class DataGatheringService {
   private async getSymbolsMax(): Promise<IDataGatheringItem[]> {
     const startDate = new Date(getUtc('2015-01-01'));
 
-    const customSymbolsToGather = await this.getCustomSymbolsToGather(
-      startDate
-    );
+    const customSymbolsToGather =
+      await this.ghostfolioScraperApi.getCustomSymbolsToGather(startDate);
 
     const currencyPairsToGather = currencyPairs.map(
       ({ dataSource, symbol }) => {
@@ -373,18 +399,13 @@ export class DataGatheringService {
   }
 
   private async isDataGatheringNeeded() {
-    const lastDataGathering = await this.prismaService.property.findUnique({
-      where: { key: 'LAST_DATA_GATHERING' }
-    });
+    const lastDataGathering = await this.getLastDataGathering();
 
     const isDataGatheringLocked = await this.prismaService.property.findUnique({
       where: { key: 'LOCKED_DATA_GATHERING' }
     });
 
-    const diffInHours = differenceInHours(
-      new Date(),
-      new Date(lastDataGathering?.value)
-    );
+    const diffInHours = differenceInHours(new Date(), lastDataGathering);
 
     return (diffInHours >= 1 || !lastDataGathering) && !isDataGatheringLocked;
   }

apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts

@@ -12,6 +12,7 @@ import { format } from 'date-fns';
 
 import { DataProviderInterface } from '../../interfaces/data-provider.interface';
 import {
+  IDataGatheringItem,
   IDataProviderHistoricalResponse,
   IDataProviderResponse,
   MarketState
@@ -55,8 +56,7 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
           marketPrice,
           currency: scraperConfig?.currency,
           dataSource: DataSource.GHOSTFOLIO,
-          marketState: MarketState.delayed,
-          name: scraperConfig?.name
+          marketState: MarketState.delayed
         }
       };
     } catch (error) {
@@ -66,6 +66,25 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
     return {};
   }
 
+  public async getCustomSymbolsToGather(
+    startDate?: Date
+  ): Promise<IDataGatheringItem[]> {
+    const ghostfolioSymbolProfiles =
+      await this.prismaService.symbolProfile.findMany({
+        where: {
+          dataSource: DataSource.GHOSTFOLIO
+        }
+      });
+
+    return ghostfolioSymbolProfiles.map(({ dataSource, symbol }) => {
+      return {
+        dataSource,
+        symbol,
+        date: startDate
+      };
+    });
+  }
+
   public async getHistorical(
     aSymbols: string[],
     aGranularity: Granularity = 'day',

apps/api/src/services/data-provider/ghostfolio-scraper-api/interfaces/scraper-config.interface.ts

@@ -2,7 +2,6 @@ import { Currency } from '@prisma/client';
 
 export interface ScraperConfig {
   currency: Currency;
-  name: string;
   selector: string;
   symbol: string;
   url: string;

apps/api/src/services/interfaces/interfaces.ts

@@ -42,7 +42,7 @@ export interface IDataProviderResponse {
   marketChangePercent?: number;
   marketPrice: number;
   marketState: MarketState;
-  name: string;
+  name?: string;
   url?: string;
 }
 

apps/client/src/app/components/positions/positions.component.ts

@@ -7,7 +7,6 @@ import {
 } from '@angular/core';
 import { MarketState } from '@ghostfolio/api/services/interfaces/interfaces';
 import { Position } from '@ghostfolio/common/interfaces';
-import { AssetClass } from '@prisma/client';
 
 @Component({
   selector: 'gf-positions',
@@ -26,8 +25,6 @@ export class PositionsComponent implements OnChanges, OnInit {
   public positionsRest: Position[] = [];
   public positionsWithPriority: Position[] = [];
 
-  private ignoreAssetClasses = [AssetClass.CASH.toString()];
-
   public constructor() {}
 
   public ngOnInit() {}
@@ -44,10 +41,6 @@ export class PositionsComponent implements OnChanges, OnInit {
       this.positionsWithPriority = [];
 
       for (const portfolioPosition of this.positions) {
-        if (this.ignoreAssetClasses.includes(portfolioPosition.assetClass)) {
-          continue;
-        }
-
         if (
           portfolioPosition.marketState === MarketState.open ||
           this.range !== '1d'

apps/client/src/app/pages/admin/admin-page.component.ts

@@ -152,7 +152,7 @@ export class AdminPageComponent implements OnDestroy, OnInit {
           } else if (lastDataGathering === 'IN_PROGRESS') {
             this.dataGatheringInProgress = true;
           } else {
-            this.lastDataGathering = '-';
+            this.lastDataGathering = 'Starting soon...';
           }
 
           this.transactionCount = transactionCount;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ghostfolio",
-  "version": "1.35.0",
+  "version": "1.36.0",
   "homepage": "https://ghostfol.io",
   "license": "AGPL-3.0",
   "scripts": {