Feature/improve timeout of data source requests (#2330)

* Improve timeout

* Update changelog

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Added support to drop a file in the import activities dialog
+- Added a timeout to all data source requests
 
 ### Changed
 
@@ -22,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Fixed the timeout in _EOD Historical Data_ requests
 - Fixed an issue with the portfolio summary caused by the language localization for Dutch (`nl`)
 
 ## 2.0.0 - 2023-09-09

apps/api/src/app/info/info.service.ts

@@ -8,6 +8,7 @@ import { PropertyService } from '@ghostfolio/api/services/property/property.serv
 import { TagService } from '@ghostfolio/api/services/tag/tag.service';
 import {
   DEFAULT_CURRENCY,
+  DEFAULT_REQUEST_TIMEOUT,
   PROPERTY_BETTER_UPTIME_MONITOR_ID,
   PROPERTY_COUNTRIES_OF_SUBSCRIBERS,
   PROPERTY_DEMO_USER_ID,
@@ -168,10 +169,18 @@ export class InfoService {
 
   private async countDockerHubPulls(): Promise<number> {
     try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
       const { pull_count } = await got(
         `https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio`,
         {
-          headers: { 'User-Agent': 'request' }
+          headers: { 'User-Agent': 'request' },
+          // @ts-ignore
+          signal: abortController.signal
         }
       ).json<any>();
 
@@ -185,7 +194,16 @@ export class InfoService {
 
   private async countGitHubContributors(): Promise<number> {
     try {
-      const { body } = await got('https://github.com/ghostfolio/ghostfolio');
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
+      const { body } = await got('https://github.com/ghostfolio/ghostfolio', {
+        // @ts-ignore
+        signal: abortController.signal
+      });
 
       const $ = cheerio.load(body);
 
@@ -203,10 +221,18 @@ export class InfoService {
 
   private async countGitHubStargazers(): Promise<number> {
     try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
       const { stargazers_count } = await got(
         `https://api.github.com/repos/ghostfolio/ghostfolio`,
         {
-          headers: { 'User-Agent': 'request' }
+          headers: { 'User-Agent': 'request' },
+          // @ts-ignore
+          signal: abortController.signal
         }
       ).json<any>();
 
@@ -323,18 +349,25 @@ export class InfoService {
           PROPERTY_BETTER_UPTIME_MONITOR_ID
         )) as string;
 
+        const abortController = new AbortController();
+
+        setTimeout(() => {
+          abortController.abort();
+        }, DEFAULT_REQUEST_TIMEOUT);
+
         const { data } = await got(
           `https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format(
             subDays(new Date(), 90),
             DATE_FORMAT
           )}&to${format(new Date(), DATE_FORMAT)}`,
-
           {
             headers: {
               Authorization: `Bearer ${this.configurationService.get(
                 'BETTER_UPTIME_API_KEY'
               )}`
-            }
+            },
+            // @ts-ignore
+            signal: abortController.signal
           }
         ).json<any>();
 

apps/api/src/app/logo/logo.service.ts

@@ -1,4 +1,5 @@
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
+import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
 import { UniqueAsset } from '@ghostfolio/common/interfaces';
 import { HttpException, Injectable } from '@nestjs/common';
 import { DataSource } from '@prisma/client';
@@ -41,10 +42,18 @@ export class LogoService {
   }
 
   private getBuffer(aUrl: string) {
+    const abortController = new AbortController();
+
+    setTimeout(() => {
+      abortController.abort();
+    }, DEFAULT_REQUEST_TIMEOUT);
+
     return got(
       `https://t0.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=${aUrl}&size=64`,
       {
-        headers: { 'User-Agent': 'request' }
+        headers: { 'User-Agent': 'request' },
+        // @ts-ignore
+        signal: abortController.signal
       }
     ).buffer();
   }

apps/api/src/services/data-provider/coingecko/coingecko.service.ts

@@ -4,7 +4,10 @@ import {
   IDataProviderHistoricalResponse,
   IDataProviderResponse
 } from '@ghostfolio/api/services/interfaces/interfaces';
-import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';
+import {
+  DEFAULT_CURRENCY,
+  DEFAULT_REQUEST_TIMEOUT
+} from '@ghostfolio/common/config';
 import { DATE_FORMAT } from '@ghostfolio/common/helper';
 import { DataProviderInfo } from '@ghostfolio/common/interfaces';
 import { Granularity } from '@ghostfolio/common/types';
@@ -40,7 +43,16 @@ export class CoinGeckoService implements DataProviderInterface {
     };
 
     try {
-      const { name } = await got(`${this.URL}/coins/${aSymbol}`).json<any>();
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
+      const { name } = await got(`${this.URL}/coins/${aSymbol}`, {
+        // @ts-ignore
+        signal: abortController.signal
+      }).json<any>();
 
       response.name = name;
     } catch (error) {
@@ -73,12 +85,22 @@ export class CoinGeckoService implements DataProviderInterface {
     [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
   }> {
     try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
       const { prices } = await got(
         `${
           this.URL
         }/coins/${aSymbol}/market_chart/range?vs_currency=${DEFAULT_CURRENCY.toLowerCase()}&from=${getUnixTime(
           from
-        )}&to=${getUnixTime(to)}`
+        )}&to=${getUnixTime(to)}`,
+        {
+          // @ts-ignore
+          signal: abortController.signal
+        }
       ).json<any>();
 
       const result: {
@@ -122,10 +144,20 @@ export class CoinGeckoService implements DataProviderInterface {
     }
 
     try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
       const response = await got(
         `${this.URL}/simple/price?ids=${aSymbols.join(
           ','
-        )}&vs_currencies=${DEFAULT_CURRENCY.toLowerCase()}`
+        )}&vs_currencies=${DEFAULT_CURRENCY.toLowerCase()}`,
+        {
+          // @ts-ignore
+          signal: abortController.signal
+        }
       ).json<any>();
 
       for (const symbol in response) {
@@ -160,9 +192,16 @@ export class CoinGeckoService implements DataProviderInterface {
     let items: LookupItem[] = [];
 
     try {
-      const { coins } = await got(
+      const abortController = new AbortController();
-        `${this.URL}/search?query=${query}`
+
-      ).json<any>();
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
+      const { coins } = await got(`${this.URL}/search?query=${query}`, {
+        // @ts-ignore
+        signal: abortController.signal
+      }).json<any>();
 
       items = coins.map(({ id: symbol, name }) => {
         return {

apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts

@@ -1,4 +1,5 @@
 import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
+import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
 import { Country } from '@ghostfolio/common/interfaces/country.interface';
 import { Sector } from '@ghostfolio/common/interfaces/sector.interface';
 import { Injectable } from '@nestjs/common';
@@ -32,15 +33,35 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
       return response;
     }
 
+    let abortController = new AbortController();
+
+    setTimeout(() => {
+      abortController.abort();
+    }, DEFAULT_REQUEST_TIMEOUT);
+
     const profile = await got(
-      `${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol}.json`
+      `${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol}.json`,
+      {
+        // @ts-ignore
+        signal: abortController.signal
+      }
     )
       .json<any>()
       .catch(() => {
+        const abortController = new AbortController();
+
+        setTimeout(() => {
+          abortController.abort();
+        }, DEFAULT_REQUEST_TIMEOUT);
+
         return got(
           `${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol.split(
             '.'
-          )?.[0]}.json`
+          )?.[0]}.json`,
+          {
+            // @ts-ignore
+            signal: abortController.signal
+          }
         )
           .json<any>()
           .catch(() => {
@@ -54,15 +75,35 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
       response.isin = isin;
     }
 
+    abortController = new AbortController();
+
+    setTimeout(() => {
+      abortController.abort();
+    }, DEFAULT_REQUEST_TIMEOUT);
+
     const holdings = await got(
-      `${TrackinsightDataEnhancerService.baseUrl}/holdings/${symbol}.json`
+      `${TrackinsightDataEnhancerService.baseUrl}/holdings/${symbol}.json`,
+      {
+        // @ts-ignore
+        signal: abortController.signal
+      }
     )
       .json<any>()
       .catch(() => {
+        const abortController = new AbortController();
+
+        setTimeout(() => {
+          abortController.abort();
+        }, DEFAULT_REQUEST_TIMEOUT);
+
         return got(
           `${TrackinsightDataEnhancerService.baseUrl}/holdings/${symbol.split(
             '.'
-          )?.[0]}.json`
+          )?.[0]}.json`,
+          {
+            // @ts-ignore
+            signal: abortController.signal
+          }
         )
           .json<any>()
           .catch(() => {

apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts

@@ -78,6 +78,12 @@ export class EodHistoricalDataService implements DataProviderInterface {
     const symbol = this.convertToEodSymbol(aSymbol);
 
     try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
       const response = await got(
         `${this.URL}/eod/${symbol}?api_token=${
           this.apiKey
@@ -86,9 +92,8 @@ export class EodHistoricalDataService implements DataProviderInterface {
           DATE_FORMAT
         )}&period={aGranularity}`,
         {
-          timeout: {
+          // @ts-ignore
-            request: DEFAULT_REQUEST_TIMEOUT
+          signal: abortController.signal
-          }
         }
       ).json<any>();
 
@@ -138,14 +143,19 @@ export class EodHistoricalDataService implements DataProviderInterface {
     }
 
     try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
       const realTimeResponse = await got(
         `${this.URL}/real-time/${symbols[0]}?api_token=${
           this.apiKey
         }&fmt=json&s=${symbols.join(',')}`,
         {
-          timeout: {
+          // @ts-ignore
-            request: DEFAULT_REQUEST_TIMEOUT
+          signal: abortController.signal
-          }
         }
       ).json<any>();
 
@@ -331,12 +341,17 @@ export class EodHistoricalDataService implements DataProviderInterface {
     let searchResult = [];
 
     try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
       const response = await got(
         `${this.URL}/search/${aQuery}?api_token=${this.apiKey}`,
         {
-          timeout: {
+          // @ts-ignore
-            request: DEFAULT_REQUEST_TIMEOUT
+          signal: abortController.signal
-          }
         }
       ).json<any>();
 

apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts

@@ -5,7 +5,10 @@ import {
   IDataProviderHistoricalResponse,
   IDataProviderResponse
 } from '@ghostfolio/api/services/interfaces/interfaces';
-import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';
+import {
+  DEFAULT_CURRENCY,
+  DEFAULT_REQUEST_TIMEOUT
+} from '@ghostfolio/common/config';
 import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper';
 import { DataProviderInfo } from '@ghostfolio/common/interfaces';
 import { Granularity } from '@ghostfolio/common/types';
@@ -63,8 +66,18 @@ export class FinancialModelingPrepService implements DataProviderInterface {
     [symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
   }> {
     try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
       const { historical } = await got(
-        `${this.URL}/historical-price-full/${aSymbol}?apikey=${this.apiKey}`
+        `${this.URL}/historical-price-full/${aSymbol}?apikey=${this.apiKey}`,
+        {
+          // @ts-ignore
+          signal: abortController.signal
+        }
       ).json<any>();
 
       const result: {
@@ -110,8 +123,18 @@ export class FinancialModelingPrepService implements DataProviderInterface {
     }
 
     try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
       const response = await got(
-        `${this.URL}/quote/${aSymbols.join(',')}?apikey=${this.apiKey}`
+        `${this.URL}/quote/${aSymbols.join(',')}?apikey=${this.apiKey}`,
+        {
+          // @ts-ignore
+          signal: abortController.signal
+        }
       ).json<any>();
 
       for (const { price, symbol } of response) {
@@ -144,8 +167,18 @@ export class FinancialModelingPrepService implements DataProviderInterface {
     let items: LookupItem[] = [];
 
     try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
       const result = await got(
-        `${this.URL}/search?query=${query}&apikey=${this.apiKey}`
+        `${this.URL}/search?query=${query}&apikey=${this.apiKey}`,
+        {
+          // @ts-ignore
+          signal: abortController.signal
+        }
       ).json<any>();
 
       items = result.map(({ currency, name, symbol }) => {

apps/api/src/services/data-provider/manual/manual.service.ts

@@ -6,6 +6,7 @@ import {
 } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
+import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
 import {
   DATE_FORMAT,
   extractNumberFromString,
@@ -95,7 +96,17 @@ export class ManualService implements DataProviderInterface {
         return {};
       }
 
-      const { body } = await got(url, { headers });
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
+      const { body } = await got(url, {
+        headers,
+        // @ts-ignore
+        signal: abortController.signal
+      });
 
       const $ = cheerio.load(body);
 

apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts

@@ -5,7 +5,10 @@ import {
   IDataProviderHistoricalResponse,
   IDataProviderResponse
 } from '@ghostfolio/api/services/interfaces/interfaces';
-import { ghostfolioFearAndGreedIndexSymbol } from '@ghostfolio/common/config';
+import {
+  DEFAULT_REQUEST_TIMEOUT,
+  ghostfolioFearAndGreedIndexSymbol
+} from '@ghostfolio/common/config';
 import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper';
 import { Granularity } from '@ghostfolio/common/types';
 import { Injectable, Logger } from '@nestjs/common';
@@ -135,6 +138,12 @@ export class RapidApiService implements DataProviderInterface {
     oneYearAgo: { value: number; valueText: string };
   }> {
     try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, DEFAULT_REQUEST_TIMEOUT);
+
       const { fgi } = await got(
         `https://fear-and-greed-index.p.rapidapi.com/v1/fgi`,
         {
@@ -142,7 +151,9 @@ export class RapidApiService implements DataProviderInterface {
             useQueryString: 'true',
             'x-rapidapi-host': 'fear-and-greed-index.p.rapidapi.com',
             'x-rapidapi-key': this.configurationService.get('RAPID_API_API_KEY')
-          }
+          },
+          // @ts-ignore
+          signal: abortController.signal
         }
       ).json<any>();
 

libs/common/src/lib/config.ts

@@ -39,7 +39,7 @@ export const DEFAULT_CURRENCY = 'USD';
 export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy';
 export const DEFAULT_LANGUAGE_CODE = 'en';
 export const DEFAULT_PAGE_SIZE = 50;
-export const DEFAULT_REQUEST_TIMEOUT = ms('3 seconds');
+export const DEFAULT_REQUEST_TIMEOUT = ms('2 seconds');
 export const DEFAULT_ROOT_URL = 'http://localhost:4200';
 
 export const EMERGENCY_FUND_TAG_ID = '4452656d-9fa4-4bd0-ba38-70492e31d180';