sudacode/ghostfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Feature/improve timeout of data source requests (#2330)

Browse Source 
* Improve timeout

* Update changelog
This commit is contained in:
Thomas Kaul 2023-09-15 16:25:01 +02:00 committed by GitHub
parent 1fe1e2fe0c
commit 8fc5676443
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 228 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Added support to drop a file in the import activities dialog
- Added a timeout to all data source requests
### Changed
@ -22,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
- Fixed the timeout in _EOD Historical Data_ requests
- Fixed an issue with the portfolio summary caused by the language localization for Dutch (`nl`)
## 2.0.0 - 2023-09-09

43
apps/api/src/app/info/info.service.ts
View File

@ -8,6 +8,7 @@ import { PropertyService } from '@ghostfolio/api/services/property/property.serv
import { TagService } from '@ghostfolio/api/services/tag/tag.service';
import {
DEFAULT_CURRENCY,
DEFAULT_REQUEST_TIMEOUT,
PROPERTY_BETTER_UPTIME_MONITOR_ID,
PROPERTY_COUNTRIES_OF_SUBSCRIBERS,
PROPERTY_DEMO_USER_ID,
@ -168,10 +169,18 @@ export class InfoService {
private async countDockerHubPulls(): Promise<number> {
try {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const { pull_count } = await got(
`https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio`,
{
headers: { 'User-Agent': 'request' }
headers: { 'User-Agent': 'request' },
// @ts-ignore
signal: abortController.signal
}
).json<any>();
@ -185,7 +194,16 @@ export class InfoService {
private async countGitHubContributors(): Promise<number> {
try {
const { body } = await got('https://github.com/ghostfolio/ghostfolio');
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const { body } = await got('https://github.com/ghostfolio/ghostfolio', {
// @ts-ignore
signal: abortController.signal
});
const $ = cheerio.load(body);
@ -203,10 +221,18 @@ export class InfoService {
private async countGitHubStargazers(): Promise<number> {
try {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const { stargazers_count } = await got(
`https://api.github.com/repos/ghostfolio/ghostfolio`,
{
headers: { 'User-Agent': 'request' }
headers: { 'User-Agent': 'request' },
// @ts-ignore
signal: abortController.signal
}
).json<any>();
@ -323,18 +349,25 @@ export class InfoService {
PROPERTY_BETTER_UPTIME_MONITOR_ID
)) as string;
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const { data } = await got(
`https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format(
subDays(new Date(), 90),
DATE_FORMAT
)}&to${format(new Date(), DATE_FORMAT)}`,
{
headers: {
Authorization: `Bearer ${this.configurationService.get(
'BETTER_UPTIME_API_KEY'
)}`
}
},
// @ts-ignore
signal: abortController.signal
}
).json<any>();

11
apps/api/src/app/logo/logo.service.ts
View File

@ -1,4 +1,5 @@
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
import { UniqueAsset } from '@ghostfolio/common/interfaces';
import { HttpException, Injectable } from '@nestjs/common';
import { DataSource } from '@prisma/client';
@ -41,10 +42,18 @@ export class LogoService {
}
private getBuffer(aUrl: string) {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
return got(
`https://t0.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=${aUrl}&size=64`,
{
headers: { 'User-Agent': 'request' }
headers: { 'User-Agent': 'request' },
// @ts-ignore
signal: abortController.signal
}
).buffer();
}

53
apps/api/src/services/data-provider/coingecko/coingecko.service.ts
View File

@ -4,7 +4,10 @@ import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';
import {
DEFAULT_CURRENCY,
DEFAULT_REQUEST_TIMEOUT
} from '@ghostfolio/common/config';
import { DATE_FORMAT } from '@ghostfolio/common/helper';
import { DataProviderInfo } from '@ghostfolio/common/interfaces';
import { Granularity } from '@ghostfolio/common/types';
@ -40,7 +43,16 @@ export class CoinGeckoService implements DataProviderInterface {
};
try {
const { name } = await got(`${this.URL}/coins/${aSymbol}`).json<any>();
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const { name } = await got(`${this.URL}/coins/${aSymbol}`, {
// @ts-ignore
signal: abortController.signal
}).json<any>();
response.name = name;
} catch (error) {
@ -73,12 +85,22 @@ export class CoinGeckoService implements DataProviderInterface {
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
}> {
try {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const { prices } = await got(
`${
this.URL
}/coins/${aSymbol}/market_chart/range?vs_currency=${DEFAULT_CURRENCY.toLowerCase()}&from=${getUnixTime(
from
)}&to=${getUnixTime(to)}`
)}&to=${getUnixTime(to)}`,
{
// @ts-ignore
signal: abortController.signal
}
).json<any>();
const result: {
@ -122,10 +144,20 @@ export class CoinGeckoService implements DataProviderInterface {
}
try {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const response = await got(
`${this.URL}/simple/price?ids=${aSymbols.join(
','
)}&vs_currencies=${DEFAULT_CURRENCY.toLowerCase()}`
)}&vs_currencies=${DEFAULT_CURRENCY.toLowerCase()}`,
{
// @ts-ignore
signal: abortController.signal
}
).json<any>();
for (const symbol in response) {
@ -160,9 +192,16 @@ export class CoinGeckoService implements DataProviderInterface {
let items: LookupItem[] = [];
try {
const { coins } = await got(
`${this.URL}/search?query=${query}`
).json<any>();
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const { coins } = await got(`${this.URL}/search?query=${query}`, {
// @ts-ignore
signal: abortController.signal
}).json<any>();
items = coins.map(({ id: symbol, name }) => {
return {

49
apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts
View File

@ -1,4 +1,5 @@
import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
import { Country } from '@ghostfolio/common/interfaces/country.interface';
import { Sector } from '@ghostfolio/common/interfaces/sector.interface';
import { Injectable } from '@nestjs/common';
@ -32,15 +33,35 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
return response;
}
let abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const profile = await got(
`${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol}.json`
`${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol}.json`,
{
// @ts-ignore
signal: abortController.signal
}
)
.json<any>()
.catch(() => {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
return got(
`${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol.split(
'.'
)?.[0]}.json`
)?.[0]}.json`,
{
// @ts-ignore
signal: abortController.signal
}
)
.json<any>()
.catch(() => {
@ -54,15 +75,35 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
response.isin = isin;
}
abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const holdings = await got(
`${TrackinsightDataEnhancerService.baseUrl}/holdings/${symbol}.json`
`${TrackinsightDataEnhancerService.baseUrl}/holdings/${symbol}.json`,
{
// @ts-ignore
signal: abortController.signal
}
)
.json<any>()
.catch(() => {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
return got(
`${TrackinsightDataEnhancerService.baseUrl}/holdings/${symbol.split(
'.'
)?.[0]}.json`
)?.[0]}.json`,
{
// @ts-ignore
signal: abortController.signal
}
)
.json<any>()
.catch(() => {

33
apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts
View File

@ -78,6 +78,12 @@ export class EodHistoricalDataService implements DataProviderInterface {
const symbol = this.convertToEodSymbol(aSymbol);
try {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const response = await got(
`${this.URL}/eod/${symbol}?api_token=${
this.apiKey
@ -86,9 +92,8 @@ export class EodHistoricalDataService implements DataProviderInterface {
DATE_FORMAT
)}&period={aGranularity}`,
{
timeout: {
request: DEFAULT_REQUEST_TIMEOUT
}
// @ts-ignore
signal: abortController.signal
}
).json<any>();
@ -138,14 +143,19 @@ export class EodHistoricalDataService implements DataProviderInterface {
}
try {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const realTimeResponse = await got(
`${this.URL}/real-time/${symbols[0]}?api_token=${
this.apiKey
}&fmt=json&s=${symbols.join(',')}`,
{
timeout: {
request: DEFAULT_REQUEST_TIMEOUT
}
// @ts-ignore
signal: abortController.signal
}
).json<any>();
@ -331,12 +341,17 @@ export class EodHistoricalDataService implements DataProviderInterface {
let searchResult = [];
try {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const response = await got(
`${this.URL}/search/${aQuery}?api_token=${this.apiKey}`,
{
timeout: {
request: DEFAULT_REQUEST_TIMEOUT
}
// @ts-ignore
signal: abortController.signal
}
).json<any>();

41
apps/api/src/services/data-provider/financial-modeling-prep/financial-modeling-prep.service.ts
View File

@ -5,7 +5,10 @@ import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';
import {
DEFAULT_CURRENCY,
DEFAULT_REQUEST_TIMEOUT
} from '@ghostfolio/common/config';
import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper';
import { DataProviderInfo } from '@ghostfolio/common/interfaces';
import { Granularity } from '@ghostfolio/common/types';
@ -63,8 +66,18 @@ export class FinancialModelingPrepService implements DataProviderInterface {
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
}> {
try {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const { historical } = await got(
`${this.URL}/historical-price-full/${aSymbol}?apikey=${this.apiKey}`
`${this.URL}/historical-price-full/${aSymbol}?apikey=${this.apiKey}`,
{
// @ts-ignore
signal: abortController.signal
}
).json<any>();
const result: {
@ -110,8 +123,18 @@ export class FinancialModelingPrepService implements DataProviderInterface {
}
try {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const response = await got(
`${this.URL}/quote/${aSymbols.join(',')}?apikey=${this.apiKey}`
`${this.URL}/quote/${aSymbols.join(',')}?apikey=${this.apiKey}`,
{
// @ts-ignore
signal: abortController.signal
}
).json<any>();
for (const { price, symbol } of response) {
@ -144,8 +167,18 @@ export class FinancialModelingPrepService implements DataProviderInterface {
let items: LookupItem[] = [];
try {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const result = await got(
`${this.URL}/search?query=${query}&apikey=${this.apiKey}`
`${this.URL}/search?query=${query}&apikey=${this.apiKey}`,
{
// @ts-ignore
signal: abortController.signal
}
).json<any>();
items = result.map(({ currency, name, symbol }) => {

13
apps/api/src/services/data-provider/manual/manual.service.ts
View File

@ -6,6 +6,7 @@ import {
} from '@ghostfolio/api/services/interfaces/interfaces';
import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
import {
DATE_FORMAT,
extractNumberFromString,
@ -95,7 +96,17 @@ export class ManualService implements DataProviderInterface {
return {};
}
const { body } = await got(url, { headers });
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const { body } = await got(url, {
headers,
// @ts-ignore
signal: abortController.signal
});
const $ = cheerio.load(body);

15
apps/api/src/services/data-provider/rapid-api/rapid-api.service.ts
View File

@ -5,7 +5,10 @@ import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
import { ghostfolioFearAndGreedIndexSymbol } from '@ghostfolio/common/config';
import {
DEFAULT_REQUEST_TIMEOUT,
ghostfolioFearAndGreedIndexSymbol
} from '@ghostfolio/common/config';
import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common';
@ -135,6 +138,12 @@ export class RapidApiService implements DataProviderInterface {
oneYearAgo: { value: number; valueText: string };
}> {
try {
const abortController = new AbortController();
setTimeout(() => {
abortController.abort();
}, DEFAULT_REQUEST_TIMEOUT);
const { fgi } = await got(
`https://fear-and-greed-index.p.rapidapi.com/v1/fgi`,
{
@ -142,7 +151,9 @@ export class RapidApiService implements DataProviderInterface {
useQueryString: 'true',
'x-rapidapi-host': 'fear-and-greed-index.p.rapidapi.com',
'x-rapidapi-key': this.configurationService.get('RAPID_API_API_KEY')
}
},
// @ts-ignore
signal: abortController.signal
}
).json<any>();

2
libs/common/src/lib/config.ts
View File

@ -39,7 +39,7 @@ export const DEFAULT_CURRENCY = 'USD';
export const DEFAULT_DATE_FORMAT_MONTH_YEAR = 'MMM yyyy';
export const DEFAULT_LANGUAGE_CODE = 'en';
export const DEFAULT_PAGE_SIZE = 50;
export const DEFAULT_REQUEST_TIMEOUT = ms('3 seconds');
export const DEFAULT_REQUEST_TIMEOUT = ms('2 seconds');
export const DEFAULT_ROOT_URL = 'http://localhost:4200';
export const EMERGENCY_FUND_TAG_ID = '4452656d-9fa4-4bd0-ba38-70492e31d180';