Feature/increase timeout in data provider and enhancer health check endpoint (#2621)
* Increase timeout in health check endpoint (data enhancer and provider) * Update changelog
This commit is contained in:
@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
- Improved the import of historical market data in the admin control panel
|
- Improved the import of historical market data in the admin control panel
|
||||||
|
- Increased the timeout in the health check endpoint for data enhancers
|
||||||
|
- Increased the timeout in the health check endpoint for data providers
|
||||||
- Removed the account type from the `Account` database schema
|
- Removed the account type from the `Account` database schema
|
||||||
|
|
||||||
## 2.19.0 - 2023-11-06
|
## 2.19.0 - 2023-11-06
|
||||||
@ -368,7 +370,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
- Added health check endpoints for data enhancers
|
- Added a health check endpoint for data enhancers
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
@ -544,7 +546,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
- Improved the usability of the login dialog
|
- Improved the usability of the login dialog
|
||||||
- Disabled the caching in the health check endpoints for data providers
|
- Disabled the caching in the health check endpoint for data providers
|
||||||
- Improved the content of the Frequently Asked Questions (FAQ) page
|
- Improved the content of the Frequently Asked Questions (FAQ) page
|
||||||
- Upgraded `prisma` from version `4.15.0` to `4.16.2`
|
- Upgraded `prisma` from version `4.15.0` to `4.16.2`
|
||||||
|
|
||||||
@ -932,7 +934,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
- Added a fallback to historical market data if a data provider does not provide live data
|
- Added a fallback to historical market data if a data provider does not provide live data
|
||||||
- Added a general health check endpoint
|
- Added a general health check endpoint
|
||||||
- Added health check endpoints for data providers
|
- Added a health check endpoint for data providers
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
|
@ -5,6 +5,7 @@ import {
|
|||||||
IDataProviderHistoricalResponse,
|
IDataProviderHistoricalResponse,
|
||||||
IDataProviderResponse
|
IDataProviderResponse
|
||||||
} from '@ghostfolio/api/services/interfaces/interfaces';
|
} from '@ghostfolio/api/services/interfaces/interfaces';
|
||||||
|
import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
|
||||||
import { DATE_FORMAT } from '@ghostfolio/common/helper';
|
import { DATE_FORMAT } from '@ghostfolio/common/helper';
|
||||||
import { Granularity } from '@ghostfolio/common/types';
|
import { Granularity } from '@ghostfolio/common/types';
|
||||||
import { Injectable } from '@nestjs/common';
|
import { Injectable } from '@nestjs/common';
|
||||||
@ -106,8 +107,10 @@ export class AlphaVantageService implements DataProviderInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async getQuotes({
|
public async getQuotes({
|
||||||
|
requestTimeout = DEFAULT_REQUEST_TIMEOUT,
|
||||||
symbols
|
symbols
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
symbols: string[];
|
symbols: string[];
|
||||||
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
||||||
return {};
|
return {};
|
||||||
|
@ -135,8 +135,10 @@ export class CoinGeckoService implements DataProviderInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async getQuotes({
|
public async getQuotes({
|
||||||
|
requestTimeout = DEFAULT_REQUEST_TIMEOUT,
|
||||||
symbols
|
symbols
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
symbols: string[];
|
symbols: string[];
|
||||||
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
||||||
const response: { [symbol: string]: IDataProviderResponse } = {};
|
const response: { [symbol: string]: IDataProviderResponse } = {};
|
||||||
@ -150,7 +152,7 @@ export class CoinGeckoService implements DataProviderInterface {
|
|||||||
|
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
abortController.abort();
|
abortController.abort();
|
||||||
}, DEFAULT_REQUEST_TIMEOUT);
|
}, requestTimeout);
|
||||||
|
|
||||||
const quotes = await got(
|
const quotes = await got(
|
||||||
`${this.URL}/simple/price?ids=${symbols.join(
|
`${this.URL}/simple/price?ids=${symbols.join(
|
||||||
|
@ -2,6 +2,7 @@ import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/in
|
|||||||
import { HttpException, Inject, Injectable } from '@nestjs/common';
|
import { HttpException, Inject, Injectable } from '@nestjs/common';
|
||||||
import { Prisma } from '@prisma/client';
|
import { Prisma } from '@prisma/client';
|
||||||
import { StatusCodes, getReasonPhrase } from 'http-status-codes';
|
import { StatusCodes, getReasonPhrase } from 'http-status-codes';
|
||||||
|
import ms from 'ms';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class DataEnhancerService {
|
export class DataEnhancerService {
|
||||||
@ -24,6 +25,7 @@ export class DataEnhancerService {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
const assetProfile = await dataEnhancer.enhance({
|
const assetProfile = await dataEnhancer.enhance({
|
||||||
|
requestTimeout: ms('30 seconds'),
|
||||||
response: {
|
response: {
|
||||||
assetClass: 'EQUITY',
|
assetClass: 'EQUITY',
|
||||||
assetSubClass: 'ETF'
|
assetSubClass: 'ETF'
|
||||||
|
@ -15,9 +15,11 @@ export class OpenFigiDataEnhancerService implements DataEnhancerInterface {
|
|||||||
) {}
|
) {}
|
||||||
|
|
||||||
public async enhance({
|
public async enhance({
|
||||||
|
requestTimeout = DEFAULT_REQUEST_TIMEOUT,
|
||||||
response,
|
response,
|
||||||
symbol
|
symbol
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
response: Partial<SymbolProfile>;
|
response: Partial<SymbolProfile>;
|
||||||
symbol: string;
|
symbol: string;
|
||||||
}): Promise<Partial<SymbolProfile>> {
|
}): Promise<Partial<SymbolProfile>> {
|
||||||
@ -45,7 +47,7 @@ export class OpenFigiDataEnhancerService implements DataEnhancerInterface {
|
|||||||
|
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
abortController.abort();
|
abortController.abort();
|
||||||
}, DEFAULT_REQUEST_TIMEOUT);
|
}, requestTimeout);
|
||||||
|
|
||||||
const mappings = await got
|
const mappings = await got
|
||||||
.post(`${OpenFigiDataEnhancerService.baseUrl}/v3/mapping`, {
|
.post(`${OpenFigiDataEnhancerService.baseUrl}/v3/mapping`, {
|
||||||
|
@ -21,9 +21,11 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
|
|||||||
};
|
};
|
||||||
|
|
||||||
public async enhance({
|
public async enhance({
|
||||||
|
requestTimeout = DEFAULT_REQUEST_TIMEOUT,
|
||||||
response,
|
response,
|
||||||
symbol
|
symbol
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
response: Partial<SymbolProfile>;
|
response: Partial<SymbolProfile>;
|
||||||
symbol: string;
|
symbol: string;
|
||||||
}): Promise<Partial<SymbolProfile>> {
|
}): Promise<Partial<SymbolProfile>> {
|
||||||
@ -37,7 +39,7 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
|
|||||||
|
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
abortController.abort();
|
abortController.abort();
|
||||||
}, DEFAULT_REQUEST_TIMEOUT);
|
}, requestTimeout);
|
||||||
|
|
||||||
const profile = await got(
|
const profile = await got(
|
||||||
`${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol}.json`,
|
`${TrackinsightDataEnhancerService.baseUrl}/funds/${symbol}.json`,
|
||||||
|
@ -1,6 +1,10 @@
|
|||||||
import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service';
|
import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service';
|
||||||
import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
|
import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
|
||||||
import { DEFAULT_CURRENCY, UNKNOWN_KEY } from '@ghostfolio/common/config';
|
import {
|
||||||
|
DEFAULT_CURRENCY,
|
||||||
|
DEFAULT_REQUEST_TIMEOUT,
|
||||||
|
UNKNOWN_KEY
|
||||||
|
} from '@ghostfolio/common/config';
|
||||||
import { isCurrency } from '@ghostfolio/common/helper';
|
import { isCurrency } from '@ghostfolio/common/helper';
|
||||||
import { Injectable, Logger } from '@nestjs/common';
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
import {
|
import {
|
||||||
@ -72,9 +76,11 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async enhance({
|
public async enhance({
|
||||||
|
requestTimeout = DEFAULT_REQUEST_TIMEOUT,
|
||||||
response,
|
response,
|
||||||
symbol
|
symbol
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
response: Partial<SymbolProfile>;
|
response: Partial<SymbolProfile>;
|
||||||
symbol: string;
|
symbol: string;
|
||||||
}): Promise<Partial<SymbolProfile>> {
|
}): Promise<Partial<SymbolProfile>> {
|
||||||
|
@ -17,6 +17,7 @@ import { Inject, Injectable, Logger } from '@nestjs/common';
|
|||||||
import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
|
import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
|
||||||
import { format, isValid } from 'date-fns';
|
import { format, isValid } from 'date-fns';
|
||||||
import { groupBy, isEmpty, isNumber } from 'lodash';
|
import { groupBy, isEmpty, isNumber } from 'lodash';
|
||||||
|
import ms from 'ms';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class DataProviderService {
|
export class DataProviderService {
|
||||||
@ -52,6 +53,7 @@ export class DataProviderService {
|
|||||||
symbol
|
symbol
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
requestTimeout: ms('30 seconds'),
|
||||||
useCache: false
|
useCache: false
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -236,9 +238,11 @@ export class DataProviderService {
|
|||||||
|
|
||||||
public async getQuotes({
|
public async getQuotes({
|
||||||
items,
|
items,
|
||||||
|
requestTimeout,
|
||||||
useCache = true
|
useCache = true
|
||||||
}: {
|
}: {
|
||||||
items: UniqueAsset[];
|
items: UniqueAsset[];
|
||||||
|
requestTimeout?: number;
|
||||||
useCache?: boolean;
|
useCache?: boolean;
|
||||||
}): Promise<{
|
}): Promise<{
|
||||||
[symbol: string]: IDataProviderResponse;
|
[symbol: string]: IDataProviderResponse;
|
||||||
@ -312,7 +316,7 @@ export class DataProviderService {
|
|||||||
);
|
);
|
||||||
|
|
||||||
const promise = Promise.resolve(
|
const promise = Promise.resolve(
|
||||||
dataProvider.getQuotes({ symbols: symbolsChunk })
|
dataProvider.getQuotes({ requestTimeout, symbols: symbolsChunk })
|
||||||
);
|
);
|
||||||
|
|
||||||
promises.push(
|
promises.push(
|
||||||
|
@ -132,8 +132,10 @@ export class EodHistoricalDataService implements DataProviderInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async getQuotes({
|
public async getQuotes({
|
||||||
|
requestTimeout = DEFAULT_REQUEST_TIMEOUT,
|
||||||
symbols
|
symbols
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
symbols: string[];
|
symbols: string[];
|
||||||
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
||||||
let response: { [symbol: string]: IDataProviderResponse } = {};
|
let response: { [symbol: string]: IDataProviderResponse } = {};
|
||||||
@ -151,7 +153,7 @@ export class EodHistoricalDataService implements DataProviderInterface {
|
|||||||
|
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
abortController.abort();
|
abortController.abort();
|
||||||
}, DEFAULT_REQUEST_TIMEOUT);
|
}, requestTimeout);
|
||||||
|
|
||||||
const realTimeResponse = await got(
|
const realTimeResponse = await got(
|
||||||
`${this.URL}/real-time/${eodHistoricalDataSymbols[0]}?api_token=${
|
`${this.URL}/real-time/${eodHistoricalDataSymbols[0]}?api_token=${
|
||||||
|
@ -114,8 +114,10 @@ export class FinancialModelingPrepService implements DataProviderInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async getQuotes({
|
public async getQuotes({
|
||||||
|
requestTimeout = DEFAULT_REQUEST_TIMEOUT,
|
||||||
symbols
|
symbols
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
symbols: string[];
|
symbols: string[];
|
||||||
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
||||||
const response: { [symbol: string]: IDataProviderResponse } = {};
|
const response: { [symbol: string]: IDataProviderResponse } = {};
|
||||||
@ -129,7 +131,7 @@ export class FinancialModelingPrepService implements DataProviderInterface {
|
|||||||
|
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
abortController.abort();
|
abortController.abort();
|
||||||
}, DEFAULT_REQUEST_TIMEOUT);
|
}, requestTimeout);
|
||||||
|
|
||||||
const response = await got(
|
const response = await got(
|
||||||
`${this.URL}/quote/${symbols.join(',')}?apikey=${this.apiKey}`,
|
`${this.URL}/quote/${symbols.join(',')}?apikey=${this.apiKey}`,
|
||||||
|
@ -7,6 +7,7 @@ import {
|
|||||||
} from '@ghostfolio/api/services/interfaces/interfaces';
|
} from '@ghostfolio/api/services/interfaces/interfaces';
|
||||||
import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
|
import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
|
||||||
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
|
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
|
||||||
|
import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
|
||||||
import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper';
|
import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper';
|
||||||
import { Granularity } from '@ghostfolio/common/types';
|
import { Granularity } from '@ghostfolio/common/types';
|
||||||
import { Injectable, Logger } from '@nestjs/common';
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
@ -100,8 +101,10 @@ export class GoogleSheetsService implements DataProviderInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async getQuotes({
|
public async getQuotes({
|
||||||
|
requestTimeout = DEFAULT_REQUEST_TIMEOUT,
|
||||||
symbols
|
symbols
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
symbols: string[];
|
symbols: string[];
|
||||||
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
||||||
const response: { [symbol: string]: IDataProviderResponse } = {};
|
const response: { [symbol: string]: IDataProviderResponse } = {};
|
||||||
|
@ -2,9 +2,11 @@ import { SymbolProfile } from '@prisma/client';
|
|||||||
|
|
||||||
export interface DataEnhancerInterface {
|
export interface DataEnhancerInterface {
|
||||||
enhance({
|
enhance({
|
||||||
|
requestTimeout,
|
||||||
response,
|
response,
|
||||||
symbol
|
symbol
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
response: Partial<SymbolProfile>;
|
response: Partial<SymbolProfile>;
|
||||||
symbol: string;
|
symbol: string;
|
||||||
}): Promise<Partial<SymbolProfile>>;
|
}): Promise<Partial<SymbolProfile>>;
|
||||||
|
@ -37,8 +37,10 @@ export interface DataProviderInterface {
|
|||||||
getName(): DataSource;
|
getName(): DataSource;
|
||||||
|
|
||||||
getQuotes({
|
getQuotes({
|
||||||
|
requestTimeout,
|
||||||
symbols
|
symbols
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
symbols: string[];
|
symbols: string[];
|
||||||
}): Promise<{ [symbol: string]: IDataProviderResponse }>;
|
}): Promise<{ [symbol: string]: IDataProviderResponse }>;
|
||||||
|
|
||||||
|
@ -134,8 +134,10 @@ export class ManualService implements DataProviderInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async getQuotes({
|
public async getQuotes({
|
||||||
|
requestTimeout = DEFAULT_REQUEST_TIMEOUT,
|
||||||
symbols
|
symbols
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
symbols: string[];
|
symbols: string[];
|
||||||
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
||||||
const response: { [symbol: string]: IDataProviderResponse } = {};
|
const response: { [symbol: string]: IDataProviderResponse } = {};
|
||||||
|
@ -88,8 +88,10 @@ export class RapidApiService implements DataProviderInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async getQuotes({
|
public async getQuotes({
|
||||||
|
requestTimeout = DEFAULT_REQUEST_TIMEOUT,
|
||||||
symbols
|
symbols
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
symbols: string[];
|
symbols: string[];
|
||||||
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
||||||
if (symbols.length <= 0) {
|
if (symbols.length <= 0) {
|
||||||
|
@ -6,7 +6,10 @@ import {
|
|||||||
IDataProviderHistoricalResponse,
|
IDataProviderHistoricalResponse,
|
||||||
IDataProviderResponse
|
IDataProviderResponse
|
||||||
} from '@ghostfolio/api/services/interfaces/interfaces';
|
} from '@ghostfolio/api/services/interfaces/interfaces';
|
||||||
import { DEFAULT_CURRENCY } from '@ghostfolio/common/config';
|
import {
|
||||||
|
DEFAULT_CURRENCY,
|
||||||
|
DEFAULT_REQUEST_TIMEOUT
|
||||||
|
} from '@ghostfolio/common/config';
|
||||||
import { DATE_FORMAT } from '@ghostfolio/common/helper';
|
import { DATE_FORMAT } from '@ghostfolio/common/helper';
|
||||||
import { Granularity } from '@ghostfolio/common/types';
|
import { Granularity } from '@ghostfolio/common/types';
|
||||||
import { Injectable, Logger } from '@nestjs/common';
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
@ -157,8 +160,10 @@ export class YahooFinanceService implements DataProviderInterface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async getQuotes({
|
public async getQuotes({
|
||||||
|
requestTimeout = DEFAULT_REQUEST_TIMEOUT,
|
||||||
symbols
|
symbols
|
||||||
}: {
|
}: {
|
||||||
|
requestTimeout?: number;
|
||||||
symbols: string[];
|
symbols: string[];
|
||||||
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
}): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
||||||
const response: { [symbol: string]: IDataProviderResponse } = {};
|
const response: { [symbol: string]: IDataProviderResponse } = {};
|
||||||
|
Reference in New Issue
Block a user