From f1e06347d348074a6307cf7d73dcadebfc270557 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Sun, 29 May 2022 15:37:40 +0200 Subject: [PATCH] Feature/add data source eod historical data (#974) * Add EOD Historical Data as a data source * Update changelog --- CHANGELOG.md | 8 + .../api/src/services/configuration.service.ts | 1 + .../data-provider/data-provider.module.ts | 7 +- .../eod-historical-data.service.ts | 138 ++++++++++++++++++ .../ghostfolio-scraper-api.service.ts | 2 +- .../rakuten-rapid-api.service.ts | 2 +- .../interfaces/environment.interface.ts | 1 + .../migration.sql | 2 + prisma/schema.prisma | 1 + 9 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts create mode 100644 prisma/migrations/20220529071429_added_eod_historical_data_to_data_source/migration.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index dca00fbc..040b771a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added `EOD_HISTORICAL_DATA` as a new data source type + ### Changed - Exposed the environment variable `REDIS_PASSWORD` +### Todo + +- Apply data migration (`yarn database:migrate`) + ## 1.154.0 - 28.05.2022 ### Added diff --git a/apps/api/src/services/configuration.service.ts b/apps/api/src/services/configuration.service.ts index 9666e7ab..525951a7 100644 --- a/apps/api/src/services/configuration.service.ts +++ b/apps/api/src/services/configuration.service.ts @@ -25,6 +25,7 @@ export class ConfigurationService { ENABLE_FEATURE_STATISTICS: bool({ default: false }), ENABLE_FEATURE_SUBSCRIPTION: bool({ default: false }), ENABLE_FEATURE_SYSTEM_MESSAGE: bool({ default: false }), + EOD_HISTORICAL_DATA_API_KEY: str({ default: '' }), GOOGLE_CLIENT_ID: str({ default: 'dummyClientId' }), GOOGLE_SECRET: str({ default: 'dummySecret' }), GOOGLE_SHEETS_ACCOUNT: str({ default: '' }), diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts index e2a77af4..dcdb7acb 100644 --- a/apps/api/src/services/data-provider/data-provider.module.ts +++ b/apps/api/src/services/data-provider/data-provider.module.ts @@ -1,5 +1,7 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration.module'; import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module'; +import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service'; +import { EodHistoricalDataService } from '@ghostfolio/api/services/data-provider/eod-historical-data/eod-historical-data.service'; import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service'; import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/google-sheets/google-sheets.service'; import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service'; @@ -9,7 +11,6 @@ import { PrismaModule } from '@ghostfolio/api/services/prisma.module'; import { SymbolProfileModule } from '@ghostfolio/api/services/symbol-profile.module'; import { Module } from '@nestjs/common'; -import { AlphaVantageService } from './alpha-vantage/alpha-vantage.service'; import { DataProviderService } from './data-provider.service'; @Module({ @@ -22,6 +23,7 @@ import { DataProviderService } from './data-provider.service'; providers: [ AlphaVantageService, DataProviderService, + EodHistoricalDataService, GhostfolioScraperApiService, GoogleSheetsService, ManualService, @@ -30,6 +32,7 @@ import { DataProviderService } from './data-provider.service'; { inject: [ AlphaVantageService, + EodHistoricalDataService, GhostfolioScraperApiService, GoogleSheetsService, ManualService, @@ -39,6 +42,7 @@ import { DataProviderService } from './data-provider.service'; provide: 'DataProviderInterfaces', useFactory: ( alphaVantageService, + eodHistoricalDataService, ghostfolioScraperApiService, googleSheetsService, manualService, @@ -46,6 +50,7 @@ import { DataProviderService } from './data-provider.service'; yahooFinanceService ) => [ alphaVantageService, + eodHistoricalDataService, ghostfolioScraperApiService, googleSheetsService, manualService, diff --git a/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts new file mode 100644 index 00000000..bb0401f0 --- /dev/null +++ b/apps/api/src/services/data-provider/eod-historical-data/eod-historical-data.service.ts @@ -0,0 +1,138 @@ +import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; +import { ConfigurationService } from '@ghostfolio/api/services/configuration.service'; +import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; +import { + IDataProviderHistoricalResponse, + IDataProviderResponse +} from '@ghostfolio/api/services/interfaces/interfaces'; +import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service'; +import { DATE_FORMAT } from '@ghostfolio/common/helper'; +import { Granularity } from '@ghostfolio/common/types'; +import { Injectable, Logger } from '@nestjs/common'; +import { DataSource, SymbolProfile } from '@prisma/client'; +import bent from 'bent'; +import { format } from 'date-fns'; + +@Injectable() +export class EodHistoricalDataService implements DataProviderInterface { + private apiKey: string; + private readonly URL = 'https://eodhistoricaldata.com/api'; + + public constructor( + private readonly configurationService: ConfigurationService, + private readonly symbolProfileService: SymbolProfileService + ) { + this.apiKey = this.configurationService.get('EOD_HISTORICAL_DATA_API_KEY'); + } + + public canHandle(symbol: string) { + return true; + } + + public async getAssetProfile( + aSymbol: string + ): Promise> { + return { + dataSource: this.getName() + }; + } + + public async getHistorical( + aSymbol: string, + aGranularity: Granularity = 'day', + from: Date, + to: Date + ): Promise<{ + [symbol: string]: { [date: string]: IDataProviderHistoricalResponse }; + }> { + try { + const get = bent( + `${this.URL}/eod/${aSymbol}?api_token=${ + this.apiKey + }&fmt=json&from=${format(from, DATE_FORMAT)}&to=${format( + to, + DATE_FORMAT + )}&period={aGranularity}`, + 'GET', + 'json', + 200 + ); + + const response = await get(); + + return response.reduce( + (result, historicalItem, index, array) => { + result[aSymbol][historicalItem.date] = { + marketPrice: historicalItem.close, + performance: historicalItem.open - historicalItem.close + }; + + return result; + }, + { [aSymbol]: {} } + ); + } catch (error) { + Logger.error(error, 'EodHistoricalDataService'); + } + + return {}; + } + + public getName(): DataSource { + return DataSource.EOD_HISTORICAL_DATA; + } + + public async getQuotes( + aSymbols: string[] + ): Promise<{ [symbol: string]: IDataProviderResponse }> { + if (aSymbols.length <= 0) { + return {}; + } + + try { + const get = bent( + `${this.URL}/real-time/${aSymbols[0]}?api_token=${ + this.apiKey + }&fmt=json&s=${aSymbols.join(',')}`, + 'GET', + 'json', + 200 + ); + + const [response, symbolProfiles] = await Promise.all([ + get(), + this.symbolProfileService.getSymbolProfiles( + aSymbols.map((symbol) => { + return { + symbol, + dataSource: DataSource.EOD_HISTORICAL_DATA + }; + }) + ) + ]); + + const quotes = aSymbols.length === 1 ? [response] : response; + + return quotes.reduce((result, item, index, array) => { + result[item.code] = { + currency: symbolProfiles.find((symbolProfile) => { + return symbolProfile.symbol === item.code; + })?.currency, + dataSource: DataSource.EOD_HISTORICAL_DATA, + marketPrice: item.close, + marketState: 'delayed' + }; + + return result; + }, {}); + } catch (error) { + Logger.error(error, 'EodHistoricalDataService'); + } + + return {}; + } + + public async search(aQuery: string): Promise<{ items: LookupItem[] }> { + return { items: [] }; + } +} diff --git a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts index 26437bcd..7186ea7e 100644 --- a/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts +++ b/apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts @@ -10,7 +10,7 @@ import { DATE_FORMAT, getYesterday } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; -import * as bent from 'bent'; +import bent from 'bent'; import * as cheerio from 'cheerio'; import { addDays, format, isBefore } from 'date-fns'; diff --git a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts index baa6591f..2a516c5e 100644 --- a/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts +++ b/apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts @@ -11,7 +11,7 @@ import { DATE_FORMAT, getToday, getYesterday } from '@ghostfolio/common/helper'; import { Granularity } from '@ghostfolio/common/types'; import { Injectable, Logger } from '@nestjs/common'; import { DataSource, SymbolProfile } from '@prisma/client'; -import * as bent from 'bent'; +import bent from 'bent'; import { format, subMonths, subWeeks, subYears } from 'date-fns'; @Injectable() diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts index 79db93f5..36e9c726 100644 --- a/apps/api/src/services/interfaces/environment.interface.ts +++ b/apps/api/src/services/interfaces/environment.interface.ts @@ -16,6 +16,7 @@ export interface Environment extends CleanedEnvAccessors { ENABLE_FEATURE_STATISTICS: boolean; ENABLE_FEATURE_SUBSCRIPTION: boolean; ENABLE_FEATURE_SYSTEM_MESSAGE: boolean; + EOD_HISTORICAL_DATA_API_KEY: string; GOOGLE_CLIENT_ID: string; GOOGLE_SECRET: string; GOOGLE_SHEETS_ACCOUNT: string; diff --git a/prisma/migrations/20220529071429_added_eod_historical_data_to_data_source/migration.sql b/prisma/migrations/20220529071429_added_eod_historical_data_to_data_source/migration.sql new file mode 100644 index 00000000..4add2802 --- /dev/null +++ b/prisma/migrations/20220529071429_added_eod_historical_data_to_data_source/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "DataSource" ADD VALUE 'EOD_HISTORICAL_DATA'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c5a03dc9..16249d51 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -201,6 +201,7 @@ enum AssetSubClass { enum DataSource { ALPHA_VANTAGE + EOD_HISTORICAL_DATA GHOSTFOLIO GOOGLE_SHEETS MANUAL