sudacode/ghostfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Feature/eliminate ghostfolio scraper api service (#1706)

Browse Source 
* Eliminate GhostfolioScraperApiService

* Update changelog
This commit is contained in:
Thomas Kaul 2023-02-16 16:25:23 +01:00 committed by GitHub
parent 0ec50819f5
commit 0cbf275a2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 10 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## Unreleased
### Changed
- Eliminated the `GhostfolioScraperApiService`
## 1.234.0 - 2023-02-15
### Added

2
apps/api/src/services/configuration.service.ts
View File

@ -19,7 +19,7 @@ export class ConfigurationService {
CACHE_TTL: num({ default: 1 }),
DATA_SOURCE_PRIMARY: str({ default: DataSource.YAHOO }),
DATA_SOURCES: json({
default: [DataSource.GHOSTFOLIO, DataSource.MANUAL, DataSource.YAHOO]
default: [DataSource.MANUAL, DataSource.YAHOO]
}),
ENABLE_FEATURE_BLOG: bool({ default: false }),
ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }),

1
apps/api/src/services/data-gathering.service.ts
View File

@ -278,7 +278,6 @@ export class DataGatheringService {
return symbolProfiles
.filter(({ dataSource }) => {
return (
dataSource !== DataSource.GHOSTFOLIO &&
dataSource !== DataSource.MANUAL &&
dataSource !== DataSource.RAPID_API
);

11
apps/api/src/services/data-provider/data-provider.module.ts
View File

@ -2,7 +2,6 @@ import { ConfigurationModule } from '@ghostfolio/api/services/configuration.modu
import { CryptocurrencyModule } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.module';
import { AlphaVantageService } from '@ghostfolio/api/services/data-provider/alpha-vantage/alpha-vantage.service';
import { EodHistoricalDataService } from '@ghostfolio/api/services/data-provider/eod-historical-data/eod-historical-data.service';
import { GhostfolioScraperApiService } from '@ghostfolio/api/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/google-sheets/google-sheets.service';
import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';
import { RapidApiService } from '@ghostfolio/api/services/data-provider/rapid-api/rapid-api.service';
@ -24,7 +23,6 @@ import { DataProviderService } from './data-provider.service';
AlphaVantageService,
DataProviderService,
EodHistoricalDataService,
GhostfolioScraperApiService,
GoogleSheetsService,
ManualService,
RapidApiService,
@ -33,7 +31,6 @@ import { DataProviderService } from './data-provider.service';
inject: [
AlphaVantageService,
EodHistoricalDataService,
GhostfolioScraperApiService,
GoogleSheetsService,
ManualService,
RapidApiService,
@ -43,7 +40,6 @@ import { DataProviderService } from './data-provider.service';
useFactory: (
alphaVantageService,
eodHistoricalDataService,
ghostfolioScraperApiService,
googleSheetsService,
manualService,
rapidApiService,
@ -51,7 +47,6 @@ import { DataProviderService } from './data-provider.service';
) => [
alphaVantageService,
eodHistoricalDataService,
ghostfolioScraperApiService,
googleSheetsService,
manualService,
rapidApiService,
@ -59,10 +54,6 @@ import { DataProviderService } from './data-provider.service';
]
}
],
exports: [
DataProviderService,
GhostfolioScraperApiService,
YahooFinanceService
]
exports: [DataProviderService, YahooFinanceService]
})
export class DataProviderModule {}

194
apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
View File

@ -1,194 +0,0 @@
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
import { DataProviderInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface';
import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
import { PrismaService } from '@ghostfolio/api/services/prisma.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.service';
import {
DATE_FORMAT,
extractNumberFromString,
getYesterday
} from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common';
import { DataSource, SymbolProfile } from '@prisma/client';
import bent from 'bent';
import * as cheerio from 'cheerio';
import { addDays, format, isBefore } from 'date-fns';
@Injectable()
export class GhostfolioScraperApiService implements DataProviderInterface {
public constructor(
private readonly prismaService: PrismaService,
private readonly symbolProfileService: SymbolProfileService
) {}
public canHandle(symbol: string) {
return true;
}
public async getAssetProfile(
aSymbol: string
): Promise<Partial<SymbolProfile>> {
return {
dataSource: this.getName()
};
}
public async getDividends({
from,
granularity = 'day',
symbol,
to
}: {
from: Date;
granularity: Granularity;
symbol: string;
to: Date;
}) {
return {};
}
public async getHistorical(
aSymbol: string,
aGranularity: Granularity = 'day',
from: Date,
to: Date
): Promise<{
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
}> {
try {
const symbol = aSymbol;
const [symbolProfile] =
await this.symbolProfileService.getSymbolProfilesBySymbols([symbol]);
const { defaultMarketPrice, selector, url } =
symbolProfile.scraperConfiguration ?? {};
if (defaultMarketPrice) {
const historical: {
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
} = {
[symbol]: {}
};
let date = from;
while (isBefore(date, to)) {
historical[symbol][format(date, DATE_FORMAT)] = {
marketPrice: defaultMarketPrice
};
date = addDays(date, 1);
}
return historical;
} else if (selector === undefined || url === undefined) {
return {};
}
const get = bent(url, 'GET', 'string', 200, {});
const html = await get();
const $ = cheerio.load(html);
const value = extractNumberFromString($(selector).text());
return {
[symbol]: {
[format(getYesterday(), DATE_FORMAT)]: {
marketPrice: value
}
}
};
} catch (error) {
throw new Error(
`Could not get historical market data for ${aSymbol} (${this.getName()}) from ${format(
from,
DATE_FORMAT
)} to ${format(to, DATE_FORMAT)}: [${error.name}] ${error.message}`
);
}
}
public getName(): DataSource {
return DataSource.GHOSTFOLIO;
}
public async getQuotes(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {
const response: { [symbol: string]: IDataProviderResponse } = {};
if (aSymbols.length <= 0) {
return response;
}
try {
const symbolProfiles =
await this.symbolProfileService.getSymbolProfilesBySymbols(aSymbols);
const marketData = await this.prismaService.marketData.findMany({
distinct: ['symbol'],
orderBy: {
date: 'desc'
},
take: aSymbols.length,
where: {
symbol: {
in: aSymbols
}
}
});
for (const symbolProfile of symbolProfiles) {
response[symbolProfile.symbol] = {
currency: symbolProfile.currency,
dataSource: this.getName(),
marketPrice: marketData.find((marketDataItem) => {
return marketDataItem.symbol === symbolProfile.symbol;
})?.marketPrice,
marketState: 'delayed'
};
}
return response;
} catch (error) {
Logger.error(error, 'GhostfolioScraperApiService');
}
return {};
}
public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
const items = await this.prismaService.symbolProfile.findMany({
select: {
currency: true,
dataSource: true,
name: true,
symbol: true
},
where: {
OR: [
{
dataSource: this.getName(),
name: {
mode: 'insensitive',
startsWith: aQuery
}
},
{
dataSource: this.getName(),
symbol: {
mode: 'insensitive',
startsWith: aQuery
}
}
]
}
});
return { items };
}
}

2
prisma/migrations/20230215174320_changed_data_source_from_ghostfolio_to_manual/migration.sql Normal file
View File

@ -0,0 +1,2 @@
UPDATE "MarketData" SET "dataSource" = 'MANUAL' WHERE "dataSource" = 'GHOSTFOLIO';
UPDATE "SymbolProfile" SET "dataSource" = 'MANUAL' WHERE "dataSource" = 'GHOSTFOLIO';