Feature/implement scraper (#25)
* Clean up imports * Implement scraper * Sort imports
This commit is contained in:
parent
0f2c8c856c
commit
a2687eacbc
@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## Unreleased
|
## Unreleased
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- Added a generic scraper
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
- Fixed an issue in the user table of the admin control panel with missing data
|
- Fixed an issue in the user table of the admin control panel with missing data
|
||||||
|
@ -4,6 +4,7 @@ import { ConfigurationService } from '../../services/configuration.service';
|
|||||||
import { DataGatheringService } from '../../services/data-gathering.service';
|
import { DataGatheringService } from '../../services/data-gathering.service';
|
||||||
import { DataProviderService } from '../../services/data-provider.service';
|
import { DataProviderService } from '../../services/data-provider.service';
|
||||||
import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
|
import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
|
||||||
|
import { GhostfolioScraperApiService } from '../../services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
|
||||||
import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
||||||
import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
|
import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
|
||||||
import { ExchangeRateDataService } from '../../services/exchange-rate-data.service';
|
import { ExchangeRateDataService } from '../../services/exchange-rate-data.service';
|
||||||
@ -21,6 +22,7 @@ import { AdminService } from './admin.service';
|
|||||||
DataGatheringService,
|
DataGatheringService,
|
||||||
DataProviderService,
|
DataProviderService,
|
||||||
ExchangeRateDataService,
|
ExchangeRateDataService,
|
||||||
|
GhostfolioScraperApiService,
|
||||||
PrismaService,
|
PrismaService,
|
||||||
RakutenRapidApiService,
|
RakutenRapidApiService,
|
||||||
YahooFinanceService
|
YahooFinanceService
|
||||||
|
@ -10,6 +10,7 @@ import { CronService } from '../services/cron.service';
|
|||||||
import { DataGatheringService } from '../services/data-gathering.service';
|
import { DataGatheringService } from '../services/data-gathering.service';
|
||||||
import { DataProviderService } from '../services/data-provider.service';
|
import { DataProviderService } from '../services/data-provider.service';
|
||||||
import { AlphaVantageService } from '../services/data-provider/alpha-vantage/alpha-vantage.service';
|
import { AlphaVantageService } from '../services/data-provider/alpha-vantage/alpha-vantage.service';
|
||||||
|
import { GhostfolioScraperApiService } from '../services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
|
||||||
import { RakutenRapidApiService } from '../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
import { RakutenRapidApiService } from '../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
||||||
import { YahooFinanceService } from '../services/data-provider/yahoo-finance/yahoo-finance.service';
|
import { YahooFinanceService } from '../services/data-provider/yahoo-finance/yahoo-finance.service';
|
||||||
import { ExchangeRateDataService } from '../services/exchange-rate-data.service';
|
import { ExchangeRateDataService } from '../services/exchange-rate-data.service';
|
||||||
@ -65,6 +66,7 @@ import { UserModule } from './user/user.module';
|
|||||||
DataGatheringService,
|
DataGatheringService,
|
||||||
DataProviderService,
|
DataProviderService,
|
||||||
ExchangeRateDataService,
|
ExchangeRateDataService,
|
||||||
|
GhostfolioScraperApiService,
|
||||||
PrismaService,
|
PrismaService,
|
||||||
RakutenRapidApiService,
|
RakutenRapidApiService,
|
||||||
YahooFinanceService
|
YahooFinanceService
|
||||||
|
@ -3,6 +3,7 @@ import { Module } from '@nestjs/common';
|
|||||||
import { ConfigurationService } from '../../services/configuration.service';
|
import { ConfigurationService } from '../../services/configuration.service';
|
||||||
import { DataProviderService } from '../../services/data-provider.service';
|
import { DataProviderService } from '../../services/data-provider.service';
|
||||||
import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
|
import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
|
||||||
|
import { GhostfolioScraperApiService } from '../../services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
|
||||||
import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
||||||
import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
|
import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
|
||||||
import { ExchangeRateDataService } from '../../services/exchange-rate-data.service';
|
import { ExchangeRateDataService } from '../../services/exchange-rate-data.service';
|
||||||
@ -20,6 +21,7 @@ import { ExperimentalService } from './experimental.service';
|
|||||||
DataProviderService,
|
DataProviderService,
|
||||||
ExchangeRateDataService,
|
ExchangeRateDataService,
|
||||||
ExperimentalService,
|
ExperimentalService,
|
||||||
|
GhostfolioScraperApiService,
|
||||||
PrismaService,
|
PrismaService,
|
||||||
RakutenRapidApiService,
|
RakutenRapidApiService,
|
||||||
RulesService,
|
RulesService,
|
||||||
|
@ -4,6 +4,7 @@ import { ConfigurationService } from '../../services/configuration.service';
|
|||||||
import { DataGatheringService } from '../../services/data-gathering.service';
|
import { DataGatheringService } from '../../services/data-gathering.service';
|
||||||
import { DataProviderService } from '../../services/data-provider.service';
|
import { DataProviderService } from '../../services/data-provider.service';
|
||||||
import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
|
import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
|
||||||
|
import { GhostfolioScraperApiService } from '../../services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
|
||||||
import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
||||||
import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
|
import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
|
||||||
import { ImpersonationService } from '../../services/impersonation.service';
|
import { ImpersonationService } from '../../services/impersonation.service';
|
||||||
@ -22,6 +23,7 @@ import { OrderService } from './order.service';
|
|||||||
ConfigurationService,
|
ConfigurationService,
|
||||||
DataGatheringService,
|
DataGatheringService,
|
||||||
DataProviderService,
|
DataProviderService,
|
||||||
|
GhostfolioScraperApiService,
|
||||||
ImpersonationService,
|
ImpersonationService,
|
||||||
OrderService,
|
OrderService,
|
||||||
PrismaService,
|
PrismaService,
|
||||||
|
@ -4,6 +4,7 @@ import { ConfigurationService } from '../../services/configuration.service';
|
|||||||
import { DataGatheringService } from '../../services/data-gathering.service';
|
import { DataGatheringService } from '../../services/data-gathering.service';
|
||||||
import { DataProviderService } from '../../services/data-provider.service';
|
import { DataProviderService } from '../../services/data-provider.service';
|
||||||
import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
|
import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
|
||||||
|
import { GhostfolioScraperApiService } from '../../services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
|
||||||
import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
||||||
import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
|
import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
|
||||||
import { ExchangeRateDataService } from '../../services/exchange-rate-data.service';
|
import { ExchangeRateDataService } from '../../services/exchange-rate-data.service';
|
||||||
@ -27,6 +28,7 @@ import { PortfolioService } from './portfolio.service';
|
|||||||
DataGatheringService,
|
DataGatheringService,
|
||||||
DataProviderService,
|
DataProviderService,
|
||||||
ExchangeRateDataService,
|
ExchangeRateDataService,
|
||||||
|
GhostfolioScraperApiService,
|
||||||
ImpersonationService,
|
ImpersonationService,
|
||||||
OrderService,
|
OrderService,
|
||||||
PortfolioService,
|
PortfolioService,
|
||||||
|
@ -3,6 +3,7 @@ import { Module } from '@nestjs/common';
|
|||||||
import { ConfigurationService } from '../../services/configuration.service';
|
import { ConfigurationService } from '../../services/configuration.service';
|
||||||
import { DataProviderService } from '../../services/data-provider.service';
|
import { DataProviderService } from '../../services/data-provider.service';
|
||||||
import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
|
import { AlphaVantageService } from '../../services/data-provider/alpha-vantage/alpha-vantage.service';
|
||||||
|
import { GhostfolioScraperApiService } from '../../services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
|
||||||
import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
import { RakutenRapidApiService } from '../../services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
||||||
import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
|
import { YahooFinanceService } from '../../services/data-provider/yahoo-finance/yahoo-finance.service';
|
||||||
import { PrismaService } from '../../services/prisma.service';
|
import { PrismaService } from '../../services/prisma.service';
|
||||||
@ -16,6 +17,7 @@ import { SymbolService } from './symbol.service';
|
|||||||
AlphaVantageService,
|
AlphaVantageService,
|
||||||
ConfigurationService,
|
ConfigurationService,
|
||||||
DataProviderService,
|
DataProviderService,
|
||||||
|
GhostfolioScraperApiService,
|
||||||
PrismaService,
|
PrismaService,
|
||||||
RakutenRapidApiService,
|
RakutenRapidApiService,
|
||||||
SymbolService,
|
SymbolService,
|
||||||
|
@ -12,6 +12,7 @@ export class ConfigurationService {
|
|||||||
ACCESS_TOKEN_SALT: str(),
|
ACCESS_TOKEN_SALT: str(),
|
||||||
ALPHA_VANTAGE_API_KEY: str({ default: '' }),
|
ALPHA_VANTAGE_API_KEY: str({ default: '' }),
|
||||||
CACHE_TTL: num({ default: 1 }),
|
CACHE_TTL: num({ default: 1 }),
|
||||||
|
ENABLE_FEATURE_CUSTOM_SYMBOLS: bool({ default: false }),
|
||||||
ENABLE_FEATURE_FEAR_AND_GREED_INDEX: bool({ default: false }),
|
ENABLE_FEATURE_FEAR_AND_GREED_INDEX: bool({ default: false }),
|
||||||
ENABLE_FEATURE_SOCIAL_LOGIN: bool({ default: false }),
|
ENABLE_FEATURE_SOCIAL_LOGIN: bool({ default: false }),
|
||||||
GOOGLE_CLIENT_ID: str({ default: 'dummyClientId' }),
|
GOOGLE_CLIENT_ID: str({ default: 'dummyClientId' }),
|
||||||
|
@ -200,10 +200,36 @@ export class DataGatheringService {
|
|||||||
return benchmarksToGather;
|
return benchmarksToGather;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async getCustomSymbolsToGather(startDate: Date) {
|
||||||
|
const customSymbolsToGather = [];
|
||||||
|
|
||||||
|
if (this.configurationService.get('ENABLE_FEATURE_CUSTOM_SYMBOLS')) {
|
||||||
|
try {
|
||||||
|
const {
|
||||||
|
value: scraperConfigString
|
||||||
|
} = await this.prisma.property.findFirst({
|
||||||
|
select: {
|
||||||
|
value: true
|
||||||
|
},
|
||||||
|
where: { key: 'SCRAPER_CONFIG' }
|
||||||
|
});
|
||||||
|
|
||||||
|
JSON.parse(scraperConfigString).forEach((item) => {
|
||||||
|
customSymbolsToGather.push({
|
||||||
|
date: startDate,
|
||||||
|
symbol: item.symbol
|
||||||
|
});
|
||||||
|
});
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
|
||||||
|
return customSymbolsToGather;
|
||||||
|
}
|
||||||
|
|
||||||
private async getSymbols7D(): Promise<{ date: Date; symbol: string }[]> {
|
private async getSymbols7D(): Promise<{ date: Date; symbol: string }[]> {
|
||||||
const startDate = subDays(resetHours(new Date()), 7);
|
const startDate = subDays(resetHours(new Date()), 7);
|
||||||
|
|
||||||
let distinctOrders = await this.prisma.order.findMany({
|
const distinctOrders = await this.prisma.order.findMany({
|
||||||
distinct: ['symbol'],
|
distinct: ['symbol'],
|
||||||
orderBy: [{ symbol: 'asc' }],
|
orderBy: [{ symbol: 'asc' }],
|
||||||
select: { symbol: true }
|
select: { symbol: true }
|
||||||
@ -223,8 +249,13 @@ export class DataGatheringService {
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const customSymbolsToGather = await this.getCustomSymbolsToGather(
|
||||||
|
startDate
|
||||||
|
);
|
||||||
|
|
||||||
return [
|
return [
|
||||||
...this.getBenchmarksToGather(startDate),
|
...this.getBenchmarksToGather(startDate),
|
||||||
|
...customSymbolsToGather,
|
||||||
...currencyPairsToGather,
|
...currencyPairsToGather,
|
||||||
...distinctOrdersWithDate
|
...distinctOrdersWithDate
|
||||||
];
|
];
|
||||||
@ -233,11 +264,9 @@ export class DataGatheringService {
|
|||||||
private async getSymbolsMax() {
|
private async getSymbolsMax() {
|
||||||
const startDate = new Date(getUtc('2000-01-01'));
|
const startDate = new Date(getUtc('2000-01-01'));
|
||||||
|
|
||||||
let distinctOrders = await this.prisma.order.findMany({
|
const customSymbolsToGather = await this.getCustomSymbolsToGather(
|
||||||
distinct: ['symbol'],
|
startDate
|
||||||
orderBy: [{ date: 'asc' }],
|
);
|
||||||
select: { date: true, symbol: true }
|
|
||||||
});
|
|
||||||
|
|
||||||
const currencyPairsToGather = currencyPairs.map((symbol) => {
|
const currencyPairsToGather = currencyPairs.map((symbol) => {
|
||||||
return {
|
return {
|
||||||
@ -246,8 +275,15 @@ export class DataGatheringService {
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const distinctOrders = await this.prisma.order.findMany({
|
||||||
|
distinct: ['symbol'],
|
||||||
|
orderBy: [{ date: 'asc' }],
|
||||||
|
select: { date: true, symbol: true }
|
||||||
|
});
|
||||||
|
|
||||||
return [
|
return [
|
||||||
...this.getBenchmarksToGather(startDate),
|
...this.getBenchmarksToGather(startDate),
|
||||||
|
...customSymbolsToGather,
|
||||||
...currencyPairsToGather,
|
...currencyPairsToGather,
|
||||||
...distinctOrders
|
...distinctOrders
|
||||||
];
|
];
|
||||||
|
@ -1,10 +1,15 @@
|
|||||||
import { isCrypto, isRakutenRapidApi } from '@ghostfolio/helper';
|
import {
|
||||||
|
isCrypto,
|
||||||
|
isGhostfolioScraperApi,
|
||||||
|
isRakutenRapidApi
|
||||||
|
} from '@ghostfolio/helper';
|
||||||
import { Injectable } from '@nestjs/common';
|
import { Injectable } from '@nestjs/common';
|
||||||
import { MarketData } from '@prisma/client';
|
import { MarketData } from '@prisma/client';
|
||||||
import { format } from 'date-fns';
|
import { format } from 'date-fns';
|
||||||
|
|
||||||
import { ConfigurationService } from './configuration.service';
|
import { ConfigurationService } from './configuration.service';
|
||||||
import { AlphaVantageService } from './data-provider/alpha-vantage/alpha-vantage.service';
|
import { AlphaVantageService } from './data-provider/alpha-vantage/alpha-vantage.service';
|
||||||
|
import { GhostfolioScraperApiService } from './data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service';
|
||||||
import { RakutenRapidApiService } from './data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
import { RakutenRapidApiService } from './data-provider/rakuten-rapid-api/rakuten-rapid-api.service';
|
||||||
import { YahooFinanceService } from './data-provider/yahoo-finance/yahoo-finance.service';
|
import { YahooFinanceService } from './data-provider/yahoo-finance/yahoo-finance.service';
|
||||||
import { DataProviderInterface } from './interfaces/data-provider.interface';
|
import { DataProviderInterface } from './interfaces/data-provider.interface';
|
||||||
@ -20,6 +25,7 @@ export class DataProviderService implements DataProviderInterface {
|
|||||||
public constructor(
|
public constructor(
|
||||||
private readonly alphaVantageService: AlphaVantageService,
|
private readonly alphaVantageService: AlphaVantageService,
|
||||||
private readonly configurationService: ConfigurationService,
|
private readonly configurationService: ConfigurationService,
|
||||||
|
private readonly ghostfolioScraperApiService: GhostfolioScraperApiService,
|
||||||
private prisma: PrismaService,
|
private prisma: PrismaService,
|
||||||
private readonly rakutenRapidApiService: RakutenRapidApiService,
|
private readonly rakutenRapidApiService: RakutenRapidApiService,
|
||||||
private readonly yahooFinanceService: YahooFinanceService
|
private readonly yahooFinanceService: YahooFinanceService
|
||||||
@ -33,7 +39,9 @@ export class DataProviderService implements DataProviderInterface {
|
|||||||
if (aSymbols.length === 1) {
|
if (aSymbols.length === 1) {
|
||||||
const symbol = aSymbols[0];
|
const symbol = aSymbols[0];
|
||||||
|
|
||||||
if (isRakutenRapidApi(symbol)) {
|
if (isGhostfolioScraperApi(symbol)) {
|
||||||
|
return this.ghostfolioScraperApiService.get(aSymbols);
|
||||||
|
} else if (isRakutenRapidApi(symbol)) {
|
||||||
return this.rakutenRapidApiService.get(aSymbols);
|
return this.rakutenRapidApiService.get(aSymbols);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -53,12 +61,12 @@ export class DataProviderService implements DataProviderInterface {
|
|||||||
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
|
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
|
||||||
} = {};
|
} = {};
|
||||||
|
|
||||||
let granularityQuery =
|
const granularityQuery =
|
||||||
aGranularity === 'month'
|
aGranularity === 'month'
|
||||||
? `AND (date_part('day', date) = 1 OR date >= TIMESTAMP 'yesterday')`
|
? `AND (date_part('day', date) = 1 OR date >= TIMESTAMP 'yesterday')`
|
||||||
: '';
|
: '';
|
||||||
|
|
||||||
let rangeQuery =
|
const rangeQuery =
|
||||||
from && to
|
from && to
|
||||||
? `AND date >= '${format(from, 'yyyy-MM-dd')}' AND date <= '${format(
|
? `AND date >= '${format(from, 'yyyy-MM-dd')}' AND date <= '${format(
|
||||||
to,
|
to,
|
||||||
@ -127,6 +135,15 @@ export class DataProviderService implements DataProviderInterface {
|
|||||||
...dataOfAlphaVantage[symbol]
|
...dataOfAlphaVantage[symbol]
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
} else if (isGhostfolioScraperApi(symbol)) {
|
||||||
|
const dataOfGhostfolioScraperApi = await this.ghostfolioScraperApiService.getHistorical(
|
||||||
|
[symbol],
|
||||||
|
undefined,
|
||||||
|
from,
|
||||||
|
to
|
||||||
|
);
|
||||||
|
|
||||||
|
return dataOfGhostfolioScraperApi;
|
||||||
} else if (
|
} else if (
|
||||||
isRakutenRapidApi(symbol) &&
|
isRakutenRapidApi(symbol) &&
|
||||||
this.configurationService.get('RAKUTEN_RAPID_API_KEY')
|
this.configurationService.get('RAKUTEN_RAPID_API_KEY')
|
||||||
|
@ -0,0 +1,103 @@
|
|||||||
|
import { getYesterday } from '@ghostfolio/helper';
|
||||||
|
import { Injectable } from '@nestjs/common';
|
||||||
|
import * as bent from 'bent';
|
||||||
|
import * as cheerio from 'cheerio';
|
||||||
|
import { format } from 'date-fns';
|
||||||
|
|
||||||
|
import { DataProviderInterface } from '../../interfaces/data-provider.interface';
|
||||||
|
import { Granularity } from '../../interfaces/granularity.type';
|
||||||
|
import {
|
||||||
|
IDataProviderHistoricalResponse,
|
||||||
|
IDataProviderResponse
|
||||||
|
} from '../../interfaces/interfaces';
|
||||||
|
import { PrismaService } from '../../prisma.service';
|
||||||
|
import { Currency } from '.prisma/client';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class GhostfolioScraperApiService implements DataProviderInterface {
|
||||||
|
public constructor(private prisma: PrismaService) {}
|
||||||
|
|
||||||
|
public async get(
|
||||||
|
aSymbols: string[]
|
||||||
|
): Promise<{ [symbol: string]: IDataProviderResponse }> {
|
||||||
|
if (aSymbols.length <= 0) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const symbol = aSymbols[0];
|
||||||
|
const { marketPrice } = await this.prisma.marketData.findFirst({
|
||||||
|
orderBy: {
|
||||||
|
date: 'desc'
|
||||||
|
},
|
||||||
|
where: {
|
||||||
|
symbol
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
[symbol]: {
|
||||||
|
marketPrice,
|
||||||
|
currency: Currency.CHF,
|
||||||
|
isMarketOpen: true,
|
||||||
|
name: symbol
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getHistorical(
|
||||||
|
aSymbols: string[],
|
||||||
|
aGranularity: Granularity = 'day',
|
||||||
|
from: Date,
|
||||||
|
to: Date
|
||||||
|
): Promise<{
|
||||||
|
[symbol: string]: { [date: string]: IDataProviderHistoricalResponse };
|
||||||
|
}> {
|
||||||
|
if (aSymbols.length <= 0) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const symbol = aSymbols[0];
|
||||||
|
|
||||||
|
const {
|
||||||
|
value: scraperConfigString
|
||||||
|
} = await this.prisma.property.findFirst({
|
||||||
|
select: {
|
||||||
|
value: true
|
||||||
|
},
|
||||||
|
where: { key: 'SCRAPER_CONFIG' }
|
||||||
|
});
|
||||||
|
|
||||||
|
const scraperConfig = JSON.parse(scraperConfigString).find((item) => {
|
||||||
|
return item.symbol === symbol;
|
||||||
|
});
|
||||||
|
|
||||||
|
const get = bent(scraperConfig.url, 'GET', 'string', 200, {});
|
||||||
|
|
||||||
|
const html = await get();
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
|
||||||
|
const string = $(scraperConfig.selector).text().replace('CHF', '').trim();
|
||||||
|
|
||||||
|
const value = parseFloat(string);
|
||||||
|
|
||||||
|
return {
|
||||||
|
[symbol]: {
|
||||||
|
[format(getYesterday(), 'yyyy-MM-dd')]: {
|
||||||
|
marketPrice: value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
@ -4,6 +4,7 @@ export interface Environment extends CleanedEnvAccessors {
|
|||||||
ACCESS_TOKEN_SALT: string;
|
ACCESS_TOKEN_SALT: string;
|
||||||
ALPHA_VANTAGE_API_KEY: string;
|
ALPHA_VANTAGE_API_KEY: string;
|
||||||
CACHE_TTL: number;
|
CACHE_TTL: number;
|
||||||
|
ENABLE_FEATURE_CUSTOM_SYMBOLS: boolean;
|
||||||
ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean;
|
ENABLE_FEATURE_FEAR_AND_GREED_INDEX: boolean;
|
||||||
ENABLE_FEATURE_SOCIAL_LOGIN: boolean;
|
ENABLE_FEATURE_SOCIAL_LOGIN: boolean;
|
||||||
GOOGLE_CLIENT_ID: string;
|
GOOGLE_CLIENT_ID: string;
|
||||||
|
@ -66,6 +66,10 @@ export function isCurrency(aSymbol = '') {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isGhostfolioScraperApi(aSymbol = '') {
|
||||||
|
return aSymbol.startsWith('[GF]');
|
||||||
|
}
|
||||||
|
|
||||||
export function isRakutenRapidApi(aSymbol = '') {
|
export function isRakutenRapidApi(aSymbol = '') {
|
||||||
return aSymbol === 'GF.FEAR_AND_GREED_INDEX';
|
return aSymbol === 'GF.FEAR_AND_GREED_INDEX';
|
||||||
}
|
}
|
||||||
|
@ -71,6 +71,7 @@
|
|||||||
"chart.js": "3.0.2",
|
"chart.js": "3.0.2",
|
||||||
"chartjs-adapter-date-fns": "1.1.0-beta.1",
|
"chartjs-adapter-date-fns": "1.1.0-beta.1",
|
||||||
"chartjs-chart-timeline": "0.4.0",
|
"chartjs-chart-timeline": "0.4.0",
|
||||||
|
"cheerio": "1.0.0-rc.6",
|
||||||
"class-transformer": "0.3.2",
|
"class-transformer": "0.3.2",
|
||||||
"class-validator": "0.13.1",
|
"class-validator": "0.13.1",
|
||||||
"countup.js": "2.0.7",
|
"countup.js": "2.0.7",
|
||||||
|
83
yarn.lock
83
yarn.lock
@ -4042,6 +4042,29 @@ check-more-types@^2.24.0:
|
|||||||
resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600"
|
resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600"
|
||||||
integrity sha1-FCD/sQ/URNz8ebQ4kbv//TKoRgA=
|
integrity sha1-FCD/sQ/URNz8ebQ4kbv//TKoRgA=
|
||||||
|
|
||||||
|
cheerio-select@^1.3.0:
|
||||||
|
version "1.4.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/cheerio-select/-/cheerio-select-1.4.0.tgz#3a16f21e37a2ef0f211d6d1aa4eff054bb22cdc9"
|
||||||
|
integrity sha512-sobR3Yqz27L553Qa7cK6rtJlMDbiKPdNywtR95Sj/YgfpLfy0u6CGJuaBKe5YE/vTc23SCRKxWSdlon/w6I/Ew==
|
||||||
|
dependencies:
|
||||||
|
css-select "^4.1.2"
|
||||||
|
css-what "^5.0.0"
|
||||||
|
domelementtype "^2.2.0"
|
||||||
|
domhandler "^4.2.0"
|
||||||
|
domutils "^2.6.0"
|
||||||
|
|
||||||
|
cheerio@1.0.0-rc.6:
|
||||||
|
version "1.0.0-rc.6"
|
||||||
|
resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.6.tgz#a5ae81ab483aeefa1280c325543c601145506240"
|
||||||
|
integrity sha512-hjx1XE1M/D5pAtMgvWwE21QClmAEeGHOIDfycgmndisdNgI6PE1cGRQkMGBcsbUbmEQyWu5PJLUcAOjtQS8DWw==
|
||||||
|
dependencies:
|
||||||
|
cheerio-select "^1.3.0"
|
||||||
|
dom-serializer "^1.3.1"
|
||||||
|
domhandler "^4.1.0"
|
||||||
|
htmlparser2 "^6.1.0"
|
||||||
|
parse5 "^6.0.1"
|
||||||
|
parse5-htmlparser2-tree-adapter "^6.0.1"
|
||||||
|
|
||||||
"chokidar@>=2.0.0 <4.0.0", chokidar@^3.0.0, chokidar@^3.3.0, chokidar@^3.4.1:
|
"chokidar@>=2.0.0 <4.0.0", chokidar@^3.0.0, chokidar@^3.3.0, chokidar@^3.4.1:
|
||||||
version "3.5.1"
|
version "3.5.1"
|
||||||
resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.1.tgz#ee9ce7bbebd2b79f49f304799d5468e31e14e68a"
|
resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.1.tgz#ee9ce7bbebd2b79f49f304799d5468e31e14e68a"
|
||||||
@ -4787,6 +4810,17 @@ css-select@^2.0.0:
|
|||||||
domutils "^1.7.0"
|
domutils "^1.7.0"
|
||||||
nth-check "^1.0.2"
|
nth-check "^1.0.2"
|
||||||
|
|
||||||
|
css-select@^4.1.2:
|
||||||
|
version "4.1.2"
|
||||||
|
resolved "https://registry.yarnpkg.com/css-select/-/css-select-4.1.2.tgz#8b52b6714ed3a80d8221ec971c543f3b12653286"
|
||||||
|
integrity sha512-nu5ye2Hg/4ISq4XqdLY2bEatAcLIdt3OYGFc9Tm9n7VSlFBcfRv0gBNksHRgSdUDQGtN3XrZ94ztW+NfzkFSUw==
|
||||||
|
dependencies:
|
||||||
|
boolbase "^1.0.0"
|
||||||
|
css-what "^5.0.0"
|
||||||
|
domhandler "^4.2.0"
|
||||||
|
domutils "^2.6.0"
|
||||||
|
nth-check "^2.0.0"
|
||||||
|
|
||||||
css-selector-tokenizer@^0.7.1:
|
css-selector-tokenizer@^0.7.1:
|
||||||
version "0.7.3"
|
version "0.7.3"
|
||||||
resolved "https://registry.yarnpkg.com/css-selector-tokenizer/-/css-selector-tokenizer-0.7.3.tgz#735f26186e67c749aaf275783405cf0661fae8f1"
|
resolved "https://registry.yarnpkg.com/css-selector-tokenizer/-/css-selector-tokenizer-0.7.3.tgz#735f26186e67c749aaf275783405cf0661fae8f1"
|
||||||
@ -4816,6 +4850,11 @@ css-what@^3.2.1:
|
|||||||
resolved "https://registry.yarnpkg.com/css-what/-/css-what-3.4.2.tgz#ea7026fcb01777edbde52124e21f327e7ae950e4"
|
resolved "https://registry.yarnpkg.com/css-what/-/css-what-3.4.2.tgz#ea7026fcb01777edbde52124e21f327e7ae950e4"
|
||||||
integrity sha512-ACUm3L0/jiZTqfzRM3Hi9Q8eZqd6IK37mMWPLz9PJxkLWllYeRf+EHUSHYEtFop2Eqytaq1FizFVh7XfBnXCDQ==
|
integrity sha512-ACUm3L0/jiZTqfzRM3Hi9Q8eZqd6IK37mMWPLz9PJxkLWllYeRf+EHUSHYEtFop2Eqytaq1FizFVh7XfBnXCDQ==
|
||||||
|
|
||||||
|
css-what@^5.0.0:
|
||||||
|
version "5.0.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/css-what/-/css-what-5.0.0.tgz#f0bf4f8bac07582722346ab243f6a35b512cfc47"
|
||||||
|
integrity sha512-qxyKHQvgKwzwDWC/rGbT821eJalfupxYW2qbSJSAtdSTimsr/MlaGONoNLllaUPZWf8QnbcKM/kPVYUQuEKAFA==
|
||||||
|
|
||||||
css@^2.0.0:
|
css@^2.0.0:
|
||||||
version "2.2.4"
|
version "2.2.4"
|
||||||
resolved "https://registry.yarnpkg.com/css/-/css-2.2.4.tgz#c646755c73971f2bba6a601e2cf2fd71b1298929"
|
resolved "https://registry.yarnpkg.com/css/-/css-2.2.4.tgz#c646755c73971f2bba6a601e2cf2fd71b1298929"
|
||||||
@ -5298,6 +5337,15 @@ dom-serializer@0:
|
|||||||
domelementtype "^2.0.1"
|
domelementtype "^2.0.1"
|
||||||
entities "^2.0.0"
|
entities "^2.0.0"
|
||||||
|
|
||||||
|
dom-serializer@^1.0.1, dom-serializer@^1.3.1:
|
||||||
|
version "1.3.1"
|
||||||
|
resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-1.3.1.tgz#d845a1565d7c041a95e5dab62184ab41e3a519be"
|
||||||
|
integrity sha512-Pv2ZluG5ife96udGgEDovOOOA5UELkltfJpnIExPrAk1LTvecolUGn6lIaoLh86d83GiB86CjzciMd9BuRB71Q==
|
||||||
|
dependencies:
|
||||||
|
domelementtype "^2.0.1"
|
||||||
|
domhandler "^4.0.0"
|
||||||
|
entities "^2.0.0"
|
||||||
|
|
||||||
domain-browser@^1.1.1:
|
domain-browser@^1.1.1:
|
||||||
version "1.2.0"
|
version "1.2.0"
|
||||||
resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.2.0.tgz#3d31f50191a6749dd1375a7f522e823d42e54eda"
|
resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.2.0.tgz#3d31f50191a6749dd1375a7f522e823d42e54eda"
|
||||||
@ -5308,7 +5356,7 @@ domelementtype@1:
|
|||||||
resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f"
|
resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f"
|
||||||
integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==
|
integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==
|
||||||
|
|
||||||
domelementtype@^2.0.1:
|
domelementtype@^2.0.1, domelementtype@^2.2.0:
|
||||||
version "2.2.0"
|
version "2.2.0"
|
||||||
resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.2.0.tgz#9a0b6c2782ed6a1c7323d42267183df9bd8b1d57"
|
resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.2.0.tgz#9a0b6c2782ed6a1c7323d42267183df9bd8b1d57"
|
||||||
integrity sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A==
|
integrity sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A==
|
||||||
@ -5320,6 +5368,13 @@ domexception@^2.0.1:
|
|||||||
dependencies:
|
dependencies:
|
||||||
webidl-conversions "^5.0.0"
|
webidl-conversions "^5.0.0"
|
||||||
|
|
||||||
|
domhandler@^4.0.0, domhandler@^4.1.0, domhandler@^4.2.0:
|
||||||
|
version "4.2.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.2.0.tgz#f9768a5f034be60a89a27c2e4d0f74eba0d8b059"
|
||||||
|
integrity sha512-zk7sgt970kzPks2Bf+dwT/PLzghLnsivb9CcxkvR8Mzr66Olr0Ofd8neSbglHJHaHa2MadfoSdNlKYAaafmWfA==
|
||||||
|
dependencies:
|
||||||
|
domelementtype "^2.2.0"
|
||||||
|
|
||||||
domutils@^1.7.0:
|
domutils@^1.7.0:
|
||||||
version "1.7.0"
|
version "1.7.0"
|
||||||
resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a"
|
resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a"
|
||||||
@ -5328,6 +5383,15 @@ domutils@^1.7.0:
|
|||||||
dom-serializer "0"
|
dom-serializer "0"
|
||||||
domelementtype "1"
|
domelementtype "1"
|
||||||
|
|
||||||
|
domutils@^2.5.2, domutils@^2.6.0:
|
||||||
|
version "2.6.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.6.0.tgz#2e15c04185d43fb16ae7057cb76433c6edb938b7"
|
||||||
|
integrity sha512-y0BezHuy4MDYxh6OvolXYsH+1EMGmFbwv5FKW7ovwMG6zTPWqNPq3WF9ayZssFq+UlKdffGLbOEaghNdaOm1WA==
|
||||||
|
dependencies:
|
||||||
|
dom-serializer "^1.0.1"
|
||||||
|
domelementtype "^2.2.0"
|
||||||
|
domhandler "^4.2.0"
|
||||||
|
|
||||||
dot-prop@^5.2.0:
|
dot-prop@^5.2.0:
|
||||||
version "5.3.0"
|
version "5.3.0"
|
||||||
resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-5.3.0.tgz#90ccce708cd9cd82cc4dc8c3ddd9abdd55b20e88"
|
resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-5.3.0.tgz#90ccce708cd9cd82cc4dc8c3ddd9abdd55b20e88"
|
||||||
@ -6847,6 +6911,16 @@ html-escaper@^2.0.0:
|
|||||||
resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453"
|
resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453"
|
||||||
integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==
|
integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==
|
||||||
|
|
||||||
|
htmlparser2@^6.1.0:
|
||||||
|
version "6.1.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7"
|
||||||
|
integrity sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==
|
||||||
|
dependencies:
|
||||||
|
domelementtype "^2.0.1"
|
||||||
|
domhandler "^4.0.0"
|
||||||
|
domutils "^2.5.2"
|
||||||
|
entities "^2.0.0"
|
||||||
|
|
||||||
http-cache-semantics@^4.1.0:
|
http-cache-semantics@^4.1.0:
|
||||||
version "4.1.0"
|
version "4.1.0"
|
||||||
resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390"
|
resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390"
|
||||||
@ -9475,6 +9549,13 @@ nth-check@^1.0.2:
|
|||||||
dependencies:
|
dependencies:
|
||||||
boolbase "~1.0.0"
|
boolbase "~1.0.0"
|
||||||
|
|
||||||
|
nth-check@^2.0.0:
|
||||||
|
version "2.0.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.0.0.tgz#1bb4f6dac70072fc313e8c9cd1417b5074c0a125"
|
||||||
|
integrity sha512-i4sc/Kj8htBrAiH1viZ0TgU8Y5XqCaV/FziYK6TBczxmeKm3AEFWqqF3195yKudrarqy7Zu80Ra5dobFjn9X/Q==
|
||||||
|
dependencies:
|
||||||
|
boolbase "^1.0.0"
|
||||||
|
|
||||||
number-is-nan@^1.0.0:
|
number-is-nan@^1.0.0:
|
||||||
version "1.0.1"
|
version "1.0.1"
|
||||||
resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d"
|
resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d"
|
||||||
|
Loading…
x
Reference in New Issue
Block a user