sudacode/ghostfolio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Feature/skip job creation for manual data source without scraper configuration (#1857)

Browse Source 
* Skip job creation for MANUAL data source without scraper configuration

* Update changelog
This commit is contained in:
Thomas Kaul 2023-04-15 11:54:47 +02:00 committed by GitHub
parent f2d206262e
commit e23ff33e6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 138 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CHANGELOG.md
View File

@ -5,12 +5,20 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## Unreleased
### Changed
- Skipped creating queue jobs for asset profiles with `MANUAL` data source not having a scraper configuration
- Reduced the execution interval of the data gathering to every hour
## 1.254.0 - 2023-04-14 ## 1.254.0 - 2023-04-14
### Changed ### Changed
- Improved the queue jobs implementation by adding in bulk - Improved the queue jobs implementation by adding in bulk
- Improved the queue jobs implementation by introducing unique job ids - Improved the queue jobs implementation by introducing unique job ids
- Reverted the execution interval of the data gathering from every 12 hours to every 4 hours
## 1.253.0 - 2023-04-14 ## 1.253.0 - 2023-04-14

66
apps/api/src/app/admin/admin.controller.ts
View File

@ -100,19 +100,21 @@ export class AdminController {
const uniqueAssets = await this.dataGatheringService.getUniqueAssets(); const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
for (const { dataSource, symbol } of uniqueAssets) { await this.dataGatheringService.addJobsToQueue(
await this.dataGatheringService.addJobToQueue( uniqueAssets.map(({ dataSource, symbol }) => {
GATHER_ASSET_PROFILE_PROCESS, return {
{ data: {
dataSource, dataSource,
symbol symbol
}, },
{ name: GATHER_ASSET_PROFILE_PROCESS,
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, opts: {
jobId: `${dataSource}-${symbol}}` ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
} jobId: `${dataSource}-${symbol}}`
); }
} };
})
);
this.dataGatheringService.gatherMax(); this.dataGatheringService.gatherMax();
} }
@ -134,19 +136,21 @@ export class AdminController {
const uniqueAssets = await this.dataGatheringService.getUniqueAssets(); const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
for (const { dataSource, symbol } of uniqueAssets) { await this.dataGatheringService.addJobsToQueue(
await this.dataGatheringService.addJobToQueue( uniqueAssets.map(({ dataSource, symbol }) => {
GATHER_ASSET_PROFILE_PROCESS, return {
{ data: {
dataSource, dataSource,
symbol symbol
}, },
{ name: GATHER_ASSET_PROFILE_PROCESS,
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, opts: {
jobId: `${dataSource}-${symbol}}` ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
} jobId: `${dataSource}-${symbol}}`
); }
} };
})
);
} }
@Post('gather/profile-data/:dataSource/:symbol') @Post('gather/profile-data/:dataSource/:symbol')
@ -167,17 +171,17 @@ export class AdminController {
); );
} }
await this.dataGatheringService.addJobToQueue( await this.dataGatheringService.addJobToQueue({
GATHER_ASSET_PROFILE_PROCESS, data: {
{
dataSource, dataSource,
symbol symbol
}, },
{ name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
jobId: `${dataSource}-${symbol}}` jobId: `${dataSource}-${symbol}}`
} }
); });
} }
@Post('gather/:dataSource/:symbol') @Post('gather/:dataSource/:symbol')

10
apps/api/src/app/order/order.service.ts
View File

@ -112,17 +112,17 @@ export class OrderService {
}; };
} }
await this.dataGatheringService.addJobToQueue( await this.dataGatheringService.addJobToQueue({
GATHER_ASSET_PROFILE_PROCESS, data: {
{
dataSource: data.SymbolProfile.connectOrCreate.create.dataSource, dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
symbol: data.SymbolProfile.connectOrCreate.create.symbol symbol: data.SymbolProfile.connectOrCreate.create.symbol
}, },
{ name: GATHER_ASSET_PROFILE_PROCESS,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
jobId: `${data.SymbolProfile.connectOrCreate.create.dataSource}-${data.SymbolProfile.connectOrCreate.create.symbol}}` jobId: `${data.SymbolProfile.connectOrCreate.create.dataSource}-${data.SymbolProfile.connectOrCreate.create.symbol}}`
} }
); });
const isDraft = isAfter(data.date as Date, endOfToday()); const isDraft = isAfter(data.date as Date, endOfToday());

32
apps/api/src/services/cron.service.ts
View File

@ -19,8 +19,8 @@ export class CronService {
private readonly twitterBotService: TwitterBotService private readonly twitterBotService: TwitterBotService
) {} ) {}
@Cron(CronExpression.EVERY_4_HOURS) @Cron(CronExpression.EVERY_HOUR)
public async runEveryFourHours() { public async runEveryHour() {
await this.dataGatheringService.gather7Days(); await this.dataGatheringService.gather7Days();
} }
@ -38,18 +38,20 @@ export class CronService {
public async runEverySundayAtTwelvePm() { public async runEverySundayAtTwelvePm() {
const uniqueAssets = await this.dataGatheringService.getUniqueAssets(); const uniqueAssets = await this.dataGatheringService.getUniqueAssets();
for (const { dataSource, symbol } of uniqueAssets) { await this.dataGatheringService.addJobsToQueue(
await this.dataGatheringService.addJobToQueue( uniqueAssets.map(({ dataSource, symbol }) => {
GATHER_ASSET_PROFILE_PROCESS, return {
{ data: {
dataSource, dataSource,
symbol symbol
}, },
{ name: GATHER_ASSET_PROFILE_PROCESS,
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS, opts: {
jobId: `${dataSource}-${symbol}}` ...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
} jobId: `${dataSource}-${symbol}}`
); }
} };
})
);
} }
} }

120
apps/api/src/services/data-gathering.service.ts
View File

@ -11,6 +11,7 @@ import { Inject, Injectable, Logger } from '@nestjs/common';
import { DataSource } from '@prisma/client'; import { DataSource } from '@prisma/client';
import { JobOptions, Queue } from 'bull'; import { JobOptions, Queue } from 'bull';
import { format, min, subDays, subYears } from 'date-fns'; import { format, min, subDays, subYears } from 'date-fns';
import { isEmpty } from 'lodash';
import { DataProviderService } from './data-provider/data-provider.service'; import { DataProviderService } from './data-provider/data-provider.service';
import { DataEnhancerInterface } from './data-provider/interfaces/data-enhancer.interface'; import { DataEnhancerInterface } from './data-provider/interfaces/data-enhancer.interface';
@ -33,7 +34,15 @@ export class DataGatheringService {
private readonly symbolProfileService: SymbolProfileService private readonly symbolProfileService: SymbolProfileService
) {} ) {}
public async addJobToQueue(name: string, data: any, opts?: JobOptions) { public async addJobToQueue({
data,
name,
opts
}: {
data: any;
name: string;
opts?: JobOptions;
}) {
return this.dataGatheringQueue.add(name, data, opts); return this.dataGatheringQueue.add(name, data, opts);
} }
@ -223,48 +232,6 @@ export class DataGatheringService {
); );
} }
public async getSymbolsMax(): Promise<IDataGatheringItem[]> {
const startDate =
(
await this.prismaService.order.findFirst({
orderBy: [{ date: 'asc' }]
})
)?.date ?? new Date();
const currencyPairsToGather = this.exchangeRateDataService
.getCurrencyPairs()
.map(({ dataSource, symbol }) => {
return {
dataSource,
symbol,
date: min([startDate, subYears(new Date(), 10)])
};
});
const symbolProfilesToGather = (
await this.prismaService.symbolProfile.findMany({
orderBy: [{ symbol: 'asc' }],
select: {
dataSource: true,
Order: {
orderBy: [{ date: 'asc' }],
select: { date: true },
take: 1
},
scraperConfiguration: true,
symbol: true
}
})
).map((symbolProfile) => {
return {
...symbolProfile,
date: symbolProfile.Order?.[0]?.date ?? startDate
};
});
return [...currencyPairsToGather, ...symbolProfilesToGather];
}
public async getUniqueAssets(): Promise<UniqueAsset[]> { public async getUniqueAssets(): Promise<UniqueAsset[]> {
const symbolProfiles = await this.prismaService.symbolProfile.findMany({ const symbolProfiles = await this.prismaService.symbolProfile.findMany({
orderBy: [{ symbol: 'asc' }] orderBy: [{ symbol: 'asc' }]
@ -299,7 +266,7 @@ export class DataGatheringService {
// Only consider symbols with incomplete market data for the last // Only consider symbols with incomplete market data for the last
// 7 days // 7 days
const symbolsNotToGather = ( const symbolsWithCompleteMarketData = (
await this.prismaService.marketData.groupBy({ await this.prismaService.marketData.groupBy({
_count: true, _count: true,
by: ['symbol'], by: ['symbol'],
@ -317,8 +284,14 @@ export class DataGatheringService {
}); });
const symbolProfilesToGather = symbolProfiles const symbolProfilesToGather = symbolProfiles
.filter(({ symbol }) => { .filter(({ dataSource, scraperConfiguration, symbol }) => {
return !symbolsNotToGather.includes(symbol); const manualDataSourceWithScraperConfiguration =
dataSource === 'MANUAL' && !isEmpty(scraperConfiguration);
return (
!symbolsWithCompleteMarketData.includes(symbol) &&
(dataSource !== 'MANUAL' || manualDataSourceWithScraperConfiguration)
);
}) })
.map((symbolProfile) => { .map((symbolProfile) => {
return { return {
@ -330,7 +303,7 @@ export class DataGatheringService {
const currencyPairsToGather = this.exchangeRateDataService const currencyPairsToGather = this.exchangeRateDataService
.getCurrencyPairs() .getCurrencyPairs()
.filter(({ symbol }) => { .filter(({ symbol }) => {
return !symbolsNotToGather.includes(symbol); return !symbolsWithCompleteMarketData.includes(symbol);
}) })
.map(({ dataSource, symbol }) => { .map(({ dataSource, symbol }) => {
return { return {
@ -342,4 +315,57 @@ export class DataGatheringService {
return [...currencyPairsToGather, ...symbolProfilesToGather]; return [...currencyPairsToGather, ...symbolProfilesToGather];
} }
private async getSymbolsMax(): Promise<IDataGatheringItem[]> {
const startDate =
(
await this.prismaService.order.findFirst({
orderBy: [{ date: 'asc' }]
})
)?.date ?? new Date();
const currencyPairsToGather = this.exchangeRateDataService
.getCurrencyPairs()
.map(({ dataSource, symbol }) => {
return {
dataSource,
symbol,
date: min([startDate, subYears(new Date(), 10)])
};
});
const symbolProfilesToGather = (
await this.prismaService.symbolProfile.findMany({
orderBy: [{ symbol: 'asc' }],
select: {
dataSource: true,
Order: {
orderBy: [{ date: 'asc' }],
select: { date: true },
take: 1
},
scraperConfiguration: true,
symbol: true
}
})
)
.filter((symbolProfile) => {
const manualDataSourceWithScraperConfiguration =
symbolProfile.dataSource === 'MANUAL' &&
!isEmpty(symbolProfile.scraperConfiguration);
return (
symbolProfile.dataSource !== 'MANUAL' ||
manualDataSourceWithScraperConfiguration
);
})
.map((symbolProfile) => {
return {
...symbolProfile,
date: symbolProfile.Order?.[0]?.date ?? startDate
};
});
return [...currencyPairsToGather, ...symbolProfilesToGather];
}
} }