Feature/restrict historical market data gathering to active asset profiles (#4483)

* Restrict historical market data gathering to active asset profiles

* Update changelog

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+
+- Restricted the historical market data gathering to active asset profiles
+
 ## 2.148.0 - 2025-03-24
 
 ### Added

apps/api/src/app/admin/admin.controller.ts

@@ -83,7 +83,7 @@ export class AdminController {
   @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
   public async gatherMax(): Promise<void> {
     const assetProfileIdentifiers =
-      await this.dataGatheringService.getAllAssetProfileIdentifiers();
+      await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();
 
     await this.dataGatheringService.addJobsToQueue(
       assetProfileIdentifiers.map(({ dataSource, symbol }) => {
@@ -110,7 +110,7 @@ export class AdminController {
   @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
   public async gatherProfileData(): Promise<void> {
     const assetProfileIdentifiers =
-      await this.dataGatheringService.getAllAssetProfileIdentifiers();
+      await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();
 
     await this.dataGatheringService.addJobsToQueue(
       assetProfileIdentifiers.map(({ dataSource, symbol }) => {

apps/api/src/services/cron.service.ts

@@ -57,7 +57,7 @@ export class CronService {
   public async runEverySundayAtTwelvePm() {
     if (await this.isDataGatheringEnabled()) {
       const assetProfileIdentifiers =
-        await this.dataGatheringService.getAllAssetProfileIdentifiers();
+        await this.dataGatheringService.getAllActiveAssetProfileIdentifiers();
 
       await this.dataGatheringService.addJobsToQueue(
         assetProfileIdentifiers.map(({ dataSource, symbol }) => {

apps/api/src/services/queues/data-gathering/data-gathering.service.ts

@@ -159,7 +159,8 @@ export class DataGatheringService {
     );
 
     if (!assetProfileIdentifiers) {
-      assetProfileIdentifiers = await this.getAllAssetProfileIdentifiers();
+      assetProfileIdentifiers =
+        await this.getAllActiveAssetProfileIdentifiers();
     }
 
     if (assetProfileIdentifiers.length <= 0) {
@@ -296,11 +297,14 @@ export class DataGatheringService {
     );
   }
 
-  public async getAllAssetProfileIdentifiers(): Promise<
+  public async getAllActiveAssetProfileIdentifiers(): Promise<
     AssetProfileIdentifier[]
   > {
     const symbolProfiles = await this.prismaService.symbolProfile.findMany({
-      orderBy: [{ symbol: 'asc' }]
+      orderBy: [{ symbol: 'asc' }],
+      where: {
+        isActive: true
+      }
     });
 
     return symbolProfiles
@@ -370,9 +374,11 @@ export class DataGatheringService {
     withUserSubscription?: boolean;
   }): Promise<IDataGatheringItem[]> {
     const symbolProfiles =
-      await this.symbolProfileService.getSymbolProfilesByUserSubscription({
+      await this.symbolProfileService.getActiveSymbolProfilesByUserSubscription(
-        withUserSubscription
+        {
-      });
+          withUserSubscription
+        }
+      );
 
     const assetProfileIdentifiersWithCompleteMarketData =
       await this.getAssetProfileIdentifiersWithCompleteMarketData();
@@ -436,6 +442,9 @@ export class DataGatheringService {
           },
           scraperConfiguration: true,
           symbol: true
+        },
+        where: {
+          isActive: true
         }
       })
     )

apps/api/src/services/symbol-profile/symbol-profile.service.ts

@@ -35,6 +35,41 @@ export class SymbolProfileService {
     });
   }
 
+  public async getActiveSymbolProfilesByUserSubscription({
+    withUserSubscription = false
+  }: {
+    withUserSubscription?: boolean;
+  }) {
+    return this.prismaService.symbolProfile.findMany({
+      include: {
+        Order: {
+          include: {
+            User: true
+          }
+        }
+      },
+      orderBy: [{ symbol: 'asc' }],
+      where: {
+        isActive: true,
+        Order: withUserSubscription
+          ? {
+              some: {
+                User: {
+                  Subscription: { some: { expiresAt: { gt: new Date() } } }
+                }
+              }
+            }
+          : {
+              every: {
+                User: {
+                  Subscription: { none: { expiresAt: { gt: new Date() } } }
+                }
+              }
+            }
+      }
+    });
+  }
+
   public async getSymbolProfiles(
     aAssetProfileIdentifiers: AssetProfileIdentifier[]
   ): Promise<EnhancedSymbolProfile[]> {
@@ -91,40 +126,6 @@ export class SymbolProfileService {
       });
   }
 
-  public async getSymbolProfilesByUserSubscription({
-    withUserSubscription = false
-  }: {
-    withUserSubscription?: boolean;
-  }) {
-    return this.prismaService.symbolProfile.findMany({
-      include: {
-        Order: {
-          include: {
-            User: true
-          }
-        }
-      },
-      orderBy: [{ symbol: 'asc' }],
-      where: {
-        Order: withUserSubscription
-          ? {
-              some: {
-                User: {
-                  Subscription: { some: { expiresAt: { gt: new Date() } } }
-                }
-              }
-            }
-          : {
-              every: {
-                User: {
-                  Subscription: { none: { expiresAt: { gt: new Date() } } }
-                }
-              }
-            }
-      }
-    });
-  }
-
   public updateSymbolProfile({
     assetClass,
     assetSubClass,
@@ -133,6 +134,7 @@ export class SymbolProfileService {
     currency,
     dataSource,
     holdings,
+    isActive,
     name,
     scraperConfiguration,
     sectors,
@@ -149,6 +151,7 @@ export class SymbolProfileService {
         countries,
         currency,
         holdings,
+        isActive,
         name,
         scraperConfiguration,
         sectors,