Feature/add button to test scraper configuration (#2808)

* Add button to test scraper configuration

* Update changelog

---------

Co-authored-by: Manushreshta B L <manushreshta27@gmail.com>
Co-authored-by: Hugo Persson <hugo.e.persson@gmail.com>

apps/api/src/app/admin/admin.controller.ts

@@ -3,6 +3,7 @@ import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard'
 import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
 import { ApiService } from '@ghostfolio/api/services/api/api.service';
 import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
+import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';
 import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { PropertyDto } from '@ghostfolio/api/services/property/property.dto';
 import {
@@ -31,6 +32,7 @@ import {
   Get,
   HttpException,
   Inject,
+  Logger,
   Param,
   Patch,
   Post,
@@ -56,6 +58,7 @@ export class AdminController {
     private readonly adminService: AdminService,
     private readonly apiService: ApiService,
     private readonly dataGatheringService: DataGatheringService,
+    private readonly manualService: ManualService,
     private readonly marketDataService: MarketDataService,
     @Inject(REQUEST) private readonly request: RequestWithUser
   ) {}
@@ -179,8 +182,8 @@ export class AdminController {
   }
 
   @Get('market-data')
-  @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
   @HasPermission(permissions.accessAdminControl)
+  @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
   public async getMarketData(
     @Query('assetSubClasses') filterByAssetSubClasses?: string,
     @Query('presetId') presetId?: MarketDataPreset,
@@ -215,6 +218,30 @@ export class AdminController {
     return this.adminService.getMarketDataBySymbol({ dataSource, symbol });
   }
 
+  @HasPermission(permissions.accessAdminControl)
+  @Post('market-data/:dataSource/:symbol/test')
+  @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
+  public async testMarketData(
+    @Body() data: { scraperConfiguration: string },
+    @Param('dataSource') dataSource: DataSource,
+    @Param('symbol') symbol: string
+  ): Promise<{ price: number }> {
+    try {
+      const { headers, selector, url } = JSON.parse(data.scraperConfiguration);
+      const price = await this.manualService.test({ headers, selector, url });
+
+      if (price) {
+        return { price };
+      }
+
+      throw new Error('Could not parse the current market price');
+    } catch (error) {
+      Logger.error(error);
+
+      throw new HttpException(error.message, StatusCodes.BAD_REQUEST);
+    }
+  }
+
   @HasPermission(permissions.accessAdminControl)
   @Post('market-data/:dataSource/:symbol')
   @UseGuards(AuthGuard('jwt'), HasPermissionGuard)

apps/api/src/services/data-provider/data-provider.module.ts

@@ -74,6 +74,6 @@ import { DataProviderService } from './data-provider.service';
     },
     YahooFinanceDataEnhancerService
   ],
-  exports: [DataProviderService, YahooFinanceService]
+  exports: [DataProviderService, ManualService, YahooFinanceService]
 })
 export class DataProviderModule {}

apps/api/src/services/data-provider/manual/manual.service.ts

@@ -18,7 +18,7 @@ import { DataSource, SymbolProfile } from '@prisma/client';
 import * as cheerio from 'cheerio';
 import { isUUID } from 'class-validator';
 import { addDays, format, isBefore } from 'date-fns';
-import got from 'got';
+import got, { Headers } from 'got';
 
 @Injectable()
 export class ManualService implements DataProviderInterface {
@@ -97,21 +97,7 @@ export class ManualService implements DataProviderInterface {
         return {};
       }
 
-      const abortController = new AbortController();
-
-      setTimeout(() => {
-        abortController.abort();
-      }, this.configurationService.get('REQUEST_TIMEOUT'));
-
-      const { body } = await got(url, {
-        headers,
-        // @ts-ignore
-        signal: abortController.signal
-      });
-
-      const $ = cheerio.load(body);
-
-      const value = extractNumberFromString($(selector).text());
+      const value = await this.scrape({ headers, selector, url });
 
       return {
         [symbol]: {
@@ -233,4 +219,42 @@ export class ManualService implements DataProviderInterface {
 
     return { items };
   }
+
+  public async test(params: any) {
+    return this.scrape({
+      headers: params.headers,
+      selector: params.selector,
+      url: params.url
+    });
+  }
+
+  private async scrape({
+    headers = {},
+    selector,
+    url
+  }: {
+    headers?: Headers;
+    selector: string;
+    url: string;
+  }): Promise<number> {
+    try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, this.configurationService.get('REQUEST_TIMEOUT'));
+
+      const { body } = await got(url, {
+        headers,
+        // @ts-ignore
+        signal: abortController.signal
+      });
+
+      const $ = cheerio.load(body);
+
+      return extractNumberFromString($(selector).first().text());
+    } catch (error) {
+      throw error;
+    }
+  }
 }