Feature/add button to test scraper configuration (#2808)

* Add button to test scraper configuration

* Update changelog

---------

Co-authored-by: Manushreshta B L <manushreshta27@gmail.com>
Co-authored-by: Hugo Persson <hugo.e.persson@gmail.com>

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Added
+
+- Added a button to test the scraper configuration in the asset profile details dialog of the admin control
+
 ## 2.33.0 - 2023-12-31
 
 ### Added

apps/api/src/app/admin/admin.controller.ts

@@ -3,6 +3,7 @@ import { HasPermissionGuard } from '@ghostfolio/api/guards/has-permission.guard'
 import { TransformDataSourceInRequestInterceptor } from '@ghostfolio/api/interceptors/transform-data-source-in-request.interceptor';
 import { ApiService } from '@ghostfolio/api/services/api/api.service';
 import { DataGatheringService } from '@ghostfolio/api/services/data-gathering/data-gathering.service';
+import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service';
 import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service';
 import { PropertyDto } from '@ghostfolio/api/services/property/property.dto';
 import {
@@ -31,6 +32,7 @@ import {
   Get,
   HttpException,
   Inject,
+  Logger,
   Param,
   Patch,
   Post,
@@ -56,6 +58,7 @@ export class AdminController {
     private readonly adminService: AdminService,
     private readonly apiService: ApiService,
     private readonly dataGatheringService: DataGatheringService,
+    private readonly manualService: ManualService,
     private readonly marketDataService: MarketDataService,
     @Inject(REQUEST) private readonly request: RequestWithUser
   ) {}
@@ -179,8 +182,8 @@ export class AdminController {
   }
 
   @Get('market-data')
-  @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
   @HasPermission(permissions.accessAdminControl)
+  @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
   public async getMarketData(
     @Query('assetSubClasses') filterByAssetSubClasses?: string,
     @Query('presetId') presetId?: MarketDataPreset,
@@ -215,6 +218,30 @@ export class AdminController {
     return this.adminService.getMarketDataBySymbol({ dataSource, symbol });
   }
 
+  @HasPermission(permissions.accessAdminControl)
+  @Post('market-data/:dataSource/:symbol/test')
+  @UseGuards(AuthGuard('jwt'), HasPermissionGuard)
+  public async testMarketData(
+    @Body() data: { scraperConfiguration: string },
+    @Param('dataSource') dataSource: DataSource,
+    @Param('symbol') symbol: string
+  ): Promise<{ price: number }> {
+    try {
+      const { headers, selector, url } = JSON.parse(data.scraperConfiguration);
+      const price = await this.manualService.test({ headers, selector, url });
+
+      if (price) {
+        return { price };
+      }
+
+      throw new Error('Could not parse the current market price');
+    } catch (error) {
+      Logger.error(error);
+
+      throw new HttpException(error.message, StatusCodes.BAD_REQUEST);
+    }
+  }
+
   @HasPermission(permissions.accessAdminControl)
   @Post('market-data/:dataSource/:symbol')
   @UseGuards(AuthGuard('jwt'), HasPermissionGuard)

apps/api/src/services/data-provider/data-provider.module.ts

@@ -74,6 +74,6 @@ import { DataProviderService } from './data-provider.service';
     },
     YahooFinanceDataEnhancerService
   ],
-  exports: [DataProviderService, YahooFinanceService]
+  exports: [DataProviderService, ManualService, YahooFinanceService]
 })
 export class DataProviderModule {}

apps/api/src/services/data-provider/manual/manual.service.ts

@@ -18,7 +18,7 @@ import { DataSource, SymbolProfile } from '@prisma/client';
 import * as cheerio from 'cheerio';
 import { isUUID } from 'class-validator';
 import { addDays, format, isBefore } from 'date-fns';
-import got from 'got';
+import got, { Headers } from 'got';
 
 @Injectable()
 export class ManualService implements DataProviderInterface {
@@ -97,21 +97,7 @@ export class ManualService implements DataProviderInterface {
         return {};
       }
 
-      const abortController = new AbortController();
+      const value = await this.scrape({ headers, selector, url });
-
-      setTimeout(() => {
-        abortController.abort();
-      }, this.configurationService.get('REQUEST_TIMEOUT'));
-
-      const { body } = await got(url, {
-        headers,
-        // @ts-ignore
-        signal: abortController.signal
-      });
-
-      const $ = cheerio.load(body);
-
-      const value = extractNumberFromString($(selector).text());
 
       return {
         [symbol]: {
@@ -233,4 +219,42 @@ export class ManualService implements DataProviderInterface {
 
     return { items };
   }
+
+  public async test(params: any) {
+    return this.scrape({
+      headers: params.headers,
+      selector: params.selector,
+      url: params.url
+    });
+  }
+
+  private async scrape({
+    headers = {},
+    selector,
+    url
+  }: {
+    headers?: Headers;
+    selector: string;
+    url: string;
+  }): Promise<number> {
+    try {
+      const abortController = new AbortController();
+
+      setTimeout(() => {
+        abortController.abort();
+      }, this.configurationService.get('REQUEST_TIMEOUT'));
+
+      const { body } = await got(url, {
+        headers,
+        // @ts-ignore
+        signal: abortController.signal
+      });
+
+      const $ = cheerio.load(body);
+
+      return extractNumberFromString($(selector).first().text());
+    } catch (error) {
+      throw error;
+    }
+  }
 }

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts

@@ -277,6 +277,34 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
       });
   }
 
+  public onTestMarketData() {
+    this.adminService
+      .testMarketData({
+        dataSource: this.data.dataSource,
+        scraperConfiguration:
+          this.assetProfileForm.controls['scraperConfiguration'].value,
+        symbol: this.data.symbol
+      })
+      .pipe(
+        catchError(({ error }) => {
+          alert(`Error: ${error?.message}`);
+          return EMPTY;
+        }),
+        takeUntil(this.unsubscribeSubject)
+      )
+      .subscribe(({ price }) => {
+        alert(
+          $localize`The current market price is` +
+            ' ' +
+            price +
+            ' ' +
+            (<Currency>(
+              (<unknown>this.assetProfileForm.controls['currency'].value)
+            ))?.value
+        );
+      });
+  }
+
   public onUnsetBenchmark({ dataSource, symbol }: UniqueAsset) {
     this.dataService
       .deleteBenchmark({ dataSource, symbol })

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html

@@ -243,12 +243,24 @@
     <div *ngIf="assetProfile?.dataSource === 'MANUAL'">
       <mat-form-field appearance="outline" class="w-100">
         <mat-label i18n>Scraper Configuration</mat-label>
-        <textarea
+        <div class="align-items-end d-flex">
-          cdkTextareaAutosize
+          <textarea
-          formControlName="scraperConfiguration"
+            cdkTextareaAutosize
-          matInput
+            formControlName="scraperConfiguration"
-          type="text"
+            matInput
-        ></textarea>
+            type="text"
+            (keyup.enter)="$event.stopPropagation()"
+          ></textarea>
+          <button
+            color="accent"
+            mat-flat-button
+            type="button"
+            [disabled]="assetProfileForm.controls['scraperConfiguration'].value === '{}'"
+            (click)="onTestMarketData()"
+          >
+            <ng-container i18n>Test</ng-container>
+          </button>
+        </div>
       </mat-form-field>
     </div>
     <div>

apps/client/src/app/services/admin.service.ts

@@ -259,4 +259,17 @@ export class AdminService {
   public putTag(aTag: UpdateTagDto) {
     return this.http.put<Tag>(`/api/v1/tag/${aTag.id}`, aTag);
   }
+
+  public testMarketData({
+    dataSource,
+    scraperConfiguration,
+    symbol
+  }: UniqueAsset & UpdateAssetProfileDto['scraperConfiguration']) {
+    return this.http.post<any>(
+      `/api/v1/admin/market-data/${dataSource}/${symbol}/test`,
+      {
+        scraperConfiguration
+      }
+    );
+  }
 }