Silence warning in accurate filtering
This commit is contained in:
@@ -6,6 +6,7 @@ from pathlib import Path
|
|||||||
import queue
|
import queue
|
||||||
import io
|
import io
|
||||||
import re
|
import re
|
||||||
|
import logging
|
||||||
import inspect
|
import inspect
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -299,6 +300,7 @@ class TextFiltering:
|
|||||||
try:
|
try:
|
||||||
from transformers import pipeline, AutoTokenizer
|
from transformers import pipeline, AutoTokenizer
|
||||||
import torch
|
import torch
|
||||||
|
logging.getLogger('transformers').setLevel(logging.ERROR)
|
||||||
|
|
||||||
model_ckpt = 'papluca/xlm-roberta-base-language-detection'
|
model_ckpt = 'papluca/xlm-roberta-base-language-detection'
|
||||||
tokenizer = AutoTokenizer.from_pretrained(
|
tokenizer = AutoTokenizer.from_pretrained(
|
||||||
|
|||||||
Reference in New Issue
Block a user