Hate speech

!pip install transformers

!pip install ipywidgets from ipywidgets import IntProgress !jupyter nbextension enable --py widgetsnbextension

# !pip install ipywidgets # !jupyter nbextension enable --py widgetsnbextension

from transformers import AutoTokenizer, AutoModelForSequenceClassification tokenizer = AutoTokenizer.from_pretrained("Hate-speech-CNERG/dehatebert-mono-arabic") model = AutoModelForSequenceClassification.from_pretrained("Hate-speech-CNERG/dehatebert-mono-arabic")

!pip install openpyxl import pandas as pd df_test = pd.read_excel('/work/test.xlsx') df_test_labels = pd.read_excel('/work/testLabels.xlsx')

from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score import torch from transformers import TrainingArguments, Trainer # Create torch dataset class Dataset(torch.utils.data.Dataset): def __init__(self, encodings, labels=None): self.encodings = encodings self.labels = labels def __getitem__(self, idx): item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()} if self.labels: item["labels"] = torch.tensor(self.labels[idx]) return item def __len__(self): return len(self.encodings["input_ids"]) X_test = list(df_test["tweets"]) X_test_tokenized = tokenizer(X_test, padding=True, truncation=True, max_length=512) # Create torch dataset test_dataset = Dataset(X_test_tokenized) test_trainer=Trainer(model) raw_pred, _, _ = test_trainer.predict(test_dataset)