from transformers import (
  BertTokenizer,
  BertModel
)
docs = ["YOUR", "DOCS", "HERE"]
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
model = BertModel.from_pretrained("bert-base-uncased").to(DEVICE)
tokenized_sentences = tokenizer( # Tokenize the docs
  docs,
  truncation=True,               # Truncate inputs overflowing context length
  padding=True,                  # Pad inputs to maximal sequence length
  return_tensors="pt"
).to(DEVICE)
outputs = model(**tokenized_sentences)  # Run (tokenized) inputs through BERT
embeddings = outputs["last_hidden_state"] # Vector Representations: N samples, Sequence length, Embedding dimensions (you could do something more fancy)
# Retrieve embeddings at the position of the tokens of interest

from sklearn.feature_extraction.text import (
  TfidfVectorizer,
  CountVectorizer
)
docs = ["YOUR", "DOCS", "HERE"]
bow_vectorizer = CountVectorizer()
X_bow = bow_vectorizer.fit_transform(docs)
tfidf_vectorizer = TfidfVectorizer()
X_tfidf = vectorizer.fit_transform(docs)

from sentence_transformers import SentenceTransformer
docs = ["YOUR", "DOCS", "HERE"]
model_name = "<YOUR/MODEL_NAME>"       # local or from HF hub
model = SentenceTransformer(model_name)
embeddings = model.encode(sentences)

from sklearn.metrics.pairwise import cosine_similarity
docs_representations = ... # pre-computed document representations
pairwise_cosine_sims = cosine_similarity(docs_representations) # shape N_docs x N_docs
pairwise_cosine_sims[0,1] # cosine sim between docs 0 and 1

from bertopic import BERTopic                    # altenatively: use yours! `from <PATH_TO_SCRIPT>.mybertopic import MyBERTopic`
docs = ["YOUR", "DOCS", "HERE"]
topic_model = BERTopic()                         # instantiate the BERTopic object (you can add arguments / choose modules)
topics, probs = topic_model.fit_transform(docs)  # represent docs -> find clusters = topics -> assign 'labels'

from gensim import corpora
from gensim.utils import simple_preprocess
from gensim.models import LdaModel

texts = ["YOUR", "DOCS", "HERE"]                     # your texts here (might be a good idea to do a bit of cleaning at some point...)
dictionary = corpora.Dictionary(texts)                # extracts vocabulary
corpus = [dictionary.doc2bow(text) for text in texts] # documents BoW representations

lda_model = LdaModel(             # Instantiate (and run) LDA algorithm
    corpus=corpus,                # pre-processed documents
    id2word=dictionary,           # corresponding dict. {id:word}
    num_topics=n_topics,          # ! THE NUMBER OF TOPICS HERE !
)

from sklearn.metrics import classification_report
from sklearn.svm import SVC # or any other ML method
X_train, X_test = ... # pre-computed document representations
y_train, y_test = ... # associated labels
clf = SVC()                # + add arguments
clf.fit(X_train, y_train)  # train the ML method
y_pred = clf.predict(X_test)
print(classification_report(y_test, y_pred))  # or the metric(s) you are interested in

from datasets import load_dataset
from transformers import (
  AutoTokenizer, AutoModelForSequenceClassification,
  TrainingArguments, Trainer,
)

raw_dataset = load_dataset("<YOUR_DATASET>")
model_name = "YOUR_MODEL_NAME"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(
  model_name,
   num_labels=2 # Number of labels for the classification task (in the 'dataset')
)

def tokenize_function(example, text_column="text", max_length=512):
    return tokenizer(ex[text_column], padding="max_length", max_length=max_length)
tokenized_dataset = raw_dataset.map(tokenize_function, batched=True)

training_args = TrainingArguments("test-trainer")     # put the training arguments here!
trainer = Trainer(
    model,
    training_args,
    train_dataset=tokenized_dataset["train"],
    eval_dataset=tokenized_dataset["validation"],
    data_collator=data_collator,
    processing_class=tokenizer,
)
trainer.train()

from transformers import pipeline
model_name = "<YOUR/MODEL_NAME>"       # local or from HF hub
prompts = ["YOUR", "PROMPTS", "HERE"]  # list of prompts
generator = pipeline(      # (note: you can use `pipeline` for diverse tasks)
    "text-generation",     # state the task
    model=model_name,      # give the name of the LM to use
    device_map="auto",     # automatically moves model to GPU if available
)
outputs = generator(
    prompts,                       # list of prompts
    max_new_tokens=max_new_tokens, # you can give the pipeline some generation arguments
    batch_size=batch_size          # pipeline handles the batch internally
)
responses = [o[0]["generated_text"] for o in outputs]