About LLM Evaluation

            
from bert_score import score
import torch
from bleurt_pytorch import BleurtConfig, BleurtForSequenceClassification, BleurtTokenizer

config = BleurtConfig.from_pretrained('lucadiliello/BLEURT-20-D12')
model = BleurtForSequenceClassification.from_pretrained('lucadiliello/BLEURT-20-D12')
tokenizer = BleurtTokenizer.from_pretrained('lucadiliello/BLEURT-20-D12')

candidate = ["The quick brown fox leaps over the lazy dog"]
reference = ["A fast brown fox jumps over the lazy dog"]

#BERTScore
P, R, F1 = score(candidate, reference, lang="en")
print(f"BERTScore F1: {F1[0]:.4f}")

#BLEURT
model.eval()
with torch.no_grad():
    inputs = tokenizer(reference, candidate, padding='longest', return_tensors='pt')
    res = model(**inputs).logits.flatten().tolist()
print(f'BLEURT: {res[0]}')

            

from google import genai
from google.genai import types
import vertexai
from vertexai.generative_models import GenerativeModel
from vertexai.generative_models import GenerationConfig, SafetySetting

import pandas as pd
import time

import json
from concurrent.futures import ThreadPoolExecutor, as_completed

# LLM config
system_instructions = """
    You are an expert at summarizing a web page. 
    You will receive a Wikipedia URL and you will summarize the topic in exactly 500 words.
"""

TEMPERATURE = 0

client = genai.Client(vertexai=True, project='datalake--dev', location='europe-west4')


response_schema = {
    "type": "OBJECT",
    "properties": {
        "summary": {"type": "STRING"},
    },
    "required": ["summary"],
}

CONFIG_SUMMARIZE = types.GenerateContentConfig(
    system_instruction=system_instructions,
    temperature=TEMPERATURE,
    response_mime_type="application/json", 
    response_schema=response_schema
)


#Function that summarizes a url
def summarize(url, llm_model_name, config):
    response = client.models.generate_content(
        model=llm_model_name,
        contents=url,
        config=config,
    )
    response_json = json.loads(response.text)

    return response_json.get("summary", "")


#Function that creates a dataframe with a summary from both models for each url
def generate_dataset(model_a, model_b, urls):
    data_list = []
    for url in urls:
        summary_a = summarize(url, model_a, CONFIG_SUMMARIZE)
        summary_b = summarize(url, model_b, CONFIG_SUMMARIZE)
        
        row_data = {
            "url": url,
            "summary_a": summary_a,
            "summary_b": summary_b
        }
        
        data_list.append(row_data)
        
    df = pd.DataFrame(data_list)
    
    return df


#Define models, urls and run the script
MODEL_A = "gemini-2.0-flash-lite"
MODEL_B = "gemini-2.5-pro"


URLS = ['https://en.wikipedia.org/wiki/Machine_learning', 
        'https://en.wikipedia.org/wiki/Generative_artificial_intelligence',
        'https://en.wikipedia.org/wiki/Large_language_model',
        'https://en.wikipedia.org/wiki/Prompt_engineering',
        'https://en.wikipedia.org/wiki/Transformer_(deep_learning)',
        'https://en.wikipedia.org/wiki/Attention_(machine_learning)',
        'https://en.wikipedia.org/wiki/LLM-as-a-Judge',
        'https://en.wikipedia.org/wiki/Reinforcement_learning',
        'https://en.wikipedia.org/wiki/Markov_decision_process',
        'https://en.wikipedia.org/wiki/Generative_adversarial_network'
       ]


df = generate_dataset(MODEL_A, MODEL_B, URLS)

from enum import Enum
from pydantic import BaseModel, Field

#response schema
class SummaryChoice(str, Enum):
    summary_a = "summary_a"
    summary_b = "summary_b"

class OutputSchema(BaseModel):
    summary_choice: SummaryChoice = Field(description="Best Summary")
    rationale: str = Field(description="Concise rationale")


#evaluate the summaries: series of LLMs that evaluate the best summary
client = genai.Client(vertexai=True, project='datalake--dev', location='europe-west4')

system_instructions = """
    You are a highly efficient Comparative Summarization Analyst. Your sole task is to determine which of two provided texts, **Summary A** or **Summary B**, is superior.
    
    You must compare them based on the following criteria:
        1.  **Factual Accuracy:** Which summary is more accurate? 
        2.  **Readability:** Which summary is grammatically correct, coherent, and flows more naturally?
        3.  **Conciseness:** Which summary is shorter while still retaining all essential information?

    Your output must only contain the chosen winner and a concise rationale for that choice.
    """

TEMPERATURE = 0


CONFIG_JUDGE = types.GenerateContentConfig(
    system_instruction=system_instructions,
    safety_settings = SAFETY_SETTINGS,
    temperature=TEMPERATURE,
    response_mime_type="application/json", 
    response_schema=OutputSchema.model_json_schema()
)

def judge(summary_a, summary_b, model, config):
    
    content = f"""
        Summary A: {summary_a}
        
        Summary B: {summary_b}
    """
    
    response = client.models.generate_content(
        model=model,
        contents=content,
        config=config,
    )
    response_json = json.loads(response.text)

    return response_json.get("summary_choice", ""), response_json.get("rationale", "")

JUDGE_MODELS = ["gemini-2.0-flash-lite", 
                "gemini-2.0-flash", 
                "gemini-2.5-flash-lite", 
                "gemini-2.5-flash", 
                "gemini-2.5-pro"
               ]

data_list = []
for i, row in df.iterrows():
    for judge_model in JUDGE_MODELS:
        sc, r = judge(row['summary_a'], row['summary_b'], judge_model, CONFIG_JUDGE)
    
        row_data = {
            "url": row['url'],
            "judge_model": judge_model,
            "summary_choice": sc,
            "rationale": r
        }

        data_list.append(row_data)

df_judge = pd.DataFrame(data_list)