evaluation/prometheus_evaluation_analysis.py

+import pandas as pd
+import matplotlib.pyplot as plt
+import glob
+
+# Adjust the file pattern/path as needed
+csv_files = sorted(glob.glob("eval_reports/llama-finetuned-responses-evaluation-pass*.csv"))[:10]
+print(csv_files)
+# Create a subplot grid: 2 rows x 5 columns for 10 plots
+fig, axes = plt.subplots(nrows=5, ncols=2, figsize=(30, 30))
+axes = axes.flatten()
+
+for i, csv_file in enumerate(csv_files):
+    # Read the CSV with the first row as header.
+    df = pd.read_csv(csv_file)
+    
+    # Ensure the 'score' column is numeric; invalid parsing will become NaN.
+    df['score'] = pd.to_numeric(df['score'], errors='coerce')
+    
+    scores = df['score']
+    
+    # Calculate mean and mode
+    mean_score = scores.mean()
+    mode_series = scores.mode()
+    mode_scores = mode_series.tolist() if not mode_series.empty else []
+    # Create a string of modes separated by commas for display
+    mode_str = ", ".join(str(m) for m in mode_scores) if mode_scores else "None"
+    
+    # Create a dictionary for score counts
+    score_counts = scores.value_counts().sort_index().to_dict()
+
+    print(f'For {csv_file} the statistics are as follows:\nMean: {mean_score} ; Mode: {mode_str}; Score Frequency: {score_counts}' )
+    
+    # Plot: Bar plot for score vs frequency
+    ax = axes[i]
+    ax.bar(list(score_counts.keys()), list(score_counts.values()),
+           color='skyblue', edgecolor='black')
+    ax.set_title(f"Mean: {mean_score:.2f}, Mode: {mode_str}", fontsize=40)
+    ax.set_xlabel("Score", fontsize=34)
+    ax.set_ylabel("Frequency", fontsize=34)
+    ax.tick_params(axis='x', rotation=45, labelsize=34) 
+    ax.tick_params(axis='y', labelsize=34) 
+
+plt.tight_layout()
+plt.savefig("Prometheus Evaluation Results - Llama Finetuned Model - spaced.pdf")
+plt.show()

evaluation/slurm_cosine_simiarity.sh

+#!/usr/bin/zsh
+
+### Add basic configuration for job
+
+#SBATCH --job-name=cosine_simiarity_evaluation
+#SBATCH --output=logs/cosine_simiarity_evaluation_analysis%j.log
+#SBATCH --error=logs/cosine_simiarity_evaluation_analysis_error_%j.log
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+#SBATCH --time=00:30:00
+
+
+###------------------------------------------------------------------------------------------------------------------------------
+
+### Run the project in work directory of the cluster (configure based on need!! 
+### RWTH File System : https://help.itc.rwth-aachen.de/en/service/rhr4fjjutttf/article/da307ec2c60940b29bd42ac483fc3ea7/
+cd $HPCWORK
+cd codebud/evaluation
+###------------------------------------------------------------------------------------------------------------------------------
+
+### JOB SCRIPT RUN
+module load GCCcore/.13.2.0
+module load Python/3.11.5
+module load CUDA
+
+source ../../venvs/codebud/bin/activate
+echo $VIRTUAL_ENV
+
+python --version
+
+python cosine_similarity_evaluation.py
+module unload CUDA
+module unload Python/3.11.5
+
+deactivate
+echo "Script ran successfully"
\ No newline at end of file