Skip to content
Snippets Groups Projects
Commit aa3b6f12 authored by Sangamithra Panneer Selvam's avatar Sangamithra Panneer Selvam
Browse files

metric display and prompt inclusion

parent 39197445
No related branches found
No related tags found
No related merge requests found
......@@ -4,6 +4,8 @@ import threading
import os
import json
from datetime import datetime
import time
import json
import user_ratings_pb2
import user_ratings_pb2_grpc
......@@ -11,6 +13,8 @@ import user_ratings_pb2_grpc
from app import app_run
from utils_calculate_metrics import process_ratings_file
import logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
......@@ -20,6 +24,16 @@ file_path = os.path.join(local_directory, 'ratings.json')
#file_path = "ratings.json"
def run_periodically():
print("[run_periodically] Starting periodic task...")
class_object = RatingsService()
while True:
print("[run_periodically] Calling CalculateMetrics")
calculated_metrics = class_object._calculate_metrics_internal()
#class_object._get_metrics_internal(calculated_metrics)
time.sleep(2) # Wait for 2 seconds
class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer):
def __init__(self):
self.has_metrics = True # Flag to indicate the presence of metrics in this node and print a message accordingly.
......@@ -45,6 +59,8 @@ class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer):
timestamp=value
elif key.lower() == "feedback" :
feedback=value
elif key.lower() == "prompt" :
prompt=value
elif "star" in key.lower() or "rating" in key.lower():
stars = value
elif isinstance(value,(float,int)):
......@@ -52,23 +68,22 @@ class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer):
print("llm_metrics: ",llm_metrics)
rating = user_ratings_pb2.Rating(filename=filename, stars=stars, feedback=feedback, timestamp=timestamp, llm_metrics=llm_metrics)
rating = user_ratings_pb2.Rating(filename=filename, stars=stars, feedback=feedback, prompt=prompt, timestamp=timestamp, llm_metrics=llm_metrics)
yield rating
except:
rating = user_ratings_pb2.Rating(filename='', stars=0.0, feedback='', timestamp='', llm_metrics={})
yield rating
def CalculateMetrics(self, request, context): # Perform calculation for average metrics
def _calculate_metrics_internal(self):
if file_path:
score_averages, average_rating, average_sentiment_score = process_ratings_file(file_path)
if os.path.exists(file_path):
score_averages, average_rating, average_sentiment_score, current_prompt = process_ratings_file(file_path)
calculated_metrics = user_ratings_pb2.CalculatedMetrics()
calculated_metrics.type = "LLM Metrics"
calculated_metrics.status_text = "success"
calculated_metrics.prompt=current_prompt
calculated_metrics.more_is_better.overall_avg_feedback_sentiment_score = average_sentiment_score
calculated_metrics.more_is_better.overall_avg_star_rating = average_rating
for key,value in score_averages.items():
......@@ -79,8 +94,10 @@ class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer):
else:
return ()
def get_metrics_metadata(self, request, context): # Get excution metadata
final_metrics_dict = {'metrics': {}}
now = datetime.now()
current_time = now.strftime("%Y-%m-%d %H:%M:%S")
......@@ -99,13 +116,20 @@ class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer):
print(final_metrics_dict)
def CalculateMetrics(self, request, context): # Perform calculation for average metrics
calculated_metrics = self._calculate_metrics_internal()
return calculated_metrics
def serve(port):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
user_ratings_pb2_grpc.add_RatingsServiceServicer_to_server(RatingsService(), server)
server.add_insecure_port("[::]:{}".format(port))
print("Starting server. Listening on port : " + str(port))
server.start()
threading.Thread(target=app_run()).start()
threading.Thread(target=app_run, daemon=True).start()
threading.Thread(target=run_periodically, daemon=True).start()
server.wait_for_termination()
if __name__ == "__main__":
......
......@@ -9,7 +9,8 @@ message Rating {
float stars = 2;
string feedback = 3;
string timestamp = 4;
map<string, double> llm_metrics = 5 ;
string prompt = 5;
map<string, double> llm_metrics = 6 ;
}
......@@ -23,6 +24,7 @@ message CalculatedMetrics {
string type = 1;
string status_text = 2;
string prompt = 3;
Moreisbetter more_is_better = 4;
}
......
......@@ -24,12 +24,17 @@ def process_ratings_file(file_path):
total_score={}
num_score={}
num_ratings = 0
current_prompt = None
for ratings_dict in ratings_list:
num_ratings+=1
num_ratings+=1
# Handle prompt
prompt = ratings_dict.get("Prompt")
if prompt:
current_prompt = prompt
for key,value in ratings_dict.items():
if isinstance(value,str) and "filename" not in key.lower():
if isinstance(value,str) and "filename" not in key.lower() and "prompt" not in key.lower():
try:
parser.parse(value)
except:
......@@ -58,4 +63,4 @@ def process_ratings_file(file_path):
average_sentiment_score = round(total_sentiment_score / num_ratings, 2) if num_ratings > 0 else 0
average_rating = round(total_rating / num_ratings , 2) if num_ratings > 0 else 0
return score_averages, average_rating, average_sentiment_score
\ No newline at end of file
return score_averages, average_rating, average_sentiment_score, current_prompt
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment