Skip to content
Snippets Groups Projects
Commit aa3b6f12 authored by Sangamithra Panneer Selvam's avatar Sangamithra Panneer Selvam
Browse files

metric display and prompt inclusion

parent 39197445
No related branches found
No related tags found
No related merge requests found
...@@ -4,6 +4,8 @@ import threading ...@@ -4,6 +4,8 @@ import threading
import os import os
import json import json
from datetime import datetime from datetime import datetime
import time
import json
import user_ratings_pb2 import user_ratings_pb2
import user_ratings_pb2_grpc import user_ratings_pb2_grpc
...@@ -11,6 +13,8 @@ import user_ratings_pb2_grpc ...@@ -11,6 +13,8 @@ import user_ratings_pb2_grpc
from app import app_run from app import app_run
from utils_calculate_metrics import process_ratings_file from utils_calculate_metrics import process_ratings_file
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG)
...@@ -20,6 +24,16 @@ file_path = os.path.join(local_directory, 'ratings.json') ...@@ -20,6 +24,16 @@ file_path = os.path.join(local_directory, 'ratings.json')
#file_path = "ratings.json" #file_path = "ratings.json"
def run_periodically():
print("[run_periodically] Starting periodic task...")
class_object = RatingsService()
while True:
print("[run_periodically] Calling CalculateMetrics")
calculated_metrics = class_object._calculate_metrics_internal()
#class_object._get_metrics_internal(calculated_metrics)
time.sleep(2) # Wait for 2 seconds
class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer): class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer):
def __init__(self): def __init__(self):
self.has_metrics = True # Flag to indicate the presence of metrics in this node and print a message accordingly. self.has_metrics = True # Flag to indicate the presence of metrics in this node and print a message accordingly.
...@@ -45,6 +59,8 @@ class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer): ...@@ -45,6 +59,8 @@ class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer):
timestamp=value timestamp=value
elif key.lower() == "feedback" : elif key.lower() == "feedback" :
feedback=value feedback=value
elif key.lower() == "prompt" :
prompt=value
elif "star" in key.lower() or "rating" in key.lower(): elif "star" in key.lower() or "rating" in key.lower():
stars = value stars = value
elif isinstance(value,(float,int)): elif isinstance(value,(float,int)):
...@@ -52,23 +68,22 @@ class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer): ...@@ -52,23 +68,22 @@ class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer):
print("llm_metrics: ",llm_metrics) print("llm_metrics: ",llm_metrics)
rating = user_ratings_pb2.Rating(filename=filename, stars=stars, feedback=feedback, timestamp=timestamp, llm_metrics=llm_metrics) rating = user_ratings_pb2.Rating(filename=filename, stars=stars, feedback=feedback, prompt=prompt, timestamp=timestamp, llm_metrics=llm_metrics)
yield rating yield rating
except: except:
rating = user_ratings_pb2.Rating(filename='', stars=0.0, feedback='', timestamp='', llm_metrics={}) rating = user_ratings_pb2.Rating(filename='', stars=0.0, feedback='', timestamp='', llm_metrics={})
yield rating yield rating
def _calculate_metrics_internal(self):
def CalculateMetrics(self, request, context): # Perform calculation for average metrics
if file_path: if os.path.exists(file_path):
score_averages, average_rating, average_sentiment_score = process_ratings_file(file_path) score_averages, average_rating, average_sentiment_score, current_prompt = process_ratings_file(file_path)
calculated_metrics = user_ratings_pb2.CalculatedMetrics() calculated_metrics = user_ratings_pb2.CalculatedMetrics()
calculated_metrics.type = "LLM Metrics" calculated_metrics.type = "LLM Metrics"
calculated_metrics.status_text = "success" calculated_metrics.status_text = "success"
calculated_metrics.prompt=current_prompt
calculated_metrics.more_is_better.overall_avg_feedback_sentiment_score = average_sentiment_score calculated_metrics.more_is_better.overall_avg_feedback_sentiment_score = average_sentiment_score
calculated_metrics.more_is_better.overall_avg_star_rating = average_rating calculated_metrics.more_is_better.overall_avg_star_rating = average_rating
for key,value in score_averages.items(): for key,value in score_averages.items():
...@@ -79,8 +94,10 @@ class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer): ...@@ -79,8 +94,10 @@ class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer):
else: else:
return () return ()
def get_metrics_metadata(self, request, context): # Get excution metadata def get_metrics_metadata(self, request, context): # Get excution metadata
final_metrics_dict = {'metrics': {}} final_metrics_dict = {'metrics': {}}
now = datetime.now() now = datetime.now()
current_time = now.strftime("%Y-%m-%d %H:%M:%S") current_time = now.strftime("%Y-%m-%d %H:%M:%S")
...@@ -99,13 +116,20 @@ class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer): ...@@ -99,13 +116,20 @@ class RatingsService(user_ratings_pb2_grpc.RatingsServiceServicer):
print(final_metrics_dict) print(final_metrics_dict)
def CalculateMetrics(self, request, context): # Perform calculation for average metrics
calculated_metrics = self._calculate_metrics_internal()
return calculated_metrics
def serve(port): def serve(port):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
user_ratings_pb2_grpc.add_RatingsServiceServicer_to_server(RatingsService(), server) user_ratings_pb2_grpc.add_RatingsServiceServicer_to_server(RatingsService(), server)
server.add_insecure_port("[::]:{}".format(port)) server.add_insecure_port("[::]:{}".format(port))
print("Starting server. Listening on port : " + str(port)) print("Starting server. Listening on port : " + str(port))
server.start() server.start()
threading.Thread(target=app_run()).start() threading.Thread(target=app_run, daemon=True).start()
threading.Thread(target=run_periodically, daemon=True).start()
server.wait_for_termination() server.wait_for_termination()
if __name__ == "__main__": if __name__ == "__main__":
......
...@@ -9,7 +9,8 @@ message Rating { ...@@ -9,7 +9,8 @@ message Rating {
float stars = 2; float stars = 2;
string feedback = 3; string feedback = 3;
string timestamp = 4; string timestamp = 4;
map<string, double> llm_metrics = 5 ; string prompt = 5;
map<string, double> llm_metrics = 6 ;
} }
...@@ -23,6 +24,7 @@ message CalculatedMetrics { ...@@ -23,6 +24,7 @@ message CalculatedMetrics {
string type = 1; string type = 1;
string status_text = 2; string status_text = 2;
string prompt = 3;
Moreisbetter more_is_better = 4; Moreisbetter more_is_better = 4;
} }
......
...@@ -24,12 +24,17 @@ def process_ratings_file(file_path): ...@@ -24,12 +24,17 @@ def process_ratings_file(file_path):
total_score={} total_score={}
num_score={} num_score={}
num_ratings = 0 num_ratings = 0
current_prompt = None
for ratings_dict in ratings_list: for ratings_dict in ratings_list:
num_ratings+=1 num_ratings+=1
# Handle prompt
prompt = ratings_dict.get("Prompt")
if prompt:
current_prompt = prompt
for key,value in ratings_dict.items(): for key,value in ratings_dict.items():
if isinstance(value,str) and "filename" not in key.lower(): if isinstance(value,str) and "filename" not in key.lower() and "prompt" not in key.lower():
try: try:
parser.parse(value) parser.parse(value)
except: except:
...@@ -58,4 +63,4 @@ def process_ratings_file(file_path): ...@@ -58,4 +63,4 @@ def process_ratings_file(file_path):
average_sentiment_score = round(total_sentiment_score / num_ratings, 2) if num_ratings > 0 else 0 average_sentiment_score = round(total_sentiment_score / num_ratings, 2) if num_ratings > 0 else 0
average_rating = round(total_rating / num_ratings , 2) if num_ratings > 0 else 0 average_rating = round(total_rating / num_ratings , 2) if num_ratings > 0 else 0
return score_averages, average_rating, average_sentiment_score return score_averages, average_rating, average_sentiment_score, current_prompt
\ No newline at end of file \ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment