Skip to content
Snippets Groups Projects
Commit 39197445 authored by Sangamithra Panneer Selvam's avatar Sangamithra Panneer Selvam
Browse files

new webUI metrics, append prompt to execution-run

parent 5f0403b4
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
RAG-pipelines/RAG-Node/app.py
+ 9
− 6
View file @ 39197445
......@@ -21,12 +21,13 @@ logger.setLevel(logging.DEBUG)
user_question = []
user_prompt = []
user_answer = []
readme_ratings = {}
#readme_ratings = {}
ratings_list = []
metrics_history = []
chat_history = []
faithfulness = None
relevancy= None
prompt_data = "In-built system prompt"
local_directory = os.getenv("SHARED_FOLDER_PATH")
# Queues to hold metrics and results for SSE
......@@ -139,7 +140,7 @@ def handle_request():
return jsonify({"response": "Question recieved"})
elif action == "prompt":
global prompt_data
request_data = request.get_json()
if not request_data:
......@@ -154,6 +155,7 @@ def handle_request():
user_prompt.clear()
user_prompt.append(prompt)
prompt_data = prompt
print(f"Prompt received and set: {prompt}")
return jsonify({"response": "Prompt received"})
......@@ -325,6 +327,7 @@ def plot_embeddings():
def rate_readme():
global faithfulness
global relevancy
global prompt_data
faithfulness_score = float(faithfulness)
relevancy_score = float(relevancy)
......@@ -332,17 +335,17 @@ def rate_readme():
data = request.json
rating = data['rating']
feedback = data.get('feedback', '') # Get the feedback string, default to empty string if not provided
readme_ratings.setdefault(local_directory, []).append({'rating': rating, 'feedback': feedback, 'faithfulness' : faithfulness_score, 'answer_Relevancy' : relevancy_score})
logger.info(readme_ratings)
# readme_ratings.setdefault(local_directory, []).append({'rating': rating, 'feedback': feedback, 'faithfulness' : faithfulness_score, 'answer_Relevancy' : relevancy_score})
# logger.info(readme_ratings)
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
ratings_filename = os.path.join(local_directory, "ratings.json")
rating_dict={"Rating": rating, "Feedback": feedback, "Faithfulness": faithfulness_score, "Answer_Relevancy": relevancy_score, "Timestamp": timestamp}
rating_dict={"Rating": rating, "Feedback": feedback, "Faithfulness": faithfulness_score, "Answer_Relevancy": relevancy_score, "Timestamp": timestamp, "Prompt":prompt_data}
ratings_list.append(rating_dict)
with open(ratings_filename, 'w') as file:
json.dump(ratings_list, file, indent=4)
logger.info(f"Rating: {rating}, Feedback: {feedback}, Faithfulness: {faithfulness_score}, Answer_Relevancy: {relevancy_score}, Timestamp: {timestamp}\n")
logger.info(f"Rating: {rating}, Feedback: {feedback}, Faithfulness: {faithfulness_score}, Answer_Relevancy: {relevancy_score}, Timestamp: {timestamp}, Prompt:{prompt_data}\n")
except Exception as e:
logger.exception("An error occurred")
......
RAG-pipelines/RAG-Node/templates/index.html
+ 64
− 50
View file @ 39197445
......@@ -358,6 +358,8 @@
}
feedbackTextarea.value = '';
feedbackTextarea.style.display = 'none';
document.getElementById('submitted-feedback').style.display = "block";
})
.catch((error) => {
console.error('Error:', error);
......@@ -512,64 +514,76 @@
}
// Polling to fetch updated metrics
async function fetchMetrics() {
const response = await fetch('/metrics');
const data = await response.json();
const chatResponse = document.getElementById("metricsResponse");
// Clear previous results
metricsResponse.innerHTML = '';
// Populate new results
// If data is not empty, show the latest metric (last dictionary in the list)
if (data.length > 0) {
const latestMetric = data[data.length - 1]; // Get the latest dictionary
const questionDiv = document.createElement('div');
questionDiv.innerHTML = `<strong>Faithfulness:</strong> ${latestMetric.Faithfulness} <br>
<strong>Answer relevancy:</strong> ${latestMetric.AnswerRelevancy}<br>`;
metricsResponse.appendChild(questionDiv);
}
}
// fetching results
let lastMetricCount = 0;
let lastResultCount = 0;
let waitingForMetric = false;
// Fetch results
async function fetchResults() {
const chatResponse = document.getElementById("chatResponse");
try {
// Fetch the results
const response = await fetch('/results');
const data = await response.json();
// Clear the chatResponse content for new data
chatResponse.innerHTML = '';
// Check if there's any data to display
if (data.length === 0) {
// No results yet, show a "No chat history" message
chatResponse.innerHTML = 'No chat history.';
} else {
// Populate the chat with questions and answers
// Only update if there is a new result
if (data.length > lastResultCount) {
chatResponse.innerHTML = ''; // Clear chat
data.forEach(item => {
const question = item.Question || "No Question"; // Access "Question"
const answer = item.Answer || "No Answer"; // Access "Answer"
const question = item.Question || "No Question";
const answer = item.Answer || "No Answer";
const questionDiv = document.createElement('div');
questionDiv.innerHTML = `<strong>Q:</strong> ${question} <br> <strong>A:</strong> ${answer} <br> `;
questionDiv.innerHTML = `<strong>Q:</strong> ${question} <br> <strong>A:</strong> ${answer} <br>`;
questionDiv.style.marginBottom = '20px';
chatResponse.appendChild(questionDiv);
});
}
} catch (error) {
// Handle any fetch error
lastResultCount = data.length;
waitingForMetric = true; // Flag to show we're waiting for metric
document.getElementById("metricsStatus").innerHTML = "<span style='color: #ce890a;'>Calculating...</span>";
document.getElementById("metricsResponse").innerText="";
document.getElementById("rateMeSection").style.display = "none"; // Remove Rate me section
document.getElementById('feedback-list').innerHTML = "";
document.getElementById('submitted-feedback').style.display = "none";
}
} catch {
chatResponse.innerHTML = 'Error loading results.';
}
}
// Set up polling to fetch results every 2 seconds
async function fetchMetrics() {
const metricsResponse = document.getElementById("metricsResponse");
const metricsStatus = document.getElementById("metricsStatus");
try {
const response = await fetch('/metrics');
const data = await response.json();
// Only update if new metric is available
if (data.length > lastMetricCount) {
const latestMetric = data[data.length - 1];
metricsResponse.innerHTML = `<strong>Faithfulness:</strong> ${latestMetric.Faithfulness}<br>
<strong>Answer relevancy:</strong> ${latestMetric.AnswerRelevancy}<br>`;
lastMetricCount = data.length;
waitingForMetric = false;
metricsStatus.innerText = ""; // Clear "Calculating..."
// Show rate section
document.getElementById("rateMeSection").style.display = "block";
}
} catch {
metricsStatus.innerText = 'Error loading metrics.';
}
}
document.addEventListener("DOMContentLoaded", function () {
setInterval(fetchResults, 1000);
setInterval(fetchMetrics, 1000);
});
function handleWorkspaceSave(event) {
event.preventDefault();
......@@ -688,9 +702,10 @@
<div class="column small-column">
<!-- Introduction Section -->
<div id="introduction" class="section">
<h2>What is RAG and FAISS?</h2>
<h2>What is RAG ?</h2>
<p><strong>RAG (Retrieval-Augmented Generation)</strong> combines a retrieval model with a generative model to improve text generation. It retrieves relevant context from a large database using a retriever model and generates a response based on that context using a generative model.</p>
<p><strong>FAISS (Facebook AI Similarity Search)</strong> is a library for efficient similarity search and clustering of dense vectors, used in recommendation systems, semantic search, and image retrieval.</p>
<p><strong>Evaluation Metrics:</strong> Faithfulness measures how accurately an answer reflects the provided information or known facts, with a higher score indicating low hallucinations. Answer relevancy score assesses how closely the answer aligns with the user's query or context.</p>
</div>
<!-- Upload Type Selection -->
......@@ -736,9 +751,15 @@
<div id="indexResponse"></div>
</div>
<!-- Rate Me -->
<!-- Metrics -->
<div class="section">
<h3>Rate ME</h3>
<h3> Submit the Evaluation Metrics</h3>
<p>Complete the metrics section to evaluate the overall efficiency of the pipeline.</p>
<div id="metricsResponse"></div>
<div id="metricsStatus"></div>
<div id="rateMeSection" style="display: none;">
<!-- <h3>Rate ME</h3> -->
<p>Assess the quality of the generated responses based on criteria such as relevance, coherence, fluency, and overall satisfaction.</p>
<div class="rating">
......@@ -768,22 +789,15 @@
<textarea id="feedback1" name="feedback" style="display: none;"></textarea>
</div>
<!-- Section to display submitted ratings and feedback -->
<div id="submitted-feedback" style="margin-top: 20px;">
<div id="submitted-feedback" style="margin-top: 20px; display: none;">
<h4>Submitted Ratings & Feedback</h4>
<div id="feedback-list"></div>
</div>
</div>
</div>
<!-- Metrics -->
<div class="section">
<h3> Evaluation Metrics</h3>
<p>Faithfulness measures how accurately an answer reflects the provided information or known facts, with a higher score indicating low hallucinations. Answer relevancy score assesses how closely the answer aligns with the user's query or context.</p>
<div id="metricsResponse"></div>
</div>
<!-- Visualization Section -->
<div class="section">
<h2>Visualizations</h2>
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment

Copyright © Eclipse Foundation, Inc. All Rights Reserved.     Privacy Policy | Terms of Use | Copyright Agent