Merge branch 'main' of gitlab.eclipse.org:eclipse/graphene/tutorials into main

RAG-pipelines/RAG two-node pipeline/Readme.md

@@ -3,7 +3,7 @@ Retrieval-augmented generation (RAG) is a technique for enhancing the accuracy a
 
 PLease refer to the issue for more details on the usecase: https://gitlab.eclipse.org/eclipse/graphene/eclipse-graphene/-/issues/17
 
-![RAG Two-Node Pipeline](two-node-pipeline.PNG)
+![RAG Two-Node Pipeline](two-node-pipeline.png)
 
 # Docker Containers
 Here we have 3 docker containers:

RAG-pipelines/Readme.md

 # RAG
 Retrieval-augmented generation (RAG) is a technique for enhancing the accuracy and reliability of generative AI models with facts fetched from external sources. It is crucial for ensuring the quality, accuracy, and relevance of the generated output.
 
-![RAG-workflow](RAG-workflow.PNG)
+
+<img src="RAG-workflow.png" width="500" />
 
 Retrieval-Augmented Generation (RAG) combines two main components:
     
@@ -9,6 +10,11 @@ Retrieval-Augmented Generation (RAG) combines two main components:
 
 2. Generation Component: This part involves generating a response or text based on the information retrieved by the first component.
 
+Currently in our DEV AI-Builder we have two published pipelines:
+
+1. RAG-Pipeline-TN 
+2. RAG-Pipeline-SN ( Script will be updated soon )
+
 
 More RAG-pipelines will be added to this tutorial folder. This tutorial is currently in the developmental stages and may undergo frequent changes. Contributions, suggestions, and feedback are welcome to help improve this project.
 

llm_docker_generator/README.md

+# Dynamic Docker Container Generation for Various LLMs in the AI-Builder
+## Overview
+
+The idea is to dynamically create and manage individual Docker containers based on model names available from the LLM hosting platform and other sources. Users can drag and drop them into their workflows upon successful creation and onboarding in the AI builder.
+

llm_docker_generator/docker_registry_update.py

+import subprocess
+import logging
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+def docker_login(registry_url, username, password):
+    """Log in to Docker registry."""
+    try:
+        subprocess.run(
+            ["docker", "login", registry_url, "-u", username, "-p", password],
+            check=True,
+            text=True,
+        )
+        logger.info(f"Successfully logged in to {registry_url}")
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Failed to log in to Docker registry: {e.stderr}")
+
+
+def docker_push_image(tag):
+    """Push a Docker image to the registry."""
+    try:
+        subprocess.run(["docker", "push", tag], check=True, text=True)
+        logger.info(f"Pushed image with tag '{tag}'")
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Failed to push Docker image: {e.stderr}")
+
+
+def process_images(models, registry_url):
+    """Tag and push Docker images."""
+    for model in models:
+        image_name = model["tag"]
+        docker_push_image(image_name)
+
+
+if __name__ == "__main__":
+    registry_url = "cicd.ai4eu-dev.eu:7444"
+    username = os.getenv("DOCKER_USERNAME")
+    password = os.getenv("DOCKER_PASSWORD")
+
+    # Example models list from the previous script
+    models = [
+        {
+            "model_name": "llama3:70b",
+            "tag": "cicd.ai4eu-dev.eu:7444/tutorials/llms/llama3-70b",
+        },
+        {
+            "model_name": "meta-llama-Meta-Llama-3-8B-Instruct",
+            "tag": "cicd.ai4eu-dev.eu:7444/tutorials/llms/llama-3-8bi",
+        },
+        {
+            "model_name": "Mistral-7B-Instruct-v0.3_t2t",
+            "tag": "cicd.ai4eu-dev.eu:7444/tutorials/llms/mistral-7bi",
+        },
+        {
+            "model_name": "mistralai-Mixtral-8x7B-Instruct-v0.1",
+            "tag": "cicd.ai4eu-dev.eu:7444/tutorials/llms/mixtral-8x7b",
+        },
+        {
+            "model_name": "OpenGPT-X-24EU-2.5T-Instruct-17",
+            "tag": "cicd.ai4eu-dev.eu:7444/tutorials/llms/opengpt-x17",
+        },
+        {
+            "model_name": "OpenGPT-X-24EU-2.5T-Instruct-ENDEFRITES",
+            "tag": "cicd.ai4eu-dev.eu:7444/tutorials/llms/opengpt-endefrites",
+        },
+    ]
+
+    docker_login(registry_url, username, password)
+    process_images(models, registry_url)

llm_docker_generator/generate_containers.py

+import os
+import shutil
+import subprocess
+import logging
+from jinja2 import Environment, FileSystemLoader, TemplateNotFound
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+# Create a Jinja2 environment and load the templates
+env = Environment(loader=FileSystemLoader("templates"))
+
+
+def create_file_from_template(template_name, context, output_path):
+    """Render a template and write the result to a file."""
+    try:
+        template = env.get_template(template_name)
+        content = template.render(context)
+        with open(output_path, "w") as f:
+            f.write(content)
+        logger.info(f"Written to {output_path}")
+    except TemplateNotFound:
+        logger.error(f"Template '{template_name}' not found.")
+    except IOError as e:
+        logger.error(f"Error writing to {output_path}: {e}")
+
+
+def create_files(model_name, temp_dir):
+    """Create files from templates in the specified directory."""
+    templates = {
+        "app.py.j2": "app.py",
+        "server.py.j2": "server.py",
+        "index.html.j2": os.path.join("templates", "index.html"),
+        "llms.proto.j2": "llms.proto",
+        "Dockerfile.j2": "Dockerfile",
+        "requirements.txt.j2": "requirements.txt",
+        "IAIS_llm.py.j2": "IAIS_llm.py",
+    }
+
+    for template_name, output_file in templates.items():
+        output_path = os.path.join(temp_dir, output_file)
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+        create_file_from_template(
+            template_name, {"model_name": model_name}, output_path
+        )
+
+
+def build_docker_image(tag, build_path):
+    """Build a Docker image with the given tag."""
+    try:
+        build_output = subprocess.run(
+            ["docker", "build", "-t", tag, build_path],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        logger.info(f"Docker image '{tag}' built successfully.")
+        return True
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Error during Docker build: {e.stderr}")
+        return False
+
+
+def list_docker_images(tags):
+    """List Docker images matching the provided tags."""
+    if not tags:
+        logger.info("No Docker images to list.")
+        return
+
+    try:
+        result = subprocess.run(
+            [
+                "docker",
+                "images",
+                "--format",
+                "{{.Repository}}:{{.Tag}} {{.ID}} {{.CreatedAt}} {{.Size}}",
+            ],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        images = result.stdout.splitlines()
+
+        if not images:
+            logger.info("No Docker images found.")
+            return
+
+        logger.info("Listing Docker images:")
+        for tag in tags:
+            matched_images = [image for image in images if tag in image]
+            if matched_images:
+                for image in matched_images:
+                    logger.info(f"Image with tag '{tag}': {image}")
+            else:
+                logger.info(f"No Docker images found with tag '{tag}'")
+
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Error occurred while listing Docker images: {e.stderr}")
+
+
+def clean_up(temp_root):
+    """Remove the temporary root directory."""
+    try:
+        shutil.rmtree(temp_root)
+        logger.info(f"Removed temporary directory {temp_root}")
+    except OSError as e:
+        logger.error(f"Error removing {temp_root}: {e}")
+
+
+def build_and_list_containers(models):
+    """Build Docker images for a list of models and list them."""
+    temp_root = "LLM_folder"
+    os.makedirs(temp_root, exist_ok=True)
+    image_tags = []
+
+    for model in models:
+        model_tag = model["tag"]
+        model_name = model["model_name"]
+        temp_dir = os.path.join(
+            temp_root, model_tag.split("/")[-1]
+        )  # Use a clean directory name
+        os.makedirs(temp_dir, exist_ok=True)
+
+        create_files(model_name, temp_dir)
+
+        if build_docker_image(model_tag, temp_dir):
+            image_tags.append(model_tag)
+
+        shutil.rmtree(temp_dir)
+
+    clean_up(temp_root)
+    list_docker_images(image_tags)
+
+
+if __name__ == "__main__":
+    docker_tag_cicd = "cicd.ai4eu-dev.eu:7444/tutorials/llms/"
+    models = [
+        {"model_name": "llama3:70b", "tag": docker_tag_cicd + "llama3-70b"},
+        {
+            "model_name": "meta-llama-Meta-Llama-3-8B-Instruct",
+            "tag": docker_tag_cicd + "llama-3-8bi",
+        },
+        {
+            "model_name": "Mistral-7B-Instruct-v0.3_t2t",
+            "tag": docker_tag_cicd + "mistral-7bi",
+        },
+        {
+            "model_name": "mistralai-Mixtral-8x7B-Instruct-v0.1",
+            "tag": docker_tag_cicd + "mixtral-8x7b",
+        },
+        {
+            "model_name": "OpenGPT-X-24EU-2.5T-Instruct-17",
+            "tag": docker_tag_cicd + "opengpt-x17",
+        },
+        {
+            "model_name": "OpenGPT-X-24EU-2.5T-Instruct-ENDEFRITES",
+            "tag": docker_tag_cicd + "opengpt-endefrites",
+        },
+    ]
+
+    build_and_list_containers(models)

llm_docker_generator/templates/Dockerfile.j2

+FROM python:3.10
+
+RUN apt-get update -y 
+RUN apt-get install -y python3-pip python3-dev 
+RUN pip3 install --upgrade pip
+COPY requirements.txt .
+RUN pip3 install -r requirements.txt
+
+ARG TIMEZONE=Europe/Berlin
+RUN ln -sf /usr/share/zoneinfo/$TIMEZONE /etc/localtime && echo "$TIMEZONE" > /etc/timezone
+
+RUN mkdir /llms
+COPY . /llms
+WORKDIR /llms
+RUN python3 -m grpc_tools.protoc --python_out=. --proto_path=. --grpc_python_out=. llms.proto
+
+ENTRYPOINT python3 -u server.py
\ No newline at end of file

llm_docker_generator/templates/IAIS_llm.py.j2

+# To be updated
\ No newline at end of file

llm_docker_generator/templates/app.py.j2

+from flask import Flask, render_template, request, jsonify
+from IAIS_llm import IAIS_LLM
+
+app = Flask(__name__)
+app.secret_key = "llms"
+
+llm = IAIS_LLM(model_name='{{ model_name }}')
+
+@app.route('/')
+def display_model():
+    model_name = "{{ model_name }}"
+    return render_template('index.html', model_name=model_name)
+
+@app.route('/invoke_llm', methods=['POST'])
+def invoke_llm():
+    data = request.get_json()
+    prompt = data.get('prompt', '')
+    response_text = llm.invoke(prompt)
+    return jsonify({'response': response_text})
+
+def app_run():
+    app.run(host="0.0.0.0", port=8062, debug=False)
\ No newline at end of file

llm_docker_generator/templates/index.html.j2

+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>LLM Invocation</title>
+    <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
+    <style>
+        body {
+            background-color: #f0f8ff; /* AliceBlue background color */
+            font-family: Arial, sans-serif;
+        }
+        h1 {
+            color: #333;
+        }
+        form {
+            margin: 20px 0;
+        }
+        label {
+            margin-right: 10px;
+        }
+        input[type="text"] {
+            padding: 5px;
+            font-size: 14px;
+        }
+        button {
+            padding: 5px 10px;
+            font-size: 14px;
+            background-color: #4CAF50;
+            color: white;
+            border: none;
+            cursor: pointer;
+        }
+        button:hover {
+            background-color: #45a049;
+        }
+        #response {
+            margin-top: 20px;
+            font-size: 16px;
+            color: #333;
+        }
+    </style>
+</head>
+<body>
+    <h1>Model: {{ model_name }}</h1>
+    <form id="llm-form">
+        <label for="prompt">Enter your prompt:</label>
+        <input type="text" id="prompt" name="prompt">
+        <button type="submit">Submit</button>
+    </form>
+    <div id="response"></div>
+
+    <script>
+        $(document).ready(function(){
+            $('#llm-form').on('submit', function(event){
+                event.preventDefault();
+                var prompt = $('#prompt').val();
+                $.ajax({
+                    url: '/invoke_llm',
+                    type: 'POST',
+                    contentType: 'application/json',
+                    data: JSON.stringify({ prompt: prompt }),
+                    success: function(response) {
+                        $('#response').text(response.response);
+                    },
+                    error: function(error) {
+                        $('#response').text('Error invoking LLM.');
+                    }
+                });
+            });
+        });
+    </script>
+</body>
+</html>

llm_docker_generator/templates/llms.proto.j2

+syntax = "proto3";
+
+message PromptInput {
+    string system = 1;
+    string user = 2;
+    string context = 3;
+    string prompt = 4;
+}
+
+message LLMConfig {
+    double temp = 1;
+}
+
+message LLMQuery {
+    LLMConfig config = 1;
+    PromptInput input = 2;
+}
+
+message LLMAnswer {
+    string text = 1;
+}
+
+service LLMService {
+    rpc instruct_llm(LLMQuery) returns(LLMAnswer);
+}
\ No newline at end of file

llm_docker_generator/templates/requirements.txt.j2

+grpcio==1.38.0
+grpcio-tools==1.38.0
+grpc-interceptor
+protobuf==3.16.0
+multithreading
+
+Bootstrap-Flask
+Flask
+Flask-WTF
+flask_session
+WTForms
+requests
+python-dotenv
+
+langchain
+langchain_community 
\ No newline at end of file

llm_docker_generator/templates/server.py.j2

+import grpc
+from concurrent import futures
+import logging
+import threading
+
+from IAIS_llm import IAIS_LLM
+import llms_pb2
+import llms_pb2_grpc
+
+from app import app_run
+
+llm = IAIS_LLM(model_name='{{ model_name }}')
+
+class LLMServiceServicer(llms_pb2_grpc.LLMServiceServicer):
+
+    def instruct_llm(self, request, context):
+        if request.input.prompt:
+            response_text = llm.invoke('List some vegetables rich in iron-')
+        else:
+            response_text = llm.invoke(request.input.user)
+            
+        return llms_pb2.LLMAnswer(text=response_text)
+
+def serve(port):
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    llms_pb2_grpc.add_LLMServiceServicer_to_server(LLMServiceServicer(), server)
+    server.add_insecure_port("[::]:{}".format(port))
+    print("Starting server. Listening on port : " + str(port))
+    server.start()
+    threading.Thread(target=app_run).start()
+    server.wait_for_termination()
+
+if __name__ == "__main__":
+    port = 8061
+    serve(port)
\ No newline at end of file