/*
 * Purpose of This File:
 * This file is required for Quarto to render both HTML and MDX files in the `website` folder.
 * These rendered files are essential for Mintlify to function correctly.
 *
 * Context:
 * - The `quarto render` command behaves differently when the `docusaurus.config.js` file is
 *   present in the `website` folder.
 * - Without this file, Quarto does not render MDX files, leading to failures
 *   when running the `process_notebook.py` script.
 * - As a result, this file is retained temporarily until a better solution is identified.
 *
 * Options Tried (and Issues Faced):
 * 1. Adding a `_quarto.yml` file in the `website` folder with the following configurations:
 *
 *    **Minimal Configuration:**
 *    ```
 *    project:
 *      type: docusaurus
 *
 *    format:
 *      html: default
 *      docusaurus-md: default
 *    ```
 *
 *    **Full Configuration from Quarto Repository:**
 *    The configuration from [Quarto's docusaurus extension](https://github.com/quarto-dev/quarto-cli/blob/main/src/resources/extensions/quarto/docusaurus/_extension.yml).
 *
 *    - Both configurations instruct Quarto to render HTML and MDX files.
 *    - However, in both cases, Mintlify fails to generate docs, raising the following error:
 *      > Warning: data for page "/[[...slug]]" (path "/docs/Home") is 56.1 MB which exceeds
 *      > the threshold of 21 MB. This amount of data can reduce performance.
 *    - More info: https://nextjs.org/docs/messages/large-page-data
 */


// Load jQuery if not present
if (typeof jQuery === "undefined") {
  const jqueryScript = document.createElement("script");
  jqueryScript.src =
    "https://cdnjs.cloudflare.com/ajax/libs/jquery/3.7.1/jquery.min.js";
  document.head.appendChild(jqueryScript);

  jqueryScript.onload = function () {
    // Load Chosen after jQuery
    const chosenScript = document.createElement("script");
    chosenScript.src =
      "https://cdnjs.cloudflare.com/ajax/libs/chosen/1.8.7/chosen.jquery.min.js";
    document.head.appendChild(chosenScript);

    // Add Chosen CSS
    const chosenStyles = document.createElement("link");
    chosenStyles.rel = "stylesheet";
    chosenStyles.href =
      "https://cdnjs.cloudflare.com/ajax/libs/chosen/1.8.7/chosen.min.css";
    document.head.appendChild(chosenStyles);

    // Initialize Chosen when everything is loaded
    chosenScript.onload = function () {
      initializeGallerySelect();
    };
  };
}

function getTagsFromURL() {
  const searchParams = new URLSearchParams(window.location.search);
  const tags = searchParams.get("tags");
  return tags ? tags.split(",") : [];
}

// Function to initialize Chosen on gallery select
function initializeGallerySelect() {
  // Add flag to check if already initialized
  const selectElement = $(".examples-gallery-container .tag-filter");
  if (selectElement.length && !selectElement.data("chosen-initialized")) {
    selectElement
      .chosen({
        width: "100%",
        placeholder_text_multiple: "Filter by tags",
      })
      .data("chosen-initialized", true);

    // Get tags from URL and update chosen
    const urlTags = getTagsFromURL();
    if (urlTags.length > 0) {
      selectElement.val(urlTags);
      selectElement.trigger("chosen:updated");

      // Trigger the gallery:tagChange event with URL tags
      const customEvent = new CustomEvent("gallery:tagChange", {
        detail: urlTags,
      });
      document.dispatchEvent(customEvent);
    }

    // Set up change handler
    selectElement.on("change", function (evt, params) {
      const selectedValues = $(this).val() || [];
      const customEvent = new CustomEvent("gallery:tagChange", {
        detail: selectedValues,
      });
      document.dispatchEvent(customEvent);
    });
  }
}

// Debounce function
function debounce(func, wait) {
  let timeout;
  return function executedFunction(...args) {
    const later = () => {
      clearTimeout(timeout);
      func(...args);
    };
    clearTimeout(timeout);
    timeout = setTimeout(later, wait);
  };
}

// Debounced version of initialize
const debouncedInitialize = debounce(initializeGallerySelect, 100);

// Watch for URL changes using MutationObserver
const observer = new MutationObserver((mutations) => {
  if (window.location.pathname.includes("/use-cases")) {
    debouncedInitialize();
  }
});

observer.observe(document.body, {
  childList: true,
  subtree: true,
  attributes: false,
  characterData: false,
});

// Initialize on page load
document.addEventListener("DOMContentLoaded", function () {
  if (window.jQuery && window.jQuery.fn.chosen) {
    initializeGallerySelect();
  }
});


gpt4v_formatter

autogen.agentchat.contrib.img_utils.gpt4v_formatter

Key Features

Quick Start

Community Gallery

Contributing to AG2

Setting up Development Environment

Documentation

File a Bug Report

Guidance for Maintainers

Pre-commit

Tests

Research

Frequently Asked Questions

Agent Monitoring and Debugging with AgentOps

Azure Cosmos DB

Composio

Databricks

Llamaindex

Mem0:Long-Term Memory and Personalization for Agents

MemGPT

Microsoft Fabric

Ollama

PGVector

Portkey Integration

Promptflow

Home

User Guide

API References

Use Cases

User Stories

Contribute

FAQs

Ecosystem

Blog

Installing AG2

ConversableAgent

Human in the loop

Tools

Code Execution

Tools with Secrets

Conversation Patterns Deep-dive

LLM Config Deep-dive

Amazon Bedrock

Anthropic

Cerebras

Cohere

DeepSeek

Google Gemini

Google Vertex AI

Groq

LM Studio

Mistral AI

OpenAI

Together AI

vLLM

Overview

CaptainAgent

Communication Agents

DocAgent

ReasoningAgent

WebSurferAgent

Browser Use

Crawl4AI

Deep Research

Customer Service

Game Design

Travel Planning

All Notebooks

Automatically build multi-agent system with AgentBuilder

AutoBuild

Tools with Dependency Injection

Learn how to persist memories across chat sessions using the Teachability capability

Chatting with a teachable agent

Solve complex tasks with a chat nested as inner monologue.

Solving Complex Tasks with Nested Chats

Use AgentOps to simplify the development process and monitor your agents in production.

Agent Tracking with AgentOps

Config loader utility functions

Websockets facilitate real-time, bidirectional communication between web clients and servers, enhancing the responsiveness and interactivity of AG2-powered applications.

Websockets: Streaming input and output using websockets

A Uniform interface to call different LLMs

OpenAI Assistant with retrieval augmentation.

RAG OpenAI Assistants in AG2

Use tools in a GPTAssistantAgent Multi-Agent System by utilizing functions such as calling an API and writing to a file.

From Dad Jokes To Sad Jokes: Function Calling with GPTAssistantAgent

Agent Chat with custom model loading

Agentic RAG workflow on tabular data from a PDF file

Introduce group chat with customized speaker selection method.

Group Chat with Customized Speaker Selection Method

RealtimeAgent in a Swarm Orchestration using WebRTC

Code generation, execution, debugging and human feedback.

Auto Generated Agent Chat: Task Solving with Code Generation, Execution, Debugging & Human Feedback

Scrapping web pages and summarizing the content using agents with tools.

Web Scraping using Apify Tools

Swarm Orchestration with AG2

Learn how to register function calls using AssistantAgent and UserProxyAgent.

Currency Calculator: Task Solving with Provided Tools as Functions

Agent with memory using Mem0

RealtimeAgent in a Swarm Orchestration

Solve complex tasks with one or more sequence chats nested as inner monologue.

Solving Complex Tasks with A Sequence of Nested Chats

Teach the agent news skills using natural language.

Auto Generated Agent Chat: Teaching AI New Skills via Natural Language Interaction

(Legacy) Implement Swarm-style orchestration with GroupChat

Perform research using a group chat with a number of specialized agents.

Perform Research with Multi-Agent Group Chat

Usage tracking with AG2

Auto Generated Agent Chat: Collaborative Task Solving with Coding and Planning Agent

DeepSeek: Adding Browsing Capabilities to AG2

This Jupyter Notebook demonstrates how to leverage OSS Insight (Open Source Software Insight) for advanced GitHub data analysis by defining `Function calls` in AG2 for the OpenAI Assistant.

Chat with OpenAI Assistant using function call in AG2: OSS Insights for Advanced GitHub Data Analysis

RealtimeAgent with gemini client using websockets

RealtimeAgent with gemini client

Use conversational agents to solve a set of tasks with a sequence of async chats.

Solving Multiple Tasks in a Sequence of Async Chats

OpenAI Assistants in AG2

AgentEval: a multi-agent system for assessing utility of LLM-powered applications

Demonstrating the `AgentEval` framework using the task of solving math problems as an example

Adding Browsing Capabilities to AG2

Implement and manage a multi-agent chat system using AG2, where AI assistants retrieve information, generate code, and interact collaboratively to solve complex tasks, especially in areas not covered by their training data.

Group Chat with Retrieval Augmented Generation

Cross-Framework LLM Tool Integration with AG2

This Jupyter Notebook showcases the integration of the Code Interpreter tool which executes Python code dynamically within applications.

Auto Generated Agent Chat: GPTAssistant with Code Interpreter

Auto Generated Agent Chat: Task Solving with Langchain Provided Tools as Functions

Neo4j GraphRAG utilises a knowledge graph and can be added as a capability to agents.

Using Neo4j's graph database with AG2 agents for Question & Answering

Run a standalone AssistantAgent, browsing the web using the BrowserUseTool

Run a standalone AssistantAgent

StateFlow: Build workflows through state-oriented actions.

StateFlow: Build Workflows through State-Oriented Actions

Explore the use of AG2's RetrieveChat for tasks like code generation from docstrings, answering complex questions with human feedback, and exploiting features like Update Context, custom prompts, and few-shot learning.

Using RetrieveChat for Retrieve Augmented Code Generation and Question Answering

Use ReasoningAgent for o1 style reasoning in Agentic workflows with LLMs using AG2

ReasoningAgent - Advanced LLM Reasoning with Multiple Search Strategies

Generate Dalle Images With Conversable Agents

DeepResearchAgent

Enhanced Swarm Orchestration with AG2

OpenAI offers a functionality for defining a structure of the messages generated by LLMs, AutoGen enables this functionality by propagating response_format passed to your agents to the underlying client.

Structured output from json configuration

Use OpenLIT to easily monitor AI agents in production with OpenTelemetry.

Agent Observability with OpenLIT

Auto Generated Agent Chat: Solving Tasks Requiring Web Info

Use conversable language learning model agents to solve tasks and provide automatic feedback through a comprehensive example of writing, executing, and debugging Python code to compare stock price changes.

Task Solving with Code Generation, Execution and Debugging

RealtimeAgent with WebRTC connection

Use conversational agents to solve a set of tasks with a sequence of chats.

Solving Multiple Tasks in a Sequence of Chats

This notebook demonstrates the usage of QdrantRetrieveUserProxyAgent for RAG.

Using RetrieveChat with Qdrant for Retrieve Augmented Code Generation and Question Answering

Equip your agent with functions that can efficiently implement features into your software application.

Writing a software application using function calls

Provide capabilities of runtime logging for debugging and performance analysis.

Runtime Logging with AG2

Using RetrieveChat Powered by MongoDB Atlas for Retrieve Augmented Code Generation and Question Answering

Using FalkorGraphRagCapability with agents for GraphRAG Question & Answering

Register function calls using AssistantAgent and UserProxyAgent to execute python or shell code in customized ways. Demonstrating two ways of registering functions.

Auto Generated Agent Chat: Task Solving with Provided Tools as Functions

Preprocessing chat history with `TransformMessages`

Preprocessing Chat History with `TransformMessages`

Introducing CaptainAgent, a powerful agent that can manage and orchestrate other agents and tools to solve complex tasks.

Explore the demonstration of the Finite State Machine implementation, which allows the user to input speaker transition constraints.

FSM - User can input speaker transition constraints

This is a nested chat re-implementation of OptiGuide which is an LLM-based supply chain optimization framework.

OptiGuide with Nested Chats in AG2

Explore a group chat example using agents such as a coder and visualization agent.

Group Chat with Coder and Visualization Critic

Use JSON mode and Agent Descriptions to mitigate prompt manipulation and control speaker transition.

Mitigating Prompt hacking with JSON Mode in Autogen

Use tools to extract and translate the transcript of a video file.

Translating Video audio using Whisper and GPT-3.5-turbo

Auto Generated Agent Chat: Group Chat with GPTAssistantAgent

Automated continual learning from new data.

Interactive LLM Agent Dealing with Data Stream

Leveraging multimodal models through two different methodologies: MultimodalConversableAgent and VisionCapability.

Engaging with Multimodal Models: GPT-4V in AG2

Using Guidance with AG2

Natural language text to SQL query using the Spider text-to-SQL benchmark.

SQL Agent for Spider text-to-SQL benchmark

LLM-backed agents playing chess with each other using nested chats.

Conversational Chess using non-OpenAI clients

Explore the utilization of large language models in automated group chat scenarios, where agents perform tasks collectively, demonstrating how they can be configured, interact with each other, and retrieve specific information from external resources.

Group Chat

Function Inception: Enable AG2 agents to update/remove functions during conversations.

Auto Generated Agent Chat: Function Inception

Making OpenAI Assistants Teachable

Use Databricks DBRX and Foundation Model APIs to build AG2 applications backed by open-source LLMs.

Use AG2 in Databricks with DBRX

Agent Chat with Async Human Inputs

Using RetrieveChat Powered by PGVector for Retrieve Augmented Code Generation and Question Answering

Integrate llamaindex agents with Autogen.

Groupchat with Llamaindex agents

Discord, Slack, and Telegram messaging tools

Chat Context Dependency Injection

OpenAI offers a functionality for defining a structure of the messages generated by LLMs, AG2 enables this functionality by propagating response_format passed to your agents to the underlying client.

Structured output

FalkorDB GraphRAG utilises a knowledge graph and can be added as a capability to agents. Together with a swarm orchestration of agents is highly effective at providing a RAG capability.

Name	Description
`prompt`	Type: str
`img_format`	Type: str Default: ‘uri’

API Reference

​gpt4v_formatter

gpt4v_formatter