"use strict";(self.webpackChunkworkshop=self.webpackChunkworkshop||[]).push([[651],{186:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-round-robin-a7c46a4330a88a33482a5ffcbca3e413.gif"},615:(e,n,i)=>{i.r(n),i.d(n,{assets:()=>l,contentTitle:()=>o,default:()=>h,frontMatter:()=>t,metadata:()=>s,toc:()=>c});const s=JSON.parse('{"id":"develop/azure-openai/dynamic-failover","title":"Ensure resiliency and optimized resource consumption with load balancer & circuit breaker","description":"When the number of users increase to the point that one region or server where the application have trouble responding to requests in a reasonable time. This creates a user experience where the app feels slow. To avoid this poor user experience, load balancing can be used. With \\"load balancing\\" you set up multiple endpoints capable of serving requests and additionaly configure a scheme for how the \\"balancing\\" should happen.","source":"@site/docs/develop/azure-openai/dynamic-failover.md","sourceDirName":"develop/azure-openai","slug":"/develop/azure-openai/dynamic-failover","permalink":"/docs/develop/azure-openai/dynamic-failover","draft":false,"unlisted":false,"tags":[],"version":"current","sidebarPosition":4,"frontMatter":{"sidebar_position":4},"sidebar":"tutorialSidebar","previous":{"title":"Keep visibility into AI consumption with model monitoring","permalink":"/docs/develop/azure-openai/track-consumption"},"next":{"title":"Agents","permalink":"/docs/category/agents"}}');var r=i(4848),a=i(8453);const t={sidebar_position:4},o="Ensure resiliency and optimized resource consumption with load balancer & circuit breaker",l={},c=[{value:"Scenario: The Challenge of LLM Resiliency",id:"scenario-the-challenge-of-llm-resiliency",level:2},{value:"Video",id:"video",level:2},{value:"Solution: Backend Pool Load Balancing in Azure API Management",id:"solution-backend-pool-load-balancing-in-azure-api-management",level:2},{value:"Understanding the Prioritized PTU with Fallback Scenario",id:"understanding-the-prioritized-ptu-with-fallback-scenario",level:3},{value:"Exercise - Set up load balancing",id:"exercise---set-up-load-balancing",level:2},{value:"-1- Configure API Management (Backend Pool)",id:"-1--configure-api-management-backend-pool",level:3},{value:"-2- Import Azure Open AI to your Azure API Management instance",id:"-2--import-azure-open-ai-to-your-azure-api-management-instance",level:3},{value:"Exercise: Test your Load Balancing Setup",id:"exercise-test-your-load-balancing-setup",level:2},{value:"-1- Test round-robin",id:"-1--test-round-robin",level:3},{value:"-2- Test Priority-Based Load Balancing",id:"-2--test-priority-based-load-balancing",level:3},{value:"-3- Test Weighted Load Balancing",id:"-3--test-weighted-load-balancing",level:3},{value:"Conclusion",id:"conclusion",level:2},{value:"Additional Resources",id:"additional-resources",level:2},{value:"Infrastructure as Code",id:"infrastructure-as-code",level:2}];function d(e){const n={a:"a",admonition:"admonition",blockquote:"blockquote",code:"code",em:"em",h1:"h1",h2:"h2",h3:"h3",header:"header",img:"img",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,a.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsx)(n.header,{children:(0,r.jsx)(n.h1,{id:"ensure-resiliency-and-optimized-resource-consumption-with-load-balancer--circuit-breaker",children:"Ensure resiliency and optimized resource consumption with load balancer & circuit breaker"})}),"\n",(0,r.jsx)(n.p,{children:'When the number of users increase to the point that one region or server where the application have trouble responding to requests in a reasonable time. This creates a user experience where the app feels slow. To avoid this poor user experience, load balancing can be used. With "load balancing" you set up multiple endpoints capable of serving requests and additionaly configure a scheme for how the "balancing" should happen.'}),"\n",(0,r.jsx)(n.h2,{id:"scenario-the-challenge-of-llm-resiliency",children:"Scenario: The Challenge of LLM Resiliency"}),"\n",(0,r.jsx)(n.p,{children:"When building AI applications that rely on Large Language Models (LLMs), ensuring reliable and cost-effective access to these models presents several challenges:"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Service Availability:"})," ",(0,r.jsx)(n.em,{children:"What happens when an Azure OpenAI service experiences downtime or throttling?"})]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Regional Reliability:"})," ",(0,r.jsx)(n.em,{children:"How do you maintain operations if a specific Azure region faces issues?"})]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Cost Optimization:"})," ",(0,r.jsx)(n.em,{children:"How can you balance between reserved capacity (Provisioned Throughput Units PTU) and consumption-based pricing?"})]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Performance:"})," ",(0,r.jsx)(n.em,{children:"How can you ensure consistent response times for users regardless of backend load?"})]}),"\n"]}),"\n",(0,r.jsx)(n.p,{children:"Without proper management, these issues can lead to application failures, inconsistent user experiences, and unexpected costs. When your primary OpenAI service fails, your application users typically receive error messages until the backend issue resolves - a frustrating experience especially for mission-critical applications."}),"\n",(0,r.jsx)(n.h2,{id:"video",children:"Video"}),"\n",(0,r.jsx)("iframe",{width:"560",height:"315",src:"https://www.youtube.com/embed/Y8bBtwb2MTs?si=Gz8tNscg73AhD2K9",title:"YouTube video player",frameborder:"0",allow:"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share",referrerpolicy:"strict-origin-when-cross-origin",allowfullscreen:!0}),"\n",(0,r.jsx)(n.h2,{id:"solution-backend-pool-load-balancing-in-azure-api-management",children:"Solution: Backend Pool Load Balancing in Azure API Management"}),"\n",(0,r.jsx)(n.p,{children:"Azure API Management (APIM) provides a powerful solution through its backend pool functionality that allows you to:"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Distribute Load:"})," Route requests across multiple Azure OpenAI services"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Implement Failover:"})," Automatically redirect traffic when primary services are unavailable"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Optimize Resource Usage:"})," Prioritize dedicated capacity before falling back to consumption-based instances"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Manage Traffic Intelligently:"})," Use priority and weight-based routing for fine-grained control"]}),"\n"]}),"\n",(0,r.jsx)(n.h3,{id:"understanding-the-prioritized-ptu-with-fallback-scenario",children:"Understanding the Prioritized PTU with Fallback Scenario"}),"\n",(0,r.jsxs)(n.p,{children:["The pattern we'll implement is a ",(0,r.jsx)(n.strong,{children:"Prioritized PTU with Fallback Consumption"})," scenario with:"]}),"\n",(0,r.jsxs)(n.ol,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Primary Tier (Priority 1):"})," A Provisioned Throughput Unit (PTU) Azure OpenAI service that offers dedicated capacity at a fixed cost"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Secondary Tier (Priority 2):"})," Multiple consumption-based Azure OpenAI services that you only pay for when used"]}),"\n"]}),"\n",(0,r.jsx)(n.p,{children:"This pattern will allow you to:"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsx)(n.li,{children:"Maximize utilization of your pre-paid PTU capacity"}),"\n",(0,r.jsx)(n.li,{children:"Automatically scale to consumption-based instances when demand exceeds PTU capacity"}),"\n",(0,r.jsx)(n.li,{children:"Maintain service availability even if one or more regions experience issues"}),"\n"]}),"\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.img,{alt:"backend pool load balancing",src:i(7356).A+"",width:"960",height:"366"})}),"\n",(0,r.jsx)(n.h2,{id:"exercise---set-up-load-balancing",children:"Exercise - Set up load balancing"}),"\n",(0,r.jsx)(n.admonition,{type:"important",children:(0,r.jsxs)(n.p,{children:["Make sure you have completed the lesson on ",(0,r.jsx)(n.a,{href:"/docs/develop/azure-openai/create-resources",children:"setting up cloud resources"})," before continuing."]})}),"\n",(0,r.jsx)(n.p,{children:'To make load balancing happen, we need at least two instances of Azure Open AI, an Azure API Management instance and configuration instructing how the "balancing" should happen.'}),"\n",(0,r.jsx)(n.h3,{id:"-1--configure-api-management-backend-pool",children:"-1- Configure API Management (Backend Pool)"}),"\n",(0,r.jsxs)(n.ol,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["First you'll need to get the ",(0,r.jsx)(n.strong,{children:"Endpoint URLs"})," for each of your Azure OpenAI services."]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["Navigate to your Azure OpenAI resource and click on ",(0,r.jsx)(n.strong,{children:"Endpoints"}),", then copy the ",(0,r.jsx)(n.strong,{children:"Endpoint"})," and save it for a later step."]}),"\n",(0,r.jsx)(n.li,{children:"Do this for all three Azure OpenAI services."}),"\n"]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Navigate to your API Management service and expand the ",(0,r.jsx)(n.strong,{children:"APIs"})," section"]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["You will now add the Azure OpenAI services as backends to your API Management service. Click on ",(0,r.jsx)(n.strong,{children:"Backends"})," and then ",(0,r.jsx)(n.strong,{children:"+ Create new backend"}),"\n",(0,r.jsx)(n.img,{alt:"backend pool load balancing create backend",src:i(6023).A+"",width:"800",height:"646"})]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["Enter a ",(0,r.jsx)(n.em,{children:"Name"})," (e.g., ",(0,r.jsx)(n.code,{children:"openai-eastus"}),")"]}),"\n",(0,r.jsxs)(n.li,{children:["Select ",(0,r.jsx)(n.strong,{children:"Custom URL"})," as the ",(0,r.jsx)(n.em,{children:"Backend hosting type"})]}),"\n",(0,r.jsxs)(n.li,{children:["Paste in the ",(0,r.jsx)(n.strong,{children:"Endpoint URL"})," you copied earlier"]}),"\n"]}),"\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.strong,{children:"Backend pool"})}),"\n",(0,r.jsx)(n.p,{children:"Skip this for now, as we will create a load balanced pool in the next step"}),"\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.strong,{children:"Circuit breaker rule"})}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["Click on ",(0,r.jsx)(n.strong,{children:"+ Create new"})," for the ",(0,r.jsx)(n.em,{children:"Add a circuit breaker rule"})," option","\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["Enter a ",(0,r.jsx)(n.em,{children:"Name"})," (e.g., ",(0,r.jsx)(n.code,{children:"openAIBreakerRule"}),")"]}),"\n",(0,r.jsxs)(n.li,{children:["Leave ",(0,r.jsx)(n.em,{children:"Failure count"})," as ",(0,r.jsx)(n.strong,{children:"1"})]}),"\n",(0,r.jsxs)(n.li,{children:["Set ",(0,r.jsx)(n.em,{children:"Failure interval"})," to ",(0,r.jsx)(n.strong,{children:"5 minutes"}),"\n",(0,r.jsx)(n.img,{alt:"backend pool load balancing circuit breaker rule name",src:i(702).A+"",width:"1024",height:"975"})]}),"\n",(0,r.jsxs)(n.li,{children:["Specify ",(0,r.jsx)(n.em,{children:"Custom range"})," as ",(0,r.jsx)(n.strong,{children:"429"})]}),"\n",(0,r.jsxs)(n.li,{children:["Set ",(0,r.jsx)(n.em,{children:"Trip duration"})," to ",(0,r.jsx)(n.strong,{children:"1 minute"})]}),"\n",(0,r.jsxs)(n.li,{children:["Check ",(0,r.jsx)(n.strong,{children:"True (Accept)"})," for the ",(0,r.jsx)(n.em,{children:"Check 'Retry-After' header in HTTP reponse"}),"\n",(0,r.jsx)(n.img,{alt:"backend pool load balancing circuit breaker range",src:i(1679).A+"",width:"1514",height:"1442"})]}),"\n"]}),"\n"]}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Click on ",(0,r.jsx)(n.strong,{children:"Create"})," to add the Azure OpenAI service as a backend"]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"Repeat the above steps for the other two Azure OpenAI services, ensuring you set the same circuit breaker rule for each backend"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.strong,{children:"Backend 2"})}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["Name: ",(0,r.jsx)(n.code,{children:"openai-westus"})]}),"\n",(0,r.jsxs)(n.li,{children:["Circuit breaker rule: ",(0,r.jsx)(n.code,{children:"openAIBreakerRule"})]}),"\n"]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.strong,{children:"Backend 3"})}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["Name: ",(0,r.jsx)(n.code,{children:"openai-swedencentral"})]}),"\n",(0,r.jsxs)(n.li,{children:["Circuit breaker rule: ",(0,r.jsx)(n.code,{children:"openAIBreakerRule"})]}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.img,{alt:"backend pool load balancing all backends",src:i(6389).A+"",width:"2796",height:"1148"})}),"\n"]}),"\n"]}),"\n",(0,r.jsxs)(n.ol,{start:"4",children:["\n",(0,r.jsxs)(n.li,{children:["Next, you will need to create a ",(0,r.jsx)(n.strong,{children:"Load Balanced Pool"})," for your Azure OpenAI services. Click on ",(0,r.jsx)(n.strong,{children:"Load balancer"})," and then ",(0,r.jsx)(n.strong,{children:"+ Create new pool"}),"\n",(0,r.jsx)(n.img,{alt:"backend pool load balancing create load balanced pool",src:i(4965).A+"",width:"1898",height:"1532"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["Enter a ",(0,r.jsx)(n.em,{children:"Name"})," (e.g., ",(0,r.jsx)(n.code,{children:"openai-backend-pool"}),")"]}),"\n",(0,r.jsxs)(n.li,{children:["Check all three backends you created earlier for the ",(0,r.jsx)(n.em,{children:"Add backends to pool"})," option"]}),"\n",(0,r.jsxs)(n.li,{children:["Leave the ",(0,r.jsx)(n.em,{children:"Backend weight and priority"})," as ",(0,r.jsx)(n.strong,{children:"Send requests evenly"})," to distribute requests evenly across all backends (Round Robin)"]}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,r.jsx)(n.h3,{id:"-2--import-azure-open-ai-to-your-azure-api-management-instance",children:"-2- Import Azure Open AI to your Azure API Management instance"}),"\n",(0,r.jsxs)(n.ol,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["In your API Management service, click on ",(0,r.jsx)(n.strong,{children:"APIs"}),", scroll to ",(0,r.jsx)(n.strong,{children:"Create from Azure resource"})," and then select ",(0,r.jsx)(n.strong,{children:"Azure OpenAI Service"}),"\n",(0,r.jsx)(n.img,{alt:"backend pool load balancing import from AOI",src:i(1400).A+"",width:"2504",height:"1426"})]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["Select an ",(0,r.jsx)(n.em,{children:"Azure OpenAI instance"})," (e.g., ",(0,r.jsx)(n.code,{children:"wrsh-openai-eastus"}),")"]}),"\n",(0,r.jsxs)(n.li,{children:["Select ",(0,r.jsx)(n.strong,{children:"2024-02-01"})," as the ",(0,r.jsx)(n.em,{children:"Azure OpenAI API version"})]}),"\n",(0,r.jsxs)(n.li,{children:["Enter a ",(0,r.jsx)(n.em,{children:"Display name"})," (e.g., ",(0,r.jsx)(n.code,{children:"OpenAI"}),")"]}),"\n",(0,r.jsxs)(n.li,{children:["Check the ",(0,r.jsx)(n.strong,{children:"Improve SDK compatibility"})," option, which will postfix the base url with ",(0,r.jsx)(n.code,{children:"/openai"}),"\n",(0,r.jsx)(n.img,{alt:"backend pool load balancing import from AOI config",src:i(9156).A+"",width:"2072",height:"1408"})]}),"\n",(0,r.jsxs)(n.li,{children:["Click on ",(0,r.jsx)(n.strong,{children:"Review + Create"})," and then ",(0,r.jsx)(n.strong,{children:"Create"})]}),"\n"]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["You now need to configure API Policies for the API you just created. Click on ",(0,r.jsx)(n.strong,{children:"APIs"})," and select the API you just created (e.g., ",(0,r.jsx)(n.code,{children:"OpenAI"}),")"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Select ",(0,r.jsx)(n.strong,{children:"Design"}),", then ",(0,r.jsx)(n.strong,{children:"Inbound processing"}),' and click on the "Policy Code Editor" icon to open the policy editor. Replace the existing policy with the following code:']}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-xml",children:'<policies>\n    <inbound>\n        <base />\n        <authentication-managed-identity resource="https://cognitiveservices.azure.com" output-token-variable-name="managed-id-access-token" ignore-error="false" />\n        <set-header name="Authorization" exists-action="override">\n            <value>@("Bearer " + (string)context.Variables["managed-id-access-token"])</value>\n        </set-header>\n        <set-backend-service backend-id="openai-backend-pool" />\n    </inbound>\n    <backend>\n        \x3c!--Set count to one less than the number of backends in the pool to try all backends until the backend pool is temporarily unavailable.--\x3e\n        <retry count="2" interval="0" first-fast-retry="true" condition="@(context.Response.StatusCode == 429 || (context.Response.StatusCode == 503 && !context.Response.StatusReason.Contains("Backend pool") && !context.Response.StatusReason.Contains("is temporarily unavailable")))">\n            <forward-request buffer-request-body="true" />\n        </retry>\n    </backend>\n    <outbound>\n        <base />\n    </outbound>\n    <on-error>\n        <base />\n        <choose>\n            \x3c!--Return a generic error that does not reveal backend pool details.--\x3e\n            <when condition="@(context.Response.StatusCode == 503)">\n                <return-response>\n                    <set-status code="503" reason="Service Unavailable" />\n                </return-response>\n            </when>\n        </choose>\n    </on-error>\n</policies>\n'})}),"\n",(0,r.jsx)(n.p,{children:"This policy does the following:"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsx)(n.li,{children:"Uses a managed identity to authenticate with the Azure OpenAI service"}),"\n",(0,r.jsx)(n.li,{children:"Sets the backend service to the load balanced pool you created earlier"}),"\n",(0,r.jsx)(n.li,{children:"Implements a retry policy that retries requests to the backend if it receives a 429 (Too Many Requests) status code or a 503 (Service Unavailable) status code"}),"\n"]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Click on ",(0,r.jsx)(n.strong,{children:"Save"})," to save the policy"]}),"\n"]}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,r.jsx)(n.h2,{id:"exercise-test-your-load-balancing-setup",children:"Exercise: Test your Load Balancing Setup"}),"\n",(0,r.jsx)(n.h3,{id:"-1--test-round-robin",children:"-1- Test round-robin"}),"\n",(0,r.jsx)(n.p,{children:"Round-robin load balancing is the default behavior of Azure API Management. API Management will distribute requests evenly across all backends in the pool. To test this,"}),"\n",(0,r.jsxs)(n.ol,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:'In your APIM instance, go to "APIs" and select your "OpenAI" API'}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Select the ",(0,r.jsx)(n.code,{children:"POST Creates a completion for the chat"})," operation"]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Select ",(0,r.jsx)(n.strong,{children:"Test"})," and for"]}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.strong,{children:"Template parameters"}),":"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"deployment-id"}),": ",(0,r.jsx)(n.code,{children:"gpt-4o-mini"})]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"api-version"}),": ",(0,r.jsx)(n.code,{children:"2024-02-01"}),"\n",(0,r.jsx)(n.img,{alt:"backend pool load balancing test template parameters",src:i(690).A+"",width:"2424",height:"1208"})]}),"\n"]}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.strong,{children:"Request body"}),":"]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-json",children:'{\n    "messages": [\n        {"role": "system", "content": "You are a sarcastic, unhelpful assistant."},\n        {"role": "user", "content": "What is the weather like today?"}\n    ]\n}\n'})}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Click on ",(0,r.jsx)(n.strong,{children:"Send"}),' to send the request. Run the test multiple times and observe the "x-ms-region" header in the response. You\'ll notice that the requests are being distributed evenly across all three backends in the pool (East US, West US, and Sweden Central).\n',(0,r.jsx)(n.img,{alt:"backend pool load balancing round robin test",src:i(186).A+"",width:"1920",height:"1080"})]}),"\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.img,{alt:"backend pool load balancing round robin result",src:i(5172).A+"",width:"1481",height:"749"})}),"\n"]}),"\n"]}),"\n",(0,r.jsx)(n.h3,{id:"-2--test-priority-based-load-balancing",children:"-2- Test Priority-Based Load Balancing"}),"\n",(0,r.jsx)(n.p,{children:"Priority-based routing ensures backends with lower priority values (high priority) get traffic first. Only when higher-priority backends are unavailable or overloaded will traffic flow to lower-priority backends."}),"\n",(0,r.jsxs)(n.blockquote,{children:["\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.strong,{children:"Note:"})}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsx)(n.li,{children:"Higher weight values receive proportionally more traffic"}),"\n",(0,r.jsx)(n.li,{children:"Used to distribute load between equally important backends"}),"\n",(0,r.jsx)(n.li,{children:"Can be adjusted to account for different backend capacities"}),"\n"]}),"\n"]}),"\n",(0,r.jsx)(n.p,{children:"For example, if you were to set the following priorities:"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"East US"}),": Priority 1"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"West US"}),": Priority 2"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Sweden Central"}),": Priority 3"]}),"\n"]}),"\n",(0,r.jsx)(n.p,{children:"The expected behavior would be that requests would be sent to the East US backend first. If it is unavailable, requests would then be sent to the West US backend, and finally to the Sweden Central backend."}),"\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.img,{alt:"backend pool load balancing priority based load balancing results",src:i(4507).A+"",width:"1470",height:"749"})}),"\n",(0,r.jsx)(n.h3,{id:"-3--test-weighted-load-balancing",children:"-3- Test Weighted Load Balancing"}),"\n",(0,r.jsx)(n.p,{children:"Weighted load balancing allows you to assign different weights to each backend in the pool. This means that some backends will receive more traffic than others based on their assigned weight. Within the same priority level, weights determine the proportion of traffic each backend receives:"}),"\n",(0,r.jsx)(n.p,{children:"To test this, you will need to update the backend pool configuration to set priorities for each backend:"}),"\n",(0,r.jsxs)(n.ol,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["In your APIM instance, go to ",(0,r.jsx)(n.strong,{children:"Backends"}),", switch to ",(0,r.jsx)(n.strong,{children:"Load balancer"})," and select the ",(0,r.jsx)(n.strong,{children:"openai-backend-pool"})," you created earlier"]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Click on ",(0,r.jsx)(n.strong,{children:"Backends"})," and select the ",(0,r.jsx)(n.strong,{children:"Customize weight and priority"})," option, then set the following:"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"East US"}),": Priority 1"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"West US"}),": Priority 2, Weight 50"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Sweden Central"}),": Priority 2, Weight 50\n",(0,r.jsx)(n.img,{alt:"backend pool load balancing weighted load balancing",src:i(4679).A+"",width:"1751",height:"1196"})]}),"\n"]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Click on ",(0,r.jsx)(n.strong,{children:"Save"})," to save the changes"]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Now return to the ",(0,r.jsx)(n.strong,{children:"APIs"})," section to test the API again. You can use the same test as before, but this time you should see that the requests are being distributed based on the weights you set. The East US backend should receive more traffic than the West US and Sweden Central backends."]}),"\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.img,{alt:"backend pool load balancing weighted load balancing test",src:i(3788).A+"",width:"1920",height:"1080"})}),"\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.img,{alt:"backend pool load balancing weighted load balancing results",src:i(996).A+"",width:"1481",height:"749"})}),"\n"]}),"\n"]}),"\n",(0,r.jsx)(n.h2,{id:"conclusion",children:"Conclusion"}),"\n",(0,r.jsx)(n.p,{children:"You learnt how to set up automated load balancing and failover for your Azure OpenAI services using Azure API Management. This setup allows you to ensure high availability, optimize resource usage, and provide a seamless experience for your users even in the face of backend failures."}),"\n",(0,r.jsx)(n.h2,{id:"additional-resources",children:"Additional Resources"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Docs: ",(0,r.jsx)(n.a,{href:"https://learn.microsoft.com/en-us/azure/api-management/backends?tabs=portal",children:"Load balancing"})]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Blog post: ",(0,r.jsx)(n.a,{href:"https://techcommunity.microsoft.com/blog/azuredevcommunityblog/improve-llm-backend-resiliency-with-load-balancer-and-circuit-breaker-rules-in-a/4394502",children:"Improve LLM backend resiliency with load balancer and circuit breaker rules in Azure API Management"})]}),"\n"]}),"\n"]}),"\n",(0,r.jsx)(n.h2,{id:"infrastructure-as-code",children:"Infrastructure as Code"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["Lab: ",(0,r.jsx)(n.a,{href:"https://github.com/Azure-Samples/AI-Gateway/blob/main/labs/backend-pool-load-balancing/README.MD",children:"load balancing"})]}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,a.R)(),...e.components};return n?(0,r.jsx)(n,{...e,children:(0,r.jsx)(d,{...e})}):d(e)}},690:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-api-template-parameters-fdfb4ae9fb7f3aee5f4cdc5d6ba72404.png"},702:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-circuit-rule-name-3ae1542538365e4034347e9eaa1f6744.png"},996:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-weighed-results-67cb8dfabade89eac99ddce323b26f96.png"},1400:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-import-from-aoi-b8f85c84aa06191bb6ad01bd34497858.png"},1679:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-circuit-range-af90fed90bb45efd439d16030fc718c2.png"},3788:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-weighted-7c8a9795ad20468745e2bb840cd07322.gif"},4507:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-priority-based-results-0e2253e9cf0e79c58634229715bb6704.png"},4679:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-customize-weights-afbe0d2dd213a29d51300b0f5dad03d3.png"},4965:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-create-pool-d714d67f933696f878feb48fefed14c8.png"},5172:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-round-robin-results-c374ac56872934ff4c7cdbb15446c134.png"},6023:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-create-backend-581edb2b78b8fe89664e95e57b3439ec.png"},6389:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-all-backends-2ccca67fca24535359287eac19f9c537.png"},7356:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/backend-pool-load-balancing-c81662b673683e59723959766820f583.gif"},8453:(e,n,i)=>{i.d(n,{R:()=>t,x:()=>o});var s=i(6540);const r={},a=s.createContext(r);function t(e){const n=s.useContext(a);return s.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function o(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(r):e.components||r:t(e.components),s.createElement(a.Provider,{value:n},e.children)}},9156:(e,n,i)=>{i.d(n,{A:()=>s});const s=i.p+"assets/images/wrsh-loadbalancing-import-config-02884f43b0b531fba29d0d4c00b38c49.png"}}]);