"use strict";(self.webpackChunkworkshop=self.webpackChunkworkshop||[]).push([[271],{737:(e,n,t)=>{t.d(n,{A:()=>i});const i=t.p+"assets/images/token-limit-6-188bedc961bd1a79061f5ee881eb4f9f.png"},3454:(e,n,t)=>{t.d(n,{A:()=>i});const i=t.p+"assets/images/rate-limited-59be534d21fabbd4ad9c75a49ae6ee2a.png"},4415:(e,n,t)=>{t.d(n,{A:()=>i});const i=t.p+"assets/images/token-limit-8-269ff401b2f4c084279a6ee29e5c97e1.png"},5767:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>c,contentTitle:()=>a,default:()=>h,frontMatter:()=>r,metadata:()=>i,toc:()=>l});const i=JSON.parse('{"id":"develop/azure-openai/rate-limit","title":"Control cost and performance with token quotas and limits","description":"Once you bring an LLM to production and exposes it as an API Endpoint, you need to consider how you \\"manage\\" such an API. There are many considerations to be made everything from caching, scaling, error management, rate limiting, monitoring and more.","source":"@site/docs/develop/azure-openai/rate-limit.md","sourceDirName":"develop/azure-openai","slug":"/develop/azure-openai/rate-limit","permalink":"/docs/develop/azure-openai/rate-limit","draft":false,"unlisted":false,"tags":[],"version":"current","sidebarPosition":2,"frontMatter":{"sidebar_position":2},"sidebar":"tutorialSidebar","previous":{"title":"Create Azure resources","permalink":"/docs/develop/azure-openai/create-resources"},"next":{"title":"Keep visibility into AI consumption with model monitoring","permalink":"/docs/develop/azure-openai/track-consumption"}}');var o=t(4848),s=t(8453);const r={sidebar_position:2},a="Control cost and performance with token quotas and limits",c={},l=[{value:"Scenario: Manage your token consumption",id:"scenario-manage-your-token-consumption",level:2},{value:"Video",id:"video",level:2},{value:"How to approach it?",id:"how-to-approach-it",level:2},{value:"Exercise: Import the Azure Open AI instance as API",id:"exercise-import-the-azure-open-ai-instance-as-api",level:2},{value:"-1- Inspect the import",id:"-1--inspect-the-import",level:3},{value:"-2- Configure and test the policy on the API",id:"-2--configure-and-test-the-policy-on-the-api",level:3},{value:"Additional Resources",id:"additional-resources",level:2},{value:"Infrastructure as Code",id:"infrastructure-as-code",level:2}];function d(e){const n={a:"a",admonition:"admonition",code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",img:"img",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,s.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(n.header,{children:(0,o.jsx)(n.h1,{id:"control-cost-and-performance-with-token-quotas-and-limits",children:"Control cost and performance with token quotas and limits"})}),"\n",(0,o.jsx)(n.p,{children:'Once you bring an LLM to production and exposes it as an API Endpoint, you need to consider how you "manage" such an API. There are many considerations to be made everything from caching, scaling, error management, rate limiting, monitoring and more.'}),"\n",(0,o.jsx)(n.p,{children:"In this lesson we will use the Azure service, Azure API Management and show how by adding one of its policies to an LLM endpoint; you can control the usage of tokens."}),"\n",(0,o.jsx)(n.h2,{id:"scenario-manage-your-token-consumption",children:"Scenario: Manage your token consumption"}),"\n",(0,o.jsx)(n.p,{children:"Manage your token consumption is important for many reasons:"}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"Cost"}),", you want to stay on top of how much token you spend as this ultimately decides how much you are charged."]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"Fairness"}),". You want to ensure your apps gets a fair amount of token. This also leads to a better user experience as the end user will be able to get a response when they expect."]}),"\n"]}),"\n",(0,o.jsx)(n.h2,{id:"video",children:"Video"}),"\n",(0,o.jsx)("iframe",{width:"560",height:"315",src:"https://www.youtube.com/embed/tc-rUS_-FN0?si=TN6V6JYoLpQ9qnAM",title:"YouTube video player",frameborder:"0",allow:"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share",referrerpolicy:"strict-origin-when-cross-origin",allowfullscreen:!0}),"\n",(0,o.jsx)(n.h2,{id:"how-to-approach-it",children:"How to approach it?"}),"\n",(0,o.jsx)(n.p,{children:"To control token spend, here's how we need to implement this at high level:"}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:["Create an Azure Open AI instance.","\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsx)(n.li,{children:"Create a deployment for above instance."}),"\n"]}),"\n"]}),"\n",(0,o.jsxs)(n.li,{children:["Create an Azure API Management instance.","\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsx)(n.li,{children:"Create an API on above Azure API Management instance (we can import it throught the UI)."}),"\n",(0,o.jsx)(n.li,{children:"Map the Aure Open instance to Azure API Management backend instances."}),"\n",(0,o.jsx)(n.li,{children:"Configure the policy on the API manage the token consumption."}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,o.jsx)(n.p,{children:"Now that we understand the plan, let's execute said plan."}),"\n",(0,o.jsx)(n.h2,{id:"exercise-import-the-azure-open-ai-instance-as-api",children:"Exercise: Import the Azure Open AI instance as API"}),"\n",(0,o.jsx)(n.admonition,{type:"important",children:(0,o.jsxs)(n.p,{children:["Make sure you have completed the lesson on ",(0,o.jsx)(n.a,{href:"/docs/develop/azure-openai/create-resources",children:"setting up cloud resources"})," before continuing."]})}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:["\n",(0,o.jsx)(n.p,{children:'Navigate to your newly created Azure API Management instance and select "API" in the menu like indicated in the image:'}),"\n",(0,o.jsx)(n.p,{children:(0,o.jsx)(n.img,{alt:"Import Azure Open AI to Azure API Management",src:t(737).A+"",width:"1476",height:"1269"})}),"\n"]}),"\n",(0,o.jsxs)(n.li,{children:["\n",(0,o.jsx)(n.p,{children:"Select your subscription and provide a name of your choosing."}),"\n",(0,o.jsx)(n.p,{children:(0,o.jsx)(n.img,{alt:"Fill in API details",src:t(9608).A+"",width:"1540",height:"1618"})}),"\n"]}),"\n",(0,o.jsxs)(n.li,{children:["\n",(0,o.jsx)(n.p,{children:'On the next tab, fill in that you want to "Manage token consumption"'}),"\n",(0,o.jsx)(n.p,{children:(0,o.jsx)(n.img,{alt:"Manage token consumption",src:t(4415).A+"",width:"1546",height:"1594"})}),"\n",(0,o.jsx)(n.p,{children:"Great, now it's time to configure the remaining parts, almost there!"}),"\n"]}),"\n"]}),"\n",(0,o.jsx)(n.h3,{id:"-1--inspect-the-import",children:"-1- Inspect the import"}),"\n",(0,o.jsx)(n.p,{children:"When we imported the Azure Open AI API, the following was done for us:"}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:["\n",(0,o.jsx)(n.p,{children:"Managed identity was setup as security. This means all the call you do are more secure and that identity management is handled by Azure. This is our best recommendation for security."}),"\n"]}),"\n",(0,o.jsxs)(n.li,{children:["\n",(0,o.jsx)(n.p,{children:"A token limit policy was set up in for our API looking like so:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-xml",children:'<azure-openai-token-limit \n  tokens-per-minute="1000" \n  counter-key="@(context.Subscription.Id)" \n  estimate-prompt-tokens="true" \n/>\n'})}),"\n",(0,o.jsx)(n.p,{children:"Here we see how:"}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:["We can spend 1000 tokens per minute ",(0,o.jsx)(n.code,{children:'tokens-per-minute="1000"'})]}),"\n",(0,o.jsxs)(n.li,{children:["How this is tracked by Subscription Id ",(0,o.jsx)(n.code,{children:'counter-key="@(context.Subscription.Id)"'})]}),"\n",(0,o.jsx)(n.li,{children:"How it estimates the number of tokens we're using for prompting."}),"\n"]}),"\n"]}),"\n",(0,o.jsxs)(n.li,{children:["\n",(0,o.jsxs)(n.p,{children:["A backend is setup pointing to our Azure Open AI API, you can find it by navigating to  APIs / Backends. The runtime url field should have a value similar to ",(0,o.jsx)(n.code,{children:"https://<name>-aoai.openai.azure.com/openai"}),"."]}),"\n"]}),"\n"]}),"\n",(0,o.jsx)(n.p,{children:"We've shown you the policy above but let's see how it works in the next section."}),"\n",(0,o.jsx)(n.h3,{id:"-2--configure-and-test-the-policy-on-the-api",children:"-2- Configure and test the policy on the API"}),"\n",(0,o.jsx)(n.p,{children:"Now that we have everything setup, let's try see how we can configure the policy so we see how it works. Let's review our configuration:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-xml",children:'<azure-openai-token-limit \n  tokens-per-minute="1000" \n  counter-key="@(context.Subscription.Id)" \n  estimate-prompt-tokens="true" \n/>\n'})}),"\n",(0,o.jsx)(n.p,{children:"In the preceding XML, we need to spend more than 1000 tokens per minute to be rate limited. To make it easy to test, let's change this to 50 tokens and make several consecutive requests, this should make the policy react and stop as from making any further requests with in that minute, so witha policy like so:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-xml",children:'<azure-openai-token-limit \ntokens-per-minute="50" \ncounter-key="@(context.Subscription.Id)" \nestimate-prompt-tokens="true" \n/>\n'})}),"\n",(0,o.jsx)(n.p,{children:"and a JSON payload like so:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{className:"language-json",children:'{"messages":[{"role":"system", "content": "you are a friendly assistant"}, { "role": "user", "content": "how is the weather in London?" }]}\n'})}),"\n",(0,o.jsx)(n.p,{children:'let\'s make a few requests, remember, we expect a 429 error to happen once we exhaust our allowed "tokens-per-minute value"'}),"\n",(0,o.jsx)(n.p,{children:(0,o.jsx)(n.img,{alt:"rate limited",src:t(3454).A+"",width:"1219",height:"852"})}),"\n",(0,o.jsx)(n.p,{children:"If you see the above image, that means the policy is working"}),"\n",(0,o.jsx)(n.h2,{id:"additional-resources",children:"Additional Resources"}),"\n",(0,o.jsx)(n.p,{children:"Here's a list of resources that you might find useful:"}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:["\n",(0,o.jsxs)(n.p,{children:["Docs: ",(0,o.jsx)(n.a,{href:"https://learn.microsoft.com/en-us/azure/api-management/azure-openai-token-limit-policy",children:"Policy docs page"})]}),"\n"]}),"\n",(0,o.jsxs)(n.li,{children:["\n",(0,o.jsxs)(n.p,{children:["Sample: ",(0,o.jsx)(n.a,{href:"https://github.com/Azure-Samples/genai-gateway-apim",children:"Azure Sample"})]}),"\n"]}),"\n",(0,o.jsxs)(n.li,{children:["\n",(0,o.jsxs)(n.p,{children:["Repo (with many demoes): ",(0,o.jsx)(n.a,{href:"https://github.com/Azure-Samples/AI-Gateway",children:"Azure Gateway"})]}),"\n"]}),"\n"]}),"\n",(0,o.jsx)(n.h2,{id:"infrastructure-as-code",children:"Infrastructure as Code"}),"\n",(0,o.jsx)(n.p,{children:"To deploy this in production, you need to specify your cloud resources in Bicep files and use either Azure CLI or Azure Developer CLI, azd. Follow below lab to learn how."}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:["Lab: ",(0,o.jsx)(n.a,{href:"https://github.com/Azure-Samples/AI-Gateway/blob/main/labs/token-rate-limiting/README.MD",children:"Token rate limit Lab"})]}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,s.R)(),...e.components};return n?(0,o.jsx)(n,{...e,children:(0,o.jsx)(d,{...e})}):d(e)}},8453:(e,n,t)=>{t.d(n,{R:()=>r,x:()=>a});var i=t(6540);const o={},s=i.createContext(o);function r(e){const n=i.useContext(s);return i.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function a(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:r(e.components),i.createElement(s.Provider,{value:n},e.children)}},9608:(e,n,t)=>{t.d(n,{A:()=>i});const i=t.p+"assets/images/token-limit-7-252ed0f12562aa04922e86c8acb32b04.png"}}]);