"use strict";(self.webpackChunkworkshop=self.webpackChunkworkshop||[]).push([[937],{315:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/monitor-enable-5a491de522560345efd6ee394acc1e55.png"},1899:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/monitor-import-setup-dimensions-b5e465316e6af3e6a3e58764c488e1db.png"},2573:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/monitor-import-select-monitor-7194b336adb27e0a569be9290534d375.png"},4176:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>l,contentTitle:()=>a,default:()=>h,frontMatter:()=>r,metadata:()=>s,toc:()=>c});const s=JSON.parse('{"id":"develop/azure-openai/track-consumption","title":"Keep visibility into AI consumption with model monitoring","description":"In this lesson we will use the Azure service, Azure API Management and show how by adding one of its policies to an LLM endpoint; you can monitor the usage of tokens.","source":"@site/docs/develop/azure-openai/track-consumption.md","sourceDirName":"develop/azure-openai","slug":"/develop/azure-openai/track-consumption","permalink":"/docs/develop/azure-openai/track-consumption","draft":false,"unlisted":false,"tags":[],"version":"current","sidebarPosition":3,"frontMatter":{"sidebar_position":3},"sidebar":"tutorialSidebar","previous":{"title":"Control cost and performance with token quotas and limits","permalink":"/docs/develop/azure-openai/rate-limit"},"next":{"title":"Ensure resiliency and optimized resource consumption with load balancer & circuit breaker","permalink":"/docs/develop/azure-openai/dynamic-failover"}}');var i=t(4848),o=t(8453);const r={sidebar_position:3},a="Keep visibility into AI consumption with model monitoring",l={},c=[{value:"Scenario: Monitor your token consumption",id:"scenario-monitor-your-token-consumption",level:2},{value:"Video",id:"video",level:2},{value:"How to approach it?",id:"how-to-approach-it",level:2},{value:"Exercise: Import Azure Open AI as API",id:"exercise-import-azure-open-ai-as-api",level:2},{value:"-1- Enable monitoring on the API",id:"-1--enable-monitoring-on-the-api",level:3},{value:"-2- Inspect the API and policy on the API",id:"-2--inspect-the-api-and-policy-on-the-api",level:3},{value:"Exercise: Test monitoring",id:"exercise-test-monitoring",level:2},{value:"Additional Resources",id:"additional-resources",level:2},{value:"Infrastructure as Code",id:"infrastructure-as-code",level:2}];function d(e){const n={a:"a",admonition:"admonition",code:"code",em:"em",h1:"h1",h2:"h2",h3:"h3",header:"header",img:"img",li:"li",ol:"ol",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,o.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(n.header,{children:(0,i.jsx)(n.h1,{id:"keep-visibility-into-ai-consumption-with-model-monitoring",children:"Keep visibility into AI consumption with model monitoring"})}),"\n",(0,i.jsx)(n.p,{children:"In this lesson we will use the Azure service, Azure API Management and show how by adding one of its policies to an LLM endpoint; you can monitor the usage of tokens."}),"\n",(0,i.jsx)(n.h2,{id:"scenario-monitor-your-token-consumption",children:"Scenario: Monitor your token consumption"}),"\n",(0,i.jsx)(n.p,{children:"Monitor your token consumption is important for many reasons:"}),"\n",(0,i.jsxs)(n.ul,{children:["\n",(0,i.jsxs)(n.li,{children:[(0,i.jsx)(n.strong,{children:"Cost"}),", by seeing how much you spend, you will be able to take decisions to reduce it."]}),"\n",(0,i.jsxs)(n.li,{children:[(0,i.jsx)(n.strong,{children:"Fairness"}),". You want to ensure your apps gets a fair amount of token. This also leads to a better user experience as the end user will be able to get a response when they expect."]}),"\n"]}),"\n",(0,i.jsx)(n.h2,{id:"video",children:"Video"}),"\n",(0,i.jsx)("iframe",{width:"560",height:"315",src:"https://www.youtube.com/embed/2pW6Z2VwHmQ?si=NwKkyTUa17IPhHMm",title:"YouTube video player",frameborder:"0",allow:"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share",referrerpolicy:"strict-origin-when-cross-origin",allowfullscreen:!0}),"\n",(0,i.jsx)(n.h2,{id:"how-to-approach-it",children:"How to approach it?"}),"\n",(0,i.jsxs)(n.ul,{children:["\n",(0,i.jsx)(n.li,{children:"Create an App Insights instance."}),"\n",(0,i.jsx)(n.li,{children:"Import an Azure Open AI instance as an API to your Azure API Management instance."}),"\n",(0,i.jsx)(n.li,{children:"Configure your Azure API Management API and its policy"}),"\n"]}),"\n",(0,i.jsx)(n.h2,{id:"exercise-import-azure-open-ai-as-api",children:"Exercise: Import Azure Open AI as API"}),"\n",(0,i.jsx)(n.admonition,{type:"important",children:(0,i.jsxs)(n.p,{children:["Make sure you have completed the lesson on ",(0,i.jsx)(n.a,{href:"/docs/develop/azure-openai/create-resources",children:"setting up cloud resources"})," before continuing."]})}),"\n",(0,i.jsxs)(n.ol,{children:["\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsx)(n.p,{children:"In the Azure portal, navigate to your API Management instance."}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsx)(n.p,{children:"In the left menu, under APIs, select APIs > + Add API."}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["Under ",(0,i.jsx)(n.strong,{children:"Create from Azure resource"}),", select Azure OpenAI Service."]}),"\n",(0,i.jsx)(n.p,{children:(0,i.jsx)(n.img,{src:"https://learn.microsoft.com/en-us/azure/api-management/media/azure-openai-api-from-specification/azure-openai-api.png",alt:"Import tile"})}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsx)(n.p,{children:"On the Basics tab:"}),"\n",(0,i.jsx)(n.p,{children:"a. Select the Azure OpenAI resource that you want to import."}),"\n",(0,i.jsx)(n.p,{children:"b. Optionally select an Azure OpenAI API version. If you don't select one, the latest production-ready REST API version is used by default. Make a note of the version you selected. You'll need it to test the API."}),"\n",(0,i.jsxs)(n.p,{children:["c. Enter a Display name and optional Description for the API, for example ",(0,i.jsx)(n.strong,{children:"aoai"})," and ",(0,i.jsx)(n.strong,{children:"My Azure Open AI"})," respectively."]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["In ",(0,i.jsx)(n.strong,{children:"Base URL"}),", append a path that your API Management instance uses to access the Azure OpenAI API endpoints. If you enable Ensure OpenAI SDK compatibility (recommended), /openai is automatically appended to the base URL."]}),"\n",(0,i.jsxs)(n.p,{children:["For example, if your API Management gateway endpoint is ",(0,i.jsx)(n.a,{href:"https://contoso.azure-api.net",children:"https://contoso.azure-api.net"}),", set a Base URL similar to ",(0,i.jsx)(n.a,{href:"https://contoso.azure-api.net/my-openai-api/openai",children:"https://contoso.azure-api.net/my-openai-api/openai"}),"."]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["Optionally select one or more products to associate with the API. Select ",(0,i.jsx)(n.strong,{children:"Next"}),"."]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["On the ",(0,i.jsx)(n.strong,{children:"Policies tab"}),", optionally enable policies to monitor and manage Azure OpenAI API token consumption. You can also set or edit policies later."]}),"\n",(0,i.jsx)(n.p,{children:(0,i.jsx)(n.img,{alt:"Select track during import",src:t(2573).A+"",width:"1750",height:"1590"})}),"\n",(0,i.jsx)(n.p,{children:"If selected, enter settings or accept defaults that define the following policies (see linked articles for prerequisites and configuration details):"}),"\n",(0,i.jsxs)(n.ul,{children:["\n",(0,i.jsx)(n.li,{children:"Manage token consumption"}),"\n",(0,i.jsx)(n.li,{children:"Track token usage"}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsx)(n.p,{children:"Add dimensions you want to track, you can also do this at a later stage. Here's how you can add dimensions:"}),"\n",(0,i.jsx)(n.p,{children:(0,i.jsx)(n.img,{alt:"Add dimensions",src:t(1899).A+"",width:"1518",height:"1497"})}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["Select ",(0,i.jsx)(n.strong,{children:"Review + Create"}),"."]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["After settings are validated, select ",(0,i.jsx)(n.strong,{children:"Create"}),"."]}),"\n"]}),"\n"]}),"\n",(0,i.jsx)(n.p,{children:"Great, now the import is complete, let's test out our API."}),"\n",(0,i.jsx)(n.h3,{id:"-1--enable-monitoring-on-the-api",children:"-1- Enable monitoring on the API"}),"\n",(0,i.jsx)(n.p,{children:"Now that we have imported our Azure Open AI instance, let's inspect what we got and test the API to make sure everything works."}),"\n",(0,i.jsxs)(n.ol,{children:["\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["Select your API ",(0,i.jsx)(n.strong,{children:"aoai"})," and select the ",(0,i.jsx)(n.strong,{children:"Settings"})," tab"]}),"\n",(0,i.jsx)(n.p,{children:(0,i.jsx)(n.img,{alt:"Select settings on the API",src:t(315).A+"",width:"2612",height:"1622"})}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["Check ",(0,i.jsx)(n.em,{children:"enable"})," checkbox and leave the rest as is."]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["Select ",(0,i.jsx)(n.strong,{children:"Save"})]}),"\n"]}),"\n"]}),"\n",(0,i.jsx)(n.h3,{id:"-2--inspect-the-api-and-policy-on-the-api",children:"-2- Inspect the API and policy on the API"}),"\n",(0,i.jsxs)(n.ol,{children:["\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["Select ",(0,i.jsx)(n.strong,{children:"Design"})," tab."]}),"\n",(0,i.jsx)(n.p,{children:"You should see a policy and all the dimensions you've select during import. You can add further dimensions if you wish."}),"\n",(0,i.jsx)(n.p,{children:(0,i.jsx)(n.img,{alt:"Inspect policy",src:t(4337).A+"",width:"2675",height:"1421"})}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["Let's test the API by navigating to ",(0,i.jsx)(n.strong,{children:"Test"})," tab."]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsx)(n.p,{children:"Fill in the following values:"}),"\n",(0,i.jsxs)(n.table,{children:[(0,i.jsx)(n.thead,{children:(0,i.jsxs)(n.tr,{children:[(0,i.jsx)(n.th,{children:"Settings"}),(0,i.jsx)(n.th,{children:"Value"}),(0,i.jsx)(n.th,{children:"Description"})]})}),(0,i.jsxs)(n.tbody,{children:[(0,i.jsxs)(n.tr,{children:[(0,i.jsx)(n.td,{children:"deployment-id"}),(0,i.jsx)(n.td,{children:"gpt-4o"}),(0,i.jsx)(n.td,{children:"your deployment ID, double check the name in Azure AI Foundry"})]}),(0,i.jsxs)(n.tr,{children:[(0,i.jsx)(n.td,{children:"api version"}),(0,i.jsx)(n.td,{children:"2024-02-01"}),(0,i.jsx)(n.td,{children:"a supported schema"})]}),(0,i.jsxs)(n.tr,{children:[(0,i.jsx)(n.td,{children:"request body"}),(0,i.jsx)(n.td,{children:(0,i.jsx)(n.code,{children:'{"messages":[{"role":"system", "content": "you are a friendly assistant"}, { "role": "user", "content": "how is the weather in London?" }]} '})}),(0,i.jsx)(n.td,{children:"a JSON request body that contains messages for the AI model."})]})]})]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["Select ",(0,i.jsx)(n.strong,{children:"Send"}),", you should see a request response coming back."]}),"\n",(0,i.jsx)(n.p,{children:(0,i.jsx)(n.img,{alt:"Send request",src:t(5878).A+"",width:"2663",height:"1573"})}),"\n"]}),"\n"]}),"\n",(0,i.jsx)(n.h2,{id:"exercise-test-monitoring",children:"Exercise: Test monitoring"}),"\n",(0,i.jsx)(n.p,{children:"To test the monitoring, we need to run a few requests, then navigate to it and inspect it."}),"\n",(0,i.jsxs)(n.ol,{children:["\n",(0,i.jsx)(n.li,{children:'Run a few requests by going to your API, select the "Test" tab and fill in values for:'}),"\n"]}),"\n",(0,i.jsxs)(n.table,{children:[(0,i.jsx)(n.thead,{children:(0,i.jsxs)(n.tr,{children:[(0,i.jsx)(n.th,{children:"Field"}),(0,i.jsx)(n.th,{children:"Value"})]})}),(0,i.jsxs)(n.tbody,{children:[(0,i.jsxs)(n.tr,{children:[(0,i.jsx)(n.td,{children:"Deployment Id"}),(0,i.jsx)(n.td,{children:"gpt-4o"})]}),(0,i.jsxs)(n.tr,{children:[(0,i.jsx)(n.td,{children:"API Version"}),(0,i.jsx)(n.td,{children:"2024-02-01"})]}),(0,i.jsxs)(n.tr,{children:[(0,i.jsx)(n.td,{children:"Request body"}),(0,i.jsx)(n.td,{children:(0,i.jsx)(n.code,{children:'{"messages":[{"role":"system", "content": "you are a friendly assistant"}, { "role": "user", "content": "how is the weather in London?" }]}'})})]})]})]}),"\n",(0,i.jsxs)(n.ol,{children:["\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsx)(n.p,{children:"In the menu, select Monitoring / Application Insights / Select your instance"}),"\n",(0,i.jsx)(n.p,{children:"That should take you to your application insights resource."}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsx)(n.p,{children:"Select Monitoring / Metrics"}),"\n",(0,i.jsx)(n.p,{children:"That takes you to your dashboard."}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["In Metrics namespace droplist, select ",(0,i.jsx)(n.strong,{children:"api management"}),", like so:"]}),"\n",(0,i.jsx)(n.p,{children:(0,i.jsx)(n.img,{alt:"api management entry in namespace",src:t(9523).A+"",width:"1582",height:"737"})}),"\n",(0,i.jsx)(n.p,{children:"Once you select that, Metrics droplist should filter down to some very interesting metrics like Completion Tokens, Prompt Tokens and Total Tokens."}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsx)(n.p,{children:"Add all three metrics and you should see something similar to below image:"}),"\n",(0,i.jsx)(n.p,{children:(0,i.jsx)(n.img,{alt:"metrics dashboard",src:t(6856).A+"",width:"1579",height:"1390"})}),"\n",(0,i.jsx)(n.p,{children:"Now you can see your prompts token (23), the number of tokens used to present a response (77.17) and the total number of tokens (100.17)"}),"\n",(0,i.jsx)(n.p,{children:"If you want, try to test some more requests with different prompts and see how they show up on the dashboard. Below here's what it can look like with a new request, note how both the second smaller request (to the left in the screen) is present and the new request (to the right in the screen)"}),"\n",(0,i.jsx)(n.p,{children:(0,i.jsx)(n.img,{alt:"metrics dashboard",src:t(6877).A+"",width:"2478",height:"889"})}),"\n"]}),"\n"]}),"\n",(0,i.jsx)(n.h2,{id:"additional-resources",children:"Additional Resources"}),"\n",(0,i.jsxs)(n.ul,{children:["\n",(0,i.jsxs)(n.li,{children:["Docs: ",(0,i.jsx)(n.a,{href:"https://learn.microsoft.com/en-us/azure/api-management/azure-openai-emit-token-metric-policy",children:"Emit token metric policy"})]}),"\n",(0,i.jsxs)(n.li,{children:["Docs: ",(0,i.jsx)(n.a,{href:"https://learn.microsoft.com/en-us/azure/api-management/api-management-howto-use-azure-monitor",children:"Set up Azure Monitor"})]}),"\n"]}),"\n",(0,i.jsx)(n.h2,{id:"infrastructure-as-code",children:"Infrastructure as Code"}),"\n",(0,i.jsxs)(n.ul,{children:["\n",(0,i.jsxs)(n.li,{children:["Lab: ",(0,i.jsx)(n.a,{href:"https://github.com/Azure-Samples/AI-Gateway/blob/main/labs/token-metrics-emitting/README.MD",children:"Token metrics emitting lab"})]}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,o.R)(),...e.components};return n?(0,i.jsx)(n,{...e,children:(0,i.jsx)(d,{...e})}):d(e)}},4337:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/monitor-inspect-policy-d52857c045e05b6611dce1e6ff954425.png"},5878:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/monitor-test-import-eb3130256a490fcb64f018fc5afe861b.png"},6856:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/monitor-dashboard-0c4eed879301f3eb3ec9aa6c5872e15f.png"},6877:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/monitor-dashboard-2-526a5042cab0c8f6fdd4934068358fd1.png"},8453:(e,n,t)=>{t.d(n,{R:()=>r,x:()=>a});var s=t(6540);const i={},o=s.createContext(i);function r(e){const n=s.useContext(o);return s.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function a(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:r(e.components),s.createElement(o.Provider,{value:n},e.children)}},9523:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/monitor-metrics-d9a4f69227ff8a97550e20526cbd97cd.png"}}]);