Azure Monitor & Application Insights

# Create a resource group for monitoring resources
az group create \
  --name rg-monitoring-prod \
  --location eastus

# Create a Log Analytics workspace with 90-day retention
az monitor log-analytics workspace create \
  --resource-group rg-monitoring-prod \
  --workspace-name law-central-prod \
  --location eastus \
  --retention-time 90 \
  --sku PerGB2018

# Verify workspace creation
az monitor log-analytics workspace show \
  --resource-group rg-monitoring-prod \
  --workspace-name law-central-prod \
  --query '{Name:name, Sku:sku.name, Retention:retentionInDays, Id:customerId}' \
  --output table

# Create a workspace-based Application Insights resource
az monitor app-insights component create \
  --app appi-mywebapp-prod \
  --location eastus \
  --resource-group rg-monitoring-prod \
  --workspace /subscriptions/<sub-id>/resourceGroups/rg-monitoring-prod/providers/Microsoft.OperationalInsights/workspaces/law-central-prod \
  --application-type web \
  --kind web

# Retrieve the connection string (used for SDK configuration)
az monitor app-insights component show \
  --app appi-mywebapp-prod \
  --resource-group rg-monitoring-prod \
  --query connectionString \
  --output tsv

using Azure.Monitor.OpenTelemetry.AspNetCore;

var builder = WebApplication.CreateBuilder(args);

// Add Azure Monitor OpenTelemetry distro
builder.Services.AddOpenTelemetry().UseAzureMonitor(options =>
{
    options.ConnectionString = builder.Configuration
        .GetConnectionString("ApplicationInsights");
});

// Add services
builder.Services.AddControllers();
builder.Services.AddHealthChecks();

var app = builder.Build();

app.MapControllers();
app.MapHealthChecks("/health");
app.Run();

{
  "ConnectionStrings": {
    "ApplicationInsights": "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://eastus-0.in.applicationinsights.azure.com/;LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/"
  },
  "Logging": {
    "LogLevel": {
      "Default": "Information",
      "Microsoft.AspNetCore": "Warning"
    }
  },
  "ApplicationInsights": {
    "EnableAdaptiveSampling": true,
    "EnableDependencyTrackingTelemetryModule": true,
    "EnablePerformanceCounterCollectionModule": true
  }
}

const { useAzureMonitor } = require("@azure/monitor-opentelemetry");

// Must be called before other imports to patch modules
useAzureMonitor({
  azureMonitorExporterOptions: {
    connectionString: process.env.APPLICATIONINSIGHTS_CONNECTION_STRING
  },
  instrumentationOptions: {
    http: { enabled: true },
    azureSdk: { enabled: true },
    mongoDb: { enabled: true },
    mySql: { enabled: true },
    postgreSql: { enabled: true },
    redis: { enabled: true }
  }
});

const express = require("express");
const app = express();

app.get("/", (req, res) => {
  res.json({ status: "healthy", timestamp: new Date().toISOString() });
});

app.get("/api/users/:id", async (req, res) => {
  // Dependency calls are automatically tracked
  const user = await fetchUserFromDatabase(req.params.id);
  res.json(user);
});

app.listen(3000, () => console.log("Server running on port 3000"));

// 1. Request failure rate by endpoint (last 24 hours)
requests
| where timestamp > ago(24h)
| summarize totalRequests = count(),
            failedRequests = countif(success == false),
            avgDuration = avg(duration),
            p95Duration = percentile(duration, 95),
            p99Duration = percentile(duration, 99)
  by name
| extend failureRate = round(100.0 * failedRequests / totalRequests, 2)
| order by failureRate desc
| project name, totalRequests, failedRequests, failureRate,
          avgDuration = round(avgDuration, 1),
          p95Duration = round(p95Duration, 1),
          p99Duration = round(p99Duration, 1)

// 2. Slowest dependency calls (database, HTTP, etc.)
dependencies
| where timestamp > ago(1h)
| summarize avgDuration = avg(duration),
            p95Duration = percentile(duration, 95),
            callCount = count(),
            failureCount = countif(success == false)
  by target, type, name
| where callCount > 10
| order by p95Duration desc
| take 20

// 3. Exception trends by type
exceptions
| where timestamp > ago(7d)
| summarize exceptionCount = count() by bin(timestamp, 1h), type
| render timechart

// 4. Unique users and sessions over time
customEvents
| where timestamp > ago(30d)
| summarize users = dcount(user_Id),
            sessions = dcount(session_Id)
  by bin(timestamp, 1d)
| render timechart

// 5. End-to-end transaction analysis
requests
| where timestamp > ago(1h) and duration > 5000
| project operation_Id, name, duration, resultCode, timestamp
| join kind=inner (
    dependencies
    | where timestamp > ago(1h)
    | project operation_Id, depName = name, depDuration = duration,
              depType = type, depTarget = target, depSuccess = success
  ) on operation_Id
| order by duration desc
| project timestamp, name, duration, resultCode,
          depName, depDuration, depType, depTarget, depSuccess

using System.Diagnostics.Metrics;

public class OrderMetrics
{
    private static readonly Meter OrderMeter = new("MyApp.Orders", "1.0.0");

    // Counter: monotonically increasing value
    private static readonly Counter<long> OrdersProcessed =
        OrderMeter.CreateCounter<long>(
            "orders.processed",
            description: "Total number of orders processed");

    // Histogram: distribution of values (e.g., latency, size)
    private static readonly Histogram<double> OrderProcessingTime =
        OrderMeter.CreateHistogram<double>(
            "orders.processing_time_ms",
            unit: "ms",
            description: "Time to process an order");

    // UpDownCounter: value that can increase or decrease
    private static readonly UpDownCounter<int> ActiveOrders =
        OrderMeter.CreateUpDownCounter<int>(
            "orders.active",
            description: "Number of orders currently being processed");

    public async Task ProcessOrderAsync(Order order)
    {
        ActiveOrders.Add(1);
        var stopwatch = Stopwatch.StartNew();

        try
        {
            await ValidateOrder(order);
            await ChargePayment(order);
            await FulfillOrder(order);

            OrdersProcessed.Add(1,
                new KeyValuePair<string, object?>("region", order.Region),
                new KeyValuePair<string, object?>("type", order.Type));
        }
        finally
        {
            stopwatch.Stop();
            OrderProcessingTime.Record(stopwatch.ElapsedMilliseconds);
            ActiveOrders.Add(-1);
        }
    }
}

# List available metrics for an App Service
az monitor metrics list-definitions \
  --resource /subscriptions/<sub-id>/resourceGroups/rg-app/providers/Microsoft.Web/sites/mywebapp \
  --query "[].{Name:name.value, Unit:unit, Aggregation:primaryAggregationType}" \
  --output table

# Query CPU percentage for the last 6 hours (5-minute intervals)
az monitor metrics list \
  --resource /subscriptions/<sub-id>/resourceGroups/rg-app/providers/Microsoft.Web/sites/mywebapp \
  --metric "CpuPercentage" \
  --aggregation Average Maximum \
  --interval PT5M \
  --start-time 2024-01-15T00:00:00Z \
  --end-time 2024-01-15T06:00:00Z \
  --output table

# Query custom metric from Application Insights
az monitor app-insights metrics show \
  --app appi-mywebapp-prod \
  --resource-group rg-monitoring-prod \
  --metrics "customMetrics/orders.processed" \
  --aggregation sum \
  --interval PT1H

# Create an action group with email and webhook notifications
az monitor action-group create \
  --resource-group rg-monitoring-prod \
  --name ag-platform-team \
  --short-name PlatTeam \
  --email-receiver name="Platform Lead" email-address="platform-lead@company.com" \
  --webhook-receiver name="PagerDuty" uri="https://events.pagerduty.com/integration/<key>/enqueue"

# Create a metric alert for high response time
az monitor metrics alert create \
  --resource-group rg-monitoring-prod \
  --name "alert-high-response-time" \
  --description "Average response time exceeds 2 seconds" \
  --scopes /subscriptions/<sub-id>/resourceGroups/rg-app/providers/Microsoft.Web/sites/mywebapp \
  --condition "avg HttpResponseTime > 2000" \
  --window-size 5m \
  --evaluation-frequency 1m \
  --severity 2 \
  --action ag-platform-team

# Create a log search alert for error spike detection
az monitor scheduled-query create \
  --resource-group rg-monitoring-prod \
  --name "alert-error-spike" \
  --description "More than 50 server errors in 5 minutes" \
  --scopes /subscriptions/<sub-id>/resourceGroups/rg-monitoring-prod/providers/Microsoft.OperationalInsights/workspaces/law-central-prod \
  --condition "count > 50" \
  --condition-query "requests | where resultCode startswith '5'" \
  --window-size 5 \
  --evaluation-frequency 5 \
  --severity 1 \
  --action /subscriptions/<sub-id>/resourceGroups/rg-monitoring-prod/providers/Microsoft.Insights/actionGroups/ag-platform-team

// Find slow end-to-end transactions spanning multiple services
let slowOperations = requests
| where timestamp > ago(1h)
| where duration > 5000
| project operation_Id, requestName = name, requestDuration = duration;

// Get all telemetry for those operations across all services
union requests, dependencies, exceptions, traces
| where timestamp > ago(1h)
| where operation_Id in ((slowOperations | project operation_Id))
| project timestamp, operation_Id, itemType,
          name = coalesce(name, type, problemId),
          duration,
          success,
          cloud_RoleName,
          cloud_RoleInstance
| order by operation_Id, timestamp asc

// Application dependency health summary
dependencies
| where timestamp > ago(24h)
| summarize totalCalls = count(),
            failedCalls = countif(success == false),
            avgDuration = avg(duration),
            p99Duration = percentile(duration, 99)
  by target, type, cloud_RoleName
| extend failureRate = round(100.0 * failedCalls / totalCalls, 2)
| order by failureRate desc, totalCalls desc

# Create a standard availability test (URL ping test)
az monitor app-insights web-test create \
  --resource-group rg-monitoring-prod \
  --name "avail-homepage-check" \
  --defined-web-test-name "Homepage Availability" \
  --location "us-fl-mia-edge" \
  --location "emea-gb-db3-azr" \
  --location "apac-sg-sin-azr" \
  --location "us-ca-sjc-azr" \
  --location "emea-nl-ams-azr" \
  --frequency 300 \
  --timeout 120 \
  --kind standard \
  --enabled true \
  --web-test-kind standard \
  --request-url "https://mywebapp.azurewebsites.net/health" \
  --expected-status-code 200 \
  --ssl-check true \
  --ssl-lifetime-check 7 \
  --tags "hidden-link:/subscriptions/<sub-id>/resourceGroups/rg-monitoring-prod/providers/Microsoft.Insights/components/appi-mywebapp-prod=Resource"

using Microsoft.ApplicationInsights;
using Microsoft.ApplicationInsights.DataContracts;
using Microsoft.Azure.Functions.Worker;
using System.Diagnostics;

public class AvailabilityTestFunction
{
    private readonly TelemetryClient _telemetry;
    private readonly HttpClient _httpClient;

    public AvailabilityTestFunction(
        TelemetryClient telemetry,
        IHttpClientFactory httpClientFactory)
    {
        _telemetry = telemetry;
        _httpClient = httpClientFactory.CreateClient("ApiClient");
    }

    [Function("CheckApiHealth")]
    public async Task Run(
        [TimerTrigger("0 */5 * * * *")] TimerInfo timer)
    {
        var availability = new AvailabilityTelemetry
        {
            Name = "API Health Check",
            RunLocation = Environment.GetEnvironmentVariable("REGION_NAME"),
            Success = false
        };

        var stopwatch = Stopwatch.StartNew();

        try
        {
            // Step 1: Authenticate
            var token = await GetAccessTokenAsync();

            // Step 2: Call protected API endpoint
            _httpClient.DefaultRequestHeaders.Authorization =
                new AuthenticationHeaderValue("Bearer", token);

            var response = await _httpClient.GetAsync("/api/health/deep");
            response.EnsureSuccessStatusCode();

            var body = await response.Content.ReadAsStringAsync();

            // Step 3: Validate response content
            if (body.Contains("\"status\":\"healthy\""))
            {
                availability.Success = true;
                availability.Message = "All health checks passed";
            }
            else
            {
                availability.Message = $"Unexpected response: {body[..200]}";
            }
        }
        catch (Exception ex)
        {
            availability.Message = ex.Message;
            _telemetry.TrackException(ex);
        }
        finally
        {
            stopwatch.Stop();
            availability.Duration = stopwatch.Elapsed;
            _telemetry.TrackAvailability(availability);
        }
    }
}

param location string = resourceGroup().location
param appInsightsId string
param workspaceName string = 'law-central-prod'

resource workspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = {
  name: workspaceName
}

resource workbook 'Microsoft.Insights/workbooks@2022-04-01' = {
  name: guid('app-health-workbook', resourceGroup().id)
  location: location
  kind: 'shared'
  properties: {
    displayName: 'Application Health Overview'
    category: 'workbook'
    sourceId: appInsightsId
    serializedData: loadTextContent('workbook-template.json')
    version: '1.0'
  }
  tags: {
    'hidden-title': 'Application Health Overview'
    environment: 'production'
  }
}

// Daily ingestion volume by table (last 30 days)
Usage
| where TimeGenerated > ago(30d)
| where IsBillable == true
| summarize IngestedGB = round(sum(Quantity) / 1024, 2) by bin(TimeGenerated, 1d), DataType
| order by TimeGenerated desc, IngestedGB desc

// Top 10 tables by cost contribution
Usage
| where TimeGenerated > ago(30d)
| where IsBillable == true
| summarize TotalGB = round(sum(Quantity) / 1024, 2) by DataType
| top 10 by TotalGB
| extend EstimatedMonthlyCost = round(TotalGB * 2.76, 2) // ~$2.76/GB pay-as-you-go
| order by EstimatedMonthlyCost desc

// Identify verbose trace sources
traces
| where timestamp > ago(1d)
| summarize traceCount = count(), estimatedSizeMB = round(sum(estimate_data_size()) / 1048576, 2)
  by cloud_RoleName, severityLevel
| order by estimatedSizeMB desc
| take 20