This blog is follow up post pertaining to blog series related to Azure Storage Analytics. To go back to the main article and understand the background of this post click- Azure Storage Analytics for Azure Storage –Blob, Queue and Tables - Metrics example
Following is the code of classes required for analytics metrics information for Azure storage – Blob, Queue and Table.
Class MetricsHelper.cs -
public class MetricsHelper
{
/// <summary>
/// Given a service, start time, end time search for, this method retrieves the metrics rows for requests and exports it to CSV file
/// </summary>
/// <param name="tableClient"></param> /// <param name="serviceName">The name of the service interested in</param>
/// <param name="startTimeForSearch">Start time for reporting</param>
/// <param name="endTimeForSearch">End time for reporting</param>
/// <param name="fileName">The file to write the report to</param>
public static void DumpTransactionsMetrics(CloudTableClient tableClient, string serviceName, DateTime startTimeForSearch, DateTime endTimeForSearch, string fileName)
{
string startingPK = startTimeForSearch.ToString("yyyyMMddTHH00");
string endingPK = endTimeForSearch.ToString("yyyyMMddTHH00");
// table names are case insensitive
Console.WriteLine("Querying table '{0}' for PartitionKey >= '{1}' and PartitionKey <= '{2}'", tableName, startingPK, endingPK);
TableServiceContext context = tableClient.GetDataServiceContext();
// turn off merge option as we only want to query and not issue deletes etc.
context.MergeOption = MergeOption.NoTracking;
CloudTableQuery<MetricsTransactionsEntity> query = (from entity in context.CreateQuery<MetricsTransactionsEntity>(tableName)
where entity.PartitionKey.CompareTo(startingPK) >= 0
&& entity.PartitionKey.CompareTo(endingPK) <= 0
select entity).AsTableServiceQuery<MetricsTransactionsEntity>();
// now we have the query set. Let us iterate over all entities and store into an output file.
// Also overwrite the file
Console.WriteLine("Writing to '{0}'", fileName);
using (StreamWriter writer = new StreamWriter(fileName))
{
// write the header
writer.Write("Time, Category, Request Type, Total Ingress, Total Egress, Total Requests, Total Billable Requests,");
writer.Write("Availability, Avg E2E Latency, Avg Server Latency, % Success, % Throttling Errors, % Timeout Errors, % Other Server Errors, % Other Client Errors, % Authorization Errors, % Network Errors, Success,");
writer.Write("Anonymous Success, SAS Success, Throttling Error, Anonymous Throttling Error, SAS ThrottlingError, Client Timeout Error, Anonymous Client Timeout Error, SAS Client Timeout Error,");
writer.Write("Server Timeout Error, Anonymous Server Timeout Error, SAS Server Timeout Error, Client Other Error, SAS Client Other Error, Anonymous Client Other Error,");
writer.Write("Server Other Errors, SAS Server Other Errors, Anonymous Server Other Errors, Authorization Errors, Anonymous Authorization Error, SAS Authorization Error,");
writer.WriteLine("Network Error, Anonymous Network Error, SAS Network Error");
foreach (MetricsTransactionsEntity entity in query)
{
string[] rowKeys = entity.RowKey.Split(';');
writer.WriteLine("{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}, {11}, {12}, {13}, {14}, {15}, {16}, {17}, {18}, {19}, {20}, {21}, {22}, {23}, {24}, {25}, {26}, {27}, {28}, {29}, {30}, {31}, {32}, {33}, {34}, {35}, {36}, {37}, {38}, {39}, {40}",
entity.PartitionKey,
rowKeys[0],
// category - user | system
rowKeys[1],
// request type is the API name (and "All" for service summary rows)
entity.TotalIngress,
entity.TotalEgress,
entity.TotalRequests,
entity.TotalBillableRequests,
entity.Availability,
entity.AverageE2ELatency,
entity.AverageServerLatency,
entity.PercentSuccess,
entity.PercentThrottlingError,
entity.PercentTimeoutError,
entity.PercentServerOtherError,
entity.PercentClientOtherError,
entity.PercentAuthorizationError,
entity.PercentNetworkError,
entity.Success,
entity.AnonymousSuccess,
entity.SASSuccess,
entity.ThrottlingError,
entity.AnonymousThrottlingError,
entity.SASThrottlingError,
entity.ClientTimeoutError,
entity.AnonymousClientTimeoutError,
entity.SASClientTimeoutError,
entity.ServerTimeoutError,
entity.AnonymousServerTimeoutError,
entity.SASServerTimeoutError,
entity.ClientOtherError,
entity.SASClientOtherError,
entity.AnonymousClientOtherError,
entity.ServerOtherError,
entity.SASServerOtherError,
entity.AnonymousServerOtherError,
entity.AuthorizationError,
entity.AnonymousAuthorizationError,
entity.SASAuthorizationError,
entity.NetworkError,
entity.AnonymousNetworkError,
entity.SASNetworkError);
}
}
}
/// <summary>
/// Given a service, start time, end time search for, this method retrieves the metrics rows for capacity and exports it to CSV file
/// </summary>
/// <param name="tableClient"></param>
/// <param name="serviceName">The name of the service interested in</param>
/// /// <param name="startTimeForSearch">Start time for reporting</param>
/// <param name="endTimeForSearch">End time for reporting</param>
/// <param name="fileName">The file to write the report to</param>
public static void DumpCapacityMetrics(CloudTableClient tableClient, string serviceName, DateTime startTimeForSearch, DateTime endTimeForSearch, string fileName)
{
string startingPK = startTimeForSearch.ToString("yyyyMMddT0000");
string endingPK = endTimeForSearch.ToString("yyyyMMddT0000");
// table names are case insensitive
string tableName = string.Format("$MetricsCapacity{0}", serviceName);
Console.WriteLine("Querying table '{0}' for PartitionKey >= '{1}' and PartitionKey <= '{2}'", tableName, startingPK, endingPK);
TableServiceContext context = tableClient.GetDataServiceContext();
// turn off merge option as we only want to query and not issue deletes etc.
context.MergeOption = MergeOption.NoTracking;
CloudTableQuery<MetricsCapacityEntity> query = (from entity in context.CreateQuery<MetricsCapacityEntity>(tableName)
where entity.PartitionKey.CompareTo(startingPK) >= 0
&& entity.PartitionKey.CompareTo(endingPK) <= 0
select entity).AsTableServiceQuery<MetricsCapacityEntity>();
// now we have the query set. Let us iterate over all entities and store into an output file.
// Also overwrite the file
Console.WriteLine("Writing to '{0}'", fileName);
using (StreamWriter writer = new StreamWriter(fileName))
{
// write the header
writer.WriteLine("Time, Category, Capacity (bytes), Container count, Object count");
foreach (MetricsCapacityEntity entity in query)
{
writer.WriteLine("{0}, {1}, {2}, {3}, {4}",
entity.PartitionKey,
entity.RowKey,
entity.Capacity,
entity.ContainerCount,
entity.ObjectCount);
}
}
}
}
Class MetricsEntities.cs –
[DataServiceKey("PartitionKey", "RowKey")]
public class MetricsCapacityEntity
{
public string PartitionKey { get; set; }
public string RowKey { get; set; }
public long Capacity { get; set; }
public long ContainerCount { get; set; }
public long ObjectCount { get; set; }
}
[DataServiceKey("PartitionKey", "RowKey")]
public class MetricsTransactionsEntity
{
public string PartitionKey { get; set; }
public string RowKey { get; set; }
public long TotalIngress { get; set; }
public long TotalEgress { get; set; }
public long TotalRequests { get; set; }
public long TotalBillableRequests { get; set; }
public double Availability { get; set; }
public double AverageE2ELatency { get; set; }
public double AverageServerLatency { get; set; }
public double PercentSuccess { get; set; }
public double PercentThrottlingError { get; set; }
public double PercentTimeoutError { get; set; }
public double PercentServerOtherError { get; set; }
public double PercentClientOtherError { get; set; }
public double PercentAuthorizationError { get; set; }
public double PercentNetworkError { get; set; }
public long Success { get; set; }
public long AnonymousSuccess { get; set; }
public long SASSuccess { get; set; }
public long ThrottlingError { get; set; }
public long AnonymousThrottlingError { get; set; }
public long SASThrottlingError { get; set; }
public long ClientTimeoutError { get; set; }
public long AnonymousClientTimeoutError { get; set; }
public long SASClientTimeoutError { get; set; }
public long ServerTimeoutError { get; set; }
public long AnonymousServerTimeoutError { get; set; }
public long SASServerTimeoutError { get; set; }
public long ClientOtherError { get; set; }
public long AnonymousClientOtherError { get; set; }
public long SASClientOtherError { get; set; }
public long ServerOtherError { get; set; }
public long AnonymousServerOtherError { get; set; }
public long SASServerOtherError { get; set; }
public long AuthorizationError { get; set; }
public long AnonymousAuthorizationError { get; set; }
public long SASAuthorizationError { get; set; }
public long NetworkError { get; set; }
public long AnonymousNetworkError { get; set; }
public long SASNetworkError { get; set; }
}
Hi, this is a great write-up and guide! I have one minor suggestion, I would add these two options to your rsync command. They aren't included in -a, and in some cases can be helpful or necessary when doing system migrations
ReplyDeleteinstalling kvm