From aa083a63ad160fbd4c0249de68df6d56462b9533 Mon Sep 17 00:00:00 2001
From: Petar Hristov <hristov@itc.rwth-aachen.de>
Date: Mon, 12 Dec 2022 09:14:59 +0100
Subject: [PATCH] Fix: Request wrapping and log levels
---
src/KPI Generator/KPI Generator.csproj | 1 +
src/KPI Generator/Program.cs | 4 +--
src/KPI Generator/Reporting.cs | 34 +++++++++++++++++--
.../ApplicationProfileReporting.cs | 2 +-
.../Reportings/Complete/CompleteReporting.cs | 10 +++---
.../Reportings/Project/ProjectReporting.cs | 4 +--
.../Reportings/Resource/ResourceReporting.cs | 2 +-
.../Reportings/User/UserReporting.cs | 4 +--
src/KPI Generator/nlog.config | 23 ++++++-------
9 files changed, 56 insertions(+), 28 deletions(-)
diff --git a/src/KPI Generator/KPI Generator.csproj b/src/KPI Generator/KPI Generator.csproj
index 3007255..fb7cf90 100644
--- a/src/KPI Generator/KPI Generator.csproj
+++ b/src/KPI Generator/KPI Generator.csproj
@@ -26,6 +26,7 @@
<PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" />
<PackageReference Include="NLog" Version="5.1.0" />
<PackageReference Include="NLog.Extensions.Logging" Version="5.2.0" />
+ <PackageReference Include="Polly.Extensions.Http" Version="3.0.0" />
</ItemGroup>
<ItemGroup>
diff --git a/src/KPI Generator/Program.cs b/src/KPI Generator/Program.cs
index 43f4036..f6866b8 100644
--- a/src/KPI Generator/Program.cs
+++ b/src/KPI Generator/Program.cs
@@ -57,7 +57,7 @@ public class Program
catch (Exception e)
{
Console.WriteLine(e.Message);
- _logger.LogError(e, e.Message);
+ _logger.LogWarning(e, e.Message);
return -1; // Exit Code -1 for Failure
}
}
@@ -129,7 +129,7 @@ public class Program
{
if (ex.InnerException is not null)
{
- _logger.LogError(ex.InnerException, "InnerException: {innerException}", ex.InnerException.Message);
+ _logger.LogWarning(ex.InnerException, "InnerException: {innerException}", ex.InnerException.Message);
LogInnerException(ex.InnerException);
}
}
diff --git a/src/KPI Generator/Reporting.cs b/src/KPI Generator/Reporting.cs
index bc617f3..f0aa22e 100644
--- a/src/KPI Generator/Reporting.cs
+++ b/src/KPI Generator/Reporting.cs
@@ -5,6 +5,7 @@ using GitLabApiClient;
using GitLabApiClient.Models.Branches.Requests;
using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
using Microsoft.Extensions.Logging;
+using Polly;
using System.Text;
using System.Web;
using VDS.RDF.Query;
@@ -154,7 +155,7 @@ public abstract class Reporting<O> where O : class
catch (Exception e)
{
Console.WriteLine(e.Message);
- _logger.LogError(e, e.Message);
+ _logger.LogWarning(e, e.Message);
return false;
}
}
@@ -173,7 +174,7 @@ public abstract class Reporting<O> where O : class
}
else
{
- var organizationTriples = RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList();
+ var organizationTriples = WrapRequest(() => RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList());
if (organizationTriples.Any())
{
result = new Organization
@@ -261,4 +262,33 @@ public abstract class Reporting<O> where O : class
}
Console.WriteLine();
}
+
+ /// <summary>
+ /// Retry Virtuoso Requests since they sometimes just fail
+ /// </summary>
+ /// <typeparam name="W"></typeparam>
+ /// <param name="function"></param>
+ /// <returns></returns>
+ public void WrapRequest(Action action)
+ {
+ Policy
+ .Handle<Exception>()
+ .WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
+ .Execute(() => action.Invoke());
+ }
+
+ /// <summary>
+ /// Retry Virtuoso Requests since they sometimes just fail
+ /// </summary>
+ /// <typeparam name="W"></typeparam>
+ /// <param name="function"></param>
+ /// <returns></returns>
+ public W WrapRequest<W>(Func<W> function)
+ {
+ return Policy
+ .Handle<Exception>()
+ .WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
+ .ExecuteAndCapture(() => function.Invoke()).Result;
+ }
+
}
\ No newline at end of file
diff --git a/src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReporting.cs b/src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReporting.cs
index 14c3520..bc19296 100644
--- a/src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReporting.cs
+++ b/src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReporting.cs
@@ -53,7 +53,7 @@ public class ApplicationProfileReporting : Reporting<ApplicationProfileReporting
OPTIONAL {{ ?{_applicationProfile} dcterms:{_license} ?{_license} . }}
}}"
};
- using var result = RdfStoreConnector.QueryEndpoint.QueryWithResultSet(queryString.ToString());
+ using var result = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(queryString.ToString()));
var grouped = result.GroupBy(ap => new
{
diff --git a/src/KPI Generator/Reportings/Complete/CompleteReporting.cs b/src/KPI Generator/Reportings/Complete/CompleteReporting.cs
index f0f1ef7..368a595 100644
--- a/src/KPI Generator/Reportings/Complete/CompleteReporting.cs
+++ b/src/KPI Generator/Reportings/Complete/CompleteReporting.cs
@@ -29,7 +29,7 @@ public class CompleteReporting : Reporting<CompleteReportingOptions>
catch (Exception e)
{
Console.WriteLine($"!! Skipping ProjectReporting: {e.Message} \n");
- _logger.LogWarning(e, "Skipping ProjectReporting: {message}", e.Message);
+ _logger.LogError(e, "Skipping ProjectReporting: {message}", e.Message);
}
// Resource Reporting
@@ -43,7 +43,7 @@ public class CompleteReporting : Reporting<CompleteReportingOptions>
catch (Exception e)
{
Console.WriteLine($"!! Skipping ResourceReporting: {e.Message} \n");
- _logger.LogWarning(e, "Skipping ResourceReporting: {message}", e.Message);
+ _logger.LogError(e, "Skipping ResourceReporting: {message}", e.Message);
}
// User Reporting
@@ -57,7 +57,7 @@ public class CompleteReporting : Reporting<CompleteReportingOptions>
catch (Exception e)
{
Console.WriteLine($"!! Skipping UserReporting: {e.Message} \n");
- _logger.LogWarning(e, "Skipping UserReporting: {message}", e.Message);
+ _logger.LogError(e, "Skipping UserReporting: {message}", e.Message);
}
// Application Profile Reporting
@@ -71,7 +71,7 @@ public class CompleteReporting : Reporting<CompleteReportingOptions>
catch (Exception e)
{
Console.WriteLine($"!! Skipping ApplicationProfileReporting: {e.Message} \n");
- _logger.LogWarning(e, "Skipping ApplicationProfileReporting: {message}", e.Message);
+ _logger.LogError(e, "Skipping ApplicationProfileReporting: {message}", e.Message);
}
// System Status Reporting
@@ -85,7 +85,7 @@ public class CompleteReporting : Reporting<CompleteReportingOptions>
catch (Exception e)
{
Console.WriteLine($"!! Skipping SystemReporting: {e.Message} \n");
- _logger.LogWarning(e, "Skipping SystemReporting: {message}", e.Message);
+ _logger.LogError(e, "Skipping SystemReporting: {message}", e.Message);
}
return result;
diff --git a/src/KPI Generator/Reportings/Project/ProjectReporting.cs b/src/KPI Generator/Reportings/Project/ProjectReporting.cs
index a3efbae..fa69a8e 100644
--- a/src/KPI Generator/Reportings/Project/ProjectReporting.cs
+++ b/src/KPI Generator/Reportings/Project/ProjectReporting.cs
@@ -164,7 +164,7 @@ public class ProjectReporting : Reporting<ProjectReportingOptions>
}
catch (Exception ex)
{
- _logger.LogError(ex, ex.Message);
+ _logger.LogWarning(ex, ex.Message);
return 0f;
}
}
@@ -198,7 +198,7 @@ public class ProjectReporting : Reporting<ProjectReportingOptions>
}
catch (Exception ex)
{
- _logger.LogError(ex, ex.Message);
+ _logger.LogWarning(ex, ex.Message);
return 0f;
}
}
diff --git a/src/KPI Generator/Reportings/Resource/ResourceReporting.cs b/src/KPI Generator/Reportings/Resource/ResourceReporting.cs
index d5b7909..aadf588 100644
--- a/src/KPI Generator/Reportings/Resource/ResourceReporting.cs
+++ b/src/KPI Generator/Reportings/Resource/ResourceReporting.cs
@@ -135,7 +135,7 @@ public class ResourceReporting : Reporting<ResourceReportingOptions>
}
catch (Exception ex)
{
- _logger.LogError(ex, "Error: {message} for {@resource}", ex.Message, resource);
+ _logger.LogWarning(ex, "Error: {message} for {@resource}", ex.Message, resource);
return null;
}
}
diff --git a/src/KPI Generator/Reportings/User/UserReporting.cs b/src/KPI Generator/Reportings/User/UserReporting.cs
index 87c23ba..4195285 100644
--- a/src/KPI Generator/Reportings/User/UserReporting.cs
+++ b/src/KPI Generator/Reportings/User/UserReporting.cs
@@ -245,7 +245,7 @@ public class UserReporting : Reporting<UserReportingOptions>
}}
"
};
- using var results = RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString());
+ using var results = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
if (!results.IsEmpty)
{
var ror = results.Select(x => x.Value("ror").ToString()); // Get the value for ?ror
@@ -279,7 +279,7 @@ public class UserReporting : Reporting<UserReportingOptions>
}}
"
};
- using var results = RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString());
+ using var results = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
if (!results.IsEmpty)
{
var ror = results.Select(x => x.Value("ror").ToString()); // Get the value for ?ror
diff --git a/src/KPI Generator/nlog.config b/src/KPI Generator/nlog.config
index 32f9d3e..95fae67 100644
--- a/src/KPI Generator/nlog.config
+++ b/src/KPI Generator/nlog.config
@@ -8,19 +8,16 @@
<targets>
<!-- Write logs to File -->
<target xsi:type="File" name="fileLog" fileName="C:/coscine/logs/${assembly-name}/${assembly-version}/log-${shortdate}.log" >
- <layout xsi:type="CompoundLayout">
- <layout xsi:type="JsonLayout" EscapeForwardSlash="true">
- <attribute layout="${longdate}" name="Timestamp"/>
- <attribute layout="${level:upperCase=true}" name="Level"/>
- <attribute layout="${message}" name="Message"/>
- <attribute layout="${exception:format=tostring,StackTrace}" name="Exception"/>
- <attribute layout="${ndlc}" name="Context"/>
- <attribute layout="${event-properties:item=Metric}" name="Alarm" encode="false"/>
- <attribute name="EventProperties" encode="false" >
- <layout xsi:type='JsonLayout' includeAllProperties="true" maxRecursionLimit="2"/>
- </attribute>
- </layout>
- <layout xsi:type='SimpleLayout' text="," />
+ <layout xsi:type="JsonLayout" EscapeForwardSlash="true">
+ <attribute layout="${longdate}" name="Timestamp"/>
+ <attribute layout="${level:upperCase=true}" name="Level"/>
+ <attribute layout="${message}" name="Message"/>
+ <attribute layout="${exception:format=tostring,StackTrace}" name="Exception"/>
+ <attribute layout="${ndlc}" name="Context"/>
+ <attribute layout="${event-properties:item=Metric}" name="Alarm" encode="false"/>
+ <attribute name="EventProperties" encode="false" >
+ <layout xsi:type='JsonLayout' includeAllProperties="true" maxRecursionLimit="2"/>
+ </attribute>
</layout>
</target>
</targets>
--
GitLab