feat(temporal): implemented date time index creation

This commit is contained in:
Dylan R. E. Moonfire 2022-06-27 23:38:50 -05:00
parent 63ff163fbf
commit ca13ae34d0
13 changed files with 530 additions and 47 deletions

View file

@ -43,6 +43,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CopyFiles", "examples\CopyF
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nitride.Slugs.Tests", "tests\Nitride.Slugs.Tests\Nitride.Slugs.Tests.csproj", "{C49E07D0-CD32-4332-90FA-07494195CAC4}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nitride.Temporal.Tests", "tests\Nitride.Temporal.Tests\Nitride.Temporal.Tests.csproj", "{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -260,6 +262,18 @@ Global
{C49E07D0-CD32-4332-90FA-07494195CAC4}.Release|x64.Build.0 = Release|Any CPU
{C49E07D0-CD32-4332-90FA-07494195CAC4}.Release|x86.ActiveCfg = Release|Any CPU
{C49E07D0-CD32-4332-90FA-07494195CAC4}.Release|x86.Build.0 = Release|Any CPU
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}.Debug|x64.ActiveCfg = Debug|Any CPU
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}.Debug|x64.Build.0 = Debug|Any CPU
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}.Debug|x86.ActiveCfg = Debug|Any CPU
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}.Debug|x86.Build.0 = Debug|Any CPU
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}.Release|Any CPU.Build.0 = Release|Any CPU
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}.Release|x64.ActiveCfg = Release|Any CPU
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}.Release|x64.Build.0 = Release|Any CPU
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}.Release|x86.ActiveCfg = Release|Any CPU
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{D480943C-764D-4A8A-B546-642ED10586BB} = {570184FD-ECE4-4EC8-86E1-C1265E17D647}
@ -279,5 +293,6 @@ Global
{29743817-A401-458F-9DD0-AF3579965953} = {251D9C68-34EB-439D-B167-688BCC47DA17}
{2C92A626-7A14-4FDB-906B-E7FA5FF18CC1} = {47461A29-E502-4B0E-AAF5-D87C4B93AB6D}
{C49E07D0-CD32-4332-90FA-07494195CAC4} = {251D9C68-34EB-439D-B167-688BCC47DA17}
{0B74A4DE-4F92-44EE-8273-E5A15EAB4266} = {251D9C68-34EB-439D-B167-688BCC47DA17}
EndGlobalSection
EndGlobal

View file

@ -0,0 +1,168 @@
using System;
using System.Collections.Generic;
using System.Linq;
using FluentValidation;
using Gallium;
using NodaTime;
namespace Nitride.Temporal;
public class CreateDateIndexesValidator : AbstractValidator<CreateDateIndexes>
{
public CreateDateIndexesValidator()
{
this.RuleFor(a => a.Timekeeper).NotNull();
this.RuleFor(a => a.CreateIndex).NotNull();
this.RuleFor(a => a.Formats).NotNull();
}
}
/// <summary>
/// Constructs indexes for any arbitrary formatting of date time to allow for
/// nested structures. This takes a list
/// of DateTime formats, ordered from most specific to least specific and then
/// organizes the results.
/// </summary>
[WithProperties]
public partial class CreateDateIndexes : OperationBase, IResolvingOperation
{
private readonly IValidator<CreateDateIndexes> validator;
public CreateDateIndexes(IValidator<CreateDateIndexes> validator, Timekeeper timekeeper)
{
this.validator = validator;
this.Timekeeper = timekeeper;
}
/// <summary>
/// Gets or sets the callback used to create a new index.
/// </summary>
public Func<DateIndex, Entity>? CreateIndex { get; set; } = null!;
/// <summary>
/// Gets or sets the ordered list of DateTime formats, such as "yyyy/MM", going
/// from most specific to least
/// specific. Indexes will be created for every applicable entry at all the levels.
/// </summary>
public List<string> Formats { get; set; } = null!;
/// <summary>
/// Gets or sets the threshold where entries will be "collapsed" and emitted a
/// higher level. For example, with a
/// threshold of 10, if there are 10 or less entities, then they will also be
/// emitted at a higher-level index.
/// </summary>
public int LessThanEqualCollapse { get; set; }
public Timekeeper Timekeeper { get; }
/// <inheritdoc />
public override IEnumerable<Entity> Run(IEnumerable<Entity> input)
{
// Validate our input.
this.validator.ValidateAndThrow(this);
// Go through all the inputs that have an Instant, get the DateTime, and then use that to categories each entity
// into all the categories they match. We make the assumption that the entity "belongs" into the most precise
// category they fit in. The `entries` variable is a list with the same indexes as the `this.Formats` property.
List<Dictionary<string, List<Entity>>> entries = new();
for (int i = 0; i < this.Formats.Count; i++)
{
entries.Add(new Dictionary<string, List<Entity>>());
}
// Go through the inputs and group each one. We also use `ToList` to force the enumeration to completely
// resolve and we can get everything we need. We will append the created indexes to the end of this list.
var output = input.ForEachEntity<Instant>((entity, instant) => this.GroupOnFormats(instant, entries, entity))
.ToList();
// Going in reverse order (most precise to less precise), we create the various indexes.
Dictionary<string, List<Entity>> indexes = new();
List<Entity> seen = new();
for (int i = 0; i < this.Formats.Count; i++)
{
foreach (KeyValuePair<string, List<Entity>> pair in entries[i])
{
// Ignore blank entries. This should not be possible, but we're being paranoid.
if (pair.Value.Count == 0)
{
continue;
}
// Get all the entities at this level and split them into ones we've seen (at a lower level) and which
// ones are new (these always go on the index).
var seenEntities = pair.Value.Where(a => seen.Contains(a)).ToList();
var newEntities = pair.Value.Where(a => !seen.Contains(a)).ToList();
seen.AddRange(newEntities);
// The new entities are going to always be added, but if the new + seen is <= the threshold, we'll be
// including both of them.
List<Entity>? childEntities = (newEntities.Count + seenEntities.Count) <= this.LessThanEqualCollapse
? pair.Value
: newEntities;
// Figure out which child indexes need to be included. If there isn't a list, then return an empty one.
List<Entity>? childIndexes = indexes.TryGetValue(pair.Key, out List<Entity>? list)
? list
: new List<Entity>();
// Create the index then add it to the output list.
var index = new DateIndex(pair.Key, childEntities, childIndexes);
Entity? indexEntity = this.CreateIndex!(index);
output.Add(indexEntity);
// Also add the index into the next level up. We don't do this if we are in the last format (-1) plus
// the zero-based index (-1).
if (i > this.Formats.Count - 2)
{
continue;
}
Entity? first = pair.Value[0];
string? nextKey = this.Timekeeper.ToDateTime(first.Get<Instant>()).ToString(this.Formats[i + 1]);
if (!indexes.ContainsKey(nextKey))
{
indexes[nextKey] = new List<Entity>();
}
indexes[nextKey].Add(indexEntity);
}
}
// We are done processing.
return output;
}
public CreateDateIndexes WithFormats(params string[] formats)
{
this.Formats = formats.ToList();
return this;
}
private Entity GroupOnFormats(Instant instant, List<Dictionary<string, List<Entity>>> grouped, Entity entity)
{
var dateTime = this.Timekeeper.ToDateTime(instant);
for (int i = 0; i < this.Formats.Count; i++)
{
string? formatted = dateTime.ToString(this.Formats[i]);
if (!grouped[i].ContainsKey(formatted))
{
grouped[i][formatted] = new List<Entity>();
}
grouped[i][formatted].Add(entity);
}
return entity;
}
}

View file

@ -0,0 +1,32 @@
using System.Collections.Generic;
using Gallium;
namespace Nitride.Temporal;
public class DateIndex
{
public DateIndex(string key, IReadOnlyList<Entity> entries, IReadOnlyList<Entity> indexes)
{
this.Key = key;
this.Entries = entries;
this.Indexes = indexes;
}
/// <summary>
/// Gets the list of entries that are in this index.
/// </summary>
public IReadOnlyList<Entity> Entries { get; }
/// <summary>
/// Gets the ordered list of nested indexes, if there are any. This will be an
/// empty list if the index has no
/// sub-index.
/// </summary>
public IReadOnlyList<Entity> Indexes { get; }
/// <summary>
/// Gets the key for the index, which is a formatted date.
/// </summary>
public string Key { get; }
}

View file

@ -18,39 +18,4 @@ public abstract class NitrideIOTestBase : TestBase<NitrideIOTestContext>
: base(output)
{
}
protected void CompareObjects<T>(T expected, T actual)
where T : class
{
CompareLogic compare = new()
{
Config =
{
MaxDifferences = int.MaxValue,
},
};
ComparisonResult comparison = compare.Compare(expected, actual);
if (comparison.AreEqual)
{
return;
}
// Format the error message.
StringBuilder message = new();
message.AppendLine("# Expected");
message.AppendLine();
message.AppendLine(JsonConvert.SerializeObject(expected, Formatting.Indented));
message.AppendLine();
message.Append("# Actual");
message.AppendLine();
message.AppendLine(JsonConvert.SerializeObject(actual, Formatting.Indented));
message.AppendLine();
message.Append("# Results");
message.AppendLine();
message.AppendLine(comparison.DifferencesString);
throw new XunitException(message.ToString());
}
}

View file

@ -11,8 +11,6 @@ namespace Nitride.IO.Tests;
public class NitrideIOTestContext : NitrideTestContext
{
private static int bob = 0;
public IFileSystem FileSystem => this.Resolve<IFileSystem>();
/// <inheritdoc />
@ -22,13 +20,5 @@ public class NitrideIOTestContext : NitrideTestContext
builder.RegisterModule<NitrideIOModule>();
builder.RegisterInstance(new MemoryFileSystem()).As<IFileSystem>().SingleInstance();
builder.RegisterBuildCallback(x => x.Resolve<ILogger>().Error("Registered!"));
builder.RegisterInstance(new Bob { Value = bob++ }).As<Bob>().SingleInstance();
}
public class Bob
{
public int Value { get; set; }
}
}

View file

@ -3,6 +3,7 @@ using System.Linq;
using Nitride.IO.Contents;
using Nitride.IO.Paths;
using Nitride.Tests;
using Xunit;
using Xunit.Abstractions;
@ -59,6 +60,6 @@ public class DirectChildPathScannerTests : NitrideIOTestBase
new KeyValuePair<string, string[]>("/a/d/", new[] { "/a/d/e/index.md" }),
};
CompareObjects(expected, actual);
TestHelper.CompareObjects(expected, actual);
}
}

View file

@ -4,6 +4,7 @@ using System.Linq;
using Nitride.Entities;
using Nitride.IO.Contents;
using Nitride.IO.Paths;
using Nitride.Tests;
using Xunit;
using Xunit.Abstractions;
@ -72,6 +73,6 @@ public class LinkDirectChildrenTests : NitrideIOTestBase
new Tuple<string, string[]?>("/index.md", new[] { "/a/index.md", "/b/index.md" }),
};
this.CompareObjects(expected, actual);
TestHelper.CompareObjects(expected, actual);
}
}

View file

@ -0,0 +1,205 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Gallium;
using Nitride.Tests;
using Xunit;
using Xunit.Abstractions;
namespace Nitride.Temporal.Tests;
public class CreateDateIndexesTests : TemporalTestBase
{
public CreateDateIndexesTests(ITestOutputHelper output)
: base(output)
{
}
[Fact]
public void MonthOnlyIndexes()
{
using TemporalTestContext context = this.CreateContext();
Timekeeper timekeeper = context.Resolve<Timekeeper>();
CreateDateIndexes op = context.Resolve<CreateDateIndexes>()
.WithFormats("yyyy-MM")
.WithCreateIndex(this.CreateIndex);
List<Entity> input = new()
{
new Entity().Add("page1").Add(timekeeper.CreateInstant(2021, 1, 2)),
new Entity().Add("page2").Add(timekeeper.CreateInstant(2021, 2, 2)),
new Entity().Add("page3").Add(timekeeper.CreateInstant(2022, 1, 2)),
};
List<Tuple<string, List<string>?, List<string>?>> actual = this.GetActual(op, input);
var expected = new List<Tuple<string, List<string>?, List<string>?>>
{
new("index-2021-01", new List<string> { "page1" }, new List<string>()),
new("index-2021-02", new List<string> { "page2" }, new List<string>()),
new("index-2022-01", new List<string> { "page3" }, new List<string>()),
new("page1", null, null),
new("page2", null, null),
new("page3", null, null),
};
TestHelper.CompareObjects(expected, actual);
}
[Fact]
public void YearMonthDayIndexes()
{
using TemporalTestContext context = this.CreateContext();
Timekeeper timekeeper = context.Resolve<Timekeeper>();
CreateDateIndexes op = context.Resolve<CreateDateIndexes>()
.WithFormats("yyyy/MM/dd", "yyyy/MM", "yyyy")
.WithCreateIndex(this.CreateIndex);
List<Entity> input = new()
{
new Entity().Add("page1").Add(timekeeper.CreateInstant(2021, 1, 2)),
new Entity().Add("page2").Add(timekeeper.CreateInstant(2021, 2, 2)),
new Entity().Add("page3").Add(timekeeper.CreateInstant(2022, 1, 2)),
};
List<Tuple<string, List<string>?, List<string>?>> actual = this.GetActual(op, input);
var expected = new List<Tuple<string, List<string>?, List<string>?>>
{
new("index-2021", new List<string>(), new List<string> { "index-2021/01", "index-2021/02" }),
new("index-2021/01", new List<string>(), new List<string> { "index-2021/01/02" }),
new("index-2021/01/02", new List<string> { "page1" }, new List<string>()),
new("index-2021/02", new List<string>(), new List<string> { "index-2021/02/02" }),
new("index-2021/02/02", new List<string> { "page2" }, new List<string>()),
new("index-2022", new List<string>(), new List<string> { "index-2022/01" }),
new("index-2022/01", new List<string>(), new List<string> { "index-2022/01/02" }),
new("index-2022/01/02", new List<string> { "page3" }, new List<string>()),
new("page1", null, null),
new("page2", null, null),
new("page3", null, null),
};
TestHelper.CompareObjects(expected, actual);
}
[Fact]
public void YearMonthDayIndexesThreshold1()
{
using TemporalTestContext context = this.CreateContext();
Timekeeper timekeeper = context.Resolve<Timekeeper>();
CreateDateIndexes op = context.Resolve<CreateDateIndexes>()
.WithFormats("yyyy/MM/dd", "yyyy/MM", "yyyy")
.WithCreateIndex(this.CreateIndex)
.WithLessThanEqualCollapse(1);
List<Entity> input = new()
{
new Entity().Add("page1").Add(timekeeper.CreateInstant(2021, 1, 2)),
new Entity().Add("page2").Add(timekeeper.CreateInstant(2021, 2, 2)),
new Entity().Add("page3").Add(timekeeper.CreateInstant(2022, 1, 2)),
};
List<Tuple<string, List<string>?, List<string>?>> actual = this.GetActual(op, input);
var expected = new List<Tuple<string, List<string>?, List<string>?>>
{
new("index-2021", new List<string>(), new List<string> { "index-2021/01", "index-2021/02" }),
new("index-2021/01", new List<string> { "page1" }, new List<string> { "index-2021/01/02" }),
new("index-2021/01/02", new List<string> { "page1" }, new List<string>()),
new("index-2021/02", new List<string> { "page2" }, new List<string> { "index-2021/02/02" }),
new("index-2021/02/02", new List<string> { "page2" }, new List<string>()),
new("index-2022", new List<string> { "page3" }, new List<string> { "index-2022/01" }),
new("index-2022/01", new List<string> { "page3" }, new List<string> { "index-2022/01/02" }),
new("index-2022/01/02", new List<string> { "page3" }, new List<string>()),
new("page1", null, null),
new("page2", null, null),
new("page3", null, null),
};
TestHelper.CompareObjects(expected, actual);
}
[Fact]
public void YearMonthIndexes()
{
using TemporalTestContext context = this.CreateContext();
Timekeeper timekeeper = context.Resolve<Timekeeper>();
CreateDateIndexes op = context.Resolve<CreateDateIndexes>()
.WithFormats("yyyy-MM", "yyyy")
.WithCreateIndex(this.CreateIndex);
List<Entity> input = new()
{
new Entity().Add("page1").Add(timekeeper.CreateInstant(2021, 1, 2)),
new Entity().Add("page2").Add(timekeeper.CreateInstant(2021, 2, 2)),
new Entity().Add("page3").Add(timekeeper.CreateInstant(2022, 1, 2)),
};
List<Tuple<string, List<string>?, List<string>?>> actual = this.GetActual(op, input);
var expected = new List<Tuple<string, List<string>?, List<string>?>>
{
new("index-2021", new List<string>(), new List<string> { "index-2021-01", "index-2021-02" }),
new("index-2021-01", new List<string> { "page1" }, new List<string>()),
new("index-2021-02", new List<string> { "page2" }, new List<string>()),
new("index-2022", new List<string>(), new List<string> { "index-2022-01" }),
new("index-2022-01", new List<string> { "page3" }, new List<string>()),
new("page1", null, null),
new("page2", null, null),
new("page3", null, null),
};
TestHelper.CompareObjects(expected, actual);
}
[Fact]
public void YearOnlyIndexes()
{
using TemporalTestContext context = this.CreateContext();
Timekeeper timekeeper = context.Resolve<Timekeeper>();
CreateDateIndexes op = context.Resolve<CreateDateIndexes>()
.WithFormats("yyyy")
.WithCreateIndex(this.CreateIndex);
List<Entity> input = new()
{
new Entity().Add("page1").Add(timekeeper.CreateInstant(2021, 1, 2)),
new Entity().Add("page2").Add(timekeeper.CreateInstant(2021, 2, 2)),
new Entity().Add("page3").Add(timekeeper.CreateInstant(2022, 1, 2)),
};
List<Tuple<string, List<string>?, List<string>?>> actual = this.GetActual(op, input);
var expected = new List<Tuple<string, List<string>?, List<string>?>>
{
new("index-2021", new List<string> { "page1", "page2" }, new List<string>()),
new("index-2022", new List<string> { "page3" }, new List<string>()),
new("page1", null, null),
new("page2", null, null),
new("page3", null, null),
};
TestHelper.CompareObjects(expected, actual);
}
private Entity CreateIndex(DateIndex a)
{
return new Entity().Add(a).Add($"index-{a.Key}");
}
private List<Tuple<string, List<string>?, List<string>?>> GetActual(CreateDateIndexes op, List<Entity> input)
{
var actual = op.Run(input)
.Select(
x => new Tuple<string, List<string>?, List<string>?>(
x.Get<string>(),
x.GetOptional<DateIndex>()?.Entries.Select(a => a.Get<string>()).OrderBy(x => x).ToList(),
x.GetOptional<DateIndex>()?.Indexes.Select(a => a.Get<string>()).OrderBy(x => x).ToList()))
.OrderBy(x => x.Item1)
.ToList();
return actual;
}
}

View file

@ -0,0 +1,30 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Nitride.Temporal\Nitride.Temporal.csproj" />
<ProjectReference Include="..\Nitride.Tests\Nitride.Tests.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="CompareNETObjects" Version="4.77.0" />
<PackageReference Include="Gallium" Version="1.0.2" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />
<PackageReference Include="JunitXml.TestLogger" Version="3.0.114" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="3.1.2">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>
</Project>

View file

@ -0,0 +1,13 @@
using MfGames.TestSetup;
using Xunit.Abstractions;
namespace Nitride.Temporal.Tests;
public abstract class TemporalTestBase : TestBase<TemporalTestContext>
{
protected TemporalTestBase(ITestOutputHelper output)
: base(output)
{
}
}

View file

@ -0,0 +1,15 @@
using Autofac;
using Nitride.Tests;
namespace Nitride.Temporal.Tests;
public class TemporalTestContext : NitrideTestContext
{
/// <inheritdoc />
protected override void ConfigureContainer(ContainerBuilder builder)
{
base.ConfigureContainer(builder);
builder.RegisterModule<NitrideTemporalModule>();
}
}

View file

@ -8,6 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="CompareNETObjects" Version="4.77.0" />
<PackageReference Include="Gallium" Version="1.0.2" />
<PackageReference Include="MfGames.TestSetup" Version="1.0.4" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />

View file

@ -0,0 +1,47 @@
using System.Text;
using KellermanSoftware.CompareNetObjects;
using Newtonsoft.Json;
using Xunit.Sdk;
namespace Nitride.Tests;
public static class TestHelper
{
public static void CompareObjects<T>(T expected, T actual)
where T : class
{
CompareLogic compare = new()
{
Config =
{
MaxDifferences = int.MaxValue,
},
};
ComparisonResult comparison = compare.Compare(expected, actual);
if (comparison.AreEqual)
{
return;
}
// Format the error message.
StringBuilder message = new();
message.AppendLine("# Expected");
message.AppendLine();
message.AppendLine(JsonConvert.SerializeObject(expected, Formatting.Indented));
message.AppendLine();
message.AppendLine("# Actual");
message.AppendLine();
message.AppendLine(JsonConvert.SerializeObject(actual, Formatting.Indented));
message.AppendLine();
message.Append("# Results");
message.AppendLine();
message.AppendLine(comparison.DifferencesString);
throw new XunitException(message.ToString());
}
}