diff --git a/tools/Elastic.CommonSchema.Generator/CodeConfiguration.cs b/tools/Elastic.CommonSchema.Generator/CodeConfiguration.cs
index 0d70f1ef..b084cd15 100644
--- a/tools/Elastic.CommonSchema.Generator/CodeConfiguration.cs
+++ b/tools/Elastic.CommonSchema.Generator/CodeConfiguration.cs
@@ -2,6 +2,7 @@
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information
+using System;
using System.IO;
namespace Elastic.CommonSchema.Generator;
@@ -10,11 +11,19 @@ public static class CodeConfiguration
{
static CodeConfiguration()
{
- //tools/Elastic.CommonSchema.Generator/bin/Debug/net6.0
- var directoryInfo = new DirectoryInfo(Directory.GetCurrentDirectory());
- var rootInfo = new DirectoryInfo(Path.Combine(directoryInfo.FullName, @"../../../../../"));
- Root = rootInfo.FullName;
+ var rootInfo = new DirectoryInfo(Directory.GetCurrentDirectory());
+ do
+ {
+ var file = new FileInfo(Path.Combine(rootInfo.FullName, "license.txt"));
+ if (file.Exists) break;
+ rootInfo = rootInfo.Parent;
+
+ } while (rootInfo != null && rootInfo != rootInfo.Root);
+ if (rootInfo == null)
+ throw new Exception("Can not resolve folder structure for ECS.NET codebase");
+
+ Root = rootInfo.FullName;
SourceFolder = Path.Combine(Root, "src");
ToolFolder = Path.Combine(Root, "tools");
ElasticCommonSchemaGeneratedFolder = Path.Combine(SourceFolder, "Elastic.CommonSchema");
@@ -22,7 +31,7 @@ static CodeConfiguration()
ViewFolder = Path.Combine(ToolFolder, "Elastic.CommonSchema.Generator", "Views");
}
- private static string Root { get; }
+ public static string Root { get; }
private static string SourceFolder { get; }
private static string ToolFolder { get; }
diff --git a/tools/Elastic.CommonSchema.Generator/Elastic.CommonSchema.Generator.csproj b/tools/Elastic.CommonSchema.Generator/Elastic.CommonSchema.Generator.csproj
index 68de7e42..fbcf7d97 100644
--- a/tools/Elastic.CommonSchema.Generator/Elastic.CommonSchema.Generator.csproj
+++ b/tools/Elastic.CommonSchema.Generator/Elastic.CommonSchema.Generator.csproj
@@ -6,16 +6,18 @@
False
False
true
+ latest
+
-
+
diff --git a/tools/Elastic.CommonSchema.Generator/Program.cs b/tools/Elastic.CommonSchema.Generator/Program.cs
index de0d12d4..5855bc4c 100644
--- a/tools/Elastic.CommonSchema.Generator/Program.cs
+++ b/tools/Elastic.CommonSchema.Generator/Program.cs
@@ -1,78 +1,85 @@
using System;
+using System.IO;
using System.Linq;
+using System.Threading.Tasks;
using Elastic.CommonSchema.Generator.Projection;
using Elastic.CommonSchema.Generator.Schema;
-namespace Elastic.CommonSchema.Generator
+namespace Elastic.CommonSchema.Generator;
+
+public static class Program
{
- public static class Program
+ private const string DefaultDownloadBranch = "v8.9.0";
+
+ // ReSharper disable once UnusedParameter.Local
+ private static async Task Main(string[] args)
{
- private const string DefaultDownloadBranch = "v8.6.0";
+ var token = args.Length > 0 ? args[0] : string.Empty;
- // ReSharper disable once UnusedParameter.Local
- private static void Main(string[] args)
- {
- var redownloadCoreSpecification = true;
- var downloadBranch = DefaultDownloadBranch;
+ Console.WriteLine($"Running from: {Directory.GetCurrentDirectory()}");
+ Console.WriteLine($"Resolved codebase root to: {CodeConfiguration.Root}");
+ Console.WriteLine();
- var answer = "invalid";
- while (answer != "y" && answer != "n" && answer != "")
- {
- Console.Write("Download online specifications? [Y/N] (default N): ");
- answer = Console.ReadLine()?.Trim().ToLowerInvariant();
- redownloadCoreSpecification = answer == "y";
- }
+ var redownloadCoreSpecification = true;
+ var downloadBranch = DefaultDownloadBranch;
- Console.Write($"Tag to use (default {downloadBranch}): ");
- var readBranch = Console.ReadLine()?.Trim();
- if (!string.IsNullOrEmpty(readBranch)) downloadBranch = readBranch;
+ var answer = "invalid";
+ while (answer != "y" && answer != "n" && answer != "")
+ {
+ Console.Write("Download online specifications? [Y/N] (default N): ");
+ answer = Console.ReadLine()?.Trim().ToLowerInvariant();
+ redownloadCoreSpecification = answer == "y";
+ }
- if (string.IsNullOrEmpty(downloadBranch))
- downloadBranch = DefaultDownloadBranch;
+ Console.Write($"Tag to use (default {downloadBranch}): ");
+ var readBranch = Console.ReadLine()?.Trim();
+ if (!string.IsNullOrEmpty(readBranch)) downloadBranch = readBranch;
- if (redownloadCoreSpecification)
- SpecificationDownloader.Download(downloadBranch);
+ if (string.IsNullOrEmpty(downloadBranch))
+ downloadBranch = DefaultDownloadBranch;
+ if (redownloadCoreSpecification)
+ await SpecificationDownloader.DownloadAsync(downloadBranch, token);
- var ecsSchema = new EcsSchemaParser(downloadBranch).Parse();
- WarnAboutSchemaValidations(ecsSchema);
- var projection = new TypeProjector(ecsSchema).CreateProjection();
- WarnAboutProjectionValidations(projection);
+ var ecsSchema = new EcsSchemaParser(downloadBranch).Parse();
+ WarnAboutSchemaValidations(ecsSchema);
- FileGenerator.Generate(projection);
- }
+ var projection = new TypeProjector(ecsSchema).CreateProjection();
+ WarnAboutProjectionValidations(projection);
- private static void WarnAboutSchemaValidations(EcsSchema ecsSchema)
+ FileGenerator.Generate(projection);
+ }
+
+ private static void WarnAboutSchemaValidations(EcsSchema ecsSchema)
+ {
+ if (ecsSchema.Warnings.Count > 0)
{
- if (ecsSchema.Warnings.Count > 0)
- {
- Console.ForegroundColor = ConsoleColor.Red;
- Console.WriteLine("Validation errors in YAML");
- foreach (var warning in ecsSchema.Warnings.Distinct().OrderBy(w => w))
- Console.WriteLine(warning);
- Console.ResetColor();
- return;
- }
- Console.ForegroundColor = ConsoleColor.Green;
- Console.WriteLine("No validation errors in YAML");
+ Console.ForegroundColor = ConsoleColor.Red;
+ Console.WriteLine("Validation errors in YAML");
+ foreach (var warning in ecsSchema.Warnings.Distinct().OrderBy(w => w))
+ Console.WriteLine(warning);
Console.ResetColor();
+ return;
}
+ Console.ForegroundColor = ConsoleColor.Green;
+ Console.WriteLine("No validation errors in YAML");
+ Console.ResetColor();
+ }
- private static void WarnAboutProjectionValidations(CommonSchemaTypesProjection projection)
+ private static void WarnAboutProjectionValidations(CommonSchemaTypesProjection projection)
+ {
+ if (projection.Warnings.Count > 0)
{
- if (projection.Warnings.Count > 0)
- {
- Console.ForegroundColor = ConsoleColor.Red;
- Console.WriteLine("Validation errors in Canonical Model");
- foreach (var warning in projection.Warnings.Distinct().OrderBy(w => w))
- Console.WriteLine(warning);
- Console.ResetColor();
- return;
- }
- Console.ForegroundColor = ConsoleColor.Green;
- Console.WriteLine("No validation errors in the Canonical Model");
+ Console.ForegroundColor = ConsoleColor.Red;
+ Console.WriteLine("Validation errors in Canonical Model");
+ foreach (var warning in projection.Warnings.Distinct().OrderBy(w => w))
+ Console.WriteLine(warning);
Console.ResetColor();
+ return;
}
+ Console.ForegroundColor = ConsoleColor.Green;
+ Console.WriteLine("No validation errors in the Canonical Model");
+ Console.ResetColor();
}
}
diff --git a/tools/Elastic.CommonSchema.Generator/SpecificationDownloader.cs b/tools/Elastic.CommonSchema.Generator/SpecificationDownloader.cs
index 846e78d0..4e1c5d59 100644
--- a/tools/Elastic.CommonSchema.Generator/SpecificationDownloader.cs
+++ b/tools/Elastic.CommonSchema.Generator/SpecificationDownloader.cs
@@ -6,27 +6,31 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
-using System.Net;
-using CsQuery;
+using System.Threading.Tasks;
+using Octokit;
using ShellProgressBar;
+using static System.Text.Encoding;
namespace Elastic.CommonSchema.Generator
{
- public class SpecificationDownloader
+ public static class SpecificationDownloader
{
private const string Core = "Core";
private const string Legacy = "legacy";
private const string Composable = "composable";
private const string Components = "component";
- private static readonly ProgressBarOptions MainProgressBarOptions = new() { BackgroundColor = ConsoleColor.DarkGray };
+ private static readonly ProgressBarOptions MainProgressBarOptions = new()
+ {
+ BackgroundColor = ConsoleColor.DarkGray, ForegroundColorError = ConsoleColor.Red
+ };
private static readonly Dictionary OnlineSpecifications = new()
{
- { Core, "https://github.com/elastic/ecs/tree/{version}/generated/ecs" },
- { Legacy, "https://github.com/elastic/ecs/tree/{version}/generated/elasticsearch/legacy" },
- { Composable, "https://github.com/elastic/ecs/tree/{version}/generated/elasticsearch/composable" },
- { Path.Combine(Composable, Components), "https://github.com/elastic/ecs/tree/{version}/generated/elasticsearch/composable/component" }
+ { Core, "generated/ecs" },
+ { Legacy, "generated/elasticsearch/legacy" },
+ { Composable, "generated/elasticsearch/composable" },
+ { Path.Combine(Composable, Components), "generated/elasticsearch/composable/component" }
};
private static readonly ProgressBarOptions SubProgressBarOptions = new()
@@ -37,54 +41,80 @@ public class SpecificationDownloader
BackgroundColor = ConsoleColor.DarkGray
};
- public static void Download(string branch)
+ public static async Task DownloadAsync(string branch, string token)
{
- var specifications =
- (from kv in OnlineSpecifications
- let url = kv.Value.Replace("{version}", branch)
- select new Specification { FolderOnDisk = Path.Combine(branch, kv.Key), Branch = branch, GithubListingUrl = url })
- .ToList();
+ var client = new GitHubClient(new ProductHeaderValue("ecs-generator"));
+ if (!string.IsNullOrEmpty(token))
+ client.Credentials = new Credentials(token);
+ using var queryProgress = new ProgressBar(OnlineSpecifications.Count, "Listing remote files", MainProgressBarOptions);
+
+ await WaitRateLimit(client, queryProgress);
+ var repo = await client.Repository.Get("elastic", "ecs");
+ var specifications = new List();
+ foreach (var (folder, remotePath) in OnlineSpecifications)
+ {
+ await WaitRateLimit(client, queryProgress);
+ var contents = await client.Repository.Content.GetAllContentsByRef(repo.Id, remotePath, branch);
+ var spec = new Specification
+ {
+ FolderOnDisk = Path.Combine(branch, folder),
+ Branch = branch,
+ RemoteFiles = contents.Select(c => new RemoteFile(c.Name, c.Path)).ToArray()
+ };
+ specifications.Add(spec);
+ }
using var progress = new ProgressBar(specifications.Count, "Downloading specifications", MainProgressBarOptions);
foreach (var spec in specifications)
{
progress.Message = $"Downloading to {spec.FolderOnDisk} for branch {branch}";
- DownloadDefinitions(spec, progress, ".yml");
- DownloadDefinitions(spec, progress, ".json");
+ await DownloadDefinitionsAsync(spec, client, progress, ".yml", ".json");
progress.Tick($"Downloaded to {spec.FolderOnDisk} for branch {branch}");
}
}
- private static void DownloadDefinitions(Specification spec, IProgressBar progress, string filenameMatch)
+ private static async Task WaitRateLimit(GitHubClient client, ProgressBar progressBar)
{
- //TODO move to HttpClient
-#pragma warning disable SYSLIB0014
-#pragma warning disable CS0618
- var client = new WebClient();
-#pragma warning restore CS0618
-#pragma warning restore SYSLIB0014
- var html = client.DownloadString(spec.GithubListingUrl);
- if (!Directory.Exists(CodeConfiguration.SpecificationFolder))
- Directory.CreateDirectory(CodeConfiguration.SpecificationFolder);
+ var apiInfo = client.GetLastApiInfo();
+ var rateLimit = apiInfo?.RateLimit ?? (await client.RateLimit.GetRateLimits()).Rate;
+ if (rateLimit.Remaining > 0) return;
+ var options = new ProgressBarOptions
+ {
+ ForegroundColor = ConsoleColor.Yellow,
+ ForegroundColorDone = ConsoleColor.DarkGreen,
+ BackgroundColor = ConsoleColor.DarkGray,
+ BackgroundCharacter = '\u2593'
+ };
+ var waitTime = rateLimit.Reset - DateTimeOffset.UtcNow;
+ using var indeterminate = progressBar.SpawnIndeterminate($"Github rate limit hit, waiting: {waitTime}", options);
+ await Task.Delay(waitTime);
+ indeterminate.Finished();
+ }
- var dom = CQ.Create(html);
- WriteToFolder(spec.FolderOnDisk, "root.html", html);
+ private static async Task DownloadDefinitionsAsync(Specification spec, GitHubClient client, ProgressBar progress,
+ params string[] filenameMatch
+ )
+ {
+ if (!Directory.Exists(CodeConfiguration.SpecificationFolder))
+ Directory.CreateDirectory(CodeConfiguration.SpecificationFolder);
- var endpoints = dom[".js-navigation-open"]
- .Select(s => s.InnerText)
- .Where(s => !string.IsNullOrEmpty(s) && s.EndsWith(filenameMatch))
- .ToList();
+ var endpoints = spec.RemoteFiles.Where(r => filenameMatch.Any(m => r.FileName.EndsWith(m))).ToArray();
- using var subBar = progress.Spawn(endpoints.Count, "fetching individual files", SubProgressBarOptions);
- endpoints.ForEach(s => {
- var rawFile = spec.GithubDownloadUrl(s);
- var fileName = rawFile.Split('/').Last();
- var contents = client.DownloadString(rawFile);
- WriteToFolder(spec.FolderOnDisk, fileName, contents);
- subBar.Tick($"Downloading {fileName}");
- });
+ if (endpoints.Length == 0)
+ {
+ progress.WriteErrorLine($"No remote files found to download to: {spec.FolderOnDisk}");
+ return;
+ }
+ using var subBar = progress.Spawn(endpoints.Length, "fetching individual files", SubProgressBarOptions);
+ foreach (var endpoint in endpoints)
+ {
+ await WaitRateLimit(client, progress);
+ var bytes = await client.Repository.Content.GetRawContentByRef("elastic", "ecs", endpoint.Path, spec.Branch);
+ WriteToFolder(spec.FolderOnDisk, endpoint.FileName, UTF8.GetString(bytes));
+ subBar.Tick($"Downloading {endpoint.FileName}");
+ }
}
private static void WriteToFolder(string folder, string filename, string contents)
@@ -95,15 +125,24 @@ private static void WriteToFolder(string folder, string filename, string content
File.WriteAllText(path, contents);
}
+ private readonly struct RemoteFile
+ {
+ public readonly string FileName;
+ public readonly string Path;
+
+ public RemoteFile(string fileName, string path)
+ {
+ FileName = fileName;
+ Path = path;
+ }
+ }
+
private class Specification
{
// ReSharper disable once UnusedAutoPropertyAccessor.Local
public string Branch { get; set; }
public string FolderOnDisk { get; set; }
- public string GithubListingUrl { get; set; }
-
- public string GithubDownloadUrl(string file) =>
- $"{GithubListingUrl.Replace("github.com", "raw.githubusercontent.com").Replace("tree/", "")}/{file}";
+ public RemoteFile[] RemoteFiles { get; set; }
}
}
}