diff --git a/tools/Elastic.CommonSchema.Generator/CodeConfiguration.cs b/tools/Elastic.CommonSchema.Generator/CodeConfiguration.cs index 0d70f1ef..b084cd15 100644 --- a/tools/Elastic.CommonSchema.Generator/CodeConfiguration.cs +++ b/tools/Elastic.CommonSchema.Generator/CodeConfiguration.cs @@ -2,6 +2,7 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System; using System.IO; namespace Elastic.CommonSchema.Generator; @@ -10,11 +11,19 @@ public static class CodeConfiguration { static CodeConfiguration() { - //tools/Elastic.CommonSchema.Generator/bin/Debug/net6.0 - var directoryInfo = new DirectoryInfo(Directory.GetCurrentDirectory()); - var rootInfo = new DirectoryInfo(Path.Combine(directoryInfo.FullName, @"../../../../../")); - Root = rootInfo.FullName; + var rootInfo = new DirectoryInfo(Directory.GetCurrentDirectory()); + do + { + var file = new FileInfo(Path.Combine(rootInfo.FullName, "license.txt")); + if (file.Exists) break; + rootInfo = rootInfo.Parent; + + } while (rootInfo != null && rootInfo != rootInfo.Root); + if (rootInfo == null) + throw new Exception("Can not resolve folder structure for ECS.NET codebase"); + + Root = rootInfo.FullName; SourceFolder = Path.Combine(Root, "src"); ToolFolder = Path.Combine(Root, "tools"); ElasticCommonSchemaGeneratedFolder = Path.Combine(SourceFolder, "Elastic.CommonSchema"); @@ -22,7 +31,7 @@ static CodeConfiguration() ViewFolder = Path.Combine(ToolFolder, "Elastic.CommonSchema.Generator", "Views"); } - private static string Root { get; } + public static string Root { get; } private static string SourceFolder { get; } private static string ToolFolder { get; } diff --git a/tools/Elastic.CommonSchema.Generator/Elastic.CommonSchema.Generator.csproj b/tools/Elastic.CommonSchema.Generator/Elastic.CommonSchema.Generator.csproj index 68de7e42..fbcf7d97 100644 --- a/tools/Elastic.CommonSchema.Generator/Elastic.CommonSchema.Generator.csproj +++ b/tools/Elastic.CommonSchema.Generator/Elastic.CommonSchema.Generator.csproj @@ -6,16 +6,18 @@ False False true + latest + - + diff --git a/tools/Elastic.CommonSchema.Generator/Program.cs b/tools/Elastic.CommonSchema.Generator/Program.cs index de0d12d4..5855bc4c 100644 --- a/tools/Elastic.CommonSchema.Generator/Program.cs +++ b/tools/Elastic.CommonSchema.Generator/Program.cs @@ -1,78 +1,85 @@ using System; +using System.IO; using System.Linq; +using System.Threading.Tasks; using Elastic.CommonSchema.Generator.Projection; using Elastic.CommonSchema.Generator.Schema; -namespace Elastic.CommonSchema.Generator +namespace Elastic.CommonSchema.Generator; + +public static class Program { - public static class Program + private const string DefaultDownloadBranch = "v8.9.0"; + + // ReSharper disable once UnusedParameter.Local + private static async Task Main(string[] args) { - private const string DefaultDownloadBranch = "v8.6.0"; + var token = args.Length > 0 ? args[0] : string.Empty; - // ReSharper disable once UnusedParameter.Local - private static void Main(string[] args) - { - var redownloadCoreSpecification = true; - var downloadBranch = DefaultDownloadBranch; + Console.WriteLine($"Running from: {Directory.GetCurrentDirectory()}"); + Console.WriteLine($"Resolved codebase root to: {CodeConfiguration.Root}"); + Console.WriteLine(); - var answer = "invalid"; - while (answer != "y" && answer != "n" && answer != "") - { - Console.Write("Download online specifications? [Y/N] (default N): "); - answer = Console.ReadLine()?.Trim().ToLowerInvariant(); - redownloadCoreSpecification = answer == "y"; - } + var redownloadCoreSpecification = true; + var downloadBranch = DefaultDownloadBranch; - Console.Write($"Tag to use (default {downloadBranch}): "); - var readBranch = Console.ReadLine()?.Trim(); - if (!string.IsNullOrEmpty(readBranch)) downloadBranch = readBranch; + var answer = "invalid"; + while (answer != "y" && answer != "n" && answer != "") + { + Console.Write("Download online specifications? [Y/N] (default N): "); + answer = Console.ReadLine()?.Trim().ToLowerInvariant(); + redownloadCoreSpecification = answer == "y"; + } - if (string.IsNullOrEmpty(downloadBranch)) - downloadBranch = DefaultDownloadBranch; + Console.Write($"Tag to use (default {downloadBranch}): "); + var readBranch = Console.ReadLine()?.Trim(); + if (!string.IsNullOrEmpty(readBranch)) downloadBranch = readBranch; - if (redownloadCoreSpecification) - SpecificationDownloader.Download(downloadBranch); + if (string.IsNullOrEmpty(downloadBranch)) + downloadBranch = DefaultDownloadBranch; + if (redownloadCoreSpecification) + await SpecificationDownloader.DownloadAsync(downloadBranch, token); - var ecsSchema = new EcsSchemaParser(downloadBranch).Parse(); - WarnAboutSchemaValidations(ecsSchema); - var projection = new TypeProjector(ecsSchema).CreateProjection(); - WarnAboutProjectionValidations(projection); + var ecsSchema = new EcsSchemaParser(downloadBranch).Parse(); + WarnAboutSchemaValidations(ecsSchema); - FileGenerator.Generate(projection); - } + var projection = new TypeProjector(ecsSchema).CreateProjection(); + WarnAboutProjectionValidations(projection); - private static void WarnAboutSchemaValidations(EcsSchema ecsSchema) + FileGenerator.Generate(projection); + } + + private static void WarnAboutSchemaValidations(EcsSchema ecsSchema) + { + if (ecsSchema.Warnings.Count > 0) { - if (ecsSchema.Warnings.Count > 0) - { - Console.ForegroundColor = ConsoleColor.Red; - Console.WriteLine("Validation errors in YAML"); - foreach (var warning in ecsSchema.Warnings.Distinct().OrderBy(w => w)) - Console.WriteLine(warning); - Console.ResetColor(); - return; - } - Console.ForegroundColor = ConsoleColor.Green; - Console.WriteLine("No validation errors in YAML"); + Console.ForegroundColor = ConsoleColor.Red; + Console.WriteLine("Validation errors in YAML"); + foreach (var warning in ecsSchema.Warnings.Distinct().OrderBy(w => w)) + Console.WriteLine(warning); Console.ResetColor(); + return; } + Console.ForegroundColor = ConsoleColor.Green; + Console.WriteLine("No validation errors in YAML"); + Console.ResetColor(); + } - private static void WarnAboutProjectionValidations(CommonSchemaTypesProjection projection) + private static void WarnAboutProjectionValidations(CommonSchemaTypesProjection projection) + { + if (projection.Warnings.Count > 0) { - if (projection.Warnings.Count > 0) - { - Console.ForegroundColor = ConsoleColor.Red; - Console.WriteLine("Validation errors in Canonical Model"); - foreach (var warning in projection.Warnings.Distinct().OrderBy(w => w)) - Console.WriteLine(warning); - Console.ResetColor(); - return; - } - Console.ForegroundColor = ConsoleColor.Green; - Console.WriteLine("No validation errors in the Canonical Model"); + Console.ForegroundColor = ConsoleColor.Red; + Console.WriteLine("Validation errors in Canonical Model"); + foreach (var warning in projection.Warnings.Distinct().OrderBy(w => w)) + Console.WriteLine(warning); Console.ResetColor(); + return; } + Console.ForegroundColor = ConsoleColor.Green; + Console.WriteLine("No validation errors in the Canonical Model"); + Console.ResetColor(); } } diff --git a/tools/Elastic.CommonSchema.Generator/SpecificationDownloader.cs b/tools/Elastic.CommonSchema.Generator/SpecificationDownloader.cs index 846e78d0..4e1c5d59 100644 --- a/tools/Elastic.CommonSchema.Generator/SpecificationDownloader.cs +++ b/tools/Elastic.CommonSchema.Generator/SpecificationDownloader.cs @@ -6,27 +6,31 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Net; -using CsQuery; +using System.Threading.Tasks; +using Octokit; using ShellProgressBar; +using static System.Text.Encoding; namespace Elastic.CommonSchema.Generator { - public class SpecificationDownloader + public static class SpecificationDownloader { private const string Core = "Core"; private const string Legacy = "legacy"; private const string Composable = "composable"; private const string Components = "component"; - private static readonly ProgressBarOptions MainProgressBarOptions = new() { BackgroundColor = ConsoleColor.DarkGray }; + private static readonly ProgressBarOptions MainProgressBarOptions = new() + { + BackgroundColor = ConsoleColor.DarkGray, ForegroundColorError = ConsoleColor.Red + }; private static readonly Dictionary OnlineSpecifications = new() { - { Core, "https://github.com/elastic/ecs/tree/{version}/generated/ecs" }, - { Legacy, "https://github.com/elastic/ecs/tree/{version}/generated/elasticsearch/legacy" }, - { Composable, "https://github.com/elastic/ecs/tree/{version}/generated/elasticsearch/composable" }, - { Path.Combine(Composable, Components), "https://github.com/elastic/ecs/tree/{version}/generated/elasticsearch/composable/component" } + { Core, "generated/ecs" }, + { Legacy, "generated/elasticsearch/legacy" }, + { Composable, "generated/elasticsearch/composable" }, + { Path.Combine(Composable, Components), "generated/elasticsearch/composable/component" } }; private static readonly ProgressBarOptions SubProgressBarOptions = new() @@ -37,54 +41,80 @@ public class SpecificationDownloader BackgroundColor = ConsoleColor.DarkGray }; - public static void Download(string branch) + public static async Task DownloadAsync(string branch, string token) { - var specifications = - (from kv in OnlineSpecifications - let url = kv.Value.Replace("{version}", branch) - select new Specification { FolderOnDisk = Path.Combine(branch, kv.Key), Branch = branch, GithubListingUrl = url }) - .ToList(); + var client = new GitHubClient(new ProductHeaderValue("ecs-generator")); + if (!string.IsNullOrEmpty(token)) + client.Credentials = new Credentials(token); + using var queryProgress = new ProgressBar(OnlineSpecifications.Count, "Listing remote files", MainProgressBarOptions); + + await WaitRateLimit(client, queryProgress); + var repo = await client.Repository.Get("elastic", "ecs"); + var specifications = new List(); + foreach (var (folder, remotePath) in OnlineSpecifications) + { + await WaitRateLimit(client, queryProgress); + var contents = await client.Repository.Content.GetAllContentsByRef(repo.Id, remotePath, branch); + var spec = new Specification + { + FolderOnDisk = Path.Combine(branch, folder), + Branch = branch, + RemoteFiles = contents.Select(c => new RemoteFile(c.Name, c.Path)).ToArray() + }; + specifications.Add(spec); + } using var progress = new ProgressBar(specifications.Count, "Downloading specifications", MainProgressBarOptions); foreach (var spec in specifications) { progress.Message = $"Downloading to {spec.FolderOnDisk} for branch {branch}"; - DownloadDefinitions(spec, progress, ".yml"); - DownloadDefinitions(spec, progress, ".json"); + await DownloadDefinitionsAsync(spec, client, progress, ".yml", ".json"); progress.Tick($"Downloaded to {spec.FolderOnDisk} for branch {branch}"); } } - private static void DownloadDefinitions(Specification spec, IProgressBar progress, string filenameMatch) + private static async Task WaitRateLimit(GitHubClient client, ProgressBar progressBar) { - //TODO move to HttpClient -#pragma warning disable SYSLIB0014 -#pragma warning disable CS0618 - var client = new WebClient(); -#pragma warning restore CS0618 -#pragma warning restore SYSLIB0014 - var html = client.DownloadString(spec.GithubListingUrl); - if (!Directory.Exists(CodeConfiguration.SpecificationFolder)) - Directory.CreateDirectory(CodeConfiguration.SpecificationFolder); + var apiInfo = client.GetLastApiInfo(); + var rateLimit = apiInfo?.RateLimit ?? (await client.RateLimit.GetRateLimits()).Rate; + if (rateLimit.Remaining > 0) return; + var options = new ProgressBarOptions + { + ForegroundColor = ConsoleColor.Yellow, + ForegroundColorDone = ConsoleColor.DarkGreen, + BackgroundColor = ConsoleColor.DarkGray, + BackgroundCharacter = '\u2593' + }; + var waitTime = rateLimit.Reset - DateTimeOffset.UtcNow; + using var indeterminate = progressBar.SpawnIndeterminate($"Github rate limit hit, waiting: {waitTime}", options); + await Task.Delay(waitTime); + indeterminate.Finished(); + } - var dom = CQ.Create(html); - WriteToFolder(spec.FolderOnDisk, "root.html", html); + private static async Task DownloadDefinitionsAsync(Specification spec, GitHubClient client, ProgressBar progress, + params string[] filenameMatch + ) + { + if (!Directory.Exists(CodeConfiguration.SpecificationFolder)) + Directory.CreateDirectory(CodeConfiguration.SpecificationFolder); - var endpoints = dom[".js-navigation-open"] - .Select(s => s.InnerText) - .Where(s => !string.IsNullOrEmpty(s) && s.EndsWith(filenameMatch)) - .ToList(); + var endpoints = spec.RemoteFiles.Where(r => filenameMatch.Any(m => r.FileName.EndsWith(m))).ToArray(); - using var subBar = progress.Spawn(endpoints.Count, "fetching individual files", SubProgressBarOptions); - endpoints.ForEach(s => { - var rawFile = spec.GithubDownloadUrl(s); - var fileName = rawFile.Split('/').Last(); - var contents = client.DownloadString(rawFile); - WriteToFolder(spec.FolderOnDisk, fileName, contents); - subBar.Tick($"Downloading {fileName}"); - }); + if (endpoints.Length == 0) + { + progress.WriteErrorLine($"No remote files found to download to: {spec.FolderOnDisk}"); + return; + } + using var subBar = progress.Spawn(endpoints.Length, "fetching individual files", SubProgressBarOptions); + foreach (var endpoint in endpoints) + { + await WaitRateLimit(client, progress); + var bytes = await client.Repository.Content.GetRawContentByRef("elastic", "ecs", endpoint.Path, spec.Branch); + WriteToFolder(spec.FolderOnDisk, endpoint.FileName, UTF8.GetString(bytes)); + subBar.Tick($"Downloading {endpoint.FileName}"); + } } private static void WriteToFolder(string folder, string filename, string contents) @@ -95,15 +125,24 @@ private static void WriteToFolder(string folder, string filename, string content File.WriteAllText(path, contents); } + private readonly struct RemoteFile + { + public readonly string FileName; + public readonly string Path; + + public RemoteFile(string fileName, string path) + { + FileName = fileName; + Path = path; + } + } + private class Specification { // ReSharper disable once UnusedAutoPropertyAccessor.Local public string Branch { get; set; } public string FolderOnDisk { get; set; } - public string GithubListingUrl { get; set; } - - public string GithubDownloadUrl(string file) => - $"{GithubListingUrl.Replace("github.com", "raw.githubusercontent.com").Replace("tree/", "")}/{file}"; + public RemoteFile[] RemoteFiles { get; set; } } } }