From dc1d06c40e5dd4b2be68cd7107c4122b26a22007 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Thu, 3 Aug 2023 10:57:52 +0100 Subject: [PATCH] [CI] Updates Buildkite configuration (#1336) --- .buildkite/Dockerfile | 42 ++++++ .buildkite/certs/ca.crt | 20 +++ .buildkite/certs/ca.key | 27 ++++ .buildkite/certs/testnode.crt | 19 +++ .buildkite/certs/testnode.key | 27 ++++ .buildkite/certs/testnode_san.crt | 20 +++ .buildkite/certs/testnode_san.key | 27 ++++ .buildkite/docker/Dockerfile | 5 + .buildkite/functions/cleanup.sh | 67 +++++++++ .buildkite/functions/imports.sh | 60 +++++++++ .buildkite/functions/wait-for-container.sh | 36 +++++ .buildkite/pipeline.yml | 21 ++- .buildkite/run-elasticsearch.sh | 150 +++++++++++++++++++++ .buildkite/run-repository.sh | 46 +++++++ .buildkite/run-tests | 23 ++++ .buildkite/yaml-tests.sh | 10 ++ 16 files changed, 598 insertions(+), 2 deletions(-) create mode 100644 .buildkite/Dockerfile create mode 100755 .buildkite/certs/ca.crt create mode 100644 .buildkite/certs/ca.key create mode 100755 .buildkite/certs/testnode.crt create mode 100755 .buildkite/certs/testnode.key create mode 100644 .buildkite/certs/testnode_san.crt create mode 100644 .buildkite/certs/testnode_san.key create mode 100644 .buildkite/docker/Dockerfile create mode 100644 .buildkite/functions/cleanup.sh create mode 100644 .buildkite/functions/imports.sh create mode 100644 .buildkite/functions/wait-for-container.sh create mode 100755 .buildkite/run-elasticsearch.sh create mode 100644 .buildkite/run-repository.sh create mode 100755 .buildkite/run-tests create mode 100755 .buildkite/yaml-tests.sh diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile new file mode 100644 index 000000000..7d267991f --- /dev/null +++ b/.buildkite/Dockerfile @@ -0,0 +1,42 @@ +ARG PHP_VERSION=8.0-cli +FROM php:${PHP_VERSION} + +WORKDIR /usr/src/app + +# Install git +RUN apt-get update -qq > /dev/null && \ + apt-get install -y --no-install-recommends git -qq > /dev/null + +# Install zip extension for PHP +RUN apt-get install -y \ + libzip-dev \ + zip \ + libyaml-dev \ + -qq > /dev/null \ + && docker-php-ext-install zip > /dev/null + +# Install yaml extension for PHP +RUN pecl channel-update pecl.php.net > /dev/null +RUN pecl install yaml > /dev/null +RUN docker-php-ext-enable yaml > /dev/null + +# Print ext YAML version +RUN php -i | grep -E "yaml|YAML|Module Version" + +# Print PHP version +RUN php -v + +# Install composer +RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer + +# Print composer version +RUN composer --version + +COPY composer.json . + +# Install dependencies with composer +RUN composer install --no-progress > /dev/null + +COPY . . + +CMD ["bash", ".buildkite/yaml-tests.sh"] diff --git a/.buildkite/certs/ca.crt b/.buildkite/certs/ca.crt new file mode 100755 index 000000000..71f9bfc81 --- /dev/null +++ b/.buildkite/certs/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSjCCAjKgAwIBAgIVAJQLm8V2LcaCTHUcoIfO+KL63nG3MA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTIwMDIyNjA1NTA1N1oXDTIzMDIyNTA1NTA1N1owNDEyMDAG +A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYyajkPvGtUOE5M1OowQfB +kWVrWjo1+LIxzgCeRHp0YztLtdVJ0sk2xoSrt2uZpxcPepdyOseLTjFJex1D2yCR +AEniIqcFif4G72nDih2LlbhpUe/+/MTryj8ZTkFTzI+eMmbQi5FFMaH+kwufmdt/ +5/w8YazO18SxxJUlzMqzfNUrhM8vvvVdxgboU7PWhk28wZHCMHQovomHmzclhRpF +N0FMktA98vHHeRjH19P7rNhifSd7hZzoH3H148HVAKoPgqnZ6vW2O2YfAWOP6ulq +cyszr57p8fS9B2wSdlWW7nVHU1JuKcYD67CxbBS23BeGFgCj4tiNrmxO8S5Yf85v +AgMBAAGjUzBRMB0GA1UdDgQWBBSWAlip9eoPmnG4p4OFZeOUBlAbNDAfBgNVHSME +GDAWgBSWAlip9eoPmnG4p4OFZeOUBlAbNDAPBgNVHRMBAf8EBTADAQH/MA0GCSqG +SIb3DQEBCwUAA4IBAQA19qqrMTWl7YyId+LR/QIHDrP4jfxmrEELrAL58q5Epc1k +XxZLzOBSXoBfBrPdv+3XklWqXrZjKWfdkux0Xmjnl4qul+srrZDLJVZG3I7IrITh +AmQUmL9MuPiMnAcxoGZp1xpijtW8Qmd2qnambbljWfkuVaa4hcVRfrAX6TciIQ21 +bS5aeLGrPqR14h30YzDp0RMmTujEa1o6ExN0+RSTkE9m89Q6WdM69az8JW7YkWqm +I+UCG3TcLd3TXmN1zNQkq4y2ObDK4Sxy/2p6yFPI1Fds5w/zLfBOvvPQY61vEqs8 +SCCcQIe7f6NDpIRIBlty1C9IaEHj7edyHjF6rtYb +-----END CERTIFICATE----- diff --git a/.buildkite/certs/ca.key b/.buildkite/certs/ca.key new file mode 100644 index 000000000..dfc41b558 --- /dev/null +++ b/.buildkite/certs/ca.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpgIBAAKCAQEA2Mmo5D7xrVDhOTNTqMEHwZFla1o6NfiyMc4AnkR6dGM7S7XV +SdLJNsaEq7drmacXD3qXcjrHi04xSXsdQ9sgkQBJ4iKnBYn+Bu9pw4odi5W4aVHv +/vzE68o/GU5BU8yPnjJm0IuRRTGh/pMLn5nbf+f8PGGsztfEscSVJczKs3zVK4TP +L771XcYG6FOz1oZNvMGRwjB0KL6Jh5s3JYUaRTdBTJLQPfLxx3kYx9fT+6zYYn0n +e4Wc6B9x9ePB1QCqD4Kp2er1tjtmHwFjj+rpanMrM6+e6fH0vQdsEnZVlu51R1NS +binGA+uwsWwUttwXhhYAo+LYja5sTvEuWH/ObwIDAQABAoIBAQC8QDGnMnmPdWJ+ +13FYY3cmwel+FXXjFDk5QpgK15A2rUz6a8XxO1d7d1wR+U84uH4v9Na6XQyWjaoD +EyPQnuJiyAtgkZLUHoY244PGR5NsePEQlBSCKmGeF5w/j1LvP/2e9EmP4wKdQYJY +nLxFNcgEBCFnFbKIU5n8fKa/klybCrwlBokenyBro02tqH4LL7h1YMRRrl97fv1V +e/y/0WcMN+KnMglfz6haimBRV2yamCCHHmBImC+wzOgT/quqlxPfI+a3ScHxuA65 +3QyCavaqlPh+T3lXnN/Na4UWqFtzMmwgJX2x1zM5qiln46/JoDiXtagvV43L3rNs +LhPRFeIRAoGBAPhEB7nNpEDNjIRUL6WpebWS9brKAVY7gYn7YQrKGhhCyftyaiBZ +zYgxPaJdqYXf+DmkWlANGoYiwEs40QwkR/FZrvO4+Xh3n3dgtl59ZmieuoQvDsG+ +RYIj+TfBaqhewhZNMMl7dxz7DeyQhyRCdsvl3VqJM0RuOsIrzrhCIEItAoGBAN+K +lgWI7swDpOEaLmu+IWMkGImh1LswXoZqIgi/ywZ7htZjPzidOIeUsMi+lrYsKojG +uU3sBxASsf9kYXDnuUuUbGT5M/N2ipXERt7klUAA/f5sg1IKlTrabaN/HGs/uNtf +Efa8v/h2VyTurdPCJ17TNpbOMDwX1qGM62tyt2CLAoGBAIHCnP8iWq18QeuQTO8b +a3/Z9hHRL22w4H4MI6aOB6GSlxuTq6CJD4IVqo9IwSg17fnCy2l3z9s4IqWuZqUf ++XJOW8ELd2jdrT2qEOfGR1Z7UCVyqxXcq1vgDYx0zZh/HpalddB5dcJx/c8do2Ty +UEE2PcHqYB9uNcvzNbLc7RtpAoGBALbuU0yePUTI6qGnajuTcQEPpeDjhRHWSFRZ +ABcG1N8uMS66Mx9iUcNp462zgeP8iqY5caUZtMHreqxT+gWKK7F0+as7386pwElF +QPXgO18QMMqHBIQb0vlBjJ1SRPBjSiSDTVEML1DljvTTOX7kEJHh6HdKrmBO5b54 +cqMQUo53AoGBAPVWRPUXCqlBz914xKna0ZUh2aesRBg5BvOoq9ey9c52EIU5PXL5 +0Isk8sWSsvhl3tjDPBH5WuL5piKgnCTqkVbEHmWu9s1T57Mw6NuxlPMLBWvyv4c6 +tB9brOxv0ui3qGMuBsBoDKbkNnwXyOXLyFg7O+H4l016A3mLQzJM+NGV +-----END RSA PRIVATE KEY----- diff --git a/.buildkite/certs/testnode.crt b/.buildkite/certs/testnode.crt new file mode 100755 index 000000000..a49dfd775 --- /dev/null +++ b/.buildkite/certs/testnode.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDIzCCAgugAwIBAgIVAMTO6uVx9dLox2t0lY4IcBKZXb5WMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTIwMDIyNjA1NTA1OVoXDTIzMDIyNTA1NTA1OVowEzERMA8G +A1UEAxMIaW5zdGFuY2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDK +YLTOikVENiN/qYupOsoXd7VYYnryyfCC/dK4FC2aozkbqjFzBdvPGAasoc4yEiH5 +CGeXMgJuOjk1maqetmdIsw00j4oHJviYsnGXzxxS5swhD7spcW4Uk4V4tAUzrbfT +vW/2WW/yYCLe5phVb2chz0jL+WYb4bBmdfs/t6RtP9RqsplYAmVp3gZ6lt2YNtvE +k9gz0TVk3DuO1TquIClfRYUjuywS6xDSvxJ8Jl91EfDWM8QU+9F+YAtiv74xl2U3 +P0wwMqNvMxf9/3ak3lTQGsgO4L6cwbKpVLMMzxSVunZz/sgl19xy3qHHz1Qr2MjJ +/2c2J7vahUL4NPRkjJClAgMBAAGjTTBLMB0GA1UdDgQWBBS2Wn8E2VZv4oenY+pR +O8G3zfQXhzAfBgNVHSMEGDAWgBSWAlip9eoPmnG4p4OFZeOUBlAbNDAJBgNVHRME +AjAAMA0GCSqGSIb3DQEBCwUAA4IBAQAvwPvCiJJ6v9jYcyvYY8I3gP0oCwrylpRL +n91UlgRSHUmuAObyOoVN5518gSV/bTU2SDrstcLkLFxHvnfpoGJoxsQEHuGxwDRI +nhYNd62EKLerehNM/F9ILKmvTh8f6QPCzjUuExTXv+63l2Sr6dBS7FHsGs6UKUYO +llM/y9wMZ1LCuZuBg9RhtgpFXRSgDM9Z7Begu0d/BPX9od/qAeZg9Arz4rwUiCN4 +IJOMEBEPi5q1tgeS0Fb1Grpqd0Uz5tZKtEHNKzLG+zSMmkneL62Nk2HsmEFZKwzg +u2pU42UaUE596G6o78s1aLn9ICcElPHTjiuZNSiyuu9IzvFDjGQw +-----END CERTIFICATE----- diff --git a/.buildkite/certs/testnode.key b/.buildkite/certs/testnode.key new file mode 100755 index 000000000..82efeecb9 --- /dev/null +++ b/.buildkite/certs/testnode.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAymC0zopFRDYjf6mLqTrKF3e1WGJ68snwgv3SuBQtmqM5G6ox +cwXbzxgGrKHOMhIh+QhnlzICbjo5NZmqnrZnSLMNNI+KByb4mLJxl88cUubMIQ+7 +KXFuFJOFeLQFM623071v9llv8mAi3uaYVW9nIc9Iy/lmG+GwZnX7P7ekbT/UarKZ +WAJlad4GepbdmDbbxJPYM9E1ZNw7jtU6riApX0WFI7ssEusQ0r8SfCZfdRHw1jPE +FPvRfmALYr++MZdlNz9MMDKjbzMX/f92pN5U0BrIDuC+nMGyqVSzDM8Ulbp2c/7I +Jdfcct6hx89UK9jIyf9nNie72oVC+DT0ZIyQpQIDAQABAoIBADAh7f7NjgnaInlD +ds8KB3SraPsbeQhzlPtiqRJU4j/MIFH/GYG03AGWQkget67a9y+GmzSvlTpoKKEh +6h2TXl9BDpv4o6ht0WRn1HJ5tM/Wyqf2WNpTew3zxCPgFPikkXsPrChYPzLTQJfp +GkP/mfTFmxfAOlPZSp4j41zVLYs53eDkAegFPVfKSr1XNNJ3QODLPcIBfxBYsiC9 +oU+jRW8xYuj31cEl5k5UqrChJ1rm3mt6cguqXKbISuoSvi13gXI6DccqhuLAU+Kr +ib2XYrRP+pWocZo/pM9WUVoNGtFxfY88sAQtvG6gDKo2AURtFyq84Ow0h9mdixV/ +gRIDPcECgYEA5nEqE3OKuG9WuUFGXvjtn4C0F6JjflYWh7AbX51S4F6LKrW6/XHL +Rg4BtF+XReT7OQ6llsV8kZeUxsUckkgDLzSaA8lysNDV5KkhAWHfRqH//QKFbqZi +JL9t3x63Qt81US8s2hQk3khPYTRM8ZB3xHiXvZYSGC/0x/DxfEO3QJECgYEA4NK5 +sxtrat8sFz6SK9nWEKimPjDVzxJ0hxdX4tRq/JdOO5RncawVqt6TNP9gTuxfBvhW +MhJYEsQj8iUoL1dxo9d1eP8HEANNV0iX5OBvJNmgBp+2OyRSyr+PA55+wAxYuAE7 +QKaitOjW57fpArNRt2hQyiSzTuqUFRWTWJHCWNUCgYAEurPTXF6vdFGCUc2g61jt +GhYYGhQSpq+lrz6Qksj9o9MVWE9zHh++21C7o+6V16I0RJGva3QoBMVf4vG4KtQt +5tV2WG8LI+4P2Ey+G4UajP6U8bVNVQrUmD0oBBhcvfn5JY+1Fg6/pRpD82/U0VMz +7AmpMWhDqNBMPiymkTk0kQKBgCuWb05cSI0ly4SOKwS5bRk5uVFhYnKNH255hh6C +FGP4acB/WzbcqC7CjEPAJ0nl5d6SExQOHmk1AcsWjR3wlCWxxiK5PwNJwJrlhh1n +reS1FKN0H36D4lFQpkeLWQOe4Sx7gKNeKzlr0w6Fx3Uwku0+Gju2tdTdAey8jB6l +08opAoGAEe1AuR/OFp2xw6V8TH9UHkkpGxy+OrXI6PX6tgk29PgB+uiMu4RwbjVz +1di1KKq2XecAilVbnyqY+edADxYGbSnci9x5wQRIebfMi3VXKtV8NQBv2as6qwtW +JDcQUWotOHjpdvmfJWWkcBhbAKrgX8ukww00ZI/lC3/rmkGnBBg= +-----END RSA PRIVATE KEY----- diff --git a/.buildkite/certs/testnode_san.crt b/.buildkite/certs/testnode_san.crt new file mode 100644 index 000000000..8abba55b5 --- /dev/null +++ b/.buildkite/certs/testnode_san.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDVjCCAj6gAwIBAgIULh42yRefYlRRl1hvt055LrUH0HwwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMjAwMjI4MDMzNzIwWhcNMjMwMjI3MDMzNzIwWjATMREwDwYD +VQQDEwhpbnN0YW5jZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIUP +t267NN21z+3ukajej8eojSXwP6zHxy7CUAp+sQ7bTq2XCKxkYX3CW9ThcS4cV9mL +ayYdWEYnbEDGYPQDo7Wk3Ih5OEXTMZb/yNEx5D4S2lGMOS5bCDdYx6GvwCMG4jNx +aMktosaxpprAJiHh2oLgQk0hQc/a9JfMo6kJKtuhjxsxjxLwcOHhuaUD7NS0Pjop +CJkSYcrL+nnQPQjKe4uLhAbSyiX914h4QX0CJ0e4z1ccdDX2PFWTrwaIf//vQhCR +wP2YKdfjR0JB4oDAlu85GsIs2cFLPysM5ufuNZO4fCr8uOwloKI8zZ2HhlIfBEcY +Gcy4g9N/9epmxMXZlGcCAwEAAaOBgDB+MB0GA1UdDgQWBBRefYm8DHHDdkTPHhS1 +HEUwTb2uiDAfBgNVHSMEGDAWgBSWAlip9eoPmnG4p4OFZeOUBlAbNDAxBgNVHREE +KjAogglsb2NhbGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAGCA2VzMTAJBgNV +HRMEAjAAMA0GCSqGSIb3DQEBCwUAA4IBAQC+pauqM2wJjQaHyHu+kIm59P4b/5Oj +IH1cYCQfMB7Y2UMLxp0ew+f7o7zzE2DA52YYFDWy6J5DVWtSBPyeFGgX+RH+aA+9 +Iv4cc9QpAs6aFjncorHrzNOrWLgCHIeRAxTR0CAkeP2dUZfDBuMpRyP6rAsYzyLH +Rb3/BfYJSI5vxgt5Ke49Y/ljDKFJTyDmAVrHQ4JWrseYE1UZ2eDkBXeiRlYE/QtB +YsrUSqdL6zvFZyUcilxDUUabNcA+GgeGZ2lAEA90F8vwi62QwRXo3Iv1Hz+6xc43 +nFofDK9D8/qkrUD9iuhpx1974QwPhwWyjn9RZRpbZA4ngRL+szdRXR4N +-----END CERTIFICATE----- diff --git a/.buildkite/certs/testnode_san.key b/.buildkite/certs/testnode_san.key new file mode 100644 index 000000000..75d19539e --- /dev/null +++ b/.buildkite/certs/testnode_san.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAhQ+3brs03bXP7e6RqN6Px6iNJfA/rMfHLsJQCn6xDttOrZcI +rGRhfcJb1OFxLhxX2YtrJh1YRidsQMZg9AOjtaTciHk4RdMxlv/I0THkPhLaUYw5 +LlsIN1jHoa/AIwbiM3FoyS2ixrGmmsAmIeHaguBCTSFBz9r0l8yjqQkq26GPGzGP +EvBw4eG5pQPs1LQ+OikImRJhysv6edA9CMp7i4uEBtLKJf3XiHhBfQInR7jPVxx0 +NfY8VZOvBoh//+9CEJHA/Zgp1+NHQkHigMCW7zkawizZwUs/Kwzm5+41k7h8Kvy4 +7CWgojzNnYeGUh8ERxgZzLiD03/16mbExdmUZwIDAQABAoIBAEwhjulLMVc9JEfV +PP/qv0cUOBYh3LzF3T/yq4slq7Z9YgnOJYdFM8aZgqNNjc09KEJvE5JOLeiNu9Ff +768Nugg+2HM5MCo7SN9FYCfZLOcbMFCCM2FDcnMAV9A512vzD08xryuT8dNPZ6yZ +DfhK2hQRrb2lrpr3gwSrcGRRu3THqvq7X1RIjpLV3teDMeP8rQPAlpj8fmP+kdVV +5y1ihiDIo87McihG9FMavJtBDXQkUEuVw6eIeir8L/zHHD/ZwhYjNHZGWbrB88sz +CkJkfWh/FlA63tCVdJzkmnERALLTVy9mR0Sq6sUlnFhFNO2BRdWgYLrcp9McfTJC +e8+WsSECgYEAuwQ3nAaFL0jqYu1AREyKT/f3WUenf2UsX7dwwV2/yFtQvkzW7ji4 +uZLnfUnZBojtHf35dRo+hDgtvhZhgZNAuPPsbOl/EIMTcbChEqV/3CSTFlhLFM1d +hfM9PoM+Bt/pyUNabjD1sWM0X7WeUhzcddshY3S4daBsNsLuOzweRRcCgYEAtiSS +4qiiGafYsY7gOHuAlOhs/00+1uWIFEHKgoHM9vzCxDN3LCmBdynHk8ZE2TAdhw+l +7xpu6LUxKQDfGmVZa9Epg0kQmVq9c54oQP57pJ3tR+68++insEkfnaZH8jblfq2s +sSkFrY3pdS19edq60nuft64kswKRUUkamCXTXTECgYBdoSfiMpV9bekC7DsPtq5M +iR3KEgi2zEViCmomNTRuL+GF1NyKWdWJ+xVwcYd5MRZdvKimyyPfeGzWTUg14i42 +KtEEWgZmkukqMz8BIeCYq6sENeIpIQQgqv3PjU+Bi5r1S4Y7wsFPNRakkD4aaB6r +1rCppWcwZMeoxwEUoO2aswKBgBdDIIdWJi3EpAY5SyWrkEZ0UMdiZC4p7nE33ddB +IJ5CtdU9BXFcc652ZYjX/58FaCABvZ2F8LhDu92SwOusGfmNIxIjWL1dO2jywA1c +8wmZKd7P/M7nbdMz45fMzs9+d1zwbWfK53C8+R4AC1BuwQF0zHc3BHTgVRLelUjt +O8thAoGAdO2gHIqEsZzTgbvLbsh52eVbumjfNGnrnEv1fjb+o+/wAol8dymcmzbL +bZCRzoyA0qwU9kdPFgX46H6so6o1tUM2GQtVFoT6kDnPv7EkLQK0C4cDh6OOHxDU +NPvr/9fHhQd9EDWDvS1JnVMAdKDO6ELp3SoKGGmCXR2QplnqWAk= +-----END RSA PRIVATE KEY----- diff --git a/.buildkite/docker/Dockerfile b/.buildkite/docker/Dockerfile new file mode 100644 index 000000000..138fb1cb6 --- /dev/null +++ b/.buildkite/docker/Dockerfile @@ -0,0 +1,5 @@ +ARG PHP_VERSION=8.0-cli +FROM php:${PHP_VERSION} + +# Create app directory +WORKDIR /usr/src/app diff --git a/.buildkite/functions/cleanup.sh b/.buildkite/functions/cleanup.sh new file mode 100644 index 000000000..4c25166fb --- /dev/null +++ b/.buildkite/functions/cleanup.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# +# Shared cleanup routines between different steps +# +# Please source .ci/functions/imports.sh as a whole not just this file +# +# Version 1.0.0 +# - Initial version after refactor + +function cleanup_volume { + if [[ "$(docker volume ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m" + (docker volume rm "$1") || true + fi +} +function container_running { + if [[ "$(docker ps -q -f name=$1)" ]]; then + return 0; + else return 1; + fi +} +function cleanup_node { + if container_running "$1"; then + echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m" + (docker container rm --force --volumes "$1") || true + fi + if [[ -n "$1" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m" + cleanup_volume "$1-${suffix}-data" + fi +} +function cleanup_network { + if [[ "$(docker network ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m" + (docker network rm "$1") || true + fi +} + +function cleanup_trap { + status=$? + set +x + if [[ "$DETACH" != "true" ]]; then + echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m" + cleanup_all_in_network "$1" + fi + # status is 0 or SIGINT + if [[ "$status" == "0" || "$status" == "130" ]]; then + echo -e "\n\033[32;1mSUCCESS run-tests\033[0m" + exit 0 + else + echo -e "\n\033[31;1mFAILURE during run-tests\033[0m" + exit ${status} + fi +}; +function cleanup_all_in_network { + + if [[ -z "$(docker network ls -q -f name="^$1\$")" ]]; then + echo -e "\033[34;1mINFO:\033[0m $1 is already deleted\033[0m" + return 0 + fi + containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' $1) + while read -r container; do + cleanup_node "$container" + done <<< "$containers" + cleanup_network $1 + echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m" +}; diff --git a/.buildkite/functions/imports.sh b/.buildkite/functions/imports.sh new file mode 100644 index 000000000..c05f36826 --- /dev/null +++ b/.buildkite/functions/imports.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# Sets up all the common variables and imports relevant functions +# +# Version 1.0.1 +# - Initial version after refactor +# - Validate STACK_VERSION asap + +function require_stack_version() { + if [[ -z $STACK_VERSION ]]; then + echo -e "\033[31;1mERROR:\033[0m Required environment variable [STACK_VERSION] not set\033[0m" + exit 1 + fi +} + +require_stack_version + +if [[ -z $es_node_name ]]; then + # only set these once + set -euo pipefail + export TEST_SUITE=${TEST_SUITE-free} + export RUNSCRIPTS=${RUNSCRIPTS-} + export DETACH=${DETACH-false} + export CLEANUP=${CLEANUP-false} + + export es_node_name=instance + export elastic_password=changeme + export elasticsearch_image=elasticsearch + export elasticsearch_scheme="https" + if [[ $TEST_SUITE != "platinum" ]]; then + export elasticsearch_scheme="http" + fi + export elasticsearch_url=${elasticsearch_scheme}://elastic:${elastic_password}@${es_node_name}:9200 + export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost} + export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}" + + export suffix=rest-test + export moniker=$(echo "$elasticsearch_container" | tr -C "[:alnum:]" '-') + export network_name=${moniker}${suffix} + + export ssl_cert="${script_path}/certs/testnode.crt" + export ssl_key="${script_path}/certs/testnode.key" + export ssl_ca="${script_path}/certs/ca.crt" + +fi + + export script_path=$(dirname $(realpath -s $0)) + source $script_path/functions/cleanup.sh + source $script_path/functions/wait-for-container.sh + trap "cleanup_trap ${network_name}" EXIT + + +if [[ "$CLEANUP" == "true" ]]; then + cleanup_all_in_network $network_name + exit 0 +fi + +echo -e "\033[34;1mINFO:\033[0m Creating network $network_name if it does not exist already \033[0m" +docker network inspect "$network_name" > /dev/null 2>&1 || docker network create "$network_name" + diff --git a/.buildkite/functions/wait-for-container.sh b/.buildkite/functions/wait-for-container.sh new file mode 100644 index 000000000..1a721b588 --- /dev/null +++ b/.buildkite/functions/wait-for-container.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Exposes a routine scripts can call to wait for a container if that container set up a health command +# +# Please source .ci/functions/imports.sh as a whole not just this file +# +# Version 1.0.1 +# - Initial version after refactor +# - Make sure wait_for_contiainer is silent + +function wait_for_container { + set +x + until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do + echo "" + docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1} + echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m" + sleep 2; + done; + + # Always show logs if the container is running, this is very useful both on CI as well as while developing + if container_running $1; then + docker logs $1 + fi + + if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then + cleanup_all_in_network $2 + echo + echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m" + echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m" + return 1 + else + echo + echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1} on docker network: ${network_name}\033[0m" + return 0 + fi +} diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 5eaa03dc9..51cc95da1 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,3 +1,20 @@ steps: - - label: ":php: Greetings" - command: "echo 'Hello, world!'" + - label: ":elasticsearch: :php: ES PHP ({{ matrix.php }}) Test Suite: {{ matrix.suite }}" + agents: + provider: "gcp" + env: + PHP_VERSION: "{{ matrix.php }}" + TEST_SUITE: "{{ matrix.suite }}" + STACK_VERSION: 8.9-SNAPSHOT + matrix: + setup: + suite: + - "free" + - "platinum" + php: + - "8.2-cli" + - "8.1-cli" + - "8.0-cli" + - "7.4-cli" + command: ./.buildkite/run-tests + artifact_paths: "*.xml" diff --git a/.buildkite/run-elasticsearch.sh b/.buildkite/run-elasticsearch.sh new file mode 100755 index 000000000..d6061571b --- /dev/null +++ b/.buildkite/run-elasticsearch.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# +# Launch one or more Elasticsearch nodes via the Docker image, +# to form a cluster suitable for running the REST API tests. +# +# Export the STACK_VERSION variable, eg. '8.0.0-SNAPSHOT'. +# Export the TEST_SUITE variable, eg. 'free' or 'platinum' defaults to 'free'. +# Export the NUMBER_OF_NODES variable to start more than 1 node + +# Version 1.6.1 +# - Initial version of the run-elasticsearch.sh script +# - Deleting the volume should not dependent on the container still running +# - Fixed `ES_JAVA_OPTS` config +# - Moved to STACK_VERSION and TEST_VERSION +# - Refactored into functions and imports +# - Support NUMBER_OF_NODES +# - Added 5 retries on docker pull for fixing transient network errors +# - Added flags to make local CCR configurations work +# - Added action.destructive_requires_name=false as the default will be true in v8 +# - Added ingest.geoip.downloader.enabled=false as it causes false positives in testing +# - Moved ELASTIC_PASSWORD and xpack.security.enabled to the base arguments for "Security On by default" +# - Use https only when TEST_SUITE is "platinum", when "free" use http +# - Set xpack.security.enabled=false for "free" and xpack.security.enabled=true for "platinum" + +script_path=$(dirname $(realpath -s $0)) +source $script_path/functions/imports.sh +set -euo pipefail + +echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m" +cleanup_node $es_node_name + +master_node_name=${es_node_name} +cluster_name=${moniker}${suffix} + +BUILDKITE=${BUILDKITE-false} + +# Set vm.max_map_count kernel setting to 262144 if we're in CI +if [[ "$BUILDKITE" == "true" ]]; then + sudo sysctl -w vm.max_map_count=262144 +fi + +declare -a volumes +environment=($(cat <<-END + --env ELASTIC_PASSWORD=$elastic_password + --env node.name=$es_node_name + --env cluster.name=$cluster_name + --env cluster.initial_master_nodes=$master_node_name + --env discovery.seed_hosts=$master_node_name + --env cluster.routing.allocation.disk.threshold_enabled=false + --env bootstrap.memory_lock=true + --env node.attr.testattr=test + --env path.repo=/tmp + --env repositories.url.allowed_urls=http://snapshot.test* + --env action.destructive_requires_name=false + --env ingest.geoip.downloader.enabled=false + --env cluster.deprecation_indexing.enabled=false +END +)) +if [[ "$TEST_SUITE" == "platinum" ]]; then + environment+=($(cat <<-END + --env xpack.security.enabled=true + --env xpack.license.self_generated.type=trial + --env xpack.security.http.ssl.enabled=true + --env xpack.security.http.ssl.verification_mode=certificate + --env xpack.security.http.ssl.key=certs/testnode.key + --env xpack.security.http.ssl.certificate=certs/testnode.crt + --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt + --env xpack.security.transport.ssl.enabled=true + --env xpack.security.transport.ssl.verification_mode=certificate + --env xpack.security.transport.ssl.key=certs/testnode.key + --env xpack.security.transport.ssl.certificate=certs/testnode.crt + --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt +END +)) + volumes+=($(cat <<-END + --volume $ssl_cert:/usr/share/elasticsearch/config/certs/testnode.crt + --volume $ssl_key:/usr/share/elasticsearch/config/certs/testnode.key + --volume $ssl_ca:/usr/share/elasticsearch/config/certs/ca.crt +END +)) +else + environment+=($(cat <<-END + --env xpack.security.enabled=false + --env xpack.security.http.ssl.enabled=false +END +)) +fi + +cert_validation_flags="" +if [[ "$TEST_SUITE" == "platinum" ]]; then + cert_validation_flags="--insecure --cacert /usr/share/elasticsearch/config/certs/ca.crt --resolve ${es_node_name}:443:127.0.0.1" +fi + +# Pull the container, retry on failures up to 5 times with +# short delays between each attempt. Fixes most transient network errors. +docker_pull_attempts=0 +until [ "$docker_pull_attempts" -ge 5 ] +do + docker pull -q docker.elastic.co/elasticsearch/"$elasticsearch_container" && break + docker_pull_attempts=$((docker_pull_attempts+1)) + echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..." + sleep 10 +done + +NUMBER_OF_NODES=${NUMBER_OF_NODES-1} +http_port=9200 +for (( i=0; i<$NUMBER_OF_NODES; i++, http_port++ )); do + node_name=${es_node_name}$i + node_url=${external_elasticsearch_url/9200/${http_port}}$i + if [[ "$i" == "0" ]]; then node_name=$es_node_name; fi + environment+=($(cat <<-END + --env node.name=$node_name +END +)) + echo "$i: $http_port $node_url " + volume_name=${node_name}-${suffix}-data + volumes+=($(cat <<-END + --volume $volume_name:/usr/share/elasticsearch/data${i} +END +)) + + # make sure we detach for all but the last node if DETACH=false (default) so all nodes are started + local_detach="true" + if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi + echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m" + set -x + docker run \ + --name "$node_name" \ + --network "$network_name" \ + --env "ES_JAVA_OPTS=-Xms1g -Xmx1g -da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions" \ + "${environment[@]}" \ + "${volumes[@]}" \ + --publish "$http_port":9200 \ + --ulimit nofile=65536:65536 \ + --ulimit memlock=-1:-1 \ + --detach="$local_detach" \ + --health-cmd="curl $cert_validation_flags --fail $elasticsearch_url/_cluster/health || exit 1" \ + --health-interval=5s \ + --health-retries=20 \ + --health-timeout=2s \ + --rm \ + docker.elastic.co/elasticsearch/"$elasticsearch_container"; + + set +x + if wait_for_container "$es_node_name" "$network_name"; then + echo -e "\033[32;1mSUCCESS:\033[0m Running on: $node_url\033[0m" + fi + +done + diff --git a/.buildkite/run-repository.sh b/.buildkite/run-repository.sh new file mode 100644 index 000000000..51a3d2bb5 --- /dev/null +++ b/.buildkite/run-repository.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# parameters are available to this script + +# STACK_VERSION -- version e.g Major.Minor.Patch(-Prelease) +# TEST_SUITE -- which test suite to run: oss or xpack +# ELASTICSEARCH_URL -- The url at which elasticsearch is reachable, a default is composed based on STACK_VERSION and TEST_SUITE +# PHP_VERSION -- PHP version (defined in test-matrix.yml, a default is hardcoded here) +script_path=$(dirname $(realpath -s $0)) +source $script_path/functions/imports.sh +set -euo pipefail + +PHP_VERSION=${PHP_VERSION-8.2-cli} +ELASTICSEARCH_URL=${ELASTICSEARCH_URL-"$elasticsearch_url"} +elasticsearch_container=${elasticsearch_container-} + +echo -e "\033[34;1mINFO:\033[0m VERSION ${STACK_VERSION}\033[0m" +echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m" +echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m" +echo -e "\033[34;1mINFO:\033[0m CONTAINER ${elasticsearch_container}\033[0m" +echo -e "\033[34;1mINFO:\033[0m PHP_VERSION ${PHP_VERSION}\033[0m" + +echo -e "\033[1m>>>>> Build docker container >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +docker build \ + --no-cache \ + --file $script_path/Dockerfile \ + --tag elastic/elasticsearch-php \ + --build-arg PHP_VERSION=${PHP_VERSION} \ + . + +echo -e "\033[1m>>>>> Run test:integration >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +repo=$(realpath $(dirname $(realpath -s $0))/../) + +docker run \ + --network=${network_name} \ + --workdir="/usr/src/app" \ + --volume=${repo}/tests:/usr/src/app/tests \ + --env STACK_VERSION=${STACK_VERSION} \ + --env TEST_SUITE=${TEST_SUITE} \ + --env PHP_VERSION=${PHP_VERSION} \ + --env ELASTICSEARCH_URL=${ELASTICSEARCH_URL} \ + --ulimit nofile=65535:65535 \ + --name elasticsearch-php \ + --rm \ + elastic/elasticsearch-php diff --git a/.buildkite/run-tests b/.buildkite/run-tests new file mode 100755 index 000000000..9856309be --- /dev/null +++ b/.buildkite/run-tests @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Version 1.1 +# - Moved to .ci folder and seperated out `run-repository.sh` +# - Add `$RUNSCRIPTS` env var for running Elasticsearch dependent products +script_path=$(dirname $(realpath -s $0)) +source $script_path/functions/imports.sh +set -euo pipefail + +echo "--- :elasticsearch: Starting Elasticsearch" +DETACH=true bash $script_path/run-elasticsearch.sh + +if [[ -n "$RUNSCRIPTS" ]]; then + for RUNSCRIPT in ${RUNSCRIPTS//,/ } ; do + echo -e "\033[1m>>>>> Running run-$RUNSCRIPT.sh >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + CONTAINER_NAME=${RUNSCRIPT} \ + DETACH=true \ + bash $script_path/run-${RUNSCRIPT}.sh + done +fi + +echo "+++ :php: Run Client" +bash $script_path/run-repository.sh diff --git a/.buildkite/yaml-tests.sh b/.buildkite/yaml-tests.sh new file mode 100755 index 000000000..0a83f93e2 --- /dev/null +++ b/.buildkite/yaml-tests.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# Checkout the YAML test from Elasticsearch tag +php util/RestSpecRunner.php + +# Generate the YAML tests for PHPUnit +php util/build_tests.php + +# Run YAML tests +vendor/bin/phpunit -c "phpunit-yaml-${TEST_SUITE}-tests.xml" \ No newline at end of file