Merge pull request #23 from wx257osn2/add-test-with-suite

Fix edge case in decoder when very first OP is QOI_OP_RUN / Add CI with official test suite
wx257osn2 · Feb 24, 2024 · 395017e · 395017e
2 parents be83dde + 9c3a254
commit 395017e
Show file tree

Hide file tree

Showing 4 changed files with 134 additions and 21 deletions.
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -29,17 +29,51 @@ jobs:
     - id: cached
       uses: andstor/file-existence-action@v3
       with:
-        files: images
-    - name: get benchmark suite
+        files: "images, qoi_test_images"
+    - name: get test/benchmark suite
       if: steps.cached.outputs.files_exists == 'false'
       shell: bash
-      run: curl https://qoiformat.org/benchmark/qoi_benchmark_suite.tar | tar x
+      run: |
+        curl -O https://qoiformat.org/qoi_test_images.zip
+        unzip qoi_test_images.zip
+        rm qoi_test_images.zip
+        curl https://qoiformat.org/benchmark/qoi_benchmark_suite.tar | tar x
+    - name: build reference qoiconv
+      shell: bash
+      run: |
+        pushd .dependencies/qoi
+        ln -s ../stb/stb_image.h .
+        ln -s ../stb/stb_image_write.h .
+        make conv
+        popd
+        mv .dependencies/qoi/qoiconv bin/qoiconv_orig
     - name: build
       shell: bash
       run: CXX=clang++-17 make -j
+    - name: test
+      shell: bash
+      run: |
+        pushd qoi_test_images
+        mkdir -p /tmp/qoixx
+        mkdir -p /tmp/qoi
+        for i in ./*.qoi; do
+          # test decode
+          ../bin/qoiconv ${i} /tmp/qoixx/${i%.*}.png
+          ../bin/qoiconv_orig ${i} /tmp/qoi/${i%.*}.png
+          diff /tmp/qoi{,xx}/${i%.*}.png
+        done
+        rm -rf /tmp/qoixx/*.png
+        rm -rf /tmp/qoi/*.png
+        for i in ./*.png; do
+          # test encode
+          ../bin/qoiconv ${i} /tmp/qoixx/${i%.*}.qoi
+          ../bin/qoiconv_orig /tmp/qoixx/${i%.*}.qoi /tmp/qoixx/${i}
+          ../bin/qoiconv_orig ${i} /tmp/qoi/${i%.*}.qoi
+          ../bin/qoiconv_orig /tmp/qoi/${i%.*}.qoi /tmp/qoi/${i}
+          diff /tmp/qoi{,xx}/${i}
+        done
+        popd
+        bin/test
     - name: run
       shell: bash
       run: bin/qoibench 1 images --noreference --nowarmup
-    - name: test
-      shell: bash
-      run: bin/test
diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
@@ -23,17 +23,51 @@ jobs:
     - id: cached
       uses: andstor/file-existence-action@v3
       with:
-        files: images
-    - name: get benchmark suite
+        files: "images, qoi_test_images"
+    - name: get test/benchmark suite
       if: steps.cached.outputs.files_exists == 'false'
       shell: bash
-      run: curl https://qoiformat.org/benchmark/qoi_benchmark_suite.tar | tar x
+      run: |
+        curl -O https://qoiformat.org/qoi_test_images.zip
+        unzip qoi_test_images.zip
+        rm qoi_test_images.zip
+        curl https://qoiformat.org/benchmark/qoi_benchmark_suite.tar | tar x
+    - name: build reference qoiconv
+      shell: bash
+      run: |
+        pushd .dependencies/qoi
+        ln -s ../stb/stb_image.h .
+        ln -s ../stb/stb_image_write.h .
+        make conv
+        popd
+        mv .dependencies/qoi/qoiconv bin/qoiconv_orig
     - name: build
       shell: bash
       run: CXX=$(brew --prefix llvm@17)/bin/clang++ make -j
+    - name: test
+      shell: bash
+      run: |
+        pushd qoi_test_images
+        mkdir -p /tmp/qoixx
+        mkdir -p /tmp/qoi
+        for i in ./*.qoi; do
+          # test decode
+          ../bin/qoiconv ${i} /tmp/qoixx/${i%.*}.png
+          ../bin/qoiconv_orig ${i} /tmp/qoi/${i%.*}.png
+          diff /tmp/qoi{,xx}/${i%.*}.png
+        done
+        rm -rf /tmp/qoixx/*.png
+        rm -rf /tmp/qoi/*.png
+        for i in ./*.png; do
+          # test encode
+          ../bin/qoiconv ${i} /tmp/qoixx/${i%.*}.qoi
+          ../bin/qoiconv_orig /tmp/qoixx/${i%.*}.qoi /tmp/qoixx/${i}
+          ../bin/qoiconv_orig ${i} /tmp/qoi/${i%.*}.qoi
+          ../bin/qoiconv_orig /tmp/qoi/${i%.*}.qoi /tmp/qoi/${i}
+          diff /tmp/qoi{,xx}/${i}
+        done
+        popd
+        bin/test
     - name: run
       shell: bash
       run: bin/qoibench 1 images --noreference --nowarmup
-    - name: test
-      shell: bash
-      run: bin/test
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -17,6 +17,8 @@ jobs:
         install: >-
           git
           curl
+          unzip
+          diffutils
           mingw-w64-ucrt-x86_64-make
           mingw-w64-ucrt-x86_64-clang
     - uses: actions/checkout@v4
@@ -29,17 +31,51 @@ jobs:
     - id: cached
       uses: andstor/file-existence-action@v3
       with:
-        files: images
-    - name: get benchmark suite
+        files: "images, qoi_test_images"
+    - name: get test/benchmark suite
       if: steps.cached.outputs.files_exists == 'false'
-      shell: bash
-      run: curl https://qoiformat.org/benchmark/qoi_benchmark_suite.tar | tar x
+      shell: msys2 {0}
+      run: |
+        curl -O https://qoiformat.org/qoi_test_images.zip
+        unzip qoi_test_images.zip
+        rm qoi_test_images.zip
+        curl https://qoiformat.org/benchmark/qoi_benchmark_suite.tar | tar x
+    - name: build reference qoiconv
+      shell: msys2 {0}
+      run: |
+        pushd .dependencies/qoi
+        ln -s ../stb/stb_image.h .
+        ln -s ../stb/stb_image_write.h .
+        mingw32-make conv
+        popd
+        mv .dependencies/qoi/qoiconv.exe bin/qoiconv_orig.exe
     - name: build
       shell: msys2 {0}
       run: CXX=clang++ mingw32-make -j
+    - name: test
+      shell: msys2 {0}
+      run: |
+        pushd qoi_test_images
+        mkdir -p /tmp/qoixx
+        mkdir -p /tmp/qoi
+        for i in ./*.qoi; do
+          # test decode
+          ../bin/qoiconv ${i} /tmp/qoixx/${i%.*}.png
+          ../bin/qoiconv_orig ${i} /tmp/qoi/${i%.*}.png
+          diff /tmp/qoi{,xx}/${i%.*}.png
+        done
+        rm -rf /tmp/qoixx/*.png
+        rm -rf /tmp/qoi/*.png
+        for i in ./*.png; do
+          # test encode
+          ../bin/qoiconv ${i} /tmp/qoixx/${i%.*}.qoi
+          ../bin/qoiconv_orig /tmp/qoixx/${i%.*}.qoi /tmp/qoixx/${i}
+          ../bin/qoiconv_orig ${i} /tmp/qoi/${i%.*}.qoi
+          ../bin/qoiconv_orig /tmp/qoi/${i%.*}.qoi /tmp/qoi/${i}
+          diff /tmp/qoi{,xx}/${i}
+        done
+        popd
+        bin/test
     - name: run
       shell: msys2 {0}
       run: bin/qoibench 1 images --noreference --nowarmup
-    - name: test
-      shell: msys2 {0}
-      run: bin/test
diff --git a/include/qoixx.hpp b/include/qoixx.hpp
@@ -11,6 +11,7 @@
 #include<bit>
 #include<numeric>
 #include<array>
+#include<utility>
 
 #ifndef QOIXX_NO_SIMD
 #if defined(__ARM_FEATURE_SVE)
@@ -1146,7 +1147,7 @@ class qoi{
     static constexpr auto hash_diff_table = luma_hash_diff_table.data() + hash_table_offset;
     )
 
-    const auto f = [&pixels, &p, &px_len, &size, &px, &index QOIXX_HPP_WITH_TABLES(, &hash)]{
+    const auto f = [&pixels, &p, &px_len, &size, &px, &index QOIXX_HPP_WITH_TABLES(, &hash)](bool first){
       static constexpr std::uint32_t mask_tail_6 = 0b0011'1111u;
       [[maybe_unused]] static constexpr std::uint32_t mask_tail_4 = 0b0000'1111u;
       [[maybe_unused]] static constexpr std::uint32_t mask_tail_2 = 0b0000'0011u;
@@ -1187,6 +1188,13 @@ class qoi{
             run = px_len;
           px_len -= run;
           QOIXX_HPP_DECODE_RUN(px, run)
+          if(first)[[unlikely]]{
+            QOIXX_HPP_WITH_TABLES(hash = (0*3+0*5+0*7+255*11) % index_size;)
+            if constexpr(std::is_same<rgba_t, qoi::rgba_t>::value)
+              index[QOIXX_HPP_WITH_TABLES(hash) QOIXX_HPP_WITHOUT_TABLES((0*3+0*5+0*7+255*11) % index_size)] = px;
+            else
+              efficient_memcpy<Channels>(index + QOIXX_HPP_WITH_TABLES(hash) QOIXX_HPP_WITHOUT_TABLES((0*3+0*5+0*7+255*11) % index_size), &px);
+          }
           return;
         }
         if(b1 == chunk_tag::rgb){
@@ -1271,8 +1279,9 @@ class qoi{
       push<Channels>(pixels, &px);
     };
 
+    bool first = true;
     while(px_len--)[[likely]]{
-      f();
+      f(std::exchange(first, false));
       if(size < sizeof(padding))[[unlikely]]{
         throw std::runtime_error("qoixx::qoi::decode: insufficient input data");
       }