Adds JPEG XL support, max_height/max_width support (#321)

* WIP JXL * Fix test * Tries to fix autobuild * Tries to fix autobuild * Add setup go in codeql * Bump actions version * Do not print curl output in CI * Do not print curl output in CI * Remove Metadata on RAW image * Update sample config * better loop * Prefetch should also respect AllowedType * Better Export params and UA handle * Only do conversion on supported formats * CONVERT_TYPES default to webp only * CONVERT_TYPES default to webp only * Add GIF to AllowedTypes * Update README
webp-sh · Mar 22, 2024 · 43c275e · 43c275e
1 parent ddcb532
commit 43c275e
Show file tree

Hide file tree

Showing 20 changed files with 405 additions and 126 deletions.
diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml
@@ -40,7 +40,7 @@ jobs:
           submodules: true
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Cache Docker layers
         uses: actions/cache@v2

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -15,7 +15,6 @@ on:
   push:
     branches: [ master ]
   pull_request:
-    # The branches below must be a subset of the branches above
     branches: [ master ]
   schedule:
     - cron: '32 20 * * 2'
@@ -33,38 +32,23 @@ jobs:
       fail-fast: false
       matrix:
         language: [ 'go' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://git.io/codeql-language-support
 
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
 
-    # Initializes the CodeQL tools for scanning.
+    - name: Install Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: '1.22'
+
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
       uses: github/codeql-action/autobuild@v2
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml
@@ -32,7 +32,7 @@ jobs:
       - name: Send Requests to Server
         run: |
           cd pics
-          find * -type f -print | xargs -I {} curl -H "Accept: image/webp" http://localhost:3333/{}
+          find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" http://localhost:3333/{}
 
       - name: Get container RAM stats
         run: |
@@ -61,7 +61,7 @@ jobs:
       - name: Send Requests to Server
         run: |
           cd pics
-          find * -type f -print | xargs -I {} curl -H "Accept: image/webp" http://localhost:3333/{}
+          find * -type f -print | xargs -I {} curl -o /dev/null -H "Accept: image/webp" http://localhost:3333/{}
 
       - name: Get container RAM stats
         run: |

diff --git a/.github/workflows/release_docker_image.yaml b/.github/workflows/release_docker_image.yaml
@@ -43,10 +43,10 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Cache Docker layers
         uses: actions/cache@v2

diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@ Currently supported image format: JPEG, PNG, BMP, GIF, SVG, HEIC, NEF, WEBP
 
 > e.g When you visit `https://your.website/pics/tsuki.jpg`，it will serve as `image/webp`/`image/avif` format without changing the URL.
 >
-> GIF image will not be converted to AVIF format even with `ENABLE_AVIF` to `true`, because the converted AVIF image is not animated.
+> GIF image will not be converted to AVIF format because the converted AVIF image is not animated.
 
 ## Usage with Docker(recommended)
 
@@ -32,8 +32,6 @@ services:
     image: webpsh/webp-server-go
     # image: ghcr.io/webp-sh/webp_server_go
     restart: always
-    environment:
-      - MALLOC_ARENA_MAX=1
     volumes:
       - ./path/to/pics:/opt/pics
       - ./exhaust:/opt/exhaust
@@ -74,8 +72,6 @@ services:
     image: webpsh/webp-server-go
     # image: ghcr.io/webp-sh/webp_server_go
     restart: always
-    environment:
-      - MALLOC_ARENA_MAX=1
     volumes:
       - ./path/to/pics:/opt/pics
       - ./path/to/exhaust:/opt/exhaust

diff --git a/config.json b/config.json
@@ -4,12 +4,13 @@
   "QUALITY": "80",
   "IMG_PATH": "./pics",
   "EXHAUST_PATH": "./exhaust",
-  "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","gif","svg","heic"],
   "IMG_MAP": {},
-  "ENABLE_AVIF": false,
+  "ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"],
+  "CONVERT_TYPES": ["webp"],
+  "STRIP_METADATA": true,
   "ENABLE_EXTRA_PARAMS": false,
   "READ_BUFFER_SIZE": 4096,
   "CONCURRENCY": 262144,
   "DISABLE_KEEPALIVE": false,
   "CACHE_TTL": 259200
-}
+}
diff --git a/config/config.go b/config/config.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"regexp"
 	"runtime"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -28,13 +29,14 @@ const (
   "IMG_PATH": "./pics",
   "EXHAUST_PATH": "./exhaust",
   "IMG_MAP": {},
-  "ALLOWED_TYPES": ["jpg","png","jpeg","bmp","svg","heic","nef"],
-  "ENABLE_AVIF": false,
-  "ENABLE_EXTRA_PARAMS": false
+  "ALLOWED_TYPES": ["jpg","png","jpeg","gif","bmp","svg","heic","nef"],
+  "CONVERT_TYPES": ["webp"],
+  "STRIP_METADATA": true,
+  "ENABLE_EXTRA_PARAMS": false,
   "READ_BUFFER_SIZE": 4096,
   "CONCURRENCY": 262144,
   "DISABLE_KEEPALIVE": false,
-  "CACHE_TTL": 259200,
+  "CACHE_TTL": 259200
 }`
 )
 
@@ -47,7 +49,7 @@ var (
 	ProxyMode      bool
 	Prefetch       bool
 	Config         = NewWebPConfig()
-	Version        = "0.10.8"
+	Version        = "0.11.0"
 	WriteLock      = cache.New(5*time.Minute, 10*time.Minute)
 	ConvertLock    = cache.New(5*time.Minute, 10*time.Minute)
 	RemoteRaw      = "./remote-raw"
@@ -63,32 +65,44 @@ type MetaFile struct {
 }
 
 type WebpConfig struct {
-	Host              string            `json:"HOST"`
-	Port              string            `json:"PORT"`
-	ImgPath           string            `json:"IMG_PATH"`
-	Quality           int               `json:"QUALITY,string"`
-	AllowedTypes      []string          `json:"ALLOWED_TYPES"`
-	ImageMap          map[string]string `json:"IMG_MAP"`
-	ExhaustPath       string            `json:"EXHAUST_PATH"`
-	EnableAVIF        bool              `json:"ENABLE_AVIF"`
-	EnableExtraParams bool              `json:"ENABLE_EXTRA_PARAMS"`
-	ReadBufferSize    int               `json:"READ_BUFFER_SIZE"`
-	Concurrency       int               `json:"CONCURRENCY"`
-	DisableKeepalive  bool              `json:"DISABLE_KEEPALIVE"`
-	CacheTTL          int               `json:"CACHE_TTL"`
+	Host         string            `json:"HOST"`
+	Port         string            `json:"PORT"`
+	ImgPath      string            `json:"IMG_PATH"`
+	Quality      int               `json:"QUALITY,string"`
+	AllowedTypes []string          `json:"ALLOWED_TYPES"`
+	ConvertTypes []string          `json:"CONVERT_TYPES"`
+	ImageMap     map[string]string `json:"IMG_MAP"`
+	ExhaustPath  string            `json:"EXHAUST_PATH"`
+
+	EnableWebP bool `json:"ENABLE_WEBP"`
+	EnableAVIF bool `json:"ENABLE_AVIF"`
+	EnableJXL  bool `json:"ENABLE_JXL"`
+
+	EnableExtraParams bool `json:"ENABLE_EXTRA_PARAMS"`
+	StripMetadata     bool `json:"STRIP_METADATA"`
+	ReadBufferSize    int  `json:"READ_BUFFER_SIZE"`
+	Concurrency       int  `json:"CONCURRENCY"`
+	DisableKeepalive  bool `json:"DISABLE_KEEPALIVE"`
+	CacheTTL          int  `json:"CACHE_TTL"`
 }
 
 func NewWebPConfig() *WebpConfig {
 	return &WebpConfig{
-		Host:              "0.0.0.0",
-		Port:              "3333",
-		ImgPath:           "./pics",
-		Quality:           80,
-		AllowedTypes:      []string{"jpg", "png", "jpeg", "bmp", "svg", "nef", "heic", "webp"},
-		ImageMap:          map[string]string{},
-		ExhaustPath:       "./exhaust",
-		EnableAVIF:        false,
+		Host:         "0.0.0.0",
+		Port:         "3333",
+		ImgPath:      "./pics",
+		Quality:      80,
+		AllowedTypes: []string{"jpg", "png", "jpeg", "bmp", "gif", "svg", "nef", "heic", "webp"},
+		ConvertTypes: []string{"webp"},
+		ImageMap:     map[string]string{},
+		ExhaustPath:  "./exhaust",
+
+		EnableWebP: false,
+		EnableAVIF: false,
+		EnableJXL:  false,
+
 		EnableExtraParams: false,
+		StripMetadata:     true,
 		ReadBufferSize:    4096,
 		Concurrency:       262144,
 		DisableKeepalive:  false,
@@ -115,6 +129,16 @@ func LoadConfig() {
 	switchProxyMode()
 	Config.ImageMap = parseImgMap(Config.ImageMap)
 
+	if slices.Contains(Config.ConvertTypes, "webp") {
+		Config.EnableWebP = true
+	}
+	if slices.Contains(Config.ConvertTypes, "avif") {
+		Config.EnableAVIF = true
+	}
+	if slices.Contains(Config.ConvertTypes, "jxl") {
+		Config.EnableJXL = true
+	}
+
 	// Read from ENV for override
 	if os.Getenv("WEBP_HOST") != "" {
 		Config.Host = os.Getenv("WEBP_HOST")
@@ -139,16 +163,24 @@ func LoadConfig() {
 	if os.Getenv("WEBP_ALLOWED_TYPES") != "" {
 		Config.AllowedTypes = strings.Split(os.Getenv("WEBP_ALLOWED_TYPES"), ",")
 	}
-	if os.Getenv("WEBP_ENABLE_AVIF") != "" {
-		enableAVIF := os.Getenv("WEBP_ENABLE_AVIF")
-		if enableAVIF == "true" {
+
+	// Override enabled convert types
+	if os.Getenv("WEBP_CONVERT_TYPES") != "" {
+		Config.ConvertTypes = strings.Split(os.Getenv("WEBP_CONVERT_TYPES"), ",")
+		Config.EnableWebP = false
+		Config.EnableAVIF = false
+		Config.EnableJXL = false
+		if slices.Contains(Config.ConvertTypes, "webp") {
+			Config.EnableWebP = true
+		}
+		if slices.Contains(Config.ConvertTypes, "avif") {
 			Config.EnableAVIF = true
-		} else if enableAVIF == "false" {
-			Config.EnableAVIF = false
-		} else {
-			log.Warnf("WEBP_ENABLE_AVIF is not a valid boolean, using value in config.json %t", Config.EnableAVIF)
+		}
+		if slices.Contains(Config.ConvertTypes, "jxl") {
+			Config.EnableJXL = true
 		}
 	}
+
 	if os.Getenv("WEBP_ENABLE_EXTRA_PARAMS") != "" {
 		enableExtraParams := os.Getenv("WEBP_ENABLE_EXTRA_PARAMS")
 		if enableExtraParams == "true" {
@@ -159,6 +191,16 @@ func LoadConfig() {
 			log.Warnf("WEBP_ENABLE_EXTRA_PARAMS is not a valid boolean, using value in config.json %t", Config.EnableExtraParams)
 		}
 	}
+	if os.Getenv("WEBP_STRIP_METADATA") != "" {
+		stripMetadata := os.Getenv("WEBP_STRIP_METADATA")
+		if stripMetadata == "true" {
+			Config.StripMetadata = true
+		} else if stripMetadata == "false" {
+			Config.StripMetadata = false
+		} else {
+			log.Warnf("WEBP_STRIP_METADATA is not a valid boolean, using value in config.json %t", Config.StripMetadata)
+		}
+	}
 	if os.Getenv("WEBP_IMG_MAP") != "" {
 		// TODO
 	}
@@ -223,8 +265,10 @@ func parseImgMap(imgMap map[string]string) map[string]string {
 }
 
 type ExtraParams struct {
-	Width  int // in px
-	Height int // in px
+	Width     int // in px
+	Height    int // in px
+	MaxWidth  int // in px
+	MaxHeight int // in px
 }
 
 func switchProxyMode() {