diff --git a/package/jpeg/Config.in b/package/jpeg/Config.in
index 0a786060d7..371f89aa49 100644
--- a/package/jpeg/Config.in
+++ b/package/jpeg/Config.in
@@ -3,7 +3,8 @@ config BR2_PACKAGE_JPEG_SIMD_SUPPORT
 	bool
 	default y if BR2_X86_CPU_HAS_MMX
 	default y if BR2_ARM_CPU_HAS_NEON
-	default y if BR2_POWERPC_CPU_HAS_ALTIVEC
+	default y if BR2_POWERPC_CPU_HAS_ALTIVEC && !BR2_powerpc64le
+	default y if BR2_POWERPC_CPU_HAS_VSX && BR2_powerpc64le
 	default y if BR2_aarch64 || BR2_aarch64_be
 
 config BR2_PACKAGE_JPEG