diff --git a/package/directfb/Config.in b/package/directfb/Config.in
index 3ab40f98dc..98acec5f04 100644
--- a/package/directfb/Config.in
+++ b/package/directfb/Config.in
@@ -45,10 +45,6 @@ config BR2_PACKAGE_DIRECTFB_TRACE
 	help
 	  Enable call tracing for DirectFB applications
 
-config BR2_PACKAGE_DIRECTFB_XSERVER
-	bool "build with X server backend"
-	depends on BR2_PACKAGE_XORG7
-
 config BR2_PACKAGE_DIRECTFB_UNIQUE
 	bool "enable unique window manager"
 	depends on BR2_UNIMPLEMENTED
diff --git a/package/directfb/directfb.mk b/package/directfb/directfb.mk
index da12acba2c..acb3e3addd 100644
--- a/package/directfb/directfb.mk
+++ b/package/directfb/directfb.mk
@@ -22,7 +22,8 @@ DIRECTFB_CONF_OPTS = \
 	--disable-osx \
 	--disable-video4linux \
 	--disable-video4linux2 \
-	--without-tools
+	--without-tools \
+	--disable-x11
 
 ifeq ($(BR2_STATIC_LIBS),y)
 DIRECTFB_CONF_OPTS += --disable-dynload
@@ -52,12 +53,6 @@ ifeq ($(BR2_PACKAGE_DIRECTFB_TRACE),y)
 DIRECTFB_CONF_OPTS += --enable-trace
 endif
 
-ifeq ($(BR2_PACKAGE_XSERVER),y)
-DIRECTFB_CONF_OPTS += --enable-x11
-else
-DIRECTFB_CONF_OPTS += --disable-x11
-endif
-
 ifeq ($(BR2_PACKAGE_DIRECTFB_UNIQUE),y)
 DIRECTFB_CONF_OPTS += --enable-unique
 else