NetCube-Systems-Austria/kumquat-buildroot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docker-engine: new package

Browse Source 
Docker is a platform to build, ship, and run applications in portable
containers.

Signed-off-by: Christian Stewart <christian@paral.in>
[Thomas: drop LIBRARY_PATH, as suggested by Christian.]
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
This commit is contained in:
Christian Stewart 2016-07-24 14:10:28 -07:00 committed by Thomas Petazzoni
parent 6b372804be
commit e099f37013
5 changed files with 471 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package/Config.in
View File

@ -1678,6 +1678,7 @@ menu "System tools"
source "package/dcron/Config.in"
source "package/debianutils/Config.in"
source "package/docker-containerd/Config.in"
source "package/docker-engine/Config.in"
source "package/dsp-tools/Config.in"
source "package/efibootmgr/Config.in"
source "package/efivar/Config.in"

297
package/docker-engine/0001-Fix-issues-with-tailing-rotated-jsonlog-file.patch Normal file
View File

@ -0,0 +1,297 @@
From 8d6f2e3fe8851b581309da25fc4c32f8be675932 Mon Sep 17 00:00:00 2001
From: Brian Goff <cpuguy83@gmail.com>
Date: Mon, 11 Jul 2016 16:31:42 -0400
Subject: [PATCH] Fix issues with tailing rotated jsonlog file
Fixes a race where the log reader would get events for both an actual
rotation as we from fsnotify (`fsnotify.Rename`).
This issue becomes extremely apparent when rotations are fast, for
example:
```
$ docker run -d --name test --log-opt max-size=1 --log-opt max-file=2
busybox sh -c 'while true; do echo hello; usleep 100000; done'
```
With this change the log reader for jsonlogs can handle rotations that
happen as above.
Instead of listening for both fs events AND rotation events
simultaneously, potentially meaning we see 2 rotations for only a single
rotation due to channel buffering, only listen for fs events (like
`Rename`) and then wait to be notified about rotation by the logger.
This makes sure that we don't see 2 rotations for 1, and that we don't
start trying to read until the logger is actually ready for us to.
Signed-off-by: Brian Goff <cpuguy83@gmail.com>
This commit is pending upstream commit fixing broken log tailing. The
original commit can be found in the PR here:
- https://github.com/docker/docker/pull/24514
Signed-off-by: Christian Stewart <christian@paral.in>
---
daemon/logger/jsonfilelog/read.go | 180 +++++++++++++++++++++++++-------------
1 file changed, 119 insertions(+), 61 deletions(-)
diff --git a/daemon/logger/jsonfilelog/read.go b/daemon/logger/jsonfilelog/read.go
index bea83dd..0cb44af 100644
--- a/daemon/logger/jsonfilelog/read.go
+++ b/daemon/logger/jsonfilelog/read.go
@@ -3,11 +3,14 @@ package jsonfilelog
import (
"bytes"
"encoding/json"
+ "errors"
"fmt"
"io"
"os"
"time"
+ "gopkg.in/fsnotify.v1"
+
"github.com/Sirupsen/logrus"
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/pkg/filenotify"
@@ -44,6 +47,10 @@ func (l *JSONFileLogger) ReadLogs(config logger.ReadConfig) *logger.LogWatcher {
func (l *JSONFileLogger) readLogs(logWatcher *logger.LogWatcher, config logger.ReadConfig) {
defer close(logWatcher.Msg)
+ // lock so the read stream doesn't get corrupted do to rotations or other log data written while we read
+ // This will block writes!!!
+ l.mu.Lock()
+
pth := l.writer.LogPath()
var files []io.ReadSeeker
for i := l.writer.MaxFiles(); i > 1; i-- {
@@ -61,6 +68,7 @@ func (l *JSONFileLogger) readLogs(logWatcher *logger.LogWatcher, config logger.R
latestFile, err := os.Open(pth)
if err != nil {
logWatcher.Err <- err
+ l.mu.Unlock()
return
}
@@ -80,6 +88,7 @@ func (l *JSONFileLogger) readLogs(logWatcher *logger.LogWatcher, config logger.R
if err := latestFile.Close(); err != nil {
logrus.Errorf("Error closing file: %v", err)
}
+ l.mu.Unlock()
return
}
@@ -87,7 +96,6 @@ func (l *JSONFileLogger) readLogs(logWatcher *logger.LogWatcher, config logger.R
latestFile.Seek(0, os.SEEK_END)
}
- l.mu.Lock()
l.readers[logWatcher] = struct{}{}
l.mu.Unlock()
@@ -128,92 +136,142 @@ func tailFile(f io.ReadSeeker, logWatcher *logger.LogWatcher, tail int, since ti
}
}
+func watchFile(name string) (filenotify.FileWatcher, error) {
+ fileWatcher, err := filenotify.New()
+ if err != nil {
+ return nil, err
+ }
+
+ if err := fileWatcher.Add(name); err != nil {
+ logrus.WithField("logger", "json-file").Warnf("falling back to file poller due to error: %v", err)
+ fileWatcher.Close()
+ fileWatcher = filenotify.NewPollingWatcher()
+
+ if err := fileWatcher.Add(name); err != nil {
+ fileWatcher.Close()
+ logrus.Debugf("error watching log file for modifications: %v", err)
+ return nil, err
+ }
+ }
+ return fileWatcher, nil
+}
+
func followLogs(f *os.File, logWatcher *logger.LogWatcher, notifyRotate chan interface{}, since time.Time) {
dec := json.NewDecoder(f)
l := &jsonlog.JSONLog{}
- fileWatcher, err := filenotify.New()
+ name := f.Name()
+ fileWatcher, err := watchFile(name)
if err != nil {
logWatcher.Err <- err
+ return
}
defer func() {
f.Close()
fileWatcher.Close()
}()
- name := f.Name()
- if err := fileWatcher.Add(name); err != nil {
- logrus.WithField("logger", "json-file").Warnf("falling back to file poller due to error: %v", err)
- fileWatcher.Close()
- fileWatcher = filenotify.NewPollingWatcher()
+ var retries int
+ handleRotate := func() error {
+ f.Close()
+ fileWatcher.Remove(name)
+ // retry when the file doesn't exist
+ for retries := 0; retries <= 5; retries++ {
+ f, err = os.Open(name)
+ if err == nil || !os.IsNotExist(err) {
+ break
+ }
+ }
+ if err != nil {
+ return err
+ }
if err := fileWatcher.Add(name); err != nil {
- logrus.Debugf("error watching log file for modifications: %v", err)
- logWatcher.Err <- err
- return
+ return err
}
+ dec = json.NewDecoder(f)
+ return nil
}
- var retries int
- for {
- msg, err := decodeLogLine(dec, l)
- if err != nil {
- if err != io.EOF {
- // try again because this shouldn't happen
- if _, ok := err.(*json.SyntaxError); ok && retries <= maxJSONDecodeRetry {
- dec = json.NewDecoder(f)
- retries++
- continue
+ errRetry := errors.New("retry")
+ errDone := errors.New("done")
+ waitRead := func() error {
+ select {
+ case e := <-fileWatcher.Events():
+ switch e.Op {
+ case fsnotify.Write:
+ dec = json.NewDecoder(f)
+ return nil
+ case fsnotify.Rename, fsnotify.Remove:
+ <-notifyRotate
+ if err := handleRotate(); err != nil {
+ return err
}
-
- // io.ErrUnexpectedEOF is returned from json.Decoder when there is
- // remaining data in the parser's buffer while an io.EOF occurs.
- // If the json logger writes a partial json log entry to the disk
- // while at the same time the decoder tries to decode it, the race condition happens.
- if err == io.ErrUnexpectedEOF && retries <= maxJSONDecodeRetry {
- reader := io.MultiReader(dec.Buffered(), f)
- dec = json.NewDecoder(reader)
- retries++
- continue
+ return nil
+ }
+ return errRetry
+ case err := <-fileWatcher.Errors():
+ logrus.Debug("logger got error watching file: %v", err)
+ // Something happened, let's try and stay alive and create a new watcher
+ if retries <= 5 {
+ fileWatcher, err = watchFile(name)
+ if err != nil {
+ return err
}
-
- return
+ retries++
+ return errRetry
}
+ return err
+ case <-logWatcher.WatchClose():
+ fileWatcher.Remove(name)
+ return errDone
+ }
+ }
- select {
- case <-fileWatcher.Events():
- dec = json.NewDecoder(f)
- continue
- case <-fileWatcher.Errors():
- logWatcher.Err <- err
- return
- case <-logWatcher.WatchClose():
- fileWatcher.Remove(name)
- return
- case <-notifyRotate:
- f.Close()
- fileWatcher.Remove(name)
-
- // retry when the file doesn't exist
- for retries := 0; retries <= 5; retries++ {
- f, err = os.Open(name)
- if err == nil || !os.IsNotExist(err) {
- break
- }
+ handleDecodeErr := func(err error) error {
+ if err == io.EOF {
+ for err := waitRead(); err != nil; {
+ if err == errRetry {
+ // retry the waitRead
+ continue
}
+ return err
+ }
+ return nil
+ }
+ // try again because this shouldn't happen
+ if _, ok := err.(*json.SyntaxError); ok && retries <= maxJSONDecodeRetry {
+ dec = json.NewDecoder(f)
+ retries++
+ return nil
+ }
+ // io.ErrUnexpectedEOF is returned from json.Decoder when there is
+ // remaining data in the parser's buffer while an io.EOF occurs.
+ // If the json logger writes a partial json log entry to the disk
+ // while at the same time the decoder tries to decode it, the race condition happens.
+ if err == io.ErrUnexpectedEOF && retries <= maxJSONDecodeRetry {
+ reader := io.MultiReader(dec.Buffered(), f)
+ dec = json.NewDecoder(reader)
+ retries++
+ return nil
+ }
+ return err
+ }
- if err = fileWatcher.Add(name); err != nil {
- logWatcher.Err <- err
- return
- }
- if err != nil {
- logWatcher.Err <- err
+ // main loop
+ for {
+ msg, err := decodeLogLine(dec, l)
+ if err != nil {
+ if err := handleDecodeErr(err); err != nil {
+ if err == errDone {
return
}
-
- dec = json.NewDecoder(f)
- continue
+ // we got an unrecoverable error, so return
+ logWatcher.Err <- err
+ return
}
+ // ready to try again
+ continue
}
retries = 0 // reset retries since we've succeeded
--
2.7.3

64
package/docker-engine/Config.in Normal file
View File

@ -0,0 +1,64 @@
config BR2_PACKAGE_DOCKER_ENGINE
bool "docker-engine"
depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS
depends on BR2_PACKAGE_HOST_GO_CGO_LINKING_SUPPORTS
depends on BR2_TOOLCHAIN_HAS_THREADS
help
Docker is a platform to build, ship,
and run applications as lightweight containers.
https://github.com/docker/docker
if BR2_PACKAGE_DOCKER_ENGINE
config BR2_PACKAGE_DOCKER_ENGINE_DAEMON
bool "docker daemon"
depends on BR2_USE_WCHAR # docker-containerd
select BR2_PACKAGE_DOCKER_CONTAINERD # runtime dependency
select BR2_PACKAGE_SQLITE # runtime dependency
default y
help
Build the Docker system daemon.
If not selected, will build client only.
config BR2_PACKAGE_DOCKER_ENGINE_EXPERIMENTAL
bool "build experimental features"
if BR2_PACKAGE_DOCKER_ENGINE_DAEMON
config BR2_PACKAGE_DOCKER_ENGINE_DRIVER_BTRFS
bool "btrfs filesystem driver"
depends on BR2_USE_WCHAR # btrfs-progs
depends on BR2_USE_MMU # btrfs-progs
depends on BR2_TOOLCHAIN_HAS_THREADS # btrfs-progs
select BR2_PACKAGE_BTRFS_PROGS
help
Build the btrfs filesystem driver for Docker.
config BR2_PACKAGE_DOCKER_ENGINE_DRIVER_DEVICEMAPPER
bool "devicemapper filesystem driver"
depends on BR2_TOOLCHAIN_HAS_THREADS # lvm2
depends on BR2_USE_MMU # lvm2
depends on !BR2_STATIC_LIBS # lvm2
select BR2_PACKAGE_LVM2
select BR2_PACKAGE_LVM2_APP_LIBRARY
help
Build the devicemapper filesystem driver for Docker.
config BR2_PACKAGE_DOCKER_ENGINE_DRIVER_VFS
bool "vfs filesystem driver"
depends on BR2_USE_WCHAR # gvfs
depends on BR2_USE_MMU # gvfs
depends on BR2_TOOLCHAIN_HAS_THREADS # gvfs
select BR2_PACKAGE_GVFS
help
Build the vfs filesystem driver for Docker.
endif
endif
comment "docker-engine needs a toolchain w/ threads"
depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS
depends on BR2_PACKAGE_HOST_GO_CGO_LINKING_SUPPORTS
depends on !BR2_TOOLCHAIN_HAS_THREADS

2
package/docker-engine/docker-engine.hash Normal file
View File

@ -0,0 +1,2 @@
# Locally calculated
sha256 bc256d2a348efbf236eab991254c925fa1917dd1e29cb40586f1696f4e24852e docker-engine-v1.12.0-rc4.tar.gz

107
package/docker-engine/docker-engine.mk Normal file
View File

@ -0,0 +1,107 @@
################################################################################
#
# docker-engine
#
################################################################################
DOCKER_ENGINE_VERSION = v1.12.0-rc4
DOCKER_ENGINE_SITE = $(call github,docker,docker,$(DOCKER_ENGINE_VERSION))
DOCKER_ENGINE_LICENSE = Apache-2.0
DOCKER_ENGINE_LICENSE_FILES = LICENSE
DOCKER_ENGINE_DEPENDENCIES = host-go
DOCKER_ENGINE_GOPATH = "$(@D)/vendor"
DOCKER_ENGINE_MAKE_ENV = $(HOST_GO_TARGET_ENV) \
CGO_ENABLED=1 \
CGO_NO_EMULATION=1 \
GOBIN="$(@D)/bin" \
GOPATH="$(DOCKER_ENGINE_GOPATH)"
DOCKER_ENGINE_GLDFLAGS = \
-X main.GitCommit=$(DOCKER_ENGINE_VERSION) \
-X main.Version=$(DOCKER_ENGINE_VERSION)
ifeq ($(BR2_STATIC_LIBS),y)
DOCKER_ENGINE_GLDFLAGS += -extldflags '-static'
endif
DOCKER_ENGINE_BUILD_TAGS = cgo exclude_graphdriver_zfs autogen
DOCKER_ENGINE_BUILD_TARGETS = docker
ifeq ($(BR2_PACKAGE_LIBSECCOMP),y)
DOCKER_ENGINE_BUILD_TAGS += seccomp
DOCKER_ENGINE_DEPENDENCIES += libseccomp
endif
ifeq ($(BR2_PACKAGE_DOCKER_ENGINE_DAEMON),y)
DOCKER_ENGINE_BUILD_TAGS += daemon
DOCKER_ENGINE_BUILD_TARGETS += dockerd
endif
ifeq ($(BR2_PACKAGE_DOCKER_ENGINE_EXPERIMENTAL),y)
DOCKER_ENGINE_BUILD_TAGS += experimental
endif
ifeq ($(BR2_PACKAGE_DOCKER_ENGINE_DRIVER_BTRFS),y)
DOCKER_ENGINE_DEPENDENCIES += btrfs-progs
else
DOCKER_ENGINE_BUILD_TAGS += exclude_graphdriver_btrfs
endif
ifeq ($(BR2_PACKAGE_DOCKER_ENGINE_DRIVER_DEVICEMAPPER),y)
DOCKER_ENGINE_DEPENDENCIES += lvm2
else
DOCKER_ENGINE_BUILD_TAGS += exclude_graphdriver_devicemapper
endif
ifeq ($(BR2_PACKAGE_DOCKER_ENGINE_DRIVER_VFS),y)
DOCKER_ENGINE_DEPENDENCIES += gvfs
else
DOCKER_ENGINE_BUILD_TAGS += exclude_graphdriver_vfs
endif
define DOCKER_ENGINE_CONFIGURE_CMDS
ln -fs $(@D) $(DOCKER_ENGINE_GOPATH)/src/github.com/docker/docker
cd $(@D) && \
GITCOMMIT="unknown" BUILDTIME="$$(date)" VERSION="$(DOCKER_ENGINE_VERSION)" \
bash ./hack/make/.go-autogen
endef
ifeq ($(BR2_PACKAGE_DOCKER_ENGINE_DAEMON),y)
define DOCKER_ENGINE_INSTALL_INIT_SYSTEMD
$(INSTALL) -D -m 0644 $(@D)/contrib/init/systemd/docker.service \
$(TARGET_DIR)/usr/lib/systemd/system/docker.service
$(INSTALL) -D -m 0644 $(@D)/contrib/init/systemd/docker.socket \
$(TARGET_DIR)/usr/lib/systemd/system/docker.socket
mkdir -p $(TARGET_DIR)/etc/systemd/system/multi-user.target.wants/
ln -fs ../../../../usr/lib/systemd/system/docker.service \
$(TARGET_DIR)/etc/systemd/system/multi-user.target.wants/docker.service
endef
define DOCKER_ENGINE_USERS
- - docker -1 * - - - Docker Application Container Framework
endef
endif
define DOCKER_ENGINE_BUILD_CMDS
$(foreach target,$(DOCKER_ENGINE_BUILD_TARGETS), \
cd $(@D); $(DOCKER_ENGINE_MAKE_ENV) \
$(HOST_DIR)/usr/bin/go build -v \
-o $(@D)/bin/$(target) \
-tags "$(DOCKER_ENGINE_BUILD_TAGS)" \
-ldflags "$(DOCKER_ENGINE_GLDFLAGS)" \
./cmd/$(target)
)
endef
define DOCKER_ENGINE_INSTALL_TARGET_CMDS
$(foreach target,$(DOCKER_ENGINE_BUILD_TARGETS), \
$(INSTALL) -D -m 0755 $(@D)/bin/$(target) $(TARGET_DIR)/usr/bin/$(target)
)
endef
$(eval $(generic-package))