Merge pull request #2065 from docker/c6024-improved_excludes

Improved .dockerignore pattern processing to better match Docker CLI behavior

docker/utils/build.py

 import io
 import os
 import re
-import six
 import tarfile
 import tempfile
 
+import six
+
+from .fnmatch import fnmatch
 from ..constants import IS_WINDOWS_PLATFORM
-from fnmatch import fnmatch
-from itertools import chain
 
 
 _SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/')
@@ -44,92 +44,9 @@ def exclude_paths(root, patterns, dockerfile=None):
     if dockerfile is None:
         dockerfile = 'Dockerfile'
 
-    def split_path(p):
-        return [pt for pt in re.split(_SEP, p) if pt and pt != '.']
-
-    def normalize(p):
-        # Leading and trailing slashes are not relevant. Yes,
-        # "foo.py/" must exclude the "foo.py" regular file. "."
-        # components are not relevant either, even if the whole
-        # pattern is only ".", as the Docker reference states: "For
-        # historical reasons, the pattern . is ignored."
-        # ".." component must be cleared with the potential previous
-        # component, regardless of whether it exists: "A preprocessing
-        # step [...]  eliminates . and .. elements using Go's
-        # filepath.".
-        i = 0
-        split = split_path(p)
-        while i < len(split):
-            if split[i] == '..':
-                del split[i]
-                if i > 0:
-                    del split[i - 1]
-                    i -= 1
-            else:
-                i += 1
-        return split
-
-    patterns = (
-        (True, normalize(p[1:]))
-        if p.startswith('!') else
-        (False, normalize(p))
-        for p in patterns)
-    patterns = list(reversed(list(chain(
-        # Exclude empty patterns such as "." or the empty string.
-        filter(lambda p: p[1], patterns),
-        # Always include the Dockerfile and .dockerignore
-        [(True, split_path(dockerfile)), (True, ['.dockerignore'])]))))
-    return set(walk(root, patterns))
-
-
-def walk(root, patterns, default=True):
-    """
-    A collection of file lying below root that should be included according to
-    patterns.
-    """
-
-    def match(p):
-        if p[1][0] == '**':
-            rec = (p[0], p[1][1:])
-            return [p] + (match(rec) if rec[1] else [rec])
-        elif fnmatch(f, p[1][0]):
-            return [(p[0], p[1][1:])]
-        else:
-            return []
-
-    for f in os.listdir(root):
-        cur = os.path.join(root, f)
-        # The patterns if recursing in that directory.
-        sub = list(chain(*(match(p) for p in patterns)))
-        # Whether this file is explicitely included / excluded.
-        hit = next((p[0] for p in sub if not p[1]), None)
-        # Whether this file is implicitely included / excluded.
-        matched = default if hit is None else hit
-        sub = list(filter(lambda p: p[1], sub))
-        if os.path.isdir(cur) and not os.path.islink(cur):
-            # Entirely skip directories if there are no chance any subfile will
-            # be included.
-            if all(not p[0] for p in sub) and not matched:
-                continue
-            # I think this would greatly speed up dockerignore handling by not
-            # recursing into directories we are sure would be entirely
-            # included, and only yielding the directory itself, which will be
-            # recursively archived anyway. However the current unit test expect
-            # the full list of subfiles and I'm not 100% sure it would make no
-            # difference yet.
-            # if all(p[0] for p in sub) and matched:
-            #     yield f
-            #     continue
-            children = False
-            for r in (os.path.join(f, p) for p in walk(cur, sub, matched)):
-                yield r
-                children = True
-            # The current unit tests expect directories only under those
-            # conditions. It might be simplifiable though.
-            if (not sub or not children) and hit or hit is None and default:
-                yield f
-        elif matched:
-            yield f
+    patterns.append('!' + dockerfile)
+    pm = PatternMatcher(patterns)
+    return set(pm.walk(root))
 
 
 def build_file_list(root):
@@ -217,3 +134,122 @@ def mkbuildcontext(dockerfile):
     t.close()
     f.seek(0)
     return f
+
+
+def split_path(p):
+    return [pt for pt in re.split(_SEP, p) if pt and pt != '.']
+
+
+def normalize_slashes(p):
+    if IS_WINDOWS_PLATFORM:
+        return '/'.join(split_path(p))
+    return p
+
+
+def walk(root, patterns, default=True):
+    pm = PatternMatcher(patterns)
+    return pm.walk(root)
+
+
+# Heavily based on
+# https://github.com/moby/moby/blob/master/pkg/fileutils/fileutils.go
+class PatternMatcher(object):
+    def __init__(self, patterns):
+        self.patterns = list(filter(
+            lambda p: p.dirs, [Pattern(p) for p in patterns]
+        ))
+        self.patterns.append(Pattern('!.dockerignore'))
+
+    def matches(self, filepath):
+        matched = False
+        parent_path = os.path.dirname(filepath)
+        parent_path_dirs = split_path(parent_path)
+
+        for pattern in self.patterns:
+            negative = pattern.exclusion
+            match = pattern.match(filepath)
+            if not match and parent_path != '':
+                if len(pattern.dirs) <= len(parent_path_dirs):
+                    match = pattern.match(
+                        os.path.sep.join(parent_path_dirs[:len(pattern.dirs)])
+                    )
+
+            if match:
+                matched = not negative
+
+        return matched
+
+    def walk(self, root):
+        def rec_walk(current_dir):
+            for f in os.listdir(current_dir):
+                fpath = os.path.join(
+                    os.path.relpath(current_dir, root), f
+                )
+                if fpath.startswith('.' + os.path.sep):
+                    fpath = fpath[2:]
+                match = self.matches(fpath)
+                if not match:
+                    yield fpath
+
+                cur = os.path.join(root, fpath)
+                if not os.path.isdir(cur) or os.path.islink(cur):
+                    continue
+
+                if match:
+                    # If we want to skip this file and it's a directory
+                    # then we should first check to see if there's an
+                    # excludes pattern (e.g. !dir/file) that starts with this
+                    # dir. If so then we can't skip this dir.
+                    skip = True
+
+                    for pat in self.patterns:
+                        if not pat.exclusion:
+                            continue
+                        if pat.cleaned_pattern.startswith(
+                                normalize_slashes(fpath)):
+                            skip = False
+                            break
+                    if skip:
+                        continue
+                for sub in rec_walk(cur):
+                    yield sub
+
+        return rec_walk(root)
+
+
+class Pattern(object):
+    def __init__(self, pattern_str):
+        self.exclusion = False
+        if pattern_str.startswith('!'):
+            self.exclusion = True
+            pattern_str = pattern_str[1:]
+
+        self.dirs = self.normalize(pattern_str)
+        self.cleaned_pattern = '/'.join(self.dirs)
+
+    @classmethod
+    def normalize(cls, p):
+
+        # Leading and trailing slashes are not relevant. Yes,
+        # "foo.py/" must exclude the "foo.py" regular file. "."
+        # components are not relevant either, even if the whole
+        # pattern is only ".", as the Docker reference states: "For
+        # historical reasons, the pattern . is ignored."
+        # ".." component must be cleared with the potential previous
+        # component, regardless of whether it exists: "A preprocessing
+        # step [...]  eliminates . and .. elements using Go's
+        # filepath.".
+        i = 0
+        split = split_path(p)
+        while i < len(split):
+            if split[i] == '..':
+                del split[i]
+                if i > 0:
+                    del split[i - 1]
+                    i -= 1
+            else:
+                i += 1
+        return split
+
+    def match(self, filepath):
+        return fnmatch(normalize_slashes(filepath), self.cleaned_pattern)

docker/utils/fnmatch.py

@@ -111,4 +111,5 @@ def translate(pat):
                 res = '%s[%s]' % (res, stuff)
         else:
             res = res + re.escape(c)
+
     return res + '$'

tests/unit/utils_build_test.py

+# -*- coding: utf-8 -*-
+
+import os
+import os.path
+import shutil
+import socket
+import tarfile
+import tempfile
+import unittest
+
+
+from docker.constants import IS_WINDOWS_PLATFORM
+from docker.utils import exclude_paths, tar
+
+import pytest
+
+from ..helpers import make_tree
+
+
+def convert_paths(collection):
+    return set(map(convert_path, collection))
+
+
+def convert_path(path):
+    return path.replace('/', os.path.sep)
+
+
+class ExcludePathsTest(unittest.TestCase):
+    dirs = [
+        'foo',
+        'foo/bar',
+        'bar',
+        'target',
+        'target/subdir',
+        'subdir',
+        'subdir/target',
+        'subdir/target/subdir',
+        'subdir/subdir2',
+        'subdir/subdir2/target',
+        'subdir/subdir2/target/subdir'
+    ]
+
+    files = [
+        'Dockerfile',
+        'Dockerfile.alt',
+        '.dockerignore',
+        'a.py',
+        'a.go',
+        'b.py',
+        'cde.py',
+        'foo/a.py',
+        'foo/b.py',
+        'foo/bar/a.py',
+        'bar/a.py',
+        'foo/Dockerfile3',
+        'target/file.txt',
+        'target/subdir/file.txt',
+        'subdir/file.txt',
+        'subdir/target/file.txt',
+        'subdir/target/subdir/file.txt',
+        'subdir/subdir2/file.txt',
+        'subdir/subdir2/target/file.txt',
+        'subdir/subdir2/target/subdir/file.txt',
+    ]
+
+    all_paths = set(dirs + files)
+
+    def setUp(self):
+        self.base = make_tree(self.dirs, self.files)
+
+    def tearDown(self):
+        shutil.rmtree(self.base)
+
+    def exclude(self, patterns, dockerfile=None):
+        return set(exclude_paths(self.base, patterns, dockerfile=dockerfile))
+
+    def test_no_excludes(self):
+        assert self.exclude(['']) == convert_paths(self.all_paths)
+
+    def test_no_dupes(self):
+        paths = exclude_paths(self.base, ['!a.py'])
+        assert sorted(paths) == sorted(set(paths))
+
+    def test_wildcard_exclude(self):
+        assert self.exclude(['*']) == set(['Dockerfile', '.dockerignore'])
+
+    def test_exclude_dockerfile_dockerignore(self):
+        """
+        Even if the .dockerignore file explicitly says to exclude
+        Dockerfile and/or .dockerignore, don't exclude them from
+        the actual tar file.
+        """
+        assert self.exclude(['Dockerfile', '.dockerignore']) == convert_paths(
+            self.all_paths
+        )
+
+    def test_exclude_custom_dockerfile(self):
+        """
+        If we're using a custom Dockerfile, make sure that's not
+        excluded.
+        """
+        assert self.exclude(['*'], dockerfile='Dockerfile.alt') == set(
+            ['Dockerfile.alt', '.dockerignore']
+        )
+
+        assert self.exclude(
+            ['*'], dockerfile='foo/Dockerfile3'
+        ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore']))
+
+        # https://github.com/docker/docker-py/issues/1956
+        assert self.exclude(
+            ['*'], dockerfile='./foo/Dockerfile3'
+        ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore']))
+
+    def test_exclude_dockerfile_child(self):
+        includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3')
+        assert convert_path('foo/Dockerfile3') in includes
+        assert convert_path('foo/a.py') not in includes
+
+    def test_single_filename(self):
+        assert self.exclude(['a.py']) == convert_paths(
+            self.all_paths - set(['a.py'])
+        )
+
+    def test_single_filename_leading_dot_slash(self):
+        assert self.exclude(['./a.py']) == convert_paths(
+            self.all_paths - set(['a.py'])
+        )
+
+    # As odd as it sounds, a filename pattern with a trailing slash on the
+    # end *will* result in that file being excluded.
+    def test_single_filename_trailing_slash(self):
+        assert self.exclude(['a.py/']) == convert_paths(
+            self.all_paths - set(['a.py'])
+        )
+
+    def test_wildcard_filename_start(self):
+        assert self.exclude(['*.py']) == convert_paths(
+            self.all_paths - set(['a.py', 'b.py', 'cde.py'])
+        )
+
+    def test_wildcard_with_exception(self):
+        assert self.exclude(['*.py', '!b.py']) == convert_paths(
+            self.all_paths - set(['a.py', 'cde.py'])
+        )
+
+    def test_wildcard_with_wildcard_exception(self):
+        assert self.exclude(['*.*', '!*.go']) == convert_paths(
+            self.all_paths - set([
+                'a.py', 'b.py', 'cde.py', 'Dockerfile.alt',
+            ])
+        )
+
+    def test_wildcard_filename_end(self):
+        assert self.exclude(['a.*']) == convert_paths(
+            self.all_paths - set(['a.py', 'a.go'])
+        )
+
+    def test_question_mark(self):
+        assert self.exclude(['?.py']) == convert_paths(
+            self.all_paths - set(['a.py', 'b.py'])
+        )
+
+    def test_single_subdir_single_filename(self):
+        assert self.exclude(['foo/a.py']) == convert_paths(
+            self.all_paths - set(['foo/a.py'])
+        )
+
+    def test_single_subdir_single_filename_leading_slash(self):
+        assert self.exclude(['/foo/a.py']) == convert_paths(
+            self.all_paths - set(['foo/a.py'])
+        )
+
+    def test_exclude_include_absolute_path(self):
+        base = make_tree([], ['a.py', 'b.py'])
+        assert exclude_paths(
+            base,
+            ['/*', '!/*.py']
+        ) == set(['a.py', 'b.py'])
+
+    def test_single_subdir_with_path_traversal(self):
+        assert self.exclude(['foo/whoops/../a.py']) == convert_paths(
+            self.all_paths - set(['foo/a.py'])
+        )
+
+    def test_single_subdir_wildcard_filename(self):
+        assert self.exclude(['foo/*.py']) == convert_paths(
+            self.all_paths - set(['foo/a.py', 'foo/b.py'])
+        )
+
+    def test_wildcard_subdir_single_filename(self):
+        assert self.exclude(['*/a.py']) == convert_paths(
+            self.all_paths - set(['foo/a.py', 'bar/a.py'])
+        )
+
+    def test_wildcard_subdir_wildcard_filename(self):
+        assert self.exclude(['*/*.py']) == convert_paths(
+            self.all_paths - set(['foo/a.py', 'foo/b.py', 'bar/a.py'])
+        )
+
+    def test_directory(self):
+        assert self.exclude(['foo']) == convert_paths(
+            self.all_paths - set([
+                'foo', 'foo/a.py', 'foo/b.py', 'foo/bar', 'foo/bar/a.py',
+                'foo/Dockerfile3'
+            ])
+        )
+
+    def test_directory_with_trailing_slash(self):
+        assert self.exclude(['foo']) == convert_paths(
+            self.all_paths - set([
+                'foo', 'foo/a.py', 'foo/b.py',
+                'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3'
+            ])
+        )
+
+    def test_directory_with_single_exception(self):
+        assert self.exclude(['foo', '!foo/bar/a.py']) == convert_paths(
+            self.all_paths - set([
+                'foo/a.py', 'foo/b.py', 'foo', 'foo/bar',
+                'foo/Dockerfile3'
+            ])
+        )
+
+    def test_directory_with_subdir_exception(self):
+        assert self.exclude(['foo', '!foo/bar']) == convert_paths(
+            self.all_paths - set([
+                'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3'
+            ])
+        )
+
+    @pytest.mark.skipif(
+        not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows'
+    )
+    def test_directory_with_subdir_exception_win32_pathsep(self):
+        assert self.exclude(['foo', '!foo\\bar']) == convert_paths(
+            self.all_paths - set([
+                'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3'
+            ])
+        )
+
+    def test_directory_with_wildcard_exception(self):
+        assert self.exclude(['foo', '!foo/*.py']) == convert_paths(
+            self.all_paths - set([
+                'foo/bar', 'foo/bar/a.py', 'foo', 'foo/Dockerfile3'
+            ])
+        )
+
+    def test_subdirectory(self):
+        assert self.exclude(['foo/bar']) == convert_paths(
+            self.all_paths - set(['foo/bar', 'foo/bar/a.py'])
+        )
+
+    @pytest.mark.skipif(
+        not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows'
+    )
+    def test_subdirectory_win32_pathsep(self):
+        assert self.exclude(['foo\\bar']) == convert_paths(
+            self.all_paths - set(['foo/bar', 'foo/bar/a.py'])
+        )
+
+    def test_double_wildcard(self):
+        assert self.exclude(['**/a.py']) == convert_paths(
+            self.all_paths - set(
+                ['a.py', 'foo/a.py', 'foo/bar/a.py', 'bar/a.py']
+            )
+        )
+
+        assert self.exclude(['foo/**/bar']) == convert_paths(
+            self.all_paths - set(['foo/bar', 'foo/bar/a.py'])
+        )
+
+    def test_single_and_double_wildcard(self):
+        assert self.exclude(['**/target/*/*']) == convert_paths(
+            self.all_paths - set(
+                ['target/subdir/file.txt',
+                 'subdir/target/subdir/file.txt',
+                 'subdir/subdir2/target/subdir/file.txt']
+            )
+        )
+
+    def test_trailing_double_wildcard(self):
+        assert self.exclude(['subdir/**']) == convert_paths(
+            self.all_paths - set(
+                ['subdir/file.txt',
+                 'subdir/target/file.txt',
+                 'subdir/target/subdir/file.txt',
+                 'subdir/subdir2/file.txt',
+                 'subdir/subdir2/target/file.txt',
+                 'subdir/subdir2/target/subdir/file.txt',
+                 'subdir/target',
+                 'subdir/target/subdir',
+                 'subdir/subdir2',
+                 'subdir/subdir2/target',
+                 'subdir/subdir2/target/subdir']
+            )
+        )
+
+    def test_double_wildcard_with_exception(self):
+        assert self.exclude(['**', '!bar', '!foo/bar']) == convert_paths(
+            set([
+                'foo/bar', 'foo/bar/a.py', 'bar', 'bar/a.py', 'Dockerfile',
+                '.dockerignore',
+            ])
+        )
+
+    def test_include_wildcard(self):
+        # This may be surprising but it matches the CLI's behavior
+        # (tested with 18.05.0-ce on linux)
+        base = make_tree(['a'], ['a/b.py'])
+        assert exclude_paths(
+            base,
+            ['*', '!*/b.py']
+        ) == set()
+
+    def test_last_line_precedence(self):
+        base = make_tree(
+            [],
+            ['garbage.md',
+             'trash.md',
+             'README.md',
+             'README-bis.md',
+             'README-secret.md'])
+        assert exclude_paths(
+            base,
+            ['*.md', '!README*.md', 'README-secret.md']
+        ) == set(['README.md', 'README-bis.md'])
+
+    def test_parent_directory(self):
+        base = make_tree(
+            [],
+            ['a.py',
+             'b.py',
+             'c.py'])
+        # Dockerignore reference stipulates that absolute paths are
+        # equivalent to relative paths, hence /../foo should be
+        # equivalent to ../foo. It also stipulates that paths are run
+        # through Go's filepath.Clean, which explicitely "replace
+        # "/.."  by "/" at the beginning of a path".
+        assert exclude_paths(
+            base,
+            ['../a.py', '/../b.py']
+        ) == set(['c.py'])
+
+
+class TarTest(unittest.TestCase):
+    def test_tar_with_excludes(self):
+        dirs = [
+            'foo',
+            'foo/bar',
+            'bar',
+        ]
+
+        files = [
+            'Dockerfile',
+            'Dockerfile.alt',
+            '.dockerignore',
+            'a.py',
+            'a.go',
+            'b.py',
+            'cde.py',
+            'foo/a.py',
+            'foo/b.py',
+            'foo/bar/a.py',
+            'bar/a.py',
+        ]
+
+        exclude = [
+            '*.py',
+            '!b.py',
+            '!a.go',
+            'foo',
+            'Dockerfile*',
+            '.dockerignore',
+        ]
+
+        expected_names = set([
+            'Dockerfile',
+            '.dockerignore',
+            'a.go',
+            'b.py',
+            'bar',
+            'bar/a.py',
+        ])
+
+        base = make_tree(dirs, files)
+        self.addCleanup(shutil.rmtree, base)
+
+        with tar(base, exclude=exclude) as archive:
+            tar_data = tarfile.open(fileobj=archive)
+            assert sorted(tar_data.getnames()) == sorted(expected_names)
+
+    def test_tar_with_empty_directory(self):
+        base = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, base)
+        for d in ['foo', 'bar']:
+            os.makedirs(os.path.join(base, d))
+        with tar(base) as archive:
+            tar_data = tarfile.open(fileobj=archive)
+            assert sorted(tar_data.getnames()) == ['bar', 'foo']
+
+    @pytest.mark.skipif(
+        IS_WINDOWS_PLATFORM or os.geteuid() == 0,
+        reason='root user always has access ; no chmod on Windows'
+    )
+    def test_tar_with_inaccessible_file(self):
+        base = tempfile.mkdtemp()
+        full_path = os.path.join(base, 'foo')
+        self.addCleanup(shutil.rmtree, base)
+        with open(full_path, 'w') as f:
+            f.write('content')
+        os.chmod(full_path, 0o222)
+        with pytest.raises(IOError) as ei:
+            tar(base)
+
+        assert 'Can not read file in context: {}'.format(full_path) in (
+            ei.exconly()
+        )
+
+    @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
+    def test_tar_with_file_symlinks(self):
+        base = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, base)
+        with open(os.path.join(base, 'foo'), 'w') as f:
+            f.write("content")
+        os.makedirs(os.path.join(base, 'bar'))
+        os.symlink('../foo', os.path.join(base, 'bar/foo'))
+        with tar(base) as archive:
+            tar_data = tarfile.open(fileobj=archive)
+            assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
+
+    @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
+    def test_tar_with_directory_symlinks(self):
+        base = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, base)
+        for d in ['foo', 'bar']:
+            os.makedirs(os.path.join(base, d))
+        os.symlink('../foo', os.path.join(base, 'bar/foo'))
+        with tar(base) as archive:
+            tar_data = tarfile.open(fileobj=archive)
+            assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
+
+    @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
+    def test_tar_with_broken_symlinks(self):
+        base = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, base)
+        for d in ['foo', 'bar']:
+            os.makedirs(os.path.join(base, d))
+
+        os.symlink('../baz', os.path.join(base, 'bar/foo'))
+        with tar(base) as archive:
+            tar_data = tarfile.open(fileobj=archive)
+            assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
+
+    @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No UNIX sockets on Win32')
+    def test_tar_socket_file(self):
+        base = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, base)
+        for d in ['foo', 'bar']:
+            os.makedirs(os.path.join(base, d))
+        sock = socket.socket(socket.AF_UNIX)
+        self.addCleanup(sock.close)
+        sock.bind(os.path.join(base, 'test.sock'))
+        with tar(base) as archive:
+            tar_data = tarfile.open(fileobj=archive)
+            assert sorted(tar_data.getnames()) == ['bar', 'foo']
+
+    def tar_test_negative_mtime_bug(self):
+        base = tempfile.mkdtemp()
+        filename = os.path.join(base, 'th.txt')
+        self.addCleanup(shutil.rmtree, base)
+        with open(filename, 'w') as f:
+            f.write('Invisible Full Moon')
+        os.utime(filename, (12345, -3600.0))
+        with tar(base) as archive:
+            tar_data = tarfile.open(fileobj=archive)
+            assert tar_data.getnames() == ['th.txt']
+            assert tar_data.getmember('th.txt').mtime == -3600
+
+    @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
+    def test_tar_directory_link(self):
+        dirs = ['a', 'b', 'a/c']
+        files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py']
+        base = make_tree(dirs, files)
+        self.addCleanup(shutil.rmtree, base)
+        os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b'))
+        with tar(base) as archive:
+            tar_data = tarfile.open(fileobj=archive)
+            names = tar_data.getnames()
+            for member in dirs + files:
+                assert member in names
+            assert 'a/c/b' in names
+            assert 'a/c/b/utils.py' not in names

tests/unit/utils_test.py

@@ -5,29 +5,25 @@ import json
 import os
 import os.path
 import shutil
-import socket
 import sys
-import tarfile
 import tempfile
 import unittest
 
-import pytest
-import six
 
 from docker.api.client import APIClient
-from docker.constants import IS_WINDOWS_PLATFORM
 from docker.errors import DockerException
 from docker.utils import (
-    parse_repository_tag, parse_host, convert_filters, kwargs_from_env,
-    parse_bytes, parse_env_file, exclude_paths, convert_volume_binds,
-    decode_json_header, tar, split_command, parse_devices, update_headers,
+    convert_filters, convert_volume_binds, decode_json_header, kwargs_from_env,
+    parse_bytes, parse_devices, parse_env_file, parse_host,
+    parse_repository_tag, split_command, update_headers,
 )
 
 from docker.utils.ports import build_port_bindings, split_port
 from docker.utils.utils import format_environment
 
-from ..helpers import make_tree
+import pytest
 
+import six
 
 TEST_CERT_DIR = os.path.join(
     os.path.dirname(__file__),
@@ -608,472 +604,6 @@ class PortsTest(unittest.TestCase):
         assert port_bindings["2000"] == [("127.0.0.1", "2000")]
 
 
-def convert_paths(collection):
-    return set(map(convert_path, collection))
-
-
-def convert_path(path):
-    return path.replace('/', os.path.sep)
-
-
-class ExcludePathsTest(unittest.TestCase):
-    dirs = [
-        'foo',
-        'foo/bar',
-        'bar',
-        'target',
-        'target/subdir',
-        'subdir',
-        'subdir/target',
-        'subdir/target/subdir',
-        'subdir/subdir2',
-        'subdir/subdir2/target',
-        'subdir/subdir2/target/subdir'
-    ]
-
-    files = [
-        'Dockerfile',
-        'Dockerfile.alt',
-        '.dockerignore',
-        'a.py',
-        'a.go',
-        'b.py',
-        'cde.py',
-        'foo/a.py',
-        'foo/b.py',
-        'foo/bar/a.py',
-        'bar/a.py',
-        'foo/Dockerfile3',
-        'target/file.txt',
-        'target/subdir/file.txt',
-        'subdir/file.txt',
-        'subdir/target/file.txt',
-        'subdir/target/subdir/file.txt',
-        'subdir/subdir2/file.txt',
-        'subdir/subdir2/target/file.txt',
-        'subdir/subdir2/target/subdir/file.txt',
-    ]
-
-    all_paths = set(dirs + files)
-
-    def setUp(self):
-        self.base = make_tree(self.dirs, self.files)
-
-    def tearDown(self):
-        shutil.rmtree(self.base)
-
-    def exclude(self, patterns, dockerfile=None):
-        return set(exclude_paths(self.base, patterns, dockerfile=dockerfile))
-
-    def test_no_excludes(self):
-        assert self.exclude(['']) == convert_paths(self.all_paths)
-
-    def test_no_dupes(self):
-        paths = exclude_paths(self.base, ['!a.py'])
-        assert sorted(paths) == sorted(set(paths))
-
-    def test_wildcard_exclude(self):
-        assert self.exclude(['*']) == set(['Dockerfile', '.dockerignore'])
-
-    def test_exclude_dockerfile_dockerignore(self):
-        """
-        Even if the .dockerignore file explicitly says to exclude
-        Dockerfile and/or .dockerignore, don't exclude them from
-        the actual tar file.
-        """
-        assert self.exclude(['Dockerfile', '.dockerignore']) == convert_paths(
-            self.all_paths
-        )
-
-    def test_exclude_custom_dockerfile(self):
-        """
-        If we're using a custom Dockerfile, make sure that's not
-        excluded.
-        """
-        assert self.exclude(['*'], dockerfile='Dockerfile.alt') == set(
-            ['Dockerfile.alt', '.dockerignore']
-        )
-
-        assert self.exclude(
-            ['*'], dockerfile='foo/Dockerfile3'
-        ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore']))
-
-        # https://github.com/docker/docker-py/issues/1956
-        assert self.exclude(
-            ['*'], dockerfile='./foo/Dockerfile3'
-        ) == convert_paths(set(['foo/Dockerfile3', '.dockerignore']))
-
-    def test_exclude_dockerfile_child(self):
-        includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3')
-        assert convert_path('foo/Dockerfile3') in includes
-        assert convert_path('foo/a.py') not in includes
-
-    def test_single_filename(self):
-        assert self.exclude(['a.py']) == convert_paths(
-            self.all_paths - set(['a.py'])
-        )
-
-    def test_single_filename_leading_dot_slash(self):
-        assert self.exclude(['./a.py']) == convert_paths(
-            self.all_paths - set(['a.py'])
-        )
-
-    # As odd as it sounds, a filename pattern with a trailing slash on the
-    # end *will* result in that file being excluded.
-    def test_single_filename_trailing_slash(self):
-        assert self.exclude(['a.py/']) == convert_paths(
-            self.all_paths - set(['a.py'])
-        )
-
-    def test_wildcard_filename_start(self):
-        assert self.exclude(['*.py']) == convert_paths(
-            self.all_paths - set(['a.py', 'b.py', 'cde.py'])
-        )
-
-    def test_wildcard_with_exception(self):
-        assert self.exclude(['*.py', '!b.py']) == convert_paths(
-            self.all_paths - set(['a.py', 'cde.py'])
-        )
-
-    def test_wildcard_with_wildcard_exception(self):
-        assert self.exclude(['*.*', '!*.go']) == convert_paths(
-            self.all_paths - set([
-                'a.py', 'b.py', 'cde.py', 'Dockerfile.alt',
-            ])
-        )
-
-    def test_wildcard_filename_end(self):
-        assert self.exclude(['a.*']) == convert_paths(
-            self.all_paths - set(['a.py', 'a.go'])
-        )
-
-    def test_question_mark(self):
-        assert self.exclude(['?.py']) == convert_paths(
-            self.all_paths - set(['a.py', 'b.py'])
-        )
-
-    def test_single_subdir_single_filename(self):
-        assert self.exclude(['foo/a.py']) == convert_paths(
-            self.all_paths - set(['foo/a.py'])
-        )
-
-    def test_single_subdir_single_filename_leading_slash(self):
-        assert self.exclude(['/foo/a.py']) == convert_paths(
-            self.all_paths - set(['foo/a.py'])
-        )
-
-    def test_exclude_include_absolute_path(self):
-        base = make_tree([], ['a.py', 'b.py'])
-        assert exclude_paths(
-            base,
-            ['/*', '!/*.py']
-        ) == set(['a.py', 'b.py'])
-
-    def test_single_subdir_with_path_traversal(self):
-        assert self.exclude(['foo/whoops/../a.py']) == convert_paths(
-            self.all_paths - set(['foo/a.py'])
-        )
-
-    def test_single_subdir_wildcard_filename(self):
-        assert self.exclude(['foo/*.py']) == convert_paths(
-            self.all_paths - set(['foo/a.py', 'foo/b.py'])
-        )
-
-    def test_wildcard_subdir_single_filename(self):
-        assert self.exclude(['*/a.py']) == convert_paths(
-            self.all_paths - set(['foo/a.py', 'bar/a.py'])
-        )
-
-    def test_wildcard_subdir_wildcard_filename(self):
-        assert self.exclude(['*/*.py']) == convert_paths(
-            self.all_paths - set(['foo/a.py', 'foo/b.py', 'bar/a.py'])
-        )
-
-    def test_directory(self):
-        assert self.exclude(['foo']) == convert_paths(
-            self.all_paths - set([
-                'foo', 'foo/a.py', 'foo/b.py', 'foo/bar', 'foo/bar/a.py',
-                'foo/Dockerfile3'
-            ])
-        )
-
-    def test_directory_with_trailing_slash(self):
-        assert self.exclude(['foo']) == convert_paths(
-            self.all_paths - set([
-                'foo', 'foo/a.py', 'foo/b.py',
-                'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3'
-            ])
-        )
-
-    def test_directory_with_single_exception(self):
-        assert self.exclude(['foo', '!foo/bar/a.py']) == convert_paths(
-            self.all_paths - set([
-                'foo/a.py', 'foo/b.py', 'foo', 'foo/bar',
-                'foo/Dockerfile3'
-            ])
-        )
-
-    def test_directory_with_subdir_exception(self):
-        assert self.exclude(['foo', '!foo/bar']) == convert_paths(
-            self.all_paths - set([
-                'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3'
-            ])
-        )
-
-    @pytest.mark.skipif(
-        not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows'
-    )
-    def test_directory_with_subdir_exception_win32_pathsep(self):
-        assert self.exclude(['foo', '!foo\\bar']) == convert_paths(
-            self.all_paths - set([
-                'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3'
-            ])
-        )
-
-    def test_directory_with_wildcard_exception(self):
-        assert self.exclude(['foo', '!foo/*.py']) == convert_paths(
-            self.all_paths - set([
-                'foo/bar', 'foo/bar/a.py', 'foo', 'foo/Dockerfile3'
-            ])
-        )
-
-    def test_subdirectory(self):
-        assert self.exclude(['foo/bar']) == convert_paths(
-            self.all_paths - set(['foo/bar', 'foo/bar/a.py'])
-        )
-
-    @pytest.mark.skipif(
-        not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows'
-    )
-    def test_subdirectory_win32_pathsep(self):
-        assert self.exclude(['foo\\bar']) == convert_paths(
-            self.all_paths - set(['foo/bar', 'foo/bar/a.py'])
-        )
-
-    def test_double_wildcard(self):
-        assert self.exclude(['**/a.py']) == convert_paths(
-            self.all_paths - set(
-                ['a.py', 'foo/a.py', 'foo/bar/a.py', 'bar/a.py']
-            )
-        )
-
-        assert self.exclude(['foo/**/bar']) == convert_paths(
-            self.all_paths - set(['foo/bar', 'foo/bar/a.py'])
-        )
-
-    def test_single_and_double_wildcard(self):
-        assert self.exclude(['**/target/*/*']) == convert_paths(
-            self.all_paths - set(
-                ['target/subdir/file.txt',
-                 'subdir/target/subdir/file.txt',
-                 'subdir/subdir2/target/subdir/file.txt']
-            )
-        )
-
-    def test_trailing_double_wildcard(self):
-        assert self.exclude(['subdir/**']) == convert_paths(
-            self.all_paths - set(
-                ['subdir/file.txt',
-                 'subdir/target/file.txt',
-                 'subdir/target/subdir/file.txt',
-                 'subdir/subdir2/file.txt',
-                 'subdir/subdir2/target/file.txt',
-                 'subdir/subdir2/target/subdir/file.txt',
-                 'subdir/target',
-                 'subdir/target/subdir',
-                 'subdir/subdir2',
-                 'subdir/subdir2/target',
-                 'subdir/subdir2/target/subdir']
-            )
-        )
-
-    def test_include_wildcard(self):
-        base = make_tree(['a'], ['a/b.py'])
-        assert exclude_paths(
-            base,
-            ['*', '!*/b.py']
-        ) == convert_paths(['a/b.py'])
-
-    def test_last_line_precedence(self):
-        base = make_tree(
-            [],
-            ['garbage.md',
-             'thrash.md',
-             'README.md',
-             'README-bis.md',
-             'README-secret.md'])
-        assert exclude_paths(
-            base,
-            ['*.md', '!README*.md', 'README-secret.md']
-        ) == set(['README.md', 'README-bis.md'])
-
-    def test_parent_directory(self):
-        base = make_tree(
-            [],
-            ['a.py',
-             'b.py',
-             'c.py'])
-        # Dockerignore reference stipulates that absolute paths are
-        # equivalent to relative paths, hence /../foo should be
-        # equivalent to ../foo. It also stipulates that paths are run
-        # through Go's filepath.Clean, which explicitely "replace
-        # "/.."  by "/" at the beginning of a path".
-        assert exclude_paths(
-            base,
-            ['../a.py', '/../b.py']
-        ) == set(['c.py'])
-
-
-class TarTest(unittest.TestCase):
-    def test_tar_with_excludes(self):
-        dirs = [
-            'foo',
-            'foo/bar',
-            'bar',
-        ]
-
-        files = [
-            'Dockerfile',
-            'Dockerfile.alt',
-            '.dockerignore',
-            'a.py',
-            'a.go',
-            'b.py',
-            'cde.py',
-            'foo/a.py',
-            'foo/b.py',
-            'foo/bar/a.py',
-            'bar/a.py',
-        ]
-
-        exclude = [
-            '*.py',
-            '!b.py',
-            '!a.go',
-            'foo',
-            'Dockerfile*',
-            '.dockerignore',
-        ]
-
-        expected_names = set([
-            'Dockerfile',
-            '.dockerignore',
-            'a.go',
-            'b.py',
-            'bar',
-            'bar/a.py',
-        ])
-
-        base = make_tree(dirs, files)
-        self.addCleanup(shutil.rmtree, base)
-
-        with tar(base, exclude=exclude) as archive:
-            tar_data = tarfile.open(fileobj=archive)
-            assert sorted(tar_data.getnames()) == sorted(expected_names)
-
-    def test_tar_with_empty_directory(self):
-        base = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, base)
-        for d in ['foo', 'bar']:
-            os.makedirs(os.path.join(base, d))
-        with tar(base) as archive:
-            tar_data = tarfile.open(fileobj=archive)
-            assert sorted(tar_data.getnames()) == ['bar', 'foo']
-
-    @pytest.mark.skipif(
-        IS_WINDOWS_PLATFORM or os.geteuid() == 0,
-        reason='root user always has access ; no chmod on Windows'
-    )
-    def test_tar_with_inaccessible_file(self):
-        base = tempfile.mkdtemp()
-        full_path = os.path.join(base, 'foo')
-        self.addCleanup(shutil.rmtree, base)
-        with open(full_path, 'w') as f:
-            f.write('content')
-        os.chmod(full_path, 0o222)
-        with pytest.raises(IOError) as ei:
-            tar(base)
-
-        assert 'Can not read file in context: {}'.format(full_path) in (
-            ei.exconly()
-        )
-
-    @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
-    def test_tar_with_file_symlinks(self):
-        base = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, base)
-        with open(os.path.join(base, 'foo'), 'w') as f:
-            f.write("content")
-        os.makedirs(os.path.join(base, 'bar'))
-        os.symlink('../foo', os.path.join(base, 'bar/foo'))
-        with tar(base) as archive:
-            tar_data = tarfile.open(fileobj=archive)
-            assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
-
-    @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
-    def test_tar_with_directory_symlinks(self):
-        base = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, base)
-        for d in ['foo', 'bar']:
-            os.makedirs(os.path.join(base, d))
-        os.symlink('../foo', os.path.join(base, 'bar/foo'))
-        with tar(base) as archive:
-            tar_data = tarfile.open(fileobj=archive)
-            assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
-
-    @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
-    def test_tar_with_broken_symlinks(self):
-        base = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, base)
-        for d in ['foo', 'bar']:
-            os.makedirs(os.path.join(base, d))
-
-        os.symlink('../baz', os.path.join(base, 'bar/foo'))
-        with tar(base) as archive:
-            tar_data = tarfile.open(fileobj=archive)
-            assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
-
-    @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No UNIX sockets on Win32')
-    def test_tar_socket_file(self):
-        base = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, base)
-        for d in ['foo', 'bar']:
-            os.makedirs(os.path.join(base, d))
-        sock = socket.socket(socket.AF_UNIX)
-        self.addCleanup(sock.close)
-        sock.bind(os.path.join(base, 'test.sock'))
-        with tar(base) as archive:
-            tar_data = tarfile.open(fileobj=archive)
-            assert sorted(tar_data.getnames()) == ['bar', 'foo']
-
-    def tar_test_negative_mtime_bug(self):
-        base = tempfile.mkdtemp()
-        filename = os.path.join(base, 'th.txt')
-        self.addCleanup(shutil.rmtree, base)
-        with open(filename, 'w') as f:
-            f.write('Invisible Full Moon')
-        os.utime(filename, (12345, -3600.0))
-        with tar(base) as archive:
-            tar_data = tarfile.open(fileobj=archive)
-            assert tar_data.getnames() == ['th.txt']
-            assert tar_data.getmember('th.txt').mtime == -3600
-
-    @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
-    def test_tar_directory_link(self):
-        dirs = ['a', 'b', 'a/c']
-        files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py']
-        base = make_tree(dirs, files)
-        self.addCleanup(shutil.rmtree, base)
-        os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b'))
-        with tar(base) as archive:
-            tar_data = tarfile.open(fileobj=archive)
-            names = tar_data.getnames()
-            for member in dirs + files:
-                assert member in names
-            assert 'a/c/b' in names
-            assert 'a/c/b/utils.py' not in names
-
-
 class FormatEnvironmentTest(unittest.TestCase):
     def test_format_env_binary_unicode_value(self):
         env_dict = {