Merge pull request #1259 from docker/1.10.4-release

1.10.4 release

docker/client.py

@@ -86,7 +86,7 @@ class Client(
                 tls.configure_client(self)
             elif tls:
                 self._custom_adapter = ssladapter.SSLAdapter(
-                    num_pools=num_pools
+                    pool_connections=num_pools
                 )
                 self.mount('https://', self._custom_adapter)
             self.base_url = base_url
@@ -218,7 +218,9 @@ class Client(
 
     def _get_raw_response_socket(self, response):
         self._raise_for_status(response)
-        if six.PY3:
+        if self.base_url == "http+docker://localnpipe":
+            sock = response.raw._fp.fp.raw.sock
+        elif six.PY3:
             sock = response.raw._fp.fp.raw
             if self.base_url.startswith("https://"):
                 sock = sock._sock

docker/transport/__init__.py

@@ -2,5 +2,6 @@
 from .unixconn import UnixAdapter
 try:
     from .npipeconn import NpipeAdapter
+    from .npipesocket import NpipeSocket
 except ImportError:
     pass
\ No newline at end of file

docker/transport/npipeconn.py

@@ -14,7 +14,6 @@ try:
 except ImportError:
     import urllib3
 
-
 RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer
 
 
@@ -46,6 +45,28 @@ class NpipeHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool):
             self.npipe_path, self.timeout
         )
 
+    # When re-using connections, urllib3 tries to call select() on our
+    # NpipeSocket instance, causing a crash. To circumvent this, we override
+    # _get_conn, where that check happens.
+    def _get_conn(self, timeout):
+        conn = None
+        try:
+            conn = self.pool.get(block=self.block, timeout=timeout)
+
+        except AttributeError:  # self.pool is None
+            raise urllib3.exceptions.ClosedPoolError(self, "Pool is closed.")
+
+        except six.moves.queue.Empty:
+            if self.block:
+                raise urllib3.exceptions.EmptyPoolError(
+                    self,
+                    "Pool reached maximum size and no more "
+                    "connections are allowed."
+                )
+            pass  # Oh well, we'll create a new connection then
+
+        return conn or self._new_conn()
+
 
 class NpipeAdapter(requests.adapters.HTTPAdapter):
     def __init__(self, base_url, timeout=60,

docker/transport/npipesocket.py

 import functools
 import io
 
+import six
 import win32file
 import win32pipe
 
@@ -94,7 +95,7 @@ class NpipeSocket(object):
         if mode.strip('b') != 'r':
             raise NotImplementedError()
         rawio = NpipeFileIOBase(self)
-        if bufsize is None or bufsize < 0:
+        if bufsize is None or bufsize <= 0:
             bufsize = io.DEFAULT_BUFFER_SIZE
         return io.BufferedReader(rawio, buffer_size=bufsize)
 
@@ -114,6 +115,9 @@ class NpipeSocket(object):
 
     @check_closed
     def recv_into(self, buf, nbytes=0):
+        if six.PY2:
+            return self._recv_into_py2(buf, nbytes)
+
         readbuf = buf
         if not isinstance(buf, memoryview):
             readbuf = memoryview(buf)
@@ -124,6 +128,12 @@ class NpipeSocket(object):
         )
         return len(data)
 
+    def _recv_into_py2(self, buf, nbytes):
+        err, data = win32file.ReadFile(self._handle, nbytes or len(buf))
+        n = len(data)
+        buf[:n] = data
+        return n
+
     @check_closed
     def send(self, string, flags=0):
         err, nbytes = win32file.WriteFile(self._handle, string)

docker/types/services.py

@@ -152,7 +152,7 @@ class UpdateConfig(dict):
 class RestartConditionTypesEnum(object):
     _values = (
         'none',
-        'on_failure',
+        'on-failure',
         'any',
     )
     NONE, ON_FAILURE, ANY = _values

docker/utils/socket.py

@@ -5,6 +5,11 @@ import struct
 
 import six
 
+try:
+    from ..transport import NpipeSocket
+except ImportError:
+    NpipeSocket = type(None)
+
 
 class SocketError(Exception):
     pass
@@ -14,10 +19,12 @@ def read(socket, n=4096):
     """
     Reads at most n bytes from socket
     """
+
     recoverable_errors = (errno.EINTR, errno.EDEADLK, errno.EWOULDBLOCK)
 
     # wait for data to become available
-    select.select([socket], [], [])
+    if not isinstance(socket, NpipeSocket):
+        select.select([socket], [], [])
 
     try:
         if hasattr(socket, 'recv'):

docker/utils/utils.py

@@ -194,8 +194,8 @@ def match_path(path, pattern):
     if pattern:
         pattern = os.path.relpath(pattern)
 
-    pattern_components = pattern.split('/')
-    path_components = path.split('/')[:len(pattern_components)]
+    pattern_components = pattern.split(os.path.sep)
+    path_components = path.split(os.path.sep)[:len(pattern_components)]
     return fnmatch('/'.join(path_components), pattern)
 
 
@@ -438,8 +438,8 @@ def parse_host(addr, is_win32=False, tls=False):
             "Bind address needs a port: {0}".format(addr))
 
     if proto == "http+unix" or proto == 'npipe':
-        return "{0}://{1}".format(proto, host)
-    return "{0}://{1}:{2}{3}".format(proto, host, port, path)
+        return "{0}://{1}".format(proto, host).rstrip('/')
+    return "{0}://{1}:{2}{3}".format(proto, host, port, path).rstrip('/')
 
 
 def parse_devices(devices):
@@ -986,6 +986,9 @@ def format_environment(environment):
     def format_env(key, value):
         if value is None:
             return key
+        if isinstance(value, six.binary_type):
+            value = value.decode('utf-8')
+
         return u'{key}={value}'.format(key=key, value=value)
     return [format_env(*var) for var in six.iteritems(environment)]
 

docker/version.py

-version = "1.10.3"
+version = "1.10.4"
 version_info = tuple([int(d) for d in version.split("-")[0].split(".")])

docs/change_log.md

 Change Log
 ==========
 
+1.10.4
+------
+
+[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/24?closed=1)
+
+### Bugfixes
+
+* Fixed an issue where `RestartPolicy.condition_types.ON_FAILURE` would yield
+  an invalid value.
+* Fixed an issue where the SSL connection adapter would receive an invalid
+  argument.
+* Fixed an issue that caused the Client to fail to reach API endpoints when
+  the provided `base_url` had a trailing slash.
+* Fixed a bug where some `environment` values in `create_container`
+  containing unicode characters would raise an encoding error.
+* Fixed a number of issues tied with named pipe transport on Windows.
+* Fixed a bug where inclusion patterns in `.dockerignore` would cause some
+  excluded files to appear in the build context on Windows.
+
+### Miscellaneous
+
+* Adjusted version requirements for the `requests` library.
+* It is now possible to run the docker-py test suite on Windows.
+
+
 1.10.3
 ------
 

requirements.txt

-requests==2.5.3
+requests==2.11.1
 six>=1.4.0
 websocket-client==0.32.0
 backports.ssl_match_hostname>=3.5 ; python_version < '3.5'
 ipaddress==1.0.16 ; python_version < '3.3'
-docker-pycreds==0.2.1
\ No newline at end of file
+docker-pycreds==0.2.1

setup.py

@@ -9,7 +9,7 @@ ROOT_DIR = os.path.dirname(__file__)
 SOURCE_DIR = os.path.join(ROOT_DIR)
 
 requirements = [
-    'requests >= 2.5.2, < 2.11',
+    'requests >= 2.5.2, != 2.11.0',
     'six >= 1.4.0',
     'websocket-client >= 0.32.0',
     'docker-pycreds >= 0.2.1'

tests/integration/build_test.py

 import io
-import json
 import os
 import shutil
 import tempfile
@@ -22,14 +21,11 @@ class BuildTest(helpers.BaseTestCase):
             'ADD https://dl.dropboxusercontent.com/u/20637798/silence.tar.gz'
             ' /tmp/silence.tar.gz'
         ]).encode('ascii'))
-        stream = self.client.build(fileobj=script, stream=True)
-        logs = ''
+        stream = self.client.build(fileobj=script, stream=True, decode=True)
+        logs = []
         for chunk in stream:
-            if six.PY3:
-                chunk = chunk.decode('utf-8')
-            json.loads(chunk)  # ensure chunk is a single, valid JSON blob
-            logs += chunk
-        self.assertNotEqual(logs, '')
+            logs.append(chunk)
+        assert len(logs) > 0
 
     def test_build_from_stringio(self):
         if six.PY3:

tests/integration/conftest.py

 from __future__ import print_function
 
-import json
 import sys
 import warnings
 
@@ -19,8 +18,7 @@ def setup_test_session():
         c.inspect_image(BUSYBOX)
     except docker.errors.NotFound:
         print("\npulling {0}".format(BUSYBOX), file=sys.stderr)
-        for data in c.pull(BUSYBOX, stream=True):
-            data = json.loads(data.decode('utf-8'))
+        for data in c.pull(BUSYBOX, stream=True, decode=True):
             status = data.get("status")
             progress = data.get("progress")
             detail = "{0} - {1}".format(status, progress)

tests/integration/container_test.py

@@ -3,6 +3,7 @@ import signal
 import tempfile
 
 import docker
+from docker.constants import IS_WINDOWS_PLATFORM
 from docker.utils.socket import next_frame_size
 from docker.utils.socket import read_exactly
 import pytest
@@ -414,6 +415,9 @@ class VolumeBindTest(helpers.BaseTestCase):
             ['touch', os.path.join(self.mount_dest, self.filename)],
         )
 
+    @pytest.mark.xfail(
+        IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform'
+    )
     def test_create_with_binds_rw(self):
 
         container = self.run_with_volume(
@@ -429,6 +433,9 @@ class VolumeBindTest(helpers.BaseTestCase):
         inspect_data = self.client.inspect_container(container)
         self.check_container_data(inspect_data, True)
 
+    @pytest.mark.xfail(
+        IS_WINDOWS_PLATFORM, reason='Test not designed for Windows platform'
+    )
     def test_create_with_binds_ro(self):
         self.run_with_volume(
             False,
@@ -524,13 +531,13 @@ class ArchiveTest(helpers.BaseTestCase):
 
     def test_copy_file_to_container(self):
         data = b'Deaf To All But The Song'
-        with tempfile.NamedTemporaryFile() as test_file:
+        with tempfile.NamedTemporaryFile(delete=False) as test_file:
             test_file.write(data)
             test_file.seek(0)
             ctnr = self.client.create_container(
                 BUSYBOX,
                 'cat {0}'.format(
-                    os.path.join('/vol1', os.path.basename(test_file.name))
+                    os.path.join('/vol1/', os.path.basename(test_file.name))
                 ),
                 volumes=['/vol1']
             )
@@ -822,11 +829,12 @@ class KillTest(helpers.BaseTestCase):
         self.assertEqual(state['Running'], False)
 
     def test_kill_with_signal(self):
-        container = self.client.create_container(BUSYBOX, ['sleep', '60'])
-        id = container['Id']
-        self.client.start(id)
+        id = self.client.create_container(BUSYBOX, ['sleep', '60'])
         self.tmp_containers.append(id)
-        self.client.kill(id, signal=signal.SIGKILL)
+        self.client.start(id)
+        self.client.kill(
+            id, signal=signal.SIGKILL if not IS_WINDOWS_PLATFORM else 9
+        )
         exitcode = self.client.wait(id)
         self.assertNotEqual(exitcode, 0)
         container_info = self.client.inspect_container(id)
@@ -902,28 +910,34 @@ class PortTest(helpers.BaseTestCase):
 class ContainerTopTest(helpers.BaseTestCase):
     def test_top(self):
         container = self.client.create_container(
-            BUSYBOX, ['sleep', '60'])
+            BUSYBOX, ['sleep', '60']
+        )
 
-        id = container['Id']
+        self.tmp_containers.append(container)
 
         self.client.start(container)
-        res = self.client.top(container['Id'])
-        self.assertEqual(
-            res['Titles'],
-            ['UID', 'PID', 'PPID', 'C', 'STIME', 'TTY', 'TIME', 'CMD']
-        )
-        self.assertEqual(len(res['Processes']), 1)
-        self.assertEqual(res['Processes'][0][7], 'sleep 60')
-        self.client.kill(id)
+        res = self.client.top(container)
+        if IS_WINDOWS_PLATFORM:
+            assert res['Titles'] == ['PID', 'USER', 'TIME', 'COMMAND']
+        else:
+            assert res['Titles'] == [
+                'UID', 'PID', 'PPID', 'C', 'STIME', 'TTY', 'TIME', 'CMD'
+            ]
+        assert len(res['Processes']) == 1
+        assert res['Processes'][0][-1] == 'sleep 60'
+        self.client.kill(container)
 
+    @pytest.mark.skipif(
+        IS_WINDOWS_PLATFORM, reason='No psargs support on windows'
+    )
     def test_top_with_psargs(self):
         container = self.client.create_container(
             BUSYBOX, ['sleep', '60'])
 
-        id = container['Id']
+        self.tmp_containers.append(container)
 
         self.client.start(container)
-        res = self.client.top(container['Id'], 'waux')
+        res = self.client.top(container, 'waux')
         self.assertEqual(
             res['Titles'],
             ['USER', 'PID', '%CPU', '%MEM', 'VSZ', 'RSS',
@@ -931,7 +945,6 @@ class ContainerTopTest(helpers.BaseTestCase):
         )
         self.assertEqual(len(res['Processes']), 1)
         self.assertEqual(res['Processes'][0][10], 'sleep 60')
-        self.client.kill(id)
 
 
 class RestartContainerTest(helpers.BaseTestCase):

tests/integration/image_test.py

@@ -57,12 +57,10 @@ class PullImageTest(helpers.BaseTestCase):
             self.client.remove_image('hello-world')
         except docker.errors.APIError:
             pass
-        stream = self.client.pull('hello-world', stream=True)
+        stream = self.client.pull('hello-world', stream=True, decode=True)
         self.tmp_imgs.append('hello-world')
         for chunk in stream:
-            if six.PY3:
-                chunk = chunk.decode('utf-8')
-            json.loads(chunk)  # ensure chunk is a single, valid JSON blob
+            assert isinstance(chunk, dict)
         self.assertGreaterEqual(
             len(self.client.images('hello-world')), 1
         )
@@ -152,7 +150,7 @@ class ImportImageTest(helpers.BaseTestCase):
     @contextlib.contextmanager
     def dummy_tar_file(self, n_bytes):
         '''Yields the name of a valid tar file of size n_bytes.'''
-        with tempfile.NamedTemporaryFile() as tar_file:
+        with tempfile.NamedTemporaryFile(delete=False) as tar_file:
             self.write_dummy_tar_content(n_bytes, tar_file)
             tar_file.seek(0)
             yield tar_file.name

tests/integration/network_test.py

@@ -69,19 +69,18 @@ class TestNetworks(helpers.BaseTestCase):
 
         assert ipam.pop('Options', None) is None
 
-        assert ipam == {
-            'Driver': 'default',
-            'Config': [{
-                'Subnet': "172.28.0.0/16",
-                'IPRange': "172.28.5.0/24",
-                'Gateway': "172.28.5.254",
-                'AuxiliaryAddresses': {
-                    "a": "172.28.1.5",
-                    "b": "172.28.1.6",
-                    "c": "172.28.1.7",
-                },
-            }],
-        }
+        assert ipam['Driver'] == 'default'
+
+        assert ipam['Config'] == [{
+            'Subnet': "172.28.0.0/16",
+            'IPRange': "172.28.5.0/24",
+            'Gateway': "172.28.5.254",
+            'AuxiliaryAddresses': {
+                "a": "172.28.1.5",
+                "b": "172.28.1.6",
+                "c": "172.28.1.7",
+            },
+        }]
 
     @requires_api_version('1.21')
     def test_create_network_with_host_driver_fails(self):

tests/unit/api_test.py

@@ -86,7 +86,7 @@ def fake_delete(self, url, *args, **kwargs):
 def fake_read_from_socket(self, response, stream):
     return six.binary_type()
 
-url_base = 'http+docker://localunixsocket/'
+url_base = '{0}/'.format(fake_api.prefix)
 url_prefix = '{0}v{1}/'.format(
     url_base,
     docker.constants.DEFAULT_DOCKER_API_VERSION)
@@ -422,6 +422,9 @@ class StreamTest(base.Cleanup, base.BaseTestCase):
 
             data += connection.recv(2048)
 
+    @pytest.mark.skipif(
+        docker.constants.IS_WINDOWS_PLATFORM, reason='Unix only'
+    )
     def test_early_stream_response(self):
         self.request_handler = self.early_response_sending_handler
         lines = []

tests/unit/container_test.py

@@ -270,8 +270,8 @@ class CreateContainerTest(DockerClientTest):
                          {'Content-Type': 'application/json'})
 
     def test_create_container_with_cpu_shares(self):
-        self.client.create_container('busybox', 'ls',
-                                     cpu_shares=5)
+        with pytest.deprecated_call():
+            self.client.create_container('busybox', 'ls', cpu_shares=5)
 
         args = fake_request.call_args
         self.assertEqual(args[0][1],
@@ -316,8 +316,8 @@ class CreateContainerTest(DockerClientTest):
                          {'Content-Type': 'application/json'})
 
     def test_create_container_with_cpuset(self):
-        self.client.create_container('busybox', 'ls',
-                                     cpuset='0,1')
+        with pytest.deprecated_call():
+            self.client.create_container('busybox', 'ls', cpuset='0,1')
 
         args = fake_request.call_args
         self.assertEqual(args[0][1],

tests/unit/fake_api.py

@@ -408,6 +408,9 @@ def post_fake_update_container():
 
 # Maps real api url to fake response callback
 prefix = 'http+docker://localunixsocket'
+if constants.IS_WINDOWS_PLATFORM:
+    prefix = 'http+docker://localnpipe'
+
 fake_responses = {
     '{0}/version'.format(prefix):
     get_fake_raw_version,