Merge pull request #2283 from ulyssessouza/3.7.1-release

3.7.1 release

docker/api/client.py

@@ -22,26 +22,26 @@ from .volume import VolumeApiMixin
 from .. import auth
 from ..constants import (
     DEFAULT_TIMEOUT_SECONDS, DEFAULT_USER_AGENT, IS_WINDOWS_PLATFORM,
-    DEFAULT_DOCKER_API_VERSION, STREAM_HEADER_SIZE_BYTES, DEFAULT_NUM_POOLS,
-    MINIMUM_DOCKER_API_VERSION
+    DEFAULT_DOCKER_API_VERSION, MINIMUM_DOCKER_API_VERSION,
+    STREAM_HEADER_SIZE_BYTES, DEFAULT_NUM_POOLS_SSH, DEFAULT_NUM_POOLS
 )
 from ..errors import (
     DockerException, InvalidVersion, TLSParameterError,
     create_api_error_from_http_exception
 )
 from ..tls import TLSConfig
-from ..transport import SSLAdapter, UnixAdapter
+from ..transport import SSLHTTPAdapter, UnixHTTPAdapter
 from ..utils import utils, check_resource, update_headers, config
 from ..utils.socket import frames_iter, consume_socket_output, demux_adaptor
 from ..utils.json_stream import json_stream
 from ..utils.proxy import ProxyConfig
 try:
-    from ..transport import NpipeAdapter
+    from ..transport import NpipeHTTPAdapter
 except ImportError:
     pass
 
 try:
-    from ..transport import SSHAdapter
+    from ..transport import SSHHTTPAdapter
 except ImportError:
     pass
 
@@ -101,7 +101,7 @@ class APIClient(
 
     def __init__(self, base_url=None, version=None,
                  timeout=DEFAULT_TIMEOUT_SECONDS, tls=False,
-                 user_agent=DEFAULT_USER_AGENT, num_pools=DEFAULT_NUM_POOLS,
+                 user_agent=DEFAULT_USER_AGENT, num_pools=None,
                  credstore_env=None):
         super(APIClient, self).__init__()
 
@@ -132,8 +132,12 @@ class APIClient(
         base_url = utils.parse_host(
             base_url, IS_WINDOWS_PLATFORM, tls=bool(tls)
         )
+        # SSH has a different default for num_pools to all other adapters
+        num_pools = num_pools or DEFAULT_NUM_POOLS_SSH if \
+            base_url.startswith('ssh://') else DEFAULT_NUM_POOLS
+
         if base_url.startswith('http+unix://'):
-            self._custom_adapter = UnixAdapter(
+            self._custom_adapter = UnixHTTPAdapter(
                 base_url, timeout, pool_connections=num_pools
             )
             self.mount('http+docker://', self._custom_adapter)
@@ -147,7 +151,7 @@ class APIClient(
                     'The npipe:// protocol is only supported on Windows'
                 )
             try:
-                self._custom_adapter = NpipeAdapter(
+                self._custom_adapter = NpipeHTTPAdapter(
                     base_url, timeout, pool_connections=num_pools
                 )
             except NameError:
@@ -158,7 +162,7 @@ class APIClient(
             self.base_url = 'http+docker://localnpipe'
         elif base_url.startswith('ssh://'):
             try:
-                self._custom_adapter = SSHAdapter(
+                self._custom_adapter = SSHHTTPAdapter(
                     base_url, timeout, pool_connections=num_pools
                 )
             except NameError:
@@ -173,7 +177,8 @@ class APIClient(
             if isinstance(tls, TLSConfig):
                 tls.configure_client(self)
             elif tls:
-                self._custom_adapter = SSLAdapter(pool_connections=num_pools)
+                self._custom_adapter = SSLHTTPAdapter(
+                    pool_connections=num_pools)
                 self.mount('https://', self._custom_adapter)
             self.base_url = base_url
 

docker/constants.py

@@ -18,4 +18,10 @@ WINDOWS_LONGPATH_PREFIX = '\\\\?\\'
 
 DEFAULT_USER_AGENT = "docker-sdk-python/{0}".format(version)
 DEFAULT_NUM_POOLS = 25
+
+# The OpenSSH server default value for MaxSessions is 10 which means we can
+# use up to 9, leaving the final session for the underlying SSH connection.
+# For more details see: https://github.com/docker/docker-py/issues/2246
+DEFAULT_NUM_POOLS_SSH = 9
+
 DEFAULT_DATA_CHUNK_SIZE = 1024 * 2048

docker/tls.py

@@ -2,7 +2,7 @@ import os
 import ssl
 
 from . import errors
-from .transport import SSLAdapter
+from .transport import SSLHTTPAdapter
 
 
 class TLSConfig(object):
@@ -105,7 +105,7 @@ class TLSConfig(object):
         if self.cert:
             client.cert = self.cert
 
-        client.mount('https://', SSLAdapter(
+        client.mount('https://', SSLHTTPAdapter(
             ssl_version=self.ssl_version,
             assert_hostname=self.assert_hostname,
             assert_fingerprint=self.assert_fingerprint,

docker/transport/__init__.py

 # flake8: noqa
-from .unixconn import UnixAdapter
-from .ssladapter import SSLAdapter
+from .unixconn import UnixHTTPAdapter
+from .ssladapter import SSLHTTPAdapter
 try:
-    from .npipeconn import NpipeAdapter
+    from .npipeconn import NpipeHTTPAdapter
     from .npipesocket import NpipeSocket
 except ImportError:
     pass
 
 try:
-    from .sshconn import SSHAdapter
+    from .sshconn import SSHHTTPAdapter
 except ImportError:
     pass

docker/transport/basehttpadapter.py

+import requests.adapters
+
+
+class BaseHTTPAdapter(requests.adapters.HTTPAdapter):
+    def close(self):
+        super(BaseHTTPAdapter, self).close()
+        if hasattr(self, 'pools'):
+            self.pools.clear()

docker/transport/npipeconn.py

 import six
 import requests.adapters
 
+from docker.transport.basehttpadapter import BaseHTTPAdapter
 from .. import constants
 from .npipesocket import NpipeSocket
 
@@ -68,7 +69,7 @@ class NpipeHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool):
         return conn or self._new_conn()
 
 
-class NpipeAdapter(requests.adapters.HTTPAdapter):
+class NpipeHTTPAdapter(BaseHTTPAdapter):
 
     __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + ['npipe_path',
                                                            'pools',
@@ -81,7 +82,7 @@ class NpipeAdapter(requests.adapters.HTTPAdapter):
         self.pools = RecentlyUsedContainer(
             pool_connections, dispose_func=lambda p: p.close()
         )
-        super(NpipeAdapter, self).__init__()
+        super(NpipeHTTPAdapter, self).__init__()
 
     def get_connection(self, url, proxies=None):
         with self.pools.lock:
@@ -103,6 +104,3 @@ class NpipeAdapter(requests.adapters.HTTPAdapter):
         # anyway, we simply return the path URL directly.
         # See also: https://github.com/docker/docker-sdk-python/issues/811
         return request.path_url
-
-    def close(self):
-        self.pools.clear()

docker/transport/sshconn.py

@@ -2,6 +2,7 @@ import paramiko
 import requests.adapters
 import six
 
+from docker.transport.basehttpadapter import BaseHTTPAdapter
 from .. import constants
 
 if six.PY3:
@@ -68,7 +69,7 @@ class SSHConnectionPool(urllib3.connectionpool.HTTPConnectionPool):
         return conn or self._new_conn()
 
 
-class SSHAdapter(requests.adapters.HTTPAdapter):
+class SSHHTTPAdapter(BaseHTTPAdapter):
 
     __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + [
         'pools', 'timeout', 'ssh_client',
@@ -79,15 +80,19 @@ class SSHAdapter(requests.adapters.HTTPAdapter):
         self.ssh_client = paramiko.SSHClient()
         self.ssh_client.load_system_host_keys()
 
-        parsed = six.moves.urllib_parse.urlparse(base_url)
-        self.ssh_client.connect(
-            parsed.hostname, parsed.port, parsed.username,
-        )
+        self.base_url = base_url
+        self._connect()
         self.timeout = timeout
         self.pools = RecentlyUsedContainer(
             pool_connections, dispose_func=lambda p: p.close()
         )
-        super(SSHAdapter, self).__init__()
+        super(SSHHTTPAdapter, self).__init__()
+
+    def _connect(self):
+        parsed = six.moves.urllib_parse.urlparse(self.base_url)
+        self.ssh_client.connect(
+            parsed.hostname, parsed.port, parsed.username,
+        )
 
     def get_connection(self, url, proxies=None):
         with self.pools.lock:
@@ -95,6 +100,10 @@ class SSHAdapter(requests.adapters.HTTPAdapter):
             if pool:
                 return pool
 
+            # Connection is closed try a reconnect
+            if not self.ssh_client.get_transport():
+                self._connect()
+
             pool = SSHConnectionPool(
                 self.ssh_client, self.timeout
             )
@@ -103,5 +112,5 @@ class SSHAdapter(requests.adapters.HTTPAdapter):
         return pool
 
     def close(self):
-        self.pools.clear()
+        super(SSHHTTPAdapter, self).close()
         self.ssh_client.close()

docker/transport/ssladapter.py

@@ -7,6 +7,8 @@ import sys
 from distutils.version import StrictVersion
 from requests.adapters import HTTPAdapter
 
+from docker.transport.basehttpadapter import BaseHTTPAdapter
+
 try:
     import requests.packages.urllib3 as urllib3
 except ImportError:
@@ -22,7 +24,7 @@ if sys.version_info[0] < 3 or sys.version_info[1] < 5:
     urllib3.connection.match_hostname = match_hostname
 
 
-class SSLAdapter(HTTPAdapter):
+class SSLHTTPAdapter(BaseHTTPAdapter):
     '''An HTTPS Transport Adapter that uses an arbitrary SSL version.'''
 
     __attrs__ = HTTPAdapter.__attrs__ + ['assert_fingerprint',
@@ -34,7 +36,7 @@ class SSLAdapter(HTTPAdapter):
         self.ssl_version = ssl_version
         self.assert_hostname = assert_hostname
         self.assert_fingerprint = assert_fingerprint
-        super(SSLAdapter, self).__init__(**kwargs)
+        super(SSLHTTPAdapter, self).__init__(**kwargs)
 
     def init_poolmanager(self, connections, maxsize, block=False):
         kwargs = {
@@ -57,7 +59,7 @@ class SSLAdapter(HTTPAdapter):
 
         But we still need to take care of when there is a proxy poolmanager
         """
-        conn = super(SSLAdapter, self).get_connection(*args, **kwargs)
+        conn = super(SSLHTTPAdapter, self).get_connection(*args, **kwargs)
         if conn.assert_hostname != self.assert_hostname:
             conn.assert_hostname = self.assert_hostname
         return conn

docker/transport/unixconn.py

@@ -3,6 +3,7 @@ import requests.adapters
 import socket
 from six.moves import http_client as httplib
 
+from docker.transport.basehttpadapter import BaseHTTPAdapter
 from .. import constants
 
 try:
@@ -69,7 +70,7 @@ class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool):
         )
 
 
-class UnixAdapter(requests.adapters.HTTPAdapter):
+class UnixHTTPAdapter(BaseHTTPAdapter):
 
     __attrs__ = requests.adapters.HTTPAdapter.__attrs__ + ['pools',
                                                            'socket_path',
@@ -85,7 +86,7 @@ class UnixAdapter(requests.adapters.HTTPAdapter):
         self.pools = RecentlyUsedContainer(
             pool_connections, dispose_func=lambda p: p.close()
         )
-        super(UnixAdapter, self).__init__()
+        super(UnixHTTPAdapter, self).__init__()
 
     def get_connection(self, url, proxies=None):
         with self.pools.lock:
@@ -107,6 +108,3 @@ class UnixAdapter(requests.adapters.HTTPAdapter):
         # anyway, we simply return the path URL directly.
         # See also: https://github.com/docker/docker-py/issues/811
         return request.path_url
-
-    def close(self):
-        self.pools.clear()

docker/version.py

-version = "3.7.0"
+version = "3.7.1"
 version_info = tuple([int(d) for d in version.split("-")[0].split(".")])

docs/change-log.md

 Change log
 ==========
 
+3.7.1
+-----
+
+[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/58?closed=1)
+
+### Bugfixes
+
+* Set a different default number (which is now 9) for SSH pools
+* Adds a BaseHTTPAdapter with a close method to ensure that the
+pools is clean on close()
+* Makes SSHHTTPAdapter reopen a closed connection when needed
+like the others
+
 3.7.0
 -----