Merge pull request #2219 from docker/2199-proxy-support

Support using proxy values from config

docker/api/build.py

@@ -19,7 +19,8 @@ class BuildApiMixin(object):
               forcerm=False, dockerfile=None, container_limits=None,
               decode=False, buildargs=None, gzip=False, shmsize=None,
               labels=None, cache_from=None, target=None, network_mode=None,
-              squash=None, extra_hosts=None, platform=None, isolation=None):
+              squash=None, extra_hosts=None, platform=None, isolation=None,
+              use_config_proxy=False):
         """
         Similar to the ``docker build`` command. Either ``path`` or ``fileobj``
         needs to be set. ``path`` can be a local path (to a directory
@@ -103,6 +104,10 @@ class BuildApiMixin(object):
             platform (str): Platform in the format ``os[/arch[/variant]]``
             isolation (str): Isolation technology used during build.
                 Default: `None`.
+            use_config_proxy (bool): If ``True``, and if the docker client
+                configuration file (``~/.docker/config.json`` by default)
+                contains a proxy configuration, the corresponding environment
+                variables will be set in the container being built.
 
         Returns:
             A generator for the build output.
@@ -168,6 +173,10 @@ class BuildApiMixin(object):
         }
         params.update(container_limits)
 
+        if use_config_proxy:
+            proxy_args = self._proxy_configs.get_environment()
+            for k, v in proxy_args.items():
+                buildargs.setdefault(k, v)
         if buildargs:
             params.update({'buildargs': json.dumps(buildargs)})
 

docker/api/client.py

@@ -34,6 +34,7 @@ from ..transport import SSLAdapter, UnixAdapter
 from ..utils import utils, check_resource, update_headers, config
 from ..utils.socket import frames_iter, consume_socket_output, demux_adaptor
 from ..utils.json_stream import json_stream
+from ..utils.proxy import ProxyConfig
 try:
     from ..transport import NpipeAdapter
 except ImportError:
@@ -114,6 +115,15 @@ class APIClient(
         self.headers['User-Agent'] = user_agent
 
         self._general_configs = config.load_general_config()
+
+        proxy_config = self._general_configs.get('proxies', {})
+        try:
+            proxies = proxy_config[base_url]
+        except KeyError:
+            proxies = proxy_config.get('default', {})
+
+        self._proxy_configs = ProxyConfig.from_dict(proxies)
+
         self._auth_configs = auth.load_config(
             config_dict=self._general_configs, credstore_env=credstore_env,
         )

docker/api/container.py

@@ -221,7 +221,8 @@ class ContainerApiMixin(object):
                          working_dir=None, domainname=None, host_config=None,
                          mac_address=None, labels=None, stop_signal=None,
                          networking_config=None, healthcheck=None,
-                         stop_timeout=None, runtime=None):
+                         stop_timeout=None, runtime=None,
+                         use_config_proxy=False):
         """
         Creates a container. Parameters are similar to those for the ``docker
         run`` command except it doesn't support the attach options (``-a``).
@@ -390,6 +391,10 @@ class ContainerApiMixin(object):
             runtime (str): Runtime to use with this container.
             healthcheck (dict): Specify a test to perform to check that the
                 container is healthy.
+            use_config_proxy (bool): If ``True``, and if the docker client
+                configuration file (``~/.docker/config.json`` by default)
+                contains a proxy configuration, the corresponding environment
+                variables will be set in the container being created.
 
         Returns:
             A dictionary with an image 'Id' key and a 'Warnings' key.
@@ -403,6 +408,14 @@ class ContainerApiMixin(object):
         if isinstance(volumes, six.string_types):
             volumes = [volumes, ]
 
+        if isinstance(environment, dict):
+            environment = utils.utils.format_environment(environment)
+
+        if use_config_proxy:
+            environment = self._proxy_configs.inject_proxy_environment(
+                environment
+            )
+
         config = self.create_container_config(
             image, command, hostname, user, detach, stdin_open, tty,
             ports, environment, volumes,

docker/api/exec_api.py

@@ -8,7 +8,8 @@ class ExecApiMixin(object):
     @utils.check_resource('container')
     def exec_create(self, container, cmd, stdout=True, stderr=True,
                     stdin=False, tty=False, privileged=False, user='',
-                    environment=None, workdir=None, detach_keys=None):
+                    environment=None, workdir=None, detach_keys=None,
+                    use_config_proxy=False):
         """
         Sets up an exec instance in a running container.
 
@@ -31,6 +32,10 @@ class ExecApiMixin(object):
                 or `ctrl-<value>` where `<value>` is one of:
                 `a-z`, `@`, `^`, `[`, `,` or `_`.
                 ~/.docker/config.json is used by default.
+            use_config_proxy (bool): If ``True``, and if the docker client
+                configuration file (``~/.docker/config.json`` by default)
+                contains a proxy configuration, the corresponding environment
+                variables will be set in the container being created.
 
         Returns:
             (dict): A dictionary with an exec ``Id`` key.
@@ -50,6 +55,9 @@ class ExecApiMixin(object):
 
         if isinstance(environment, dict):
             environment = utils.utils.format_environment(environment)
+        if use_config_proxy:
+            environment = \
+                self._proxy_configs.inject_proxy_environment(environment)
 
         data = {
             'Container': container,

docker/models/containers.py

@@ -144,7 +144,8 @@ class Container(Model):
 
     def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False,
                  privileged=False, user='', detach=False, stream=False,
-                 socket=False, environment=None, workdir=None, demux=False):
+                 socket=False, environment=None, workdir=None, demux=False,
+                 use_config_proxy=False):
         """
         Run a command inside this container. Similar to
         ``docker exec``.
@@ -167,6 +168,10 @@ class Container(Model):
                 ``{"PASSWORD": "xxx"}``.
             workdir (str): Path to working directory for this exec session
             demux (bool): Return stdout and stderr separately
+            use_config_proxy (bool): If ``True``, and if the docker client
+                configuration file (``~/.docker/config.json`` by default)
+                contains a proxy configuration, the corresponding environment
+                variables will be set in the command's environment.
 
         Returns:
             (ExecResult): A tuple of (exit_code, output)
@@ -185,7 +190,7 @@ class Container(Model):
         resp = self.client.api.exec_create(
             self.id, cmd, stdout=stdout, stderr=stderr, stdin=stdin, tty=tty,
             privileged=privileged, user=user, environment=environment,
-            workdir=workdir
+            workdir=workdir, use_config_proxy=use_config_proxy,
         )
         exec_output = self.client.api.exec_start(
             resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket,

docker/utils/proxy.py

+from .utils import format_environment
+
+
+class ProxyConfig(dict):
+    '''
+    Hold the client's proxy configuration
+    '''
+    @property
+    def http(self):
+        return self.get('http')
+
+    @property
+    def https(self):
+        return self.get('https')
+
+    @property
+    def ftp(self):
+        return self.get('ftp')
+
+    @property
+    def no_proxy(self):
+        return self.get('no_proxy')
+
+    @staticmethod
+    def from_dict(config):
+        '''
+        Instantiate a new ProxyConfig from a dictionary that represents a
+        client configuration, as described in `the documentation`_.
+
+        .. _the documentation:
+            https://docs.docker.com/network/proxy/#configure-the-docker-client
+        '''
+        return ProxyConfig(
+            http=config.get('httpProxy'),
+            https=config.get('httpsProxy'),
+            ftp=config.get('ftpProxy'),
+            no_proxy=config.get('noProxy'),
+        )
+
+    def get_environment(self):
+        '''
+        Return a dictionary representing the environment variables used to
+        set the proxy settings.
+        '''
+        env = {}
+        if self.http:
+            env['http_proxy'] = env['HTTP_PROXY'] = self.http
+        if self.https:
+            env['https_proxy'] = env['HTTPS_PROXY'] = self.https
+        if self.ftp:
+            env['ftp_proxy'] = env['FTP_PROXY'] = self.ftp
+        if self.no_proxy:
+            env['no_proxy'] = env['NO_PROXY'] = self.no_proxy
+        return env
+
+    def inject_proxy_environment(self, environment):
+        '''
+        Given a list of strings representing environment variables, prepend the
+        environment variables corresponding to the proxy settings.
+        '''
+        if not self:
+            return environment
+
+        proxy_env = format_environment(self.get_environment())
+        if not environment:
+            return proxy_env
+        # It is important to prepend our variables, because we want the
+        # variables defined in "environment" to take precedence.
+        return proxy_env + environment
+
+    def __str__(self):
+        return 'ProxyConfig(http={}, https={}, ftp={}, no_proxy={})'.format(
+            self.http, self.https, self.ftp, self.no_proxy)

tests/integration/api_build_test.py

@@ -4,6 +4,7 @@ import shutil
 import tempfile
 
 from docker import errors
+from docker.utils.proxy import ProxyConfig
 
 import pytest
 import six
@@ -13,6 +14,48 @@ from ..helpers import random_name, requires_api_version, requires_experimental
 
 
 class BuildTest(BaseAPIIntegrationTest):
+    def test_build_with_proxy(self):
+        self.client._proxy_configs = ProxyConfig(
+            ftp='a', http='b', https='c', no_proxy='d'
+        )
+
+        script = io.BytesIO('\n'.join([
+            'FROM busybox',
+            'RUN env | grep "FTP_PROXY=a"',
+            'RUN env | grep "ftp_proxy=a"',
+            'RUN env | grep "HTTP_PROXY=b"',
+            'RUN env | grep "http_proxy=b"',
+            'RUN env | grep "HTTPS_PROXY=c"',
+            'RUN env | grep "https_proxy=c"',
+            'RUN env | grep "NO_PROXY=d"',
+            'RUN env | grep "no_proxy=d"',
+        ]).encode('ascii'))
+
+        self.client.build(fileobj=script, decode=True)
+
+    def test_build_with_proxy_and_buildargs(self):
+        self.client._proxy_configs = ProxyConfig(
+            ftp='a', http='b', https='c', no_proxy='d'
+        )
+
+        script = io.BytesIO('\n'.join([
+            'FROM busybox',
+            'RUN env | grep "FTP_PROXY=XXX"',
+            'RUN env | grep "ftp_proxy=xxx"',
+            'RUN env | grep "HTTP_PROXY=b"',
+            'RUN env | grep "http_proxy=b"',
+            'RUN env | grep "HTTPS_PROXY=c"',
+            'RUN env | grep "https_proxy=c"',
+            'RUN env | grep "NO_PROXY=d"',
+            'RUN env | grep "no_proxy=d"',
+        ]).encode('ascii'))
+
+        self.client.build(
+            fileobj=script,
+            decode=True,
+            buildargs={'FTP_PROXY': 'XXX', 'ftp_proxy': 'xxx'}
+        )
+
     def test_build_streaming(self):
         script = io.BytesIO('\n'.join([
             'FROM busybox',

tests/integration/api_exec_test.py

 from docker.utils.socket import next_frame_header
 from docker.utils.socket import read_exactly
+from docker.utils.proxy import ProxyConfig
 
 from .base import BaseAPIIntegrationTest, BUSYBOX
 from ..helpers import (
@@ -8,6 +9,45 @@ from ..helpers import (
 
 
 class ExecTest(BaseAPIIntegrationTest):
+    def test_execute_command_with_proxy_env(self):
+        # Set a custom proxy config on the client
+        self.client._proxy_configs = ProxyConfig(
+            ftp='a', https='b', http='c', no_proxy='d'
+        )
+
+        container = self.client.create_container(
+            BUSYBOX, 'cat', detach=True, stdin_open=True,
+            use_config_proxy=True,
+        )
+        self.client.start(container)
+        self.tmp_containers.append(container)
+
+        cmd = 'sh -c "env | grep -i proxy"'
+
+        # First, just make sure the environment variables from the custom
+        # config are set
+
+        res = self.client.exec_create(container, cmd=cmd)
+        output = self.client.exec_start(res).decode('utf-8').split('\n')
+        expected = [
+            'ftp_proxy=a', 'https_proxy=b', 'http_proxy=c', 'no_proxy=d',
+            'FTP_PROXY=a', 'HTTPS_PROXY=b', 'HTTP_PROXY=c', 'NO_PROXY=d'
+        ]
+        for item in expected:
+            assert item in output
+
+        # Overwrite some variables with a custom environment
+        env = {'https_proxy': 'xxx', 'HTTPS_PROXY': 'XXX'}
+
+        res = self.client.exec_create(container, cmd=cmd, environment=env)
+        output = self.client.exec_start(res).decode('utf-8').split('\n')
+        expected = [
+            'ftp_proxy=a', 'https_proxy=xxx', 'http_proxy=c', 'no_proxy=d',
+            'FTP_PROXY=a', 'HTTPS_PROXY=XXX', 'HTTP_PROXY=c', 'NO_PROXY=d'
+        ]
+        for item in expected:
+            assert item in output
+
     def test_execute_command(self):
         container = self.client.create_container(BUSYBOX, 'cat',
                                                  detach=True, stdin_open=True)

tests/unit/api_test.py

@@ -106,8 +106,6 @@ class BaseAPIClientTest(unittest.TestCase):
         )
         self.patcher.start()
         self.client = APIClient()
-        # Force-clear authconfig to avoid tampering with the tests
-        self.client._cfg = {'Configs': {}}
 
     def tearDown(self):
         self.client.close()

tests/unit/models_containers_test.py

@@ -416,7 +416,7 @@ class ContainerTest(unittest.TestCase):
         client.api.exec_create.assert_called_with(
             FAKE_CONTAINER_ID, "echo hello world", stdout=True, stderr=True,
             stdin=False, tty=False, privileged=True, user='', environment=None,
-            workdir=None
+            workdir=None, use_config_proxy=False,
         )
         client.api.exec_start.assert_called_with(
             FAKE_EXEC_ID, detach=False, tty=False, stream=True, socket=False,
@@ -430,7 +430,7 @@ class ContainerTest(unittest.TestCase):
         client.api.exec_create.assert_called_with(
             FAKE_CONTAINER_ID, "docker ps", stdout=True, stderr=True,
             stdin=False, tty=False, privileged=True, user='', environment=None,
-            workdir=None
+            workdir=None, use_config_proxy=False,
         )
         client.api.exec_start.assert_called_with(
             FAKE_EXEC_ID, detach=False, tty=False, stream=False, socket=False,

tests/unit/utils_proxy_test.py

+# -*- coding: utf-8 -*-
+
+import unittest
+import six
+
+from docker.utils.proxy import ProxyConfig
+
+HTTP = 'http://test:80'
+HTTPS = 'https://test:443'
+FTP = 'ftp://user:password@host:23'
+NO_PROXY = 'localhost,.localdomain'
+CONFIG = ProxyConfig(http=HTTP, https=HTTPS, ftp=FTP, no_proxy=NO_PROXY)
+ENV = {
+    'http_proxy': HTTP,
+    'HTTP_PROXY': HTTP,
+    'https_proxy': HTTPS,
+    'HTTPS_PROXY': HTTPS,
+    'ftp_proxy': FTP,
+    'FTP_PROXY': FTP,
+    'no_proxy': NO_PROXY,
+    'NO_PROXY': NO_PROXY,
+}
+
+
+class ProxyConfigTest(unittest.TestCase):
+
+    def test_from_dict(self):
+        config = ProxyConfig.from_dict({
+            'httpProxy': HTTP,
+            'httpsProxy': HTTPS,
+            'ftpProxy': FTP,
+            'noProxy': NO_PROXY
+        })
+        self.assertEqual(CONFIG.http, config.http)
+        self.assertEqual(CONFIG.https, config.https)
+        self.assertEqual(CONFIG.ftp, config.ftp)
+        self.assertEqual(CONFIG.no_proxy, config.no_proxy)
+
+    def test_new(self):
+        config = ProxyConfig()
+        self.assertIsNone(config.http)
+        self.assertIsNone(config.https)
+        self.assertIsNone(config.ftp)
+        self.assertIsNone(config.no_proxy)
+
+        config = ProxyConfig(http='a', https='b', ftp='c', no_proxy='d')
+        self.assertEqual(config.http, 'a')
+        self.assertEqual(config.https, 'b')
+        self.assertEqual(config.ftp, 'c')
+        self.assertEqual(config.no_proxy, 'd')
+
+    def test_truthiness(self):
+        assert not ProxyConfig()
+        assert ProxyConfig(http='non-zero')
+        assert ProxyConfig(https='non-zero')
+        assert ProxyConfig(ftp='non-zero')
+        assert ProxyConfig(no_proxy='non-zero')
+
+    def test_environment(self):
+        self.assertDictEqual(CONFIG.get_environment(), ENV)
+        empty = ProxyConfig()
+        self.assertDictEqual(empty.get_environment(), {})
+
+    def test_inject_proxy_environment(self):
+        # Proxy config is non null, env is None.
+        self.assertSetEqual(
+            set(CONFIG.inject_proxy_environment(None)),
+            set(['{}={}'.format(k, v) for k, v in six.iteritems(ENV)]))
+
+        # Proxy config is null, env is None.
+        self.assertIsNone(ProxyConfig().inject_proxy_environment(None), None)
+
+        env = ['FOO=BAR', 'BAR=BAZ']
+
+        # Proxy config is non null, env is non null
+        actual = CONFIG.inject_proxy_environment(env)
+        expected = ['{}={}'.format(k, v) for k, v in six.iteritems(ENV)] + env
+        # It's important that the first 8 variables are the ones from the proxy
+        # config, and the last 2 are the ones from the input environment
+        self.assertSetEqual(set(actual[:8]), set(expected[:8]))
+        self.assertSetEqual(set(actual[-2:]), set(expected[-2:]))
+
+        # Proxy is null, and is non null
+        self.assertListEqual(ProxyConfig().inject_proxy_environment(env), env)