Merge pull request #2222 from docker/vendor_dockerpycreds

Make dockerpycreds part of the SDK under docker.credentials

Jenkinsfile

@@ -24,8 +24,8 @@ def buildImages = { ->
       imageNamePy2 = "${imageNameBase}:py2-${gitCommit()}"
       imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}"
 
-      buildImage(imageNamePy2, ".", "py2.7")
-      buildImage(imageNamePy3, "-f Dockerfile-py3 .", "py3.6")
+      buildImage(imageNamePy2, "-f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 .", "py2.7")
+      buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.6 .", "py3.6")
     }
   }
 }

MANIFEST.in

@@ -6,3 +6,4 @@ include LICENSE
 recursive-include tests *.py
 recursive-include tests/unit/testdata *
 recursive-include tests/integration/testdata *
+recursive-include tests/gpg-keys *

Makefile

@@ -8,11 +8,11 @@ clean:
 
 .PHONY: build
 build:
-	docker build -t docker-sdk-python .
+	docker build -t docker-sdk-python -f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 .
 
 .PHONY: build-py3
 build-py3:
-	docker build -t docker-sdk-python3 -f Dockerfile-py3 .
+	docker build -t docker-sdk-python3 -f tests/Dockerfile .
 
 .PHONY: build-docs
 build-docs:
@@ -39,7 +39,7 @@ integration-test: build
 
 .PHONY: integration-test-py3
 integration-test-py3: build-py3
-	docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test tests/integration/${file}
+	docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -v tests/integration/${file}
 
 TEST_API_VERSION ?= 1.35
 TEST_ENGINE_VERSION ?= 17.12.0-ce

docker/auth.py

@@ -2,9 +2,9 @@ import base64
 import json
 import logging
 
-import dockerpycreds
 import six
 
+from . import credentials
 from . import errors
 from .utils import config
 
@@ -273,17 +273,17 @@ class AuthConfig(dict):
                     'Password': data['Secret'],
                 })
             return res
-        except dockerpycreds.CredentialsNotFound:
+        except credentials.CredentialsNotFound:
             log.debug('No entry found')
             return None
-        except dockerpycreds.StoreError as e:
+        except credentials.StoreError as e:
             raise errors.DockerException(
                 'Credentials store error: {0}'.format(repr(e))
             )
 
     def _get_store_instance(self, name):
         if name not in self._stores:
-            self._stores[name] = dockerpycreds.Store(
+            self._stores[name] = credentials.Store(
                 name, environment=self._credstore_env
             )
         return self._stores[name]

docker/credentials/__init__.py

+# flake8: noqa
+from .store import Store
+from .errors import StoreError, CredentialsNotFound
+from .constants import *

docker/credentials/constants.py

+PROGRAM_PREFIX = 'docker-credential-'
+DEFAULT_LINUX_STORE = 'secretservice'
+DEFAULT_OSX_STORE = 'osxkeychain'
+DEFAULT_WIN32_STORE = 'wincred'

docker/credentials/errors.py

+class StoreError(RuntimeError):
+    pass
+
+
+class CredentialsNotFound(StoreError):
+    pass
+
+
+class InitializationError(StoreError):
+    pass
+
+
+def process_store_error(cpe, program):
+    message = cpe.output.decode('utf-8')
+    if 'credentials not found in native keychain' in message:
+        return CredentialsNotFound(
+            'No matching credentials in {}'.format(
+                program
+            )
+        )
+    return StoreError(
+        'Credentials store {} exited with "{}".'.format(
+            program, cpe.output.decode('utf-8').strip()
+        )
+    )

docker/credentials/store.py

+import json
+import os
+import subprocess
+
+import six
+
+from . import constants
+from . import errors
+from .utils import create_environment_dict
+from .utils import find_executable
+
+
+class Store(object):
+    def __init__(self, program, environment=None):
+        """ Create a store object that acts as an interface to
+            perform the basic operations for storing, retrieving
+            and erasing credentials using `program`.
+        """
+        self.program = constants.PROGRAM_PREFIX + program
+        self.exe = find_executable(self.program)
+        self.environment = environment
+        if self.exe is None:
+            raise errors.InitializationError(
+                '{} not installed or not available in PATH'.format(
+                    self.program
+                )
+            )
+
+    def get(self, server):
+        """ Retrieve credentials for `server`. If no credentials are found,
+            a `StoreError` will be raised.
+        """
+        if not isinstance(server, six.binary_type):
+            server = server.encode('utf-8')
+        data = self._execute('get', server)
+        result = json.loads(data.decode('utf-8'))
+
+        # docker-credential-pass will return an object for inexistent servers
+        # whereas other helpers will exit with returncode != 0. For
+        # consistency, if no significant data is returned,
+        # raise CredentialsNotFound
+        if result['Username'] == '' and result['Secret'] == '':
+            raise errors.CredentialsNotFound(
+                'No matching credentials in {}'.format(self.program)
+            )
+
+        return result
+
+    def store(self, server, username, secret):
+        """ Store credentials for `server`. Raises a `StoreError` if an error
+            occurs.
+        """
+        data_input = json.dumps({
+            'ServerURL': server,
+            'Username': username,
+            'Secret': secret
+        }).encode('utf-8')
+        return self._execute('store', data_input)
+
+    def erase(self, server):
+        """ Erase credentials for `server`. Raises a `StoreError` if an error
+            occurs.
+        """
+        if not isinstance(server, six.binary_type):
+            server = server.encode('utf-8')
+        self._execute('erase', server)
+
+    def list(self):
+        """ List stored credentials. Requires v0.4.0+ of the helper.
+        """
+        data = self._execute('list', None)
+        return json.loads(data.decode('utf-8'))
+
+    def _execute(self, subcmd, data_input):
+        output = None
+        env = create_environment_dict(self.environment)
+        try:
+            if six.PY3:
+                output = subprocess.check_output(
+                    [self.exe, subcmd], input=data_input, env=env,
+                )
+            else:
+                process = subprocess.Popen(
+                    [self.exe, subcmd], stdin=subprocess.PIPE,
+                    stdout=subprocess.PIPE, env=env,
+                )
+                output, err = process.communicate(data_input)
+                if process.returncode != 0:
+                    raise subprocess.CalledProcessError(
+                        returncode=process.returncode, cmd='', output=output
+                    )
+        except subprocess.CalledProcessError as e:
+            raise errors.process_store_error(e, self.program)
+        except OSError as e:
+            if e.errno == os.errno.ENOENT:
+                raise errors.StoreError(
+                    '{} not installed or not available in PATH'.format(
+                        self.program
+                    )
+                )
+            else:
+                raise errors.StoreError(
+                    'Unexpected OS error "{}", errno={}'.format(
+                        e.strerror, e.errno
+                    )
+                )
+        return output

docker/credentials/utils.py

+import distutils.spawn
+import os
+import sys
+
+
+def find_executable(executable, path=None):
+    """
+    As distutils.spawn.find_executable, but on Windows, look up
+    every extension declared in PATHEXT instead of just `.exe`
+    """
+    if sys.platform != 'win32':
+        return distutils.spawn.find_executable(executable, path)
+
+    if path is None:
+        path = os.environ['PATH']
+
+    paths = path.split(os.pathsep)
+    extensions = os.environ.get('PATHEXT', '.exe').split(os.pathsep)
+    base, ext = os.path.splitext(executable)
+
+    if not os.path.isfile(executable):
+        for p in paths:
+            for ext in extensions:
+                f = os.path.join(p, base + ext)
+                if os.path.isfile(f):
+                    return f
+        return None
+    else:
+        return executable
+
+
+def create_environment_dict(overrides):
+    """
+    Create and return a copy of os.environ with the specified overrides
+    """
+    result = os.environ.copy()
+    result.update(overrides or {})
+    return result

requirements.txt

@@ -3,7 +3,6 @@ asn1crypto==0.22.0
 backports.ssl-match-hostname==3.5.0.1
 cffi==1.10.0
 cryptography==2.3
-docker-pycreds==0.4.0
 enum34==1.1.6
 idna==2.5
 ipaddress==1.0.18

setup.py

@@ -12,7 +12,6 @@ SOURCE_DIR = os.path.join(ROOT_DIR)
 requirements = [
     'six >= 1.4.0',
     'websocket-client >= 0.32.0',
-    'docker-pycreds >= 0.4.0',
     'requests >= 2.14.2, != 2.18.0',
 ]
 

tests/Dockerfile

+ARG PYTHON_VERSION=3.6
+FROM python:$PYTHON_VERSION-jessie
+RUN apt-get update && apt-get -y install \
+    gnupg2 \
+    pass \
+    curl
+
+COPY ./tests/gpg-keys /gpg-keys
+RUN gpg2 --import gpg-keys/secret
+RUN gpg2 --import-ownertrust gpg-keys/ownertrust
+RUN yes | pass init $(gpg2 --no-auto-check-trustdb --list-secret-keys | grep ^sec | cut -d/ -f2 | cut -d" " -f1)
+RUN gpg2 --check-trustdb
+ARG CREDSTORE_VERSION=v0.6.0
+RUN curl -sSL -o /opt/docker-credential-pass.tar.gz \
+    https://github.com/docker/docker-credential-helpers/releases/download/$CREDSTORE_VERSION/docker-credential-pass-$CREDSTORE_VERSION-amd64.tar.gz && \
+    tar -xf /opt/docker-credential-pass.tar.gz -O > /usr/local/bin/docker-credential-pass && \
+    rm -rf /opt/docker-credential-pass.tar.gz && \
+    chmod +x /usr/local/bin/docker-credential-pass
+
+WORKDIR /src
+COPY requirements.txt /src/requirements.txt
+RUN pip install -r requirements.txt
+
+COPY test-requirements.txt /src/test-requirements.txt
+RUN pip install -r test-requirements.txt
+
+COPY . /src
+RUN pip install .

tests/gpg-keys/ownertrust

+# List of assigned trustvalues, created Wed 25 Apr 2018 01:28:17 PM PDT
+# (Use "gpg --import-ownertrust" to restore them)
+9781B87DAB042E6FD51388A5464ED987A7B21401:6:

tests/integration/credentials/create_gpg_key.sh

+#!/usr/bin/sh
+haveged
+gpg --batch --gen-key <<-EOF
+%echo Generating a standard key
+Key-Type: DSA
+Key-Length: 1024
+Subkey-Type: ELG-E
+Subkey-Length: 1024
+Name-Real: Sakuya Izayoi
+Name-Email: sakuya@gensokyo.jp
+Expire-Date: 0
+EOF
\ No newline at end of file

tests/integration/credentials/store_test.py

+import os
+import random
+import sys
+
+import pytest
+import six
+from distutils.spawn import find_executable
+
+from docker.credentials import (
+    CredentialsNotFound, Store, StoreError, DEFAULT_LINUX_STORE,
+    DEFAULT_OSX_STORE
+)
+
+
+class TestStore(object):
+    def teardown_method(self):
+        for server in self.tmp_keys:
+            try:
+                self.store.erase(server)
+            except StoreError:
+                pass
+
+    def setup_method(self):
+        self.tmp_keys = []
+        if sys.platform.startswith('linux'):
+            if find_executable('docker-credential-' + DEFAULT_LINUX_STORE):
+                self.store = Store(DEFAULT_LINUX_STORE)
+            elif find_executable('docker-credential-pass'):
+                self.store = Store('pass')
+            else:
+                raise Exception('No supported docker-credential store in PATH')
+        elif sys.platform.startswith('darwin'):
+            self.store = Store(DEFAULT_OSX_STORE)
+
+    def get_random_servername(self):
+        res = 'pycreds_test_{:x}'.format(random.getrandbits(32))
+        self.tmp_keys.append(res)
+        return res
+
+    def test_store_and_get(self):
+        key = self.get_random_servername()
+        self.store.store(server=key, username='user', secret='pass')
+        data = self.store.get(key)
+        assert data == {
+            'ServerURL': key,
+            'Username': 'user',
+            'Secret': 'pass'
+        }
+
+    def test_get_nonexistent(self):
+        key = self.get_random_servername()
+        with pytest.raises(CredentialsNotFound):
+            self.store.get(key)
+
+    def test_store_and_erase(self):
+        key = self.get_random_servername()
+        self.store.store(server=key, username='user', secret='pass')
+        self.store.erase(key)
+        with pytest.raises(CredentialsNotFound):
+            self.store.get(key)
+
+    def test_unicode_strings(self):
+        key = self.get_random_servername()
+        key = six.u(key)
+        self.store.store(server=key, username='user', secret='pass')
+        data = self.store.get(key)
+        assert data
+        self.store.erase(key)
+        with pytest.raises(CredentialsNotFound):
+            self.store.get(key)
+
+    def test_list(self):
+        names = (self.get_random_servername(), self.get_random_servername())
+        self.store.store(names[0], username='sakuya', secret='izayoi')
+        self.store.store(names[1], username='reimu', secret='hakurei')
+        data = self.store.list()
+        assert names[0] in data
+        assert data[names[0]] == 'sakuya'
+        assert names[1] in data
+        assert data[names[1]] == 'reimu'
+
+    def test_execute_with_env_override(self):
+        self.store.exe = 'env'
+        self.store.environment = {'FOO': 'bar'}
+        data = self.store._execute('--null', '')
+        assert b'\0FOO=bar\0' in data
+        assert 'FOO' not in os.environ

tests/integration/credentials/utils_test.py

+import os
+
+from docker.credentials.utils import create_environment_dict
+
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+
+@mock.patch.dict(os.environ)
+def test_create_environment_dict():
+    base = {'FOO': 'bar', 'BAZ': 'foobar'}
+    os.environ = base
+    assert create_environment_dict({'FOO': 'baz'}) == {
+        'FOO': 'baz', 'BAZ': 'foobar',
+    }
+    assert create_environment_dict({'HELLO': 'world'}) == {
+        'FOO': 'bar', 'BAZ': 'foobar', 'HELLO': 'world',
+    }
+
+    assert os.environ == base

tests/unit/auth_test.py

@@ -9,8 +9,7 @@ import shutil
 import tempfile
 import unittest
 
-from docker import auth, errors
-import dockerpycreds
+from docker import auth, credentials, errors
 import pytest
 
 try:
@@ -661,7 +660,7 @@ class CredstoreTest(unittest.TestCase):
         }
 
 
-class InMemoryStore(dockerpycreds.Store):
+class InMemoryStore(credentials.Store):
     def __init__(self, *args, **kwargs):
         self.__store = {}
 
@@ -669,7 +668,7 @@ class InMemoryStore(dockerpycreds.Store):
         try:
             return self.__store[server]
         except KeyError:
-            raise dockerpycreds.errors.CredentialsNotFound()
+            raise credentials.errors.CredentialsNotFound()
 
     def store(self, server, username, secret):
         self.__store[server] = {