Kaydet (Commit) 6dd45d75 authored tarafından Joffrey F's avatar Joffrey F

Merge branch 'ssssam-sam/import-improvements'

...@@ -43,24 +43,29 @@ class Client(requests.Session): ...@@ -43,24 +43,29 @@ class Client(requests.Session):
def __init__(self, base_url=None, version=None, def __init__(self, base_url=None, version=None,
timeout=DEFAULT_TIMEOUT_SECONDS, tls=False): timeout=DEFAULT_TIMEOUT_SECONDS, tls=False):
super(Client, self).__init__() super(Client, self).__init__()
base_url = utils.parse_host(base_url)
if 'http+unix:///' in base_url:
base_url = base_url.replace('unix:/', 'unix:')
if tls and not base_url.startswith('https://'): if tls and not base_url.startswith('https://'):
raise errors.TLSParameterError( raise errors.TLSParameterError(
'If using TLS, the base_url argument must begin with ' 'If using TLS, the base_url argument must begin with '
'"https://".') '"https://".')
self.base_url = base_url self.base_url = base_url
self.timeout = timeout self.timeout = timeout
self._auth_configs = auth.load_config() self._auth_configs = auth.load_config()
# Use SSLAdapter for the ability to specify SSL version base_url = utils.parse_host(base_url)
if isinstance(tls, TLSConfig): if base_url.startswith('http+unix://'):
tls.configure_client(self) unix_socket_adapter = unixconn.UnixAdapter(base_url, timeout)
elif tls: self.mount('http+docker://', unix_socket_adapter)
self.mount('https://', ssladapter.SSLAdapter()) self.base_url = 'http+docker://localunixsocket'
else: else:
self.mount('http+unix://', unixconn.UnixAdapter(base_url, timeout)) # Use SSLAdapter for the ability to specify SSL version
if isinstance(tls, TLSConfig):
tls.configure_client(self)
elif tls:
self.mount('https://', ssladapter.SSLAdapter())
self.base_url = base_url
# version detection needs to be after unix adapter mounting # version detection needs to be after unix adapter mounting
if version is None: if version is None:
...@@ -576,33 +581,86 @@ class Client(requests.Session): ...@@ -576,33 +581,86 @@ class Client(requests.Session):
return res return res
def import_image(self, src=None, repository=None, tag=None, image=None): def import_image(self, src=None, repository=None, tag=None, image=None):
if src:
if isinstance(src, six.string_types):
try:
result = self.import_image_from_file(
src, repository=repository, tag=tag)
except IOError:
result = self.import_image_from_url(
src, repository=repository, tag=tag)
else:
result = self.import_image_from_data(
src, repository=repository, tag=tag)
elif image:
result = self.import_image_from_image(
image, repository=repository, tag=tag)
else:
raise Exception("Must specify a src or image")
return result
def import_image_from_data(self, data, repository=None, tag=None):
u = self._url("/images/create") u = self._url("/images/create")
params = { params = {
'fromSrc': '-',
'repo': repository, 'repo': repository,
'tag': tag 'tag': tag
} }
headers = {
'Content-Type': 'application/tar',
}
return self._result(
self._post(u, data=data, params=params, headers=headers))
if src: def import_image_from_file(self, filename, repository=None, tag=None):
try: u = self._url("/images/create")
# XXX: this is ways not optimal but the only way params = {
# for now to import tarballs through the API 'fromSrc': '-',
fic = open(src) 'repo': repository,
data = fic.read() 'tag': tag
fic.close() }
src = "-" headers = {
except IOError: 'Content-Type': 'application/tar',
# file does not exists or not a file (URL) }
data = None with open(filename, 'rb') as f:
if isinstance(src, six.string_types): return self._result(
params['fromSrc'] = src self._post(u, data=f, params=params, headers=headers,
return self._result(self._post(u, data=data, params=params)) timeout=None))
return self._result(self._post(u, data=src, params=params))
if image: def import_image_from_stream(self, stream, repository=None, tag=None):
params['fromImage'] = image u = self._url("/images/create")
return self._result(self._post(u, data=None, params=params)) params = {
'fromSrc': '-',
'repo': repository,
'tag': tag
}
headers = {
'Content-Type': 'application/tar',
'Transfer-Encoding': 'chunked',
}
return self._result(
self._post(u, data=stream, params=params, headers=headers))
def import_image_from_url(self, url, repository=None, tag=None):
u = self._url("/images/create")
params = {
'fromSrc': url,
'repo': repository,
'tag': tag
}
return self._result(
self._post(u, data=None, params=params))
raise Exception("Must specify a src or image") def import_image_from_image(self, image, repository=None, tag=None):
u = self._url("/images/create")
params = {
'fromImage': image,
'repo': repository,
'tag': tag
}
return self._result(
self._post(u, data=None, params=params))
def info(self): def info(self):
return self._result(self._get(self._url("/info")), return self._result(self._get(self._url("/info")),
......
...@@ -25,6 +25,8 @@ try: ...@@ -25,6 +25,8 @@ try:
except ImportError: except ImportError:
import urllib3 import urllib3
RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer
class UnixHTTPConnection(httplib.HTTPConnection, object): class UnixHTTPConnection(httplib.HTTPConnection, object):
def __init__(self, base_url, unix_socket, timeout=60): def __init__(self, base_url, unix_socket, timeout=60):
...@@ -36,17 +38,9 @@ class UnixHTTPConnection(httplib.HTTPConnection, object): ...@@ -36,17 +38,9 @@ class UnixHTTPConnection(httplib.HTTPConnection, object):
def connect(self): def connect(self):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(self.timeout) sock.settimeout(self.timeout)
sock.connect(self.base_url.replace("http+unix:/", "")) sock.connect(self.unix_socket)
self.sock = sock self.sock = sock
def _extract_path(self, url):
# remove the base_url entirely..
return url.replace(self.base_url, "")
def request(self, method, url, **kwargs):
url = self._extract_path(self.unix_socket)
super(UnixHTTPConnection, self).request(method, url, **kwargs)
class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool):
def __init__(self, base_url, socket_path, timeout=60): def __init__(self, base_url, socket_path, timeout=60):
...@@ -63,24 +57,26 @@ class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): ...@@ -63,24 +57,26 @@ class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool):
class UnixAdapter(requests.adapters.HTTPAdapter): class UnixAdapter(requests.adapters.HTTPAdapter):
def __init__(self, base_url, timeout=60): def __init__(self, socket_url, timeout=60):
RecentlyUsedContainer = urllib3._collections.RecentlyUsedContainer socket_path = socket_url.replace('http+unix://', '')
self.base_url = base_url if not socket_path.startswith('/'):
socket_path = '/' + socket_path
self.socket_path = socket_path
self.timeout = timeout self.timeout = timeout
self.pools = RecentlyUsedContainer(10, self.pools = RecentlyUsedContainer(10,
dispose_func=lambda p: p.close()) dispose_func=lambda p: p.close())
super(UnixAdapter, self).__init__() super(UnixAdapter, self).__init__()
def get_connection(self, socket_path, proxies=None): def get_connection(self, url, proxies=None):
with self.pools.lock: with self.pools.lock:
pool = self.pools.get(socket_path) pool = self.pools.get(url)
if pool: if pool:
return pool return pool
pool = UnixHTTPConnectionPool( pool = UnixHTTPConnectionPool(url,
self.base_url, socket_path, self.timeout self.socket_path,
) self.timeout)
self.pools[socket_path] = pool self.pools[url] = pool
return pool return pool
......
...@@ -345,20 +345,66 @@ layers) ...@@ -345,20 +345,66 @@ layers)
## import_image ## import_image
Identical to the `docker import` command. If `src` is a string or unicode Similar to the `docker import` command.
string, it will be treated as a URL to fetch the image from. To import an image
from the local machine, `src` needs to be a file-like object or bytes If `src` is a string or unicode string, it will first be treated as a path to
collection. To import from a tarball use your absolute path to your tarball. a tarball on the local system. If there is an error reading from that file,
To load arbitrary data as tarball use whatever you want as src and your src will be treated as a URL instead to fetch the image from. You can also pass
tarball content in data. an open file handle as 'src', in which case the data will be read from that
file.
If `src` is unset but `image` is set, the `image` paramater will be taken as
the name of an existing image to import from.
**Params**: **Params**:
* src (str or file): Path to tarfile or URL * src (str or file): Path to tarfile, URL, or file-like object
* repository (str): The repository to create * repository (str): The repository to create
* tag (str): The tag to apply * tag (str): The tag to apply
* image (str): Use another image like the `FROM` Dockerfile parameter * image (str): Use another image like the `FROM` Dockerfile parameter
## import_image_from_data
Like `.import_image()`, but allows importing in-memory bytes data.
**Params**:
* data (bytes collection): Bytes collection containing valid tar data
* repository (str): The repository to create
* tag (str): The tag to apply
## import_image_from_file
Like `.import_image()`, but only supports importing from a tar file on
disk. If the file doesn't exist it will raise `IOError`.
**Params**:
* filename (str): Full path to a tar file.
* repository (str): The repository to create
* tag (str): The tag to apply
## import_image_from_url
Like `.import_image()`, but only supports importing from a URL.
**Params**:
* url (str): A URL pointing to a tar file.
* repository (str): The repository to create
* tag (str): The tag to apply
## import_image_from_image
Like `.import_image()`, but only supports importing from another image,
like the `FROM` Dockerfile parameter.
**Params**:
* image (str): Image name to import from
* repository (str): The repository to create
* tag (str): The tag to apply
## info ## info
Display system-wide information. Identical to the `docker info` command. Display system-wide information. Identical to the `docker info` command.
......
...@@ -356,7 +356,7 @@ def get_fake_stats(): ...@@ -356,7 +356,7 @@ def get_fake_stats():
return status_code, response return status_code, response
# Maps real api url to fake response callback # Maps real api url to fake response callback
prefix = 'http+unix://var/run/docker.sock' prefix = 'http+docker://localunixsocket'
fake_responses = { fake_responses = {
'{0}/version'.format(prefix): '{0}/version'.format(prefix):
get_fake_raw_version, get_fake_raw_version,
......
...@@ -12,25 +12,31 @@ ...@@ -12,25 +12,31 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import time
import base64 import base64
import contextlib
import json import json
import io import io
import os import os
import shutil import shutil
import signal import signal
import socket
import tarfile
import tempfile import tempfile
import threading
import time
import unittest import unittest
import warnings import warnings
import docker import docker
import six import six
from six.moves import BaseHTTPServer
from six.moves import socketserver
from test import Cleanup from test import Cleanup
# FIXME: missing tests for # FIXME: missing tests for
# export; history; import_image; insert; port; push; tag; get; load; stats; # export; history; insert; port; push; tag; get; load; stats
DEFAULT_BASE_URL = os.environ.get('DOCKER_HOST') DEFAULT_BASE_URL = os.environ.get('DOCKER_HOST')
EXEC_DRIVER_IS_NATIVE = True EXEC_DRIVER_IS_NATIVE = True
...@@ -1229,6 +1235,157 @@ class TestRemoveImage(BaseTestCase): ...@@ -1229,6 +1235,157 @@ class TestRemoveImage(BaseTestCase):
res = [x for x in images if x['Id'].startswith(img_id)] res = [x for x in images if x['Id'].startswith(img_id)]
self.assertEqual(len(res), 0) self.assertEqual(len(res), 0)
##################
# IMPORT TESTS #
##################
class ImportTestCase(BaseTestCase):
'''Base class for `docker import` test cases.'''
# Use a large file size to increase the chance of triggering any
# MemoryError exceptions we might hit.
TAR_SIZE = 512 * 1024 * 1024
def write_dummy_tar_content(self, n_bytes, tar_fd):
def extend_file(f, n_bytes):
f.seek(n_bytes - 1)
f.write(bytearray([65]))
f.seek(0)
tar = tarfile.TarFile(fileobj=tar_fd, mode='w')
with tempfile.NamedTemporaryFile() as f:
extend_file(f, n_bytes)
tarinfo = tar.gettarinfo(name=f.name, arcname='testdata')
tar.addfile(tarinfo, fileobj=f)
tar.close()
@contextlib.contextmanager
def dummy_tar_stream(self, n_bytes):
'''Yields a stream that is valid tar data of size n_bytes.'''
with tempfile.NamedTemporaryFile() as tar_file:
self.write_dummy_tar_content(n_bytes, tar_file)
tar_file.seek(0)
yield tar_file
@contextlib.contextmanager
def dummy_tar_file(self, n_bytes):
'''Yields the name of a valid tar file of size n_bytes.'''
with tempfile.NamedTemporaryFile() as tar_file:
self.write_dummy_tar_content(n_bytes, tar_file)
tar_file.seek(0)
yield tar_file.name
class TestImportFromBytes(ImportTestCase):
'''Tests importing an image from in-memory byte data.'''
def runTest(self):
with self.dummy_tar_stream(n_bytes=500) as f:
content = f.read()
# The generic import_image() function cannot import in-memory bytes
# data that happens to be represented as a string type, because
# import_image() will try to use it as a filename and usually then
# trigger an exception. So we test the import_image_from_data()
# function instead.
statuses = self.client.import_image_from_data(
content, repository='test/import-from-bytes')
result_text = statuses.splitlines()[-1]
result = json.loads(result_text)
self.assertNotIn('error', result)
img_id = result['status']
self.tmp_imgs.append(img_id)
class TestImportFromFile(ImportTestCase):
'''Tests importing an image from a tar file on disk.'''
def runTest(self):
with self.dummy_tar_file(n_bytes=self.TAR_SIZE) as tar_filename:
# statuses = self.client.import_image(
# src=tar_filename, repository='test/import-from-file')
statuses = self.client.import_image_from_file(
tar_filename, repository='test/import-from-file')
result_text = statuses.splitlines()[-1]
result = json.loads(result_text)
self.assertNotIn('error', result)
self.assertIn('status', result)
img_id = result['status']
self.tmp_imgs.append(img_id)
class TestImportFromStream(ImportTestCase):
'''Tests importing an image from a stream containing tar data.'''
def runTest(self):
with self.dummy_tar_stream(n_bytes=self.TAR_SIZE) as tar_stream:
statuses = self.client.import_image(
src=tar_stream, repository='test/import-from-stream')
# statuses = self.client.import_image_from_stream(
# tar_stream, repository='test/import-from-stream')
result_text = statuses.splitlines()[-1]
result = json.loads(result_text)
self.assertNotIn('error', result)
self.assertIn('status', result)
img_id = result['status']
self.tmp_imgs.append(img_id)
class TestImportFromURL(ImportTestCase):
'''Tests downloading an image over HTTP.'''
@contextlib.contextmanager
def temporary_http_file_server(self, stream):
'''Serve data from an IO stream over HTTP.'''
class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header('Content-Type', 'application/x-tar')
self.end_headers()
shutil.copyfileobj(stream, self.wfile)
server = socketserver.TCPServer(('', 0), Handler)
thread = threading.Thread(target=server.serve_forever)
thread.setDaemon(True)
thread.start()
yield 'http://%s:%s' % (socket.gethostname(), server.server_address[1])
server.shutdown()
def runTest(self):
# The crappy test HTTP server doesn't handle large files well, so use
# a small file.
TAR_SIZE = 10240
with self.dummy_tar_stream(n_bytes=TAR_SIZE) as tar_data:
with self.temporary_http_file_server(tar_data) as url:
statuses = self.client.import_image(
src=url, repository='test/import-from-url')
result_text = statuses.splitlines()[-1]
result = json.loads(result_text)
self.assertNotIn('error', result)
self.assertIn('status', result)
img_id = result['status']
self.tmp_imgs.append(img_id)
################# #################
# BUILDER TESTS # # BUILDER TESTS #
################# #################
......
...@@ -71,7 +71,7 @@ def fake_resp(url, data=None, **kwargs): ...@@ -71,7 +71,7 @@ def fake_resp(url, data=None, **kwargs):
fake_request = mock.Mock(side_effect=fake_resp) fake_request = mock.Mock(side_effect=fake_resp)
url_prefix = 'http+unix://var/run/docker.sock/v{0}/'.format( url_prefix = 'http+docker://localunixsocket/v{0}/'.format(
docker.client.DEFAULT_DOCKER_API_VERSION) docker.client.DEFAULT_DOCKER_API_VERSION)
...@@ -1412,20 +1412,24 @@ class DockerClientTest(Cleanup, unittest.TestCase): ...@@ -1412,20 +1412,24 @@ class DockerClientTest(Cleanup, unittest.TestCase):
timeout=None timeout=None
) )
def _socket_path_for_client_session(self, client):
socket_adapter = client.get_adapter('http+docker://')
return socket_adapter.socket_path
def test_url_compatibility_unix(self): def test_url_compatibility_unix(self):
c = docker.Client(base_url="unix://socket") c = docker.Client(base_url="unix://socket")
assert c.base_url == "http+unix://socket" assert self._socket_path_for_client_session(c) == '/socket'
def test_url_compatibility_unix_triple_slash(self): def test_url_compatibility_unix_triple_slash(self):
c = docker.Client(base_url="unix:///socket") c = docker.Client(base_url="unix:///socket")
assert c.base_url == "http+unix://socket" assert self._socket_path_for_client_session(c) == '/socket'
def test_url_compatibility_http_unix_triple_slash(self): def test_url_compatibility_http_unix_triple_slash(self):
c = docker.Client(base_url="http+unix:///socket") c = docker.Client(base_url="http+unix:///socket")
assert c.base_url == "http+unix://socket" assert self._socket_path_for_client_session(c) == '/socket'
def test_url_compatibility_http(self): def test_url_compatibility_http(self):
c = docker.Client(base_url="http://hostname:1234") c = docker.Client(base_url="http://hostname:1234")
...@@ -1853,12 +1857,11 @@ class DockerClientTest(Cleanup, unittest.TestCase): ...@@ -1853,12 +1857,11 @@ class DockerClientTest(Cleanup, unittest.TestCase):
timeout=docker.client.DEFAULT_TIMEOUT_SECONDS timeout=docker.client.DEFAULT_TIMEOUT_SECONDS
) )
def test_import_image_from_file(self): def test_import_image_from_bytes(self):
buf = tempfile.NamedTemporaryFile(delete=False) stream = (i for i in range(0, 100))
try: try:
# pretent the buffer is a file
self.client.import_image( self.client.import_image(
buf.name, stream,
repository=fake_api.FAKE_REPO_NAME, repository=fake_api.FAKE_REPO_NAME,
tag=fake_api.FAKE_TAG_NAME tag=fake_api.FAKE_TAG_NAME
) )
...@@ -1870,13 +1873,14 @@ class DockerClientTest(Cleanup, unittest.TestCase): ...@@ -1870,13 +1873,14 @@ class DockerClientTest(Cleanup, unittest.TestCase):
params={ params={
'repo': fake_api.FAKE_REPO_NAME, 'repo': fake_api.FAKE_REPO_NAME,
'tag': fake_api.FAKE_TAG_NAME, 'tag': fake_api.FAKE_TAG_NAME,
'fromSrc': '-' 'fromSrc': '-',
},
headers={
'Content-Type': 'application/tar',
}, },
data='', data=stream,
timeout=docker.client.DEFAULT_TIMEOUT_SECONDS timeout=docker.client.DEFAULT_TIMEOUT_SECONDS
) )
buf.close()
os.remove(buf.name)
def test_import_image_from_image(self): def test_import_image_from_image(self):
try: try:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment