add support for external tarballs to gbuild

Change-Id: Ic6645e9943b2445ebb37bb99114f777527c69af9

solenv/gbuild/Executable.mk

@@ -127,6 +127,7 @@ $(eval $(foreach method,\
 	add_package_headers \
 	use_package \
 	use_packages \
+	use_unpacked \
 	add_sdi_headers \
 	add_nativeres \
 	set_warnings_not_errors \

solenv/gbuild/Library.mk

@@ -212,6 +212,7 @@ $(eval $(foreach method,\
 	add_external_headers \
 	use_package \
 	use_packages \
+	use_unpacked \
 	add_package_headers \
 	add_sdi_headers \
 	export_objects_list \

solenv/gbuild/LinkTarget.mk

@@ -1094,6 +1094,12 @@ define gb_LinkTarget_use_packages
 $(foreach package,$(2),$(call gb_LinkTarget_use_package,$(1),$(package)))
 endef
 
+# Use sources from unpacked tarball of an external project
+define gb_LinkTarget_use_unpacked
+$(call gb_LinkTarget_get_external_headers_target,$(1)) :| $(call gb_UnpackedTarball_get_target,$(2))
+
+endef
+
 # this forwards to functions that must be defined in RepositoryExternal.mk.
 # $(eval $(call gb_LinkTarget_use_external,library,external))
 define gb_LinkTarget_use_external

solenv/gbuild/Package.mk

@@ -114,4 +114,10 @@ $(foreach file,$(3),$(call gb_Package_add_file,$(1),$(2)/$(file),$(file)))
 
 endef
 
+# Package files from unpacked tarball of an external project
+define gb_Package_use_unpacked
+$(call gb_Package_get_preparation_target,$(1)) :| $(call gb_UnpackedTarball_get_target,$(2))
+
+endef
+
 # vim: set noet sw=4:

solenv/gbuild/StaticLibrary.mk

@@ -124,6 +124,7 @@ $(eval $(foreach method,\
 	add_package_headers \
 	use_package \
 	use_packages \
+	use_unpacked \
 	add_sdi_headers \
 	set_warnings_not_errors \
 ,\

solenv/gbuild/TargetLocations.mk

@@ -153,6 +153,10 @@ gb_UnoApiHeadersTarget_get_bootstrap_target = $(WORKDIR)/UnoApiHeadersTarget/$(1
 gb_UnoApiHeadersTarget_get_comprehensive_target = $(WORKDIR)/UnoApiHeadersTarget/$(1)/comprehensive.done
 gb_UnoApiHeadersTarget_get_target = $(WORKDIR)/UnoApiHeadersTarget/$(1)/normal.done
 gb_UnoApiPartTarget_get_target = $(WORKDIR)/UnoApiPartTarget/$(1)
+gb_UnpackedTarball_get_dir = $(WORKDIR)/UnpackedTarball/$(1)
+gb_UnpackedTarball_get_target = $(WORKDIR)/UnpackedTarball/$(1).done
+gb_UnpackedTarball_get_preparation_target = $(WORKDIR)/UnpackedTarball/$(1).prepare
+gb_UnpackedTarget_get_target = $(WORKDIR)/UnpackedTarget/$(1)
 gb_WinResTarget_get_target = $(WORKDIR)/WinResTarget/$(1)$(gb_WinResTarget_POSTFIX)
 # workdir targets: $(1) is prefix/path
 gb_Configuration_get_target = $(WORKDIR)/Configuration/$(1).done
@@ -223,6 +227,8 @@ $(eval $(call gb_Helper_make_clean_targets,\
 	ExternalLib \
 	UnoApiHeadersTarget \
 	UnoApiTarget \
+	UnpackedTarball \
+	UnpackedTarget \
 	WinResTarget \
 	YaccTarget \
 	LexTarget \

solenv/gbuild/UnpackedTarball.mk

+# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+# UnpackedTarget class
+
+# Handles unpacking of a tarball
+
+# platform
+#  gb_UnpackedTarget_TARFILE_LOCATION
+
+gb_UnpackedTarget_STRIP_COMPONENTS_TAR_DEFAULT := 1
+gb_UnpackedTarget_STRIP_COMPONENTS_ZIP_DEFAULT := 0
+
+# gb_UnpackedTarget__get_strip_components target strip-components?
+define gb_UnpackedTarget__get_strip_components
+$(strip $(if $(2),\
+	$(2),\
+	$(if $(filter zip,$(suffix $(1))),\
+		$(gb_UnpackedTarget_STRIP_COMPONENTS_ZIP_DEFAULT),\
+		$(gb_UnpackedTarget_STRIP_COMPONENTS_TAR_DEFAULT) \
+	) \
+))
+endef
+
+define gb_UnpackedTarget__command_untar
+$(GNUTAR) \
+	-x \
+	$(3) \
+	-C $(UNPACKED_DIR) \
+	$(STRIP_COMPONENTS)=$(UNPACKED_STRIP_COMPONENTS) \
+	-f $(UNPACKED_TARBALL)
+endef
+
+define gb_UnpackedTarget__command_unzip
+unzip \
+	-qq \
+	-DD  \
+	-d $(UNPACKED_DIR) $(UNPACKED_TARBALL) && \
+$(if $(filter-out 0,$(UNPACKED_STRIP_COMPONENTS)),\
+	UNZIP_DIR=`ls $(UNPACKED_DIR)` && \
+	mv $(UNPACKED_DIR)/$$UNZIP_DIR/* $(UNPACKED_DIR) && \
+	rmdir $(UNPACKED_DIR)/$$UNZIP_DIR \
+)
+endef
+
+define gb_UnpackedTarget__command
+$(call gb_Output_announce,$(notdir $(2)),$(true),UPK,1)
+$(call gb_Helper_abbreviate_dirs,\
+	$(if $(wildcard $(UNPACKED_DIR)),rm -rf $(UNPACKED_DIR) &&) \
+	mkdir -p $(UNPACKED_DIR) && \
+	$(call gb_UnpackedTarget__command_$(1),$(2),$(3),$(4)) && \
+	touch $(2) \
+)
+endef
+
+$(dir $(call gb_UnpackedTarget_get_target,%)).dir :
+	$(if $(wildcard $(dir $@)),,mkdir -p $(dir $@))
+
+$(call gb_UnpackedTarget_get_target,%).tar.bz2 :
+	$(call gb_UnpackedTarget__command,untar,$@,$*,-j)
+
+$(call gb_UnpackedTarget_get_target,%).tar.gz :
+	$(call gb_UnpackedTarget__command,untar,$@,$*,-z)
+
+$(call gb_UnpackedTarget_get_target,%).tgz :
+	$(call gb_UnpackedTarget__command,untar,$@,$*,-z)
+
+$(call gb_UnpackedTarget_get_target,%).zip :
+	$(call gb_UnpackedTarget__command,unzip,$@,$*)
+
+.PHONY : $(call gb_UnpackedTarget_get_clean_target,%)
+$(call gb_UnpackedTarget_get_clean_target,%) :
+	$(call gb_Output_announce,$*,$(false),UPK,1)
+	$(call gb_Helper_abbreviate_dirs,\
+		rm -f $(call gb_UnpackedTarget_get_target,$*) \
+	)
+
+# gb_UnpackedTarget_UnpackedTarget target outdir strip-components?
+define gb_UnpackedTarget_UnpackedTarget
+$(call gb_UnpackedTarget_get_target,$(1)) : UNPACKED_DIR := $(2)
+$(call gb_UnpackedTarget_get_target,$(1)) : UNPACKED_TARBALL := $(gb_UnpackedTarget_TARFILE_LOCATION)/$(1)
+$(call gb_UnpackedTarget_get_target,$(1)) : UNPACKED_STRIP_COMPONENTS := $(call gb_UnpackedTarget__get_strip_components,$(1),$(3))
+
+$(call gb_UnpackedTarget_get_target,$(1)) : $(gb_UnpackedTarget_TARFILE_LOCATION)/$(1)
+$(call gb_UnpackedTarget_get_target,$(1)) :| $(dir $(call gb_UnpackedTarget_get_target,$(1))).dir
+
+endef
+
+# UnpackedTarball class
+
+# Handles unpacking and patching of an external project
+#
+# The unpacked dir is recreated every time one of the patches, copied
+# files or the makefile changes.
+
+# This is what dmake patches use. Once all external modules are
+# converted, it is better to be changed to 1.
+gb_UnpackedTarball_PATCHLEVEL_DEFAULT := 3
+
+define gb_UnpackedTarball__copy_files_impl
+$(if $(1),\
+	&& cp $(firstword $(1)) $(firstword $(2)) \
+	$(call gb_UnpackedTarball__copy_files_impl,$(wordlist 2,$(words $(1)),$(1)),$(wordlist 2,$(words $(2)),$(2))) \
+)
+endef
+
+# Drop leading &&
+define gb_UnpackedTarball__copy_files_fix
+$(wordlist 2,$(words $(1)),$(1))
+endef
+
+define gb_UnpackedTarball__copy_files
+$(call gb_UnpackedTarball__copy_files_fix,$(call gb_UnpackedTarball__copy_files_impl,$(1),$(2)))
+endef
+
+define gb_UnpackedTarball__command
+$(call gb_Output_announce,$(2),$(true),PAT,2)
+$(call gb_Helper_abbreviate_dirs,\
+	cd $(3) && \
+	$(if $(UNPACKED_PATCHES),\
+		for p in $(UNPACKED_PATCHES); do \
+			$(GNUPATCH) -s -p$(UNPACKED_PATCHLEVEL) < "$$p" || exit 1;\
+		done && \
+	) \
+	$(if $(UNPACKED_CXX_SUFFIX),\
+		for c in `find $(3) -type f -name "*.$(UNPACKED_CXX_SUFFIX)"`; do \
+			mv "$$c" "$${c%.$(UNPACKED_CXX_SUFFIX)}.cxx" || exit 1;\
+		done && \
+	) \
+	$(if $(UNPACKED_FILES),\
+		mkdir -p $(sort $(dir $(UNPACKED_DESTFILES))) && \
+		$(call gb_UnpackedTarball__copy_files,$(UNPACKED_FILES),$(UNPACKED_DESTFILES)) && \
+	) \
+	$(if $(UNPACKED_POST_ACTION),\
+		$(UNPACKED_POST_ACTION) && \
+	) \
+	touch $(1) \
+)
+endef
+
+$(dir $(call gb_UnpackedTarball_get_target,%)).dir :
+	$(if $(wildcard $(dir $@)),,mkdir -p $(dir $@))
+
+$(call gb_UnpackedTarball_get_preparation_target,%) :
+	touch $@
+
+$(call gb_UnpackedTarball_get_target,%) :
+	$(call gb_UnpackedTarball__command,$@,$*,$(call gb_UnpackedTarball_get_dir,$*))
+
+.PHONY : $(call gb_UnpackedTarball_get_clean_target,%)
+$(call gb_UnpackedTarball_get_clean_target,%) :
+	$(call gb_Output_announce,$*,$(false),PAT,2)
+	$(call gb_Helper_abbreviate_dirs,\
+		rm -rf \
+			$(call gb_UnpackedTarball_get_target,$*) \
+			$(call gb_UnpackedTarball_get_preparation_target,$*) \
+			$(call gb_UnpackedTarball_get_dir,$*) \
+	)
+
+# Initialize unpacked tarball
+define gb_UnpackedTarball_UnpackedTarball_internal
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_CXX_SUFFIX :=
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_DESTFILES :=
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_FILES :=
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_PATCHES :=
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_PATCHLEVEL := $(gb_UnpackedTarball_PATCHLEVEL_DEFAULT)
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_POST_ACTION :=
+
+$(call gb_UnpackedTarball_get_preparation_target,$(1)) : $(lastword $(MAKEFILE_LIST))
+$(call gb_UnpackedTarball_get_preparation_target,$(1)) :| $(dir $(call gb_UnpackedTarball_get_target,$(1))).dir
+$(call gb_UnpackedTarball_get_target,$(1)) : $(call gb_UnpackedTarball_get_preparation_target,$(1))
+$(call gb_UnpackedTarball_get_target,$(1)) :| $(dir $(call gb_UnpackedTarball_get_target,$(1))).dir
+
+endef
+
+# Define a new unpacked tarball
+define gb_UnpackedTarball_UnpackedTarball
+$(call gb_UnpackedTarball_UnpackedTarball_internal,$(1))
+
+$$(eval $$(call gb_Module_register_target,$(call gb_UnpackedTarball_get_target,$(1)),$(call gb_UnpackedTarball_get_clean_target,$(1))))
+
+endef
+
+# Set suffix of C++ files, if different from 'cxx'
+#
+# All files with that extension will be renamed to .cxx . This is because
+# LinkTarget requires .cxx extension for C++ files.
+#
+# This is done after applying patches.
+#
+# gb_UnpackedTarball_fix_cxx_suffix unpacked used-suffix
+define gb_UnpackedTarball_fix_cxx_suffix
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_CXX_SUFFIX := $(2)
+
+endef
+
+# Set tarball name
+#
+# gb_UnpackedTarball_set_tarball unpacked tarball-name
+define gb_UnpackedTarball_set_tarball
+$(call gb_UnpackedTarget_UnpackedTarget,$(2),$(call gb_UnpackedTarball_get_dir,$(1)),$(3))
+$(call gb_UnpackedTarball_get_target,$(1)) : $(call gb_UnpackedTarget_get_target,$(2))
+$(call gb_UnpackedTarball_get_clean_target,$(1)) : $(call gb_UnpackedTarget_get_clean_target,$(2))
+$(call gb_UnpackedTarget_get_target,$(2)) : $(call gb_UnpackedTarball_get_preparation_target,$(1))
+
+endef
+
+# Set patch level to be used for all patches
+#
+# The default value is 3 to be able to work with current dmake patches.
+#
+# gb_UnpackedTarball_set_patchlevel unpacked level
+define gb_UnpackedTarball_set_patchlevel
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_PATCHLEVEL := $(2)
+
+endef
+
+# Add a patch to be applied on the unpacked files
+#
+# gb_UnpackedTarball_add_patch unpacked patch
+define gb_UnpackedTarball_add_patch
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_PATCHES += $(SRCDIR)/$(2)
+$(call gb_UnpackedTarball_get_preparation_target,$(1)) : $(SRCDIR)/$(2)
+
+endef
+
+# Add several patches at once
+#
+# gb_UnpackedTarball_add_patches unpacked patch(es)
+define gb_UnpackedTarball_add_patches
+$(foreach patch,$(2),$(call gb_UnpackedTarball_add_patch,$(1),$(patch)))
+
+endef
+
+# Add a file from source dir to the unpacked dir
+#
+# This function should not be used for overwriting existing files--use a
+# patch for that purpose.
+#
+# gb_UnpackedTarball_add_file unpacked destfile file
+define gb_UnpackedTarball_add_file
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_FILES += $(SRCDIR)/$(3)
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_DESTFILES += $(call gb_UnpackedTarball_get_dir,$(1))/$(2)
+$(call gb_UnpackedTarball_get_preparation_target,$(1)) : $(SRCDIR)/$(3)
+
+endef
+
+# Add several files(s) from source dir to the unpacked dir
+#
+# The files are added into the specified subdir.
+#
+# gb_UnpackedTarball_add_files unpacked subdir file(s)
+define gb_UnpackedTarball_add_files
+$(foreach file,$(3),$(call gb_UnpackedTarball_add_file,$(1),$(2)/$(notdir $(file)),$(file)))
+
+endef
+
+# Set arbitrary shell command to be run during unpack
+#
+# The command is run at the very end: after patching, C++ extension
+# mangling and copying additional files in. The command is run in the
+# unpacked directory. If more than one command is used, care should be
+# taken that the whole command fails if either of the sub-commands
+# fails.
+#
+# NOTE: This is a bit hackish, but it is the easiest way to move files
+# around or delete files (typically because the file causes build
+# problems in the original location, c.f. clucene). This is doable by
+# using -E with patch (we use GNU patch anyway), but it would mean an
+# additional patch to maintain....
+#
+# gb_UnpackedTarball_set_post_action unpacked shell-command
+define gb_UnpackedTarball_set_post_action
+$(call gb_UnpackedTarball_get_target,$(1)) : UNPACKED_POST_ACTION := $(strip $(2))
+
+endef
+
+# vim: set noet sw=4 ts=4:

solenv/gbuild/gbuild.mk

@@ -317,6 +317,7 @@ include $(foreach class, \
 	UnoApiTarget \
 	UnoApi \
 	UnoApiMerge \
+	UnpackedTarball \
 	InternalUnoApi \
 	Zip \
 	Configuration \

solenv/gbuild/platform/IOS_ARM_GCC.mk

@@ -323,6 +323,10 @@ gb_InstallScript_EXT := .ins
 
 gb_ExtensionTarget_LICENSEFILE_DEFAULT := $(OUTDIR)/bin/osl/LICENSE
 
+# UnpackedTarget class
+
+gb_UnpackedTarget_TARFILE_LOCATION := $(TARFILE_LOCATION)
+
 # Python
 gb_PYTHON_PRECOMMAND := DYLD_LIBRARY_PATH=$(OUTDIR_FOR_BUILD)/lib
 

solenv/gbuild/platform/WNT_INTEL_GCC.mk

@@ -464,6 +464,10 @@ gb_InstallScript_EXT := .inf
 
 gb_ExtensionTarget_LICENSEFILE_DEFAULT := $(OUTDIR)/bin/osl/license.txt
 
+# UnpackedTarget class
+
+gb_UnpackedTarget_TARFILE_LOCATION := $(TARFILE_LOCATION)
+
 # Python
 gb_PYTHON_PRECOMMAND := PATH="$${PATH}:$(OUTDIR_FOR_BUILD)/bin" PYTHONHOME="$(OUTDIR_FOR_BUILD)/lib/python" PYTHONPATH="$(OUTDIR_FOR_BUILD)/lib/python;$(OUTDIR_FOR_BUILD)/lib/python/lib-dynload"
 

solenv/gbuild/platform/WNT_INTEL_MSC.mk

@@ -741,6 +741,10 @@ gb_InstallScript_EXT := .inf
 
 gb_ExtensionTarget_LICENSEFILE_DEFAULT := $(OUTDIR)/bin/osl/license.txt
 
+# UnpackedTarget class
+
+gb_UnpackedTarget_TARFILE_LOCATION := $(shell cygpath -u $(TARFILE_LOCATION))
+
 # Python
 gb_PYTHON_PRECOMMAND := $(gb_Helper_set_ld_path) PYTHONHOME="$(OUTDIR_FOR_BUILD)/lib/python" PYTHONPATH="$(OUTDIR_FOR_BUILD)/lib/python;$(OUTDIR_FOR_BUILD)/lib/python/lib-dynload"
 

solenv/gbuild/platform/macosx.mk

@@ -435,6 +435,10 @@ gb_InstallScript_EXT := .ins
 
 gb_ExtensionTarget_LICENSEFILE_DEFAULT := $(OUTDIR)/bin/osl/LICENSE
 
+# UnpackedTarget class
+
+gb_UnpackedTarget_TARFILE_LOCATION := $(TARFILE_LOCATION)
+
 # Python
 gb_PYTHON_PRECOMMAND := DYLD_LIBRARY_PATH=$(OUTDIR)/lib
 

solenv/gbuild/platform/solaris.mk

@@ -432,6 +432,10 @@ gb_InstallScript_EXT := .ins
 
 gb_ExtensionTarget_LICENSEFILE_DEFAULT := $(OUTDIR)/bin/osl/LICENSE
 
+# UnpackedTarget class
+
+gb_UnpackedTarget_TARFILE_LOCATION := $(TARFILE_LOCATION)
+
 # Python
 gb_PYTHON_PRECOMMAND := $(gb_Helper_set_ld_path) PYTHONHOME=$(OUTDIR)/lib/python PYTHONPATH=$(OUTDIR)/lib/python:$(OUTDIR)/lib/python/lib-dynload
 

solenv/gbuild/platform/unxgcc.mk

@@ -434,6 +434,10 @@ gb_InstallScript_EXT := .ins
 
 gb_ExtensionTarget_LICENSEFILE_DEFAULT := $(OUTDIR)/bin/osl/LICENSE
 
+# UnpackedTarget class
+
+gb_UnpackedTarget_TARFILE_LOCATION := $(TARFILE_LOCATION)
+
 # Python
 gb_PYTHON_PRECOMMAND := $(gb_Helper_set_ld_path) PYTHONHOME=$(OUTDIR)/lib/python PYTHONPATH=$(OUTDIR)/lib/python:$(OUTDIR)/lib/python/lib-dynload