allow to put files listed in file into installation

With this in place, we can replace most of our Zips by Packages.
Extensions and Dictionaries are on the radar as well.

To move an installed file from zip to filelist, do:
1. Convert the Zip_foo.mk to Package_foo.mk :
   - change destination paths of all files to the same ones they have in
     the installation (you can find that in scp2)
   - use gb_Package_set_outdir to place the files under $INSTDIR, where
     they are expected.
2. Change the scp2 record:
   - change filename to <package-name>.filelist
   - replace ARCHIVE by FILELIST in Styles
   - change Dir to FILELIST_DIR.

Change-Id: Ie17d0765406081b03dcd44a6a23cf517f2067dd3
Reviewed-on: https://gerrit.libreoffice.org/3149Reviewed-by: Fridrich Strba <fridrich@documentfoundation.org>
Tested-by: Fridrich Strba <fridrich@documentfoundation.org>

instsetoo_native/util/openoffice.lst.in

@@ -83,7 +83,7 @@ LibreOffice
         downloadname    LibreOffice_{productversion}_{os}_install_{languages}
         langpackdownloadname    LibreOffice_{productversion}_languagepack_{os}_install_{languages}
         helppackdownloadname    LibreOffice_{productversion}_helppack_{os}_install_{languages}
-        include         {solarenvpath}/{os}/loader2,.,{localpath}/bin,{solarpath}/bin.{minor}/ooowoure,{solarpath}/bin.{minor}/osl,{solarpath}/bin.{minor}/desktop-integration/{pkgtype},{solarpath}/bin.{minor},{solarpath}/lib.{minor},{solarpath}/pck.{minor}/openoffice,{solarpath}/pck.{minor},{solarpath}/xml.{minor}/office/instance,{solarpath}/xml.{minor},../../external/common,{solarenvpath}/{os}/OOo_external
+        include         {solarenvpath}/{os}/loader2,.,{localpath}/bin,{solarpath}/bin.{minor}/ooowoure,{solarpath}/bin.{minor}/osl,{solarpath}/bin.{minor}/desktop-integration/{pkgtype},{solarpath}/bin.{minor},{solarpath}/lib.{minor},{solarpath}/pck.{minor}/openoffice,{solarpath}/pck.{minor},{solarpath}/xml.{minor}/office/instance,{solarpath}/xml.{minor},../../external/common,{solarenvpath}/{os}/OOo_external,{filelistpath}
     }
 }
 
@@ -140,7 +140,7 @@ LibreOffice_Dev
         downloadname    LibreOfficeDev_{productversion}_{os}_install_{languages}
         langpackdownloadname    LibreOfficeDev_{productversion}_languagepack_{os}_install_{languages}
         helppackdownloadname    LibreOfficeDev_{productversion}_helppack_{os}_install_{languages}
-        include         {solarenvpath}/{os}/loader2,.,{localpath}/bin/dev,{localpath}/bin,{solarpath}/bin.{minor}/osl,{solarpath}/bin.{minor}/desktop-integration/{pkgtype},{solarpath}/bin.{minor},{solarpath}/lib.{minor},{solarpath}/pck.{minor}/openoffice_dev,{solarpath}/pck.{minor}/openoffice,{solarpath}/pck.{minor},{solarpath}/xml.{minor}/office/instance,{solarpath}/xml.{minor},../../external/common,{solarenvpath}/{os}/OOo_external
+        include         {solarenvpath}/{os}/loader2,.,{localpath}/bin/dev,{localpath}/bin,{solarpath}/bin.{minor}/osl,{solarpath}/bin.{minor}/desktop-integration/{pkgtype},{solarpath}/bin.{minor},{solarpath}/lib.{minor},{solarpath}/pck.{minor}/openoffice_dev,{solarpath}/pck.{minor}/openoffice,{solarpath}/pck.{minor},{solarpath}/xml.{minor}/office/instance,{solarpath}/xml.{minor},../../external/common,{solarenvpath}/{os}/OOo_external,{filelistpath}
     }
 }
 

scp2/inc/macros.inc

@@ -425,6 +425,12 @@ End
 #define SCP2_URE_JDL_NORMAL(n) SCP2_URE_DL_NORMAL(n)
 #endif
 
+#if defined MACOSX
+#define FILELIST_DIR gid_Dir_Bundle_Contents
+#else
+#define FILELIST_DIR gid_Dir_Brand_Root
+#endif
+
 #include <langmacros.inc>
 
 

solenv/bin/modules/installer.pm

@@ -41,6 +41,7 @@ use installer::download;
 use installer::environment;
 use installer::epmfile;
 use installer::files;
+use installer::filelists;
 use installer::globals;
 use installer::helppack;
 use installer::languagepack;
@@ -617,6 +618,14 @@ sub run {
             installer::scriptitems::quoting_illegal_filenames($filesinproductlanguageresolvedarrayref);
         }
 
+        ######################################################################################
+        # Processing files with flag FILELIST and putting listed files into the file list
+        ######################################################################################
+
+        installer::logger::print_message( "... analyzing files with flag FILELIST ...\n" );
+
+        $filesinproductlanguageresolvedarrayref = installer::filelists::resolve_filelist_flag($filesinproductlanguageresolvedarrayref, $ENV{'INSTDIR'});
+
         #####################################
         # Files with flag SUBST_FILENAME
         #####################################

solenv/bin/modules/installer/control.pm

@@ -280,6 +280,7 @@ sub check_system_environment
         OUTPATH
         LOCAL_OUT
         LOCAL_COMMON_OUT
+        WORKDIR
     );
 
     for my $key ( @environmentvariables )

solenv/bin/modules/installer/environment.pm

@@ -67,6 +67,9 @@ sub create_pathvariables
     my $platformname  = $environment->{'OUTPATH'};
     $variables{'platformname'} = $platformname;
 
+    my $filelistpath = $environment->{'WORKDIR'} . $installer::globals::separator . 'Package';
+    $variables{'filelistpath'} = $filelistpath;
+
     return \%variables;
 }
 

solenv/bin/modules/installer/filelists.pm

+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+package installer::filelists;
+
+use installer::files;
+use installer::globals;
+use installer::logger;
+use installer::pathanalyzer;
+
+sub resolve_filelist_flag
+{
+    my ($files, $outdir) = @_;
+    my @newfiles = ();
+
+    foreach my $file (@{$files})
+    {
+        my $is_filelist = 0;
+        if ($file->{'Styles'})
+        {
+            if ($file->{'Styles'} =~ /\bFILELIST\b/)
+            {
+                $is_filelist = 1;
+            }
+        }
+
+        if ($is_filelist)
+        {
+            my $filelist_path = $file->{'sourcepath'};
+            my $filelist = read_filelist($filelist_path);
+            if (@{$filelist})
+            {
+                my $destination = $file->{'destination'};
+                installer::pathanalyzer::get_path_from_fullqualifiedname(\$destination);
+
+                foreach my $path (@{$filelist})
+                {
+                    # TODO: check that the file is really under $outdir
+                    # TODO: check existence of the file
+                    my $subpath = substr $path, ((length $outdir) + 1); # drop separator too
+
+                    my %newfile = ();
+                    %newfile = %{$file};
+                    $newfile{'Name'} = $subpath;
+                    $newfile{'sourcepath'} = $path;
+                    $newfile{'destination'} = $destination . $subpath;
+                    $newfile{'filelistname'} = $file->{'Name'};
+                    $newfile{'filelistpath'} = $file->{'sourcepath'};
+
+                    push @newfiles, \%newfile;
+                }
+            }
+            else
+            {
+                installer::logger::print_message("filelist $filelist_path is empty\n");
+            }
+        }
+        else # not a filelist, just pass the current file over
+        {
+            push @newfiles, $file;
+        }
+    }
+
+    return \@newfiles;
+}
+
+sub read_filelist
+{
+    my ($path) = @_;
+    my $content = installer::files::read_file($path);
+    my @filelist = ();
+
+    foreach my $line (@{$content})
+    {
+        chomp $line;
+        foreach my $file (split /\s+/, $line)
+        {
+            if ($file ne "")
+            {
+                push @filelist, $file;
+            }
+        }
+    }
+
+    return \@filelist;
+}
+
+1;
+
+# vim: set expandtab shiftwidth=4 tabstop=4:

solenv/gbuild/Package.mk

@@ -56,6 +56,12 @@ endef
 
 # Package class
 
+$(dir $(call gb_Package_get_target,%)).dir :
+	$(if $(wildcard $(dir $@)),,mkdir -p $(dir $@))
+
+$(dir $(call gb_Package_get_target,%))%/.dir :
+	$(if $(wildcard $(dir $@)),,mkdir -p $(dir $@))
+
 .PHONY : $(call gb_Package_get_clean_target,%)
 $(call gb_Package_get_clean_target,%) :
 	$(call gb_Output_announce,$*,$(false),PKG,2)
@@ -68,14 +74,17 @@ $(call gb_Package_get_preparation_target,%) :
 
 $(call gb_Package_get_target,%) :
 	$(call gb_Output_announce,$*,$(true),PKG,2)
-	mkdir -p $(dir $@) && touch $@
+	rm -f $@ && \
+	mv $(call var2file,$@.tmp,100,$(FILES)) $@
 
 # for other targets that want to create Packages, does not register at Module
 define gb_Package_Package_internal
 gb_Package_SOURCEDIR_$(1) := $(2)
 gb_Package_OUTDIR_$(1) := $(OUTDIR)
+$(call gb_Package_get_target,$(1)) : FILES :=
 $(call gb_Package_get_clean_target,$(1)) : FILES := $(call gb_Package_get_target,$(1)) $(call gb_Package_get_preparation_target,$(1))
 $(call gb_Package_get_target,$(1)) : $(call gb_Package_get_preparation_target,$(1))
+$(call gb_Package_get_target,$(1)) :| $(dir $(call gb_Package_get_target,$(1))).dir
 
 endef
 
@@ -99,6 +108,7 @@ endef
 
 define gb_Package_add_file
 $(call gb_Package_get_target,$(1)) : $$(gb_Package_OUTDIR_$(1))/$(2)
+$(call gb_Package_get_target,$(1)) : FILES += $$(gb_Package_OUTDIR_$(1))/$(2)
 $(call gb_Package_get_clean_target,$(1)) : FILES += $$(gb_Package_OUTDIR_$(1))/$(2)
 $(call gb_PackagePart_PackagePart,$(2),$$(gb_Package_SOURCEDIR_$(1))/$(3),$(call gb_Package_get_preparation_target,$(1)),$$(gb_Package_OUTDIR_$(1)))
 

solenv/gbuild/TargetLocations.mk

@@ -163,8 +163,8 @@ gb_ObjCxxObject_get_target = $(WORKDIR)/ObjCxxObject/$(1).o
 gb_ObjCObject_get_target = $(WORKDIR)/ObjCObject/$(1).o
 gb_Pagein_get_target = $(WORKDIR)/Pagein/$(1)
 gb_Package_get_preparation_target = $(WORKDIR)/Package/prepared/$(1)
-gb_Package_get_target = $(WORKDIR)/Package/$(1)
-gb_Package_get_target_for_build = $(WORKDIR_FOR_BUILD)/Package/$(1)
+gb_Package_get_target = $(WORKDIR)/Package/$(1).filelist
+gb_Package_get_target_for_build = $(WORKDIR_FOR_BUILD)/Package/$(1).filelist
 gb_Postprocess_get_target = $(WORKDIR)/Postprocess/$(1)
 gb_PrecompiledHeader_get_dep_target = $(WORKDIR)/Dep/PrecompiledHeader/$(gb_PrecompiledHeader_DEBUGDIR)/$(1).hxx.gch.d
 gb_PrecompiledHeader_get_target = $(WORKDIR)/PrecompiledHeader/$(gb_PrecompiledHeader_DEBUGDIR)/$(1).hxx.gch