[Bf-blender-cvs] [d4835b88b23] master: Buildbot: Migrate package archive format for Linux from tar.bz2 to tar.xz

[Jens]

Commit: d4835b88b23bce6c28d8514e35d771408d5ef18b
Author: Jens
Date:   Thu Nov 28 11:40:59 2019 +0100
Branches: master
https://developer.blender.org/rBd4835b88b23bce6c28d8514e35d771408d5ef18b

Buildbot: Migrate package archive format for Linux from tar.bz2 to tar.xz

xz compresses 25% better than bz2, reducing download times and server load.
The numbers:

blender-2.80-linux-glibc217-x86_64.tar.bz2 (release): 134 886 174 bytes
with xz:    96 181 604 bytes (-28.7%)
with xz -9: 93 871 548 bytes (-30.4%)

blender-2.81-7c1fbe24ca33-linux-glibc217-x86_64.tar.bz2 (beta): 173 600 363 bytes
with xz:    133 100 664 bytes (-23.3%)
with xz -9: 129 534 124 bytes (-25.4%)

xz also decompresses more than twice as fast as bz2, however compression needs
four times as long (on my 7-year-old laptop 3-4 minutes instead of <1).

Also xz has become more common than bz2, e.g. Debian/Ubuntu deb packages have
been xz-compressed for years, so the dpkg package manager as well as systemd
and grub all depend on liblzma being present, whereas bz2 is becoming more and
more optional.

Current Linux archives also include the UID/GID of whatever user account
happens to be used for building by the blender.org infrastructure. If someone
then installs these archives as root e.g. to /usr/local/... and doesn't pay
full attention the files remain owned by a regular user, which is a serious
security issue. This patch fixes that by setting the UID/GID to 0.

Differential Revision: https://developer.blender.org/D6138

===================================================================

M	build_files/buildbot/slave_pack.py
M	build_files/cmake/packaging.cmake
M	build_files/package_spec/build_archive.py

===================================================================

diff --git a/build_files/buildbot/slave_pack.py b/build_files/buildbot/slave_pack.py
index 19dac236762..bbacedca0ce 100644
--- a/build_files/buildbot/slave_pack.py
+++ b/build_files/buildbot/slave_pack.py
@@ -64,7 +64,7 @@ def create_buildbot_upload_zip(builder, package_files):
         sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n')
         sys.exit(1)
 
-def create_tar_bz2(src, dest, package_name):
+def create_tar_xz(src, dest, package_name):
     # One extra to remove leading os.sep when cleaning root for package_root
     ln = len(src) + 1
     flist = list()
@@ -75,9 +75,20 @@ def create_tar_bz2(src, dest, package_name):
         flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files])
 
     import tarfile
-    package = tarfile.open(dest, 'w:bz2')
+
+    # Set UID/GID of archived files to 0, otherwise they'd be owned by whatever
+    # user compiled the package. If root then unpacks it to /usr/local/ you get
+    # a security issue.
+    def _fakeroot(tarinfo):
+        tarinfo.gid = 0
+        tarinfo.gname = "root"
+        tarinfo.uid = 0
+        tarinfo.uname = "root"
+        return tarinfo
+
+    package = tarfile.open(dest, 'w:xz', preset=9)
     for entry in flist:
-        package.add(entry[0], entry[1], recursive=False)
+        package.add(entry[0], entry[1], recursive=False, filter=_fakeroot)
     package.close()
 
 def cleanup_files(dirpath, extension):
@@ -163,11 +174,11 @@ def pack_linux(builder):
     # Construct package name
     platform_name = 'linux-' + blender_glibc + '-' + blender_arch
     package_name = get_package_name(builder, platform_name)
-    package_filename = package_name + ".tar.bz2"
+    package_filename = package_name + ".tar.xz"
 
-    print("Creating .tar.bz2 archive")
-    package_filepath = builder.install_dir + '.tar.bz2'
-    create_tar_bz2(builder.install_dir, package_filepath, package_name)
+    print("Creating .tar.xz archive")
+    package_filepath = builder.install_dir + '.tar.xz'
+    create_tar_xz(builder.install_dir, package_filepath, package_name)
 
     # Create buildbot_upload.zip
     create_buildbot_upload_zip(builder, [(package_filepath, package_filename)])
diff --git a/build_files/cmake/packaging.cmake b/build_files/cmake/packaging.cmake
index 5ace42646c5..0e530463659 100644
--- a/build_files/cmake/packaging.cmake
+++ b/build_files/cmake/packaging.cmake
@@ -135,7 +135,7 @@ elseif(UNIX)
 
   add_package_archive(
     "${PROJECT_NAME}-${BLENDER_VERSION}-${BUILD_REV}-${PACKAGE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}"
-    "tar.bz2")
+    "tar.xz")
 endif()
 
 unset(MAJOR_VERSION)
diff --git a/build_files/package_spec/build_archive.py b/build_files/package_spec/build_archive.py
index 5ca2f319d87..58a970bca31 100755
--- a/build_files/package_spec/build_archive.py
+++ b/build_files/package_spec/build_archive.py
@@ -51,8 +51,9 @@ try:
 
     if extension == 'zip':
         archive_cmd = ['zip', '-9', '-r', package_archive, package_dir]
-    elif extension == 'tar.bz2':
-        archive_cmd = ['tar', 'cjf', package_archive, package_dir]
+    elif extension == 'tar.xz':
+        archive_cmd = ['tar', '-cf', package_archive, '--owner=0', '--group=0',
+                       '--use-compress-program=xz -9', package_dir]
     else:
         sys.stderr.write('Unknown archive extension: ' + extension)
         sys.exit(-1)