[Bf-blender-cvs] [735469fea36] codesign: Codesign: Implement buildbot worker side DMG bundling
Sergey Sharybin
noreply at git.blender.org
Fri Dec 13 16:00:41 CET 2019
Commit: 735469fea36d98281af33f32222f08b1931bbf80
Author: Sergey Sharybin
Date: Fri Dec 13 15:58:07 2019 +0100
Branches: codesign
https://developer.blender.org/rB735469fea36d98281af33f32222f08b1931bbf80
Codesign: Implement buildbot worker side DMG bundling
Is based on bundle.sh but is adopted to an environment where
codesigning is happening in a dedicated VM.
Also fixed some code from previous commits:
- Removed debug only early output when running commands
- Ensured that path to git is always valid
(wasn't a case when __file__ is relative to current directory).
- Fixed simple code signer which couldn't have imported util.
===================================================================
M build_files/buildbot/codesign/base_code_signer.py
M build_files/buildbot/codesign/config_builder.py
M build_files/buildbot/codesign/config_common.py
M build_files/buildbot/codesign/config_server_template.py
A build_files/buildbot/slave_bundle_dmg.py
===================================================================
diff --git a/build_files/buildbot/codesign/base_code_signer.py b/build_files/buildbot/codesign/base_code_signer.py
index 0481bcf2564..0505905c6f4 100644
--- a/build_files/buildbot/codesign/base_code_signer.py
+++ b/build_files/buildbot/codesign/base_code_signer.py
@@ -410,7 +410,7 @@ class BaseCodeSigner(metaclass=abc.ABCMeta):
to verify logic of the code signing process.
"""
- if platform != self.platform or True:
+ if platform != self.platform:
logger_server.info(
f'Will run command for {platform}: {command}')
return
diff --git a/build_files/buildbot/codesign/config_builder.py b/build_files/buildbot/codesign/config_builder.py
index 0cb83aba41e..1fa725ed28b 100644
--- a/build_files/buildbot/codesign/config_builder.py
+++ b/build_files/buildbot/codesign/config_builder.py
@@ -25,7 +25,7 @@ import sys
from pathlib import Path
-import util
+import codesign.util as util
from codesign.config_common import *
@@ -34,6 +34,8 @@ if platform == util.Platform.LINUX:
SHARED_STORAGE_DIR = Path('/data/codesign')
elif platform == util.Platform.WINDOWS:
SHARED_STORAGE_DIR = Path('Z:\\codesign')
+elif platform == util.Platform.MACOS:
+ SHARED_STORAGE_DIR = Path('/Users/sergey/Developer/blender/codesign')
# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema
LOGGING = {
diff --git a/build_files/buildbot/codesign/config_common.py b/build_files/buildbot/codesign/config_common.py
index 3710286c777..a37bc731dc0 100644
--- a/build_files/buildbot/codesign/config_common.py
+++ b/build_files/buildbot/codesign/config_common.py
@@ -24,7 +24,10 @@ from pathlib import Path
#
# This is how long buildbot packing step will wait signing server to
# perform signing.
-TIMEOUT_IN_SECONDS = 240
+#
+# NOTE: Notarization could take a long time, hence the rather high value
+# here. Might consider using different timeout for different platforms.
+TIMEOUT_IN_SECONDS = 45 * 60 * 60
# Directory which is shared across buildbot worker and signing server.
#
diff --git a/build_files/buildbot/codesign/config_server_template.py b/build_files/buildbot/codesign/config_server_template.py
index 1d6ddc54380..ff97ed15fa5 100644
--- a/build_files/buildbot/codesign/config_server_template.py
+++ b/build_files/buildbot/codesign/config_server_template.py
@@ -27,7 +27,7 @@ from pathlib import Path
from codesign.config_common import *
-CODESIGN_DIRECTORY = Path(__file__).parent
+CODESIGN_DIRECTORY = Path(__file__).absolute().parent
BLENDER_GIT_ROOT_DIRECTORY = CODESIGN_DIRECTORY.parent.parent.parent
################################################################################
diff --git a/build_files/buildbot/slave_bundle_dmg.py b/build_files/buildbot/slave_bundle_dmg.py
new file mode 100755
index 00000000000..8a632d2ff3b
--- /dev/null
+++ b/build_files/buildbot/slave_bundle_dmg.py
@@ -0,0 +1,529 @@
+#!/usr/bin/env python3
+
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ##### END GPL LICENSE BLOCK #####
+
+import argparse
+import re
+import shutil
+import subprocess
+import sys
+import time
+
+from pathlib import Path
+from tempfile import TemporaryDirectory, NamedTemporaryFile
+from typing import List
+
+BUILDBOT_DIRECTORY = Path(__file__).absolute().parent
+CODESIGN_SCRIPT = BUILDBOT_DIRECTORY / 'slave_codesign.py'
+BLENDER_GIT_ROOT_DIRECTORY = BUILDBOT_DIRECTORY.parent.parent
+DARWIN_DIRECTORY = BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin'
+
+
+# Extra size which is added on top of actual files size when estimating size
+# of destination DNG.
+EXTRA_DMG_SIZE_IN_BYTES = 800 * 1024 * 1024
+
+################################################################################
+# Common utilities
+
+
+def get_directory_size(root_directory: Path) -> int:
+ """
+ Get size of directory on disk
+ """
+
+ total_size = 0
+ for file in root_directory.glob('**/*'):
+ total_size += file.lstat().st_size
+ return total_size
+
+
+################################################################################
+# DMG bundling specific logic
+
+def create_argument_parser():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ 'source_dir',
+ type=Path,
+ help='Source directory which points to either existing .app bundle'
+ 'or to a directory with .app bundles.')
+ parser.add_argument(
+ '--background-image',
+ type=Path,
+ help="Optional background picture which will be set on the DMG."
+ "If not provided default Blender's one is used.")
+ parser.add_argument(
+ '--volume-name',
+ type=str,
+ help='Optional name of a volume which will be used for DMG.')
+ parser.add_argument(
+ '--dmg',
+ type=Path,
+ help='Optional argument which points to a final DMG file name.')
+ parser.add_argument(
+ '--applescript',
+ type=Path,
+ help="Optional path to applescript to set up folder looks of DMG."
+ "If not provided default Blender's one is used.")
+ return parser
+
+
+def collect_app_bundles(source_dir: Path) -> List[Path]:
+ """
+ Collect all app bundles which are to be put into DMG
+
+ If the source directory points to FOO.app it will be the only app bundle
+ packed.
+
+ Otherwise all .app bundles from given directory are placed to a single
+ DMG.
+ """
+
+ if source_dir.name.endswith('.app'):
+ return [source_dir]
+
+ app_bundles = []
+ for filename in source_dir.glob('*'):
+ if not filename.is_dir():
+ continue
+ if not filename.name.endswith('.app'):
+ continue
+
+ app_bundles.append(filename)
+
+ return app_bundles
+
+
+def collect_and_log_app_bundles(source_dir: Path) -> List[Path]:
+ app_bundles = collect_app_bundles(source_dir)
+
+ if not app_bundles:
+ print('No app bundles found for packing')
+ return
+
+ print(f'Found {len(app_bundles)} to pack:')
+ for app_bundle in app_bundles:
+ print(f'- {app_bundle}')
+
+ return app_bundles
+
+
+def estimate_dmg_size(app_bundles: List[Path]) -> int:
+ """
+ Estimate size of DMG to hold requested app bundles
+
+ The size is based on actual size of all files in all bundles plus some
+ space to compensate for different size-on-disk plus some space to hold
+ codesign signatures.
+
+ Is better to be on a high side since the empty space is compressed, but
+ lack of space might cause silent failures later on.
+ """
+
+ app_bundles_size = 0
+ for app_bundle in app_bundles:
+ app_bundles_size += get_directory_size(app_bundle)
+
+ return app_bundles_size + EXTRA_DMG_SIZE_IN_BYTES
+
+
+def copy_app_bundles_to_directory(app_bundles: List[Path],
+ directory: Path) -> None:
+ """
+ Copy all bundles to a given directory
+
+ This directory is what the DMG will be created from.
+ """
+ for app_bundle in app_bundles:
+ print(f'Copying {app_bundle.name}...')
+ shutil.copytree(app_bundle, directory / app_bundle.name)
+
+
+def create_dmg_image(app_bundles: List[Path],
+ dmg_filepath: Path,
+ volume_name: str) -> None:
+ """
+ Create DMG disk image and put app bundles in it
+
+ No DMG configuration or codesigning is happening here.
+ """
+
+ if dmg_filepath.exists():
+ print(f'Removing existing writable DMG {dmg_filepath}...')
+ dmg_filepath.unlink()
+
+ print('Preparing directory with app bundles for the DMG...')
+ with TemporaryDirectory(prefix='blender-dmg-content-') as content_dir_str:
+ # Copy all bundles to a clean directory.
+ content_dir = Path(content_dir_str)
+ copy_app_bundles_to_directory(app_bundles, content_dir)
+
+ # Estimate size of the DMG.
+ dmg_size = estimate_dmg_size(app_bundles)
+ print(f'Estimated DMG size: {dmg_size:,} bytes.')
+
+ # Create the DMG.
+ print(f'Creating writable DMG {dmg_filepath}')
+ command = ('hdiutil',
+ 'create',
+ '-size', str(dmg_size),
+ '-fs', 'HFS+',
+ '-srcfolder', content_dir,
+ '-volname', volume_name,
+ '-format', 'UDRW',
+ dmg_filepath)
+ subprocess.run(command)
+
+
+def get_writable_dmg_filepath(dmg_filepath: Path):
+ """
+ Get file path for writable DMG image
+ """
+ parent = dmg_filepath.parent
+ return parent / (dmg_filepath.stem + '-temp.dmg')
+
+
+def mount_readwrite_dmg(dmg_filepath: Path) -> None:
+ """
+ Mount writable DMG
+
+ Mounting point would be /Volumes/<volume name>
+ """
+
+ print(f'Mounting read-write DMG ${dmg_filepath}')
+ command = ('hdiutil',
+ 'attach', '-readwrite',
+ '-noverify',
+ '-noautoopen',
+ dmg_filepath)
+ subprocess.run(command)
+
+
+def get_mount_directory_for_volume_name(volume_name: str) -> Path:
+ """
+ Get directory under which the volume will be mounted
+ """
+
+ return Path('/Volumes') / volume_name
+
+
+def eject_volume(volume_name: str) -> None:
+ """
+ Eject given volume, if mounted
+ """
+ mount_directory = get_mount_directory_for_volume_name(volume_name)
+ if not mount_directory.exists():
+ return
+ mount_directory_str = str(mount_
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list