[Bf-blender-cvs] [3125cfceec3] blender-v2.82-release: Codesign: Add codesign for macOS worker
Sergey Sharybin
noreply at git.blender.org
Mon Feb 3 17:31:15 CET 2020
Commit: 3125cfceec35ef14374ae98f9a400e7a678287a2
Author: Sergey Sharybin
Date: Mon Feb 3 17:03:51 2020 +0100
Branches: blender-v2.82-release
https://developer.blender.org/rB3125cfceec35ef14374ae98f9a400e7a678287a2
Codesign: Add codesign for macOS worker
Works similarly to Windows configuration where buildbot worker and
codesign machines are communicating with each other using network
drive.
===================================================================
M build_files/buildbot/codesign/absolute_and_relative_filename.py
M build_files/buildbot/codesign/base_code_signer.py
M build_files/buildbot/codesign/config_builder.py
M build_files/buildbot/codesign/config_common.py
M build_files/buildbot/codesign/config_server_template.py
M build_files/buildbot/codesign/linux_code_signer.py
A build_files/buildbot/codesign/macos_code_signer.py
M build_files/buildbot/codesign/simple_code_signer.py
M build_files/buildbot/codesign/util.py
M build_files/buildbot/codesign/windows_code_signer.py
A build_files/buildbot/codesign_server_macos.py
M build_files/buildbot/codesign_server_windows.py
A build_files/buildbot/slave_bundle_dmg.py
M build_files/buildbot/slave_codesign.py
M build_files/buildbot/slave_pack.py
===================================================================
diff --git a/build_files/buildbot/codesign/absolute_and_relative_filename.py b/build_files/buildbot/codesign/absolute_and_relative_filename.py
index bea9ea7e8d0..cb42710e785 100644
--- a/build_files/buildbot/codesign/absolute_and_relative_filename.py
+++ b/build_files/buildbot/codesign/absolute_and_relative_filename.py
@@ -65,10 +65,14 @@ class AbsoluteAndRelativeFileName:
"""
Create list of AbsoluteAndRelativeFileName for all the files in the
given directory.
+
+ NOTE: Result will be pointing to a resolved paths.
"""
assert base_dir.is_absolute()
assert base_dir.is_dir()
+ base_dir = base_dir.resolve()
+
result = []
for filename in base_dir.glob('**/*'):
if not filename.is_file():
diff --git a/build_files/buildbot/codesign/base_code_signer.py b/build_files/buildbot/codesign/base_code_signer.py
index ff4b4539658..0505905c6f4 100644
--- a/build_files/buildbot/codesign/base_code_signer.py
+++ b/build_files/buildbot/codesign/base_code_signer.py
@@ -45,13 +45,16 @@
import abc
import logging
import shutil
+import subprocess
import time
-import zipfile
+import tarfile
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Iterable, List
+import codesign.util as util
+
from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName
from codesign.archive_with_indicator import ArchiveWithIndicator
@@ -64,14 +67,14 @@ logger_server = logger.getChild('server')
def pack_files(files: Iterable[AbsoluteAndRelativeFileName],
archive_filepath: Path) -> None:
"""
- Create zip archive from given files for the signing pipeline.
+ Create tar archive from given files for the signing pipeline.
Is used by buildbot worker to create an archive of files which are to be
signed, and by signing server to send signed files back to the worker.
"""
- with zipfile.ZipFile(archive_filepath, 'w') as zip_file_handle:
+ with tarfile.TarFile.open(archive_filepath, 'w') as tar_file_handle:
for file_info in files:
- zip_file_handle.write(file_info.absolute_filepath,
- arcname=file_info.relative_filepath)
+ tar_file_handle.add(file_info.absolute_filepath,
+ arcname=file_info.relative_filepath)
def extract_files(archive_filepath: Path,
@@ -82,8 +85,8 @@ def extract_files(archive_filepath: Path,
# TODO(sergey): Verify files in the archive have relative path.
- with zipfile.ZipFile(archive_filepath, mode='r') as zip_file_handle:
- zip_file_handle.extractall(path=extraction_dir)
+ with tarfile.TarFile.open(archive_filepath, mode='r') as tar_file_handle:
+ tar_file_handle.extractall(path=extraction_dir)
class BaseCodeSigner(metaclass=abc.ABCMeta):
@@ -133,6 +136,9 @@ class BaseCodeSigner(metaclass=abc.ABCMeta):
# This archive is created by the code signing server.
signed_archive_info: ArchiveWithIndicator
+ # Platform the code is currently executing on.
+ platform: util.Platform
+
def __init__(self, config):
self.config = config
@@ -141,12 +147,14 @@ class BaseCodeSigner(metaclass=abc.ABCMeta):
# Unsigned (signing server input) configuration.
self.unsigned_storage_dir = absolute_shared_storage_dir / 'unsigned'
self.unsigned_archive_info = ArchiveWithIndicator(
- self.unsigned_storage_dir, 'unsigned_files.zip', 'ready.stamp')
+ self.unsigned_storage_dir, 'unsigned_files.tar', 'ready.stamp')
# Signed (signing server output) configuration.
self.signed_storage_dir = absolute_shared_storage_dir / 'signed'
self.signed_archive_info = ArchiveWithIndicator(
- self.signed_storage_dir, 'signed_files.zip', 'ready.stamp')
+ self.signed_storage_dir, 'signed_files.tar', 'ready.stamp')
+
+ self.platform = util.get_current_platform()
"""
General note on cleanup environment functions.
@@ -383,3 +391,61 @@ class BaseCodeSigner(metaclass=abc.ABCMeta):
logger_server.info(
'Got signing request, beging signign procedure.')
self.run_signing_pipeline()
+
+ ############################################################################
+ # Command executing.
+ #
+ # Abstracted to a degree that allows to run commands from a foreign
+ # platform.
+ # The goal with this is to allow performing dry-run tests of code signer
+ # server from other platforms (for example, to test that macOS code signer
+ # does what it is supposed to after doing a refactor on Linux).
+
+ # TODO(sergey): What is the type annotation for the command?
+ def run_command_or_mock(self, command, platform: util.Platform) -> None:
+ """
+ Run given command if current platform matches given one
+
+ If the platform is different then it will only be printed allowing
+ to verify logic of the code signing process.
+ """
+
+ if platform != self.platform:
+ logger_server.info(
+ f'Will run command for {platform}: {command}')
+ return
+
+ logger_server.info(f'Running command: {command}')
+ subprocess.run(command)
+
+ # TODO(sergey): What is the type annotation for the command?
+ def check_output_or_mock(self, command,
+ platform: util.Platform,
+ allow_nonzero_exit_code=False) -> str:
+ """
+ Run given command if current platform matches given one
+
+ If the platform is different then it will only be printed allowing
+ to verify logic of the code signing process.
+
+ If allow_nonzero_exit_code is truth then the output will be returned
+ even if application quit with non-zero exit code.
+ Otherwise an subprocess.CalledProcessError exception will be raised
+ in such case.
+ """
+
+ if platform != self.platform:
+ logger_server.info(
+ f'Will run command for {platform}: {command}')
+ return
+
+ if allow_nonzero_exit_code:
+ process = subprocess.Popen(command,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ output = process.communicate()[0]
+ return output.decode()
+
+ logger_server.info(f'Running command: {command}')
+ return subprocess.check_output(
+ command, stderr=subprocess.STDOUT).decode()
diff --git a/build_files/buildbot/codesign/config_builder.py b/build_files/buildbot/codesign/config_builder.py
index e1e3913b99e..1f41619ba13 100644
--- a/build_files/buildbot/codesign/config_builder.py
+++ b/build_files/buildbot/codesign/config_builder.py
@@ -25,13 +25,16 @@ import sys
from pathlib import Path
+import codesign.util as util
+
from codesign.config_common import *
-if sys.platform == 'linux':
+platform = util.get_current_platform()
+if platform == util.Platform.LINUX:
SHARED_STORAGE_DIR = Path('/data/codesign')
-elif sys.platform == 'win32':
+elif platform == util.Platform.WINDOWS:
SHARED_STORAGE_DIR = Path('Z:\\codesign')
-elif sys.platform == 'darwin':
+elif platform == util.Platform.MACOS:
SHARED_STORAGE_DIR = Path('/Volumes/codesign_macos/codesign')
# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema
diff --git a/build_files/buildbot/codesign/config_common.py b/build_files/buildbot/codesign/config_common.py
index 3710286c777..a37bc731dc0 100644
--- a/build_files/buildbot/codesign/config_common.py
+++ b/build_files/buildbot/codesign/config_common.py
@@ -24,7 +24,10 @@ from pathlib import Path
#
# This is how long buildbot packing step will wait signing server to
# perform signing.
-TIMEOUT_IN_SECONDS = 240
+#
+# NOTE: Notarization could take a long time, hence the rather high value
+# here. Might consider using different timeout for different platforms.
+TIMEOUT_IN_SECONDS = 45 * 60 * 60
# Directory which is shared across buildbot worker and signing server.
#
diff --git a/build_files/buildbot/codesign/config_server_template.py b/build_files/buildbot/codesign/config_server_template.py
index dc164634cef..ff97ed15fa5 100644
--- a/build_files/buildbot/codesign/config_server_template.py
+++ b/build_files/buildbot/codesign/config_server_template.py
@@ -27,8 +27,43 @@ from pathlib import Path
from codesign.config_common import *
+CODESIGN_DIRECTORY = Path(__file__).absolute().parent
+BLENDER_GIT_ROOT_DIRECTORY = CODESIGN_DIRECTORY.parent.parent.parent
+
+################################################################################
+# Common configuration.
+
+# Directory where folders for codesign requests and signed result are stored.
+# For example, /data/codesign
+SHARED_STORAGE_DIR: Path
+
+################################################################################
+# macOS-specific configuration.
+
+MACOS_ENTITLEMENTS_FILE = \
+ BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' / 'entitlements.plist'
+
+# Identity of the Developer ID Application certificate which is to be used for
+# codesign tool.
+# Use `security find-identity -v -p codesigning` to find the identity.
+#
+# NOTE: This identity is just an example from release/darwin/README.txt.
+MACOS_CODESIGN_IDENTITY = 'AE825E26F12D08B692F360133210AF46F4CF7B97'
+
+# User name (Apple ID) which will be used to request notarization.
+MACOS_XCRUN_USERNAME = 'me at example.com'
+
+# One-time application password which will be used to request notarization.
+MACOS_XCRUN_PASSWORD = '@keychain:altool-password'
+
+# Timeout in seconds within which the notarial office is supposed to reply.
+MACOS_NOTARIZE_TIMEOUT_IN_SECONDS = 60 * 60
+
+################################################################################
+# Windows-specific configuration.
+
# URL to the timestamping authority.
-TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com'
+WIN_TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com'
# Full path to the certificate used for signing.
#
@@ -36,7 +71
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list