blob: 1b3e6defeffcd0f1fe7cc119a5ced3f5fff1386a [file] [log] [blame]
#!/usr/bin/env python2.7
# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Derivative work of
"""Creates a archive manifest used for Fuchsia package generation."""
import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
# File extension of a component manifest for each Component Framework version
MANIFEST_VERSION_EXTENSIONS = {"v1": ".cmx", "v2": ".cm"}
def make_package_path(file_path, roots):
"""Computes a path for |file_path| relative to one of the |roots|.
file_path: The file path to relativize.
roots: A list of directory paths which may serve as a relative root for
For example:
* make_package_path('/foo/bar.txt', ['/foo/']) 'bar.txt'
* make_package_path('/foo/dir/bar.txt', ['/foo/']) 'dir/bar.txt'
* make_package_path('/foo/out/Debug/bar.exe', ['/foo/', '/foo/out/Debug/']) 'bar.exe'
# Prevents greedily matching against a shallow path when a deeper, better
# matching path exists.
roots.sort(key=len, reverse=True)
for next_root in roots:
if not next_root.endswith(os.sep):
next_root += os.sep
if file_path.startswith(next_root):
relative_path = file_path[len(next_root):]
return relative_path
return file_path
def _get_stripped_path(bin_path):
"""Finds the stripped version of |bin_path| in the build output directory.
returns |bin_path| if no stripped path is found.
stripped_path = bin_path.replace('lib.unstripped/',
'lib/').replace('exe.unstripped/', '')
if os.path.exists(stripped_path):
return stripped_path
return bin_path
def _is_binary(path):
"""Checks if the file at |path| is an ELF executable.
This is done by inspecting its FourCC header.
with open(path, 'rb') as f:
file_tag =
return file_tag == '\x7fELF'
def _write_build_ids_txt(binary_paths, ids_txt_path):
"""Writes an index text file mapping build IDs to unstripped binaries."""
# List of binaries whose build IDs are awaiting processing by readelf.
# Entries are removed as readelf's output is parsed.
unprocessed_binary_paths = {os.path.basename(p): p for p in binary_paths}
with open(ids_txt_path, 'w') as ids_file:
# Create a set to dedupe stripped binary paths in case both the stripped and
# unstripped versions of a binary are specified.
stripped_binary_paths = set(map(_get_stripped_path, binary_paths))
readelf_stdout = subprocess.check_output(
['readelf', '-n'] + list(stripped_binary_paths))
if len(binary_paths) == 1:
# Readelf won't report a binary's path if only one was provided to the
# tool.
binary_shortname = os.path.basename(binary_paths[0])
binary_shortname = None
for line in readelf_stdout.split('\n'):
line = line.strip()
if line.startswith(READELF_FILE_PREFIX):
binary_shortname = os.path.basename(
assert binary_shortname in unprocessed_binary_paths
elif line.startswith(READELF_BUILD_ID_PREFIX):
# Paths to the unstripped executables listed in "ids.txt" are specified
# as relative paths to that file.
unstripped_rel_path = os.path.relpath(
build_id = line[len(READELF_BUILD_ID_PREFIX):]
ids_file.write(build_id + ' ' + unstripped_rel_path + '\n')
del unprocessed_binary_paths[binary_shortname]
# Did readelf forget anything? Make sure that all binaries are accounted for.
assert not unprocessed_binary_paths
def _parse_component(component_info_file):
component_info = json.load(open(component_info_file, 'r'))
return component_info
def _get_app_filename(component_info):
for c in component_info:
if c.get('type') == 'dep':
pos = c.get('source').find(':')
return c.get('source')[pos + 1:]
def _get_component_manifests(component_info):
return [c for c in component_info if c.get('type') == 'manifest']
def _get_expanded_files(runtime_deps_file):
""" Process the runtime deps file for file paths, recursively walking
directories as needed.
Returns a set of expanded files referenced by the runtime deps file.
# runtime_deps may contain duplicate paths, so use a set for
# de-duplication.
expanded_files = set()
for next_path in open(runtime_deps_file, 'r'):
next_path = next_path.strip()
if os.path.isdir(next_path):
for root, _, files in os.walk(next_path):
for current_file in files:
if current_file.startswith('.'):
os.path.normpath(os.path.join(root, current_file)))
return expanded_files
def _write_gn_deps_file(depfile_path, manifest_path, out_dir, expanded_files):
with open(depfile_path, 'w') as depfile:
manifest_path = os.path.relpath(manifest_path, out_dir)
deps_list = []
for f in expanded_files:
deps_list.append(os.path.relpath(f, out_dir))
deps_string = ' '.join(deps_list)
depfile.write('%s: %s' % (manifest_path, deps_string))
def _write_meta_package_manifest(
manifest, manifest_path, app_name, out_dir, package_version):
# Write meta/package manifest file and add to archive manifest.
meta_package = os.path.join(os.path.dirname(manifest_path), 'package')
with open(meta_package, 'w') as package_json:
json_payload = {'version': package_version, 'name': app_name}
json.dump(json_payload, package_json)
package_json_filepath = os.path.relpath(, out_dir)
manifest.write('meta/package=%s\n' % package_json_filepath)
def _write_component_manifest(manifest, component_info, manifest_path, out_dir):
"""Copy component manifest files and add to archive manifest.
Raises an exception if a component uses a unknown manifest version.
for component_manifest in _get_component_manifests(component_info):
manifest_version = component_manifest.get('manifest_version')
if manifest_version not in MANIFEST_VERSION_EXTENSIONS:
raise Exception(
'Unknown manifest_version: {}'.format(manifest_version))
extension = MANIFEST_VERSION_EXTENSIONS.get(manifest_version)
manifest_dest_file_path = os.path.join(
component_manifest.get('output_name') + extension)
shutil.copy(component_manifest.get('source'), manifest_dest_file_path)
'meta/%s=%s\n' % (
os.path.relpath(manifest_dest_file_path, out_dir)))
def _write_package_manifest(
manifest, expanded_files, out_dir, exclude_file, root_dir,
component_info, binaries):
"""Writes the package manifest for a Fuchsia package
Returns a list of binaries in the package.
Raises an exception if the app filename does not match the package path.
Raises an exception if excluded files are not found."""
app_filename = _get_app_filename(component_info)
gen_dir = os.path.normpath(os.path.join(out_dir, 'gen'))
app_found = False
excluded_files_set = set(exclude_file)
for current_file in expanded_files:
if _is_binary(current_file):
current_file = _get_stripped_path(current_file)
# make_package_path() may relativize to either the source root or
# output directory.
in_package_path = make_package_path(
current_file, [gen_dir, root_dir, out_dir])
if in_package_path == app_filename:
app_found = True
if in_package_path in excluded_files_set:
manifest.write('%s=%s\n' % (in_package_path, current_file))
if len(excluded_files_set) > 0:
raise Exception(
'Some files were excluded with --exclude-file, but '
'not found in the deps list: %s' % ', '.join(excluded_files_set))
if not app_found:
raise Exception('Could not locate executable inside runtime_deps.')
def _build_manifest(args):
expanded_files = _get_expanded_files(args.runtime_deps_file)
args.depfile_path, args.manifest_path, args.out_dir, expanded_files)
binaries = [] # keep track of binaries to write build IDs
with open(args.manifest_path, 'w') as manifest:
manifest, args.manifest_path, args.app_name, args.out_dir,
for component_info in _parse_component(args.json_file):
manifest, expanded_files, args.out_dir, args.exclude_file,
args.root_dir, component_info, binaries)
manifest, component_info, args.manifest_path, args.out_dir)
_write_build_ids_txt(binaries, args.build_ids_file)
return 0
def main():
parser = argparse.ArgumentParser()
'--root-dir', required=True, help='Build root directory')
'--out-dir', required=True, help='Build output directory')
parser.add_argument('--app-name', required=True, help='Package name')
help='File with the list of runtime dependencies.')
'--depfile-path', required=True, help='Path to write GN deps file.')
help='Package-relative file path to exclude from the package.')
'--manifest-path', required=True, help='Manifest output path.')
'--build-ids-file', required=True, help='Debug symbol index path.')
parser.add_argument('--json-file', required=True)
'--package-version', default='0', help='Version of the package')
args = parser.parse_args()
return _build_manifest(args)
if __name__ == '__main__':