blob: 6d41d3f9995d5dfb9df8afe8e8496109ed982a92 [file] [log] [blame]
# Copyright (c) 2016 The Chromium Embedded Framework Authors. All rights
# reserved. Use of this source code is governed by a BSD-style license that
# can be found in the LICENSE file.
from __future__ import absolute_import
from __future__ import print_function
import datetime
import json
import os
import re
import sys
if sys.version_info.major == 2:
from urllib2 import urlopen
from urllib.request import urlopen
# Class used to build the cefbuilds JSON file. See
# for example usage. See for unit tests.
# Example JSON structure:
# {
# "linux32": {
# "versions": [
# {
# "cef_version": "3.2704.1414.g185cd6c",
# "chromium_version": "51.0.2704.47"
# "files": [
# {
# "last_modified": "2016-05-18T22:42:14.066Z"
# "name": "cef_binary_3.2704.1414.g185cd6c_linux32.tar.bz2",
# "sha1": "47c5cfea43912a1d1771f343de35b205f388415f"
# "size": "48549450",
# "type": "standard",
# }, ...
# ],
# }, ...
# ]
# }, ...
# }
# Notes:
# - "files" in a given version will be sorted from newest to oldest based on the
# "last_modified" value.
# - "versions" in a given platform will be sorted from newest to oldest based on
# the "last_modified" value of the first (newest) "file" sub-value.
# - There will be at most one record at the "files" level for each "type".
# This date format intentionally matches the format used in Artifactory
# directory listings.
_CEF_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
def parse_date(date):
return datetime.datetime.strptime(date, _CEF_DATE_FORMAT)
def format_date(date):
return date.strftime(_CEF_DATE_FORMAT)
# Helpers to format datetime values on JSON read/write.
def cef_from_json(json_object):
if 'last_modified' in json_object:
json_object['last_modified'] = parse_date(json_object['last_modified'])
return json_object
class cef_json_encoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, datetime.datetime):
return format_date(o)
return o
_chromium_version_regex = '[1-9]{1}[0-9]{1,2}\.0\.[1-9]{1}[0-9]{2,4}\.(0|[1-9]{1}[0-9]{0,2})'
_cef_hash_regex = 'g[0-9a-f]{7}'
_cef_number_regex = '[0-9]{1,5}\.[0-9]{1,5}\.[0-9]{1,5}'
# Example: 3.2704.1414.g185cd6c
_cef_old_version_regex = _cef_number_regex + '\.' + _cef_hash_regex
# Example: 74.0.1+g62d140e+chromium-74.0.3729.6
_cef_version_release_regex = _cef_number_regex + '\+' + _cef_hash_regex + '\+chromium\-' + _chromium_version_regex
# Example: 74.0.0-master.1920+g725ed88+chromium-74.0.3729.0
_cef_version_dev_regex = _cef_number_regex + '\-\w+\.[0-9]{1,7}\+' + _cef_hash_regex + '\+chromium\-' + _chromium_version_regex
class cef_json_builder:
""" Class used to build the cefbuilds JSON file. """
def __init__(self, prettyprint=False, silent=True):
""" Create a new cef_json_builder object. """
self._prettyprint = prettyprint
self._silent = silent
self._fatalerrors = False
def get_platforms():
""" Returns the list of supported platforms. """
return ('linux32', 'linux64', 'linuxarm', 'linuxarm64', 'macosx64',
'windows32', 'windows64')
def get_distrib_types():
""" Returns the list of supported distribution types. """
return ('standard', 'minimal', 'client', 'release_symbols', 'debug_symbols')
def is_valid_version(version):
""" Returns true if the specified CEF version is fully qualified and valid. """
if version is None:
return False
return bool(re.compile('^' + _cef_old_version_regex + '$').match(version)) \
or bool(re.compile('^' + _cef_version_release_regex + '$').match(version)) \
or bool(re.compile('^' + _cef_version_dev_regex + '$').match(version))
def is_valid_chromium_version(version):
""" Returns true if the specified Chromium version is fully qualified and valid. """
if version is None:
return False
return version == 'master' or \
bool(re.compile('^' + _chromium_version_regex + '$').match(version))
def get_file_name(version, platform, type):
""" Returns the expected distribution file name excluding extension based on
the input parameters. """
if type != 'standard':
type_str = '_' + type
type_str = ''
return 'cef_binary_%s_%s%s' % (version, platform, type_str)
def clear(self):
""" Clear the contents of this object. """
self._data = {}
for platform in self.get_platforms():
self._data[platform] = {'versions': []}
self._versions = {}
self._queryct = 0
def __repr__(self):
# Return a string representation of this object.
if self._prettyprint:
return json.dumps(
separators=(',', ': '))
return json.dumps(self._data, cls=cef_json_encoder, sort_keys=True)
def _print(self, msg):
if self._fatalerrors:
raise Exception(msg)
if not self._silent:
def get_query_count(self):
""" Returns the number of queries sent while building. """
return self._queryct
def _query_chromium_version(self, cef_version):
""" Try to remotely query the Chromium version for old-style CEF version
numbers. """
chromium_version = 'master'
git_hash = cef_version[-7:]
query_url = '' % git_hash
self._queryct = self._queryct + 1
if not self._silent:
print('Reading %s' % query_url)
# Read the remote URL contents.
handle = urlopen(query_url)
compat_value =
# Parse the contents.
config = eval(compat_value, {'__builtins__': None}, None)
if not 'chromium_checkout' in config:
raise Exception('Unexpected contents')
val = config['chromium_checkout']
if val.find('refs/tags/') == 0:
chromium_version = val[10:]
except Exception as e:
print('Failed to read Chromium version information')
return chromium_version
def set_chromium_version(self, cef_version, chromium_version=None):
""" Set the matching Chromium version. If the specified Chromium version is
invalid then it will be queried remotely. """
if not self.is_valid_version(cef_version):
raise Exception('Invalid CEF version: %s' % cef_version)
if not self.is_valid_chromium_version(chromium_version):
if cef_version in self._versions:
# Keep the Chromium version that we already know about.
return self._versions[cef_version]
if cef_version.find('+chromium') > 0:
# New-style CEF version numbers include the Chromium version number.
# Example: 74.0.1+g62d140e+chromium-74.0.3729.6
chromium_version = cef_version[cef_version.rfind('-') + 1:]
chromium_version = self._query_chromium_version(cef_version)
if not self.is_valid_chromium_version(chromium_version):
raise Exception('Invalid Chromium version: %s' % chromium_version)
self._versions[cef_version] = chromium_version
return chromium_version
def get_chromium_version(self, cef_version):
""" Return the matching Chromium version. If not currently known it will
be parsed from the CEF version or queried remotely. """
if cef_version in self._versions:
return self._versions[cef_version]
# Identify the Chromium version.
return self.set_chromium_version(cef_version)
def has_chromium_version(self, cef_version):
""" Return True if a matching Chromium version is known. """
return cef_version in self._versions
def load(self, json_string, fatalerrors=True):
""" Load new JSON into this object. Any existing contents will be cleared.
If |fatalerrors| is True then any errors while loading the JSON file
will cause an Exception to be thrown. Otherwise, malformed entries will
will be discarded. Unrecognized keys will always be discarded silently.
self._fatalerrors = fatalerrors
new_data = json.JSONDecoder(object_hook=cef_from_json).decode(json_string)
# Validate the new data's structure.
for platform in self._data.keys():
if not platform in new_data:
self._print('load: Platform %s not found' % platform)
if not 'versions' in new_data[platform]:
self._print('load: Missing platform key(s) for %s' % platform)
valid_versions = []
for version in new_data[platform]['versions']:
if not 'cef_version' in version or \
not 'chromium_version' in version or \
not 'files' in version:
self._print('load: Missing version key(s) for %s' % platform)
valid_files = []
found_types = []
for file in version['files']:
if not 'type' in file or \
not 'name' in file or \
not 'size' in file or \
not 'last_modified' in file or \
not 'sha1' in file:
self._print('load: Missing file key(s) for %s %s' %
(platform, version['cef_version']))
(expected_platform, expected_version,
expected_type) = self._parse_name(file['name'])
if expected_platform != platform or \
expected_version != version['cef_version'] or \
expected_type != file['type']:
self._print('load: File name/attribute mismatch for %s %s %s' %
(platform, version['cef_version'], file['name']))
self._validate_args(platform, version['cef_version'], file['type'],
file['size'], file['last_modified'], file['sha1'])
if file['type'] in found_types:
self._print('load: Duplicate %s type for %s %s' %
(file['type'], platform, version['cef_version']))
'type': file['type'],
'name': file['name'],
'size': file['size'],
'last_modified': file['last_modified'],
'sha1': file['sha1'],
if len(valid_files) > 0:
if len(valid_versions) > 0:
self._data[platform]['versions'] = valid_versions
self._fatalerrors = False
def _sort_versions(self):
# Sort version records by first (newest) file last_modified value.
for platform in self._data.keys():
for i in range(0, len(self._data[platform]['versions'])):
self._data[platform]['versions'] = \
key=lambda k: k['files'][0]['last_modified'],
def _sort_files(files):
# Sort file records by last_modified.
return sorted(files, key=lambda k: k['last_modified'], reverse=True)
def _parse_name(name):
# Remove file extension.
name_no_ext = os.path.splitext(name)[0]
if name_no_ext[-4:] == '.tar':
name_no_ext = name_no_ext[:-4]
name_parts = name_no_ext.split('_')
if len(
name_parts) < 4 or name_parts[0] != 'cef' or name_parts[1] != 'binary':
raise Exception('Invalid filename: %s' % name)
# Remove 'cef' and 'binary'.
del name_parts[0]
del name_parts[0]
type = None
# Might be '<version>_<platform>_[debug|release]_symbols'.
if name_parts[-1] == 'symbols':
del name_parts[-1]
if name_parts[-1] == 'debug' or name_parts[-1] == 'release':
type = name_parts[-1] + '_symbols'
del name_parts[-1]
# Might be '<version>_<platform>_minimal'.
if name_parts[-1] == 'minimal':
type = 'minimal'
del name_parts[-1]
# Might be '<version>_<platform>_client'.
if name_parts[-1] == 'client':
type = 'client'
del name_parts[-1]
# Remainder must be '<version>_<platform>'.
if len(name_parts) != 2:
raise Exception('Invalid filename: %s' % name)
if type is None:
type = 'standard'
version = name_parts[0]
platform = name_parts[1]
return [platform, version, type]
def _validate_args(platform, version, type, size, last_modified, sha1):
# Validate input arguments.
if not platform in cef_json_builder.get_platforms():
raise Exception('Unsupported platform: %s' % platform)
if not cef_json_builder.is_valid_version(version):
raise Exception('Invalid version: %s' % version)
if not type in cef_json_builder.get_distrib_types():
raise Exception('Unsupported distribution type: %s' % type)
if int(size) <= 0:
raise Exception('Invalid size: %s' % size)
if not isinstance(last_modified, datetime.datetime):
# datetime will throw a ValueException if it doesn't parse.
if not re.compile('^[0-9a-f]{40}$').match(sha1):
raise Exception('Invalid sha1: %s' % sha1)
def add_file(self, name, size, last_modified, sha1):
""" Add a file record with the specified attributes. Returns True if the
file is added or False if a file with the same |name| and |sha1|
already exists. """
# Parse the file name.
(platform, version, type) = self._parse_name(name)
if not isinstance(size, int):
size = int(size)
if not isinstance(last_modified, datetime.datetime):
last_modified = parse_date(last_modified)
# Validate arguments.
self._validate_args(platform, version, type, size, last_modified, sha1)
# Find the existing version record.
version_idx = -1
for i in range(0, len(self._data[platform]['versions'])):
if self._data[platform]['versions'][i]['cef_version'] == version:
# Check the version record.
self._print('add_file: Check %s %s' % (platform, version))
version_idx = i
if version_idx == -1:
# Add a new version record.
self._print('add_file: Add %s %s' % (platform, version))
'cef_version': version,
'chromium_version': self.get_chromium_version(version),
'files': []
version_idx = len(self._data[platform]['versions']) - 1
# Find the existing file record with matching type.
file_changed = True
for i in range(0,
if self._data[platform]['versions'][version_idx]['files'][i][
'type'] == type:
existing_sha1 = self._data[platform]['versions'][version_idx]['files'][
if existing_sha1 != sha1:
# Remove the existing file record.
self._print(' Remove %s %s' % (name, existing_sha1))
del self._data[platform]['versions'][version_idx]['files'][i]
file_changed = False
if file_changed:
# Add a new file record.
self._print(' Add %s %s' % (name, sha1))
'type': type,
'name': name,
'size': size,
'last_modified': last_modified,
'sha1': sha1
# Sort file records by last_modified.
# This is necessary for _sort_versions() to function correctly.
self._data[platform]['versions'][version_idx]['files'] = \
return file_changed
def get_files(self, platform=None, version=None, type=None):
""" Return the files that match the input parameters.
All parameters are optional. Version will do partial matching. """
results = []
if platform is None:
platforms = self._data.keys()
platforms = [platform]
for platform in platforms:
for version_obj in self._data[platform]['versions']:
if version is None or version_obj['cef_version'].find(version) == 0:
for file_obj in version_obj['files']:
if type is None or type == file_obj['type']:
result_obj = file_obj
# Add additional metadata.
result_obj['platform'] = platform
result_obj['cef_version'] = version_obj['cef_version']
result_obj['chromium_version'] = version_obj['chromium_version']
return results
def get_versions(self, platform):
""" Return all versions for the specified |platform|. """
return self._data[platform]['versions']