#! /usr/bin/env python3 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. assert __name__ == "__main__" r""" To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come before python3 in your path. Upstream: https://chromium.googlesource.com/angle/angle Our repo: https://github.com/mozilla/angle It has branches like 'firefox-60' which is the branch we use for pulling into Gecko with this script. This script leaves a record of the merge-base and cherry-picks that we pull into Gecko. (gfx/angle/cherries.log) ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/ An easy choice is to grab Chrome's Beta's ANGLE branch. ## Usage Prepare your env: * If in `cmd`: `export PATH="$PATH:/path/to/depot_tools"` * If in `powershell`: `$env:Path += ";C:\path\to\depot_tools"` If this is a new repo, don't forget: ~~~ # In the angle repo: ./scripts/bootstrap.py gclient sync ~~~ Update: (in the angle repo) ~~~ # In the angle repo: /path/to/gecko/gfx/angle/update-angle.py origin git push moz # Push the firefox-XX branch to github.com/mozilla/angle ~~~~ """ import json import os import pathlib import re import shutil import subprocess import sys from typing import * # mypy annotations REPO_DIR = pathlib.Path.cwd() GECKO_ANGLE_DIR = pathlib.Path(__file__).parent OUT_DIR = pathlib.Path("out") COMMON_HEADER = [ "# Generated by update-angle.py", "", 'include("../../moz.build.common")', ] ROOTS = ["//:translator", "//:libEGL", "//:libGLESv2"] CHECK_ONLY = False args = sys.argv[1:] while True: arg = args.pop(0) if arg == "--check": CHECK_ONLY = True continue args.insert(0, arg) break GN_ENV = dict(os.environ) GN_ENV["DEPOT_TOOLS_WIN_TOOLCHAIN"] = "0" (GIT_REMOTE,) = args # Not always 'origin'! # ------------------------------------------------------------------------------ def run_checked(*args, **kwargs): print(" ", args) sys.stdout.flush() return subprocess.run(args, check=True, **kwargs) def sorted_items(x): for k in sorted(x.keys()): yield (k, x[k]) def collapse_dotdots(path): split = path.split("/") ret = [] for x in split: if x == ".." and ret: ret.pop() continue ret.append(x) continue return "/".join(ret) def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]): visited_keys: Set[str] = set() def recurse(key): if key in visited_keys: return visited_keys.add(key) t = pre_recurse_func(key) try: (next_keys, post_recurse_func) = t except ValueError: (next_keys,) = t post_recurse_func = None for x in next_keys: recurse(x) if post_recurse_func: post_recurse_func(key) return for x in root_keys: recurse(x) return # ------------------------------------------------------------------------------ print("Importing graph") # shutil.rmtree(str(OUT_DIR), True) OUT_DIR.mkdir(exist_ok=True) GN_ARGS = b""" # Build arguments go here. # See "gn args --list" for available build arguments. is_clang = true is_debug = false angle_build_all = false angle_enable_abseil = false angle_enable_apple_translator_workarounds = true angle_enable_essl = true angle_enable_gl = false angle_enable_gl_desktop_frontend = false angle_enable_glsl = true angle_enable_null = false angle_enable_share_context_lock = true angle_enable_vulkan = false angle_has_astc_encoder = false use_custom_libcxx = false """[ 1: ] args_gn_path = OUT_DIR / "args.gn" args_gn_path.write_bytes(GN_ARGS) try: run_checked("gn", "gen", str(OUT_DIR), shell=True, env=GN_ENV) except subprocess.CalledProcessError: sys.stderr.buffer.write(b"`gn` failed. Is depot_tools in your PATH?\n") exit(1) p = run_checked( "python3", "scripts/export_targets.py", str(OUT_DIR), *ROOTS, stdout=subprocess.PIPE, shell=True, env=GN_ENV, ) # - print("\nProcessing graph") libraries = json.loads(p.stdout.decode()) # - # HACKHACKHACK: Inject linux/mac sources instead of trying to merge graphs of different # platforms. # descs["//:angle_common"]["sources"] += EXTRA_ANGLE_COMMON_SOURCES = [ "//src/common/system_utils_apple.cpp", "//src/common/system_utils_linux.cpp", "//src/common/system_utils_mac.cpp", "//src/common/system_utils_posix.cpp", ] angle_common = libraries["//:angle_common"] angle_common["sources"] += EXTRA_ANGLE_COMMON_SOURCES angle_common["sources"] = sorted(angle_common["sources"]) # - # Reuse our own zlib del libraries["//third_party/zlib:zlib"] # - if CHECK_ONLY: print("\n--check complete.") exit(0) # ------------------------------------------------------------------------------ # Output to moz.builds import vendor_from_git print("") vendor_from_git.record_cherry_picks(GECKO_ANGLE_DIR, GIT_REMOTE) # -- def sortedi(x): return sorted(x, key=str.lower) def append_arr(dest, name, vals, indent=0): if not vals: return dest.append("{}{} += [".format(" " * 4 * indent, name)) for x in sortedi(vals): dest.append('{}"{}",'.format(" " * 4 * (indent + 1), x)) dest.append("{}]".format(" " * 4 * indent)) dest.append("") return REGISTERED_DEFINES = { "ADLER32_SIMD_SSSE3": False, "ANGLE_CAPTURE_ENABLED": True, "ANGLE_DISABLE_POOL_ALLOC": True, "ANGLE_EGL_LIBRARY_NAME": False, "ANGLE_ENABLE_APPLE_WORKAROUNDS": True, "ANGLE_ENABLE_D3D11": True, "ANGLE_ENABLE_D3D11_COMPOSITOR_NATIVE_WINDOW": True, "ANGLE_ENABLE_D3D9": True, "ANGLE_ENABLE_DEBUG_ANNOTATIONS": True, "ANGLE_ENABLE_NULL": False, "ANGLE_ENABLE_OPENGL": False, "ANGLE_ENABLE_OPENGL_NULL": False, "ANGLE_ENABLE_ESSL": True, "ANGLE_ENABLE_GLSL": True, "ANGLE_ENABLE_HLSL": True, "ANGLE_ENABLE_SHARE_CONTEXT_LOCK": True, "ANGLE_GENERATE_SHADER_DEBUG_INFO": True, "ANGLE_GLESV2_LIBRARY_NAME": True, "ANGLE_HAS_ASTCENC": True, "ANGLE_HAS_VULKAN_SYSTEM_INFO": False, "ANGLE_IS_64_BIT_CPU": False, "ANGLE_IS_WIN": False, "ANGLE_PRELOADED_D3DCOMPILER_MODULE_NAMES": False, "ANGLE_SHARED_LIBVULKAN": True, "ANGLE_USE_CUSTOM_LIBVULKAN": True, "ANGLE_USE_EGL_LOADER": True, "ANGLE_VK_LAYERS_DIR": True, "ANGLE_VK_MOCK_ICD_JSON": True, "ANGLE_VMA_VERSION": True, "ASTCENC_DECOMPRESS_ONLY": True, "CERT_CHAIN_PARA_HAS_EXTRA_FIELDS": False, "CHROMIUM_BUILD": False, "COMPONENT_BUILD": False, "CRC32_SIMD_SSE42_PCLMUL": False, "DEFLATE_FILL_WINDOW_SSE2": False, "DYNAMIC_ANNOTATIONS_ENABLED": True, "EGL_EGL_PROTOTYPES": True, "EGL_EGLEXT_PROTOTYPES": True, "EGLAPI": True, "FIELDTRIAL_TESTING_ENABLED": False, "FULL_SAFE_BROWSING": False, "GL_API": True, "GL_APICALL": True, "GL_GLES_PROTOTYPES": True, "GL_GLEXT_PROTOTYPES": True, "GPU_INFO_USE_SETUPAPI": True, "INFLATE_CHUNK_READ_64LE": False, "INFLATE_CHUNK_SIMD_SSE2": False, "LIBANGLE_IMPLEMENTATION": True, "LIBEGL_IMPLEMENTATION": True, "LIBGLESV2_IMPLEMENTATION": True, "NOMINMAX": True, "NO_TCMALLOC": False, # Else: gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/win32/NativeWindow11Win32.cpp(89): error C2787: 'IDCompositionDevice': no GUID has been associated with this object "NTDDI_VERSION": False, "PSAPI_VERSION": False, "SAFE_BROWSING_CSD": False, "SAFE_BROWSING_DB_LOCAL": False, "UNICODE": True, "USE_AURA": False, "V8_DEPRECATION_WARNINGS": False, "VK_USE_PLATFORM_WIN32_KHR": False, "WIN32": False, "WIN32_LEAN_AND_MEAN": False, "WINAPI_FAMILY": False, "WINVER": True, # Otherwise: # gfx/angle/targets/libANGLE # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.cpp:10: # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src\libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.h:17: # C:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt\Windows.ui.composition.interop.h(103,20): error: unknown type name 'POINTER_INFO' # _In_ const POINTER_INFO& pointerInfo # ^ "WTF_USE_DYNAMIC_ANNOTATIONS": False, "X86_WINDOWS": False, "ZLIB_IMPLEMENTATION": True, "_ATL_NO_OPENGL": True, "_CRT_NONSTDC_NO_DEPRECATE": True, "_CRT_NONSTDC_NO_WARNINGS": True, "_CRT_RAND_S": True, "_CRT_SECURE_NO_DEPRECATE": True, "_DEBUG": False, "_HAS_EXCEPTIONS": True, "_HAS_ITERATOR_DEBUGGING": False, "_SCL_SECURE_NO_DEPRECATE": True, "_SECURE_ATL": True, "_UNICODE": True, "_USING_V110_SDK71_": False, "_WIN32_WINNT": False, "_WINDOWS": False, "_WINSOCK_DEPRECATED_NO_WARNINGS": True, "__STD_C": False, # clang specific "CR_CLANG_REVISION": True, "NDEBUG": False, "NVALGRIND": False, "_HAS_NODISCARD": False, } # - print("\nRun actions") required_files: Set[str] = set() required_files.add("//LICENSE") run_checked("ninja", "-C", str(OUT_DIR), ":angle_commit_id", shell=True) required_files.add("//out/gen/angle/angle_commit.h") # - # Export our targets print("\nExport targets") # Clear our dest directories targets_dir = pathlib.Path(GECKO_ANGLE_DIR, "targets") checkout_dir = pathlib.Path(GECKO_ANGLE_DIR, "checkout") shutil.rmtree(targets_dir, True) shutil.rmtree(checkout_dir, True) targets_dir.mkdir(exist_ok=True) checkout_dir.mkdir(exist_ok=True) # - RE_TARGET_NAME = re.compile("//(.*):(.+)") def export_target(target_full_name) -> Set[str]: # print(' ', target_full_name) descs = libraries desc = descs[target_name] flat = desc m = RE_TARGET_NAME.match(target_name) assert m, target_name name = m.group(2) required_files: Set[str] = set(flat["sources"]) # Create our manifest lines target_dir = targets_dir / name target_dir.mkdir(exist_ok=True) lines = list(COMMON_HEADER) lines.append("") for x in sorted(set(desc["defines"])): try: (k, v) = x.split("=", 1) if v.startswith('"'): v = f"'{v}'" else: v = f'"{v}"' except ValueError: (k, v) = (x, "True") line = f'DEFINES["{k}"] = {v}' try: if REGISTERED_DEFINES[k] == False: line = "# " + line except KeyError: print(f"[{name}] Unrecognized define: {k}") line = "# Unrecognized: " + line lines.append(line) lines.append("") cxxflags = set(desc["cflags"] + desc["cflags_cc"]) def fixup_paths(listt): for x in set(listt): assert x.startswith("//"), x yield "../../checkout/" + x[2:] sources_by_config: Dict[str, List[str]] = {} extras: Dict[str, str] = dict() for x in fixup_paths(flat["sources"]): # print(' '*5, x) (b, e) = x.rsplit(".", 1) if e in ["h", "hpp", "y", "l", "inc", "inl"]: continue elif e in ["cpp", "cc", "c"]: if b.endswith("_win") or b.endswith("_win32"): config = 'CONFIG["OS_ARCH"] == "WINNT"' elif b.endswith("_linux"): # Include these on BSDs too. config = 'CONFIG["OS_ARCH"] not in ("Darwin", "WINNT")' elif b.endswith("_apple") or b.endswith("_mac"): config = 'CONFIG["OS_ARCH"] == "Darwin"' elif b.endswith("_posix"): config = 'CONFIG["OS_ARCH"] != "WINNT"' else: config = "" # None can't compare against str. sources_by_config.setdefault(config, []).append(x) continue elif e == "rc": assert "RCFILE" not in extras, (target_name, extras["RCFILE"], x) extras["RCFILE"] = f'"{x}"' continue elif e == "def": assert "DEFFILE" not in extras, (target_name, extras["DEFFILE"], x) extras["DEFFILE"] = f'"{x}"' continue else: assert False, ("Unhandled ext:", x) ldflags = set(desc["ldflags"]) DEF_PREFIX = "/DEF:" for x in set(ldflags): if x.startswith(DEF_PREFIX): def_path = x[len(DEF_PREFIX) :] required_files.add(def_path) assert "DEFFILE" not in extras ldflags.remove(x) def_path = str(OUT_DIR) + "/" + def_path def_path = "//" + collapse_dotdots(def_path) def_rel_path = list(fixup_paths([def_path]))[0] extras["DEFFILE"] = '"{}"'.format(def_rel_path) elif x.startswith("/PDBSourcePath:"): ldflags.remove(x) os_libs = list(map(lambda x: x[: -len(".lib")], set(desc.get("libs", [])))) def append_arr_commented(dest, name, src): lines = [] append_arr(lines, name, src) def comment(x): if x: x = "# " + x return x lines = map(comment, lines) dest += lines append_arr(lines, "LOCAL_INCLUDES", fixup_paths(desc["include_dirs"])) append_arr_commented(lines, "CXXFLAGS", cxxflags) for config, v in sorted_items(sources_by_config): indent = 0 if config: lines.append("if {}:".format(config)) indent = 1 append_arr(lines, "SOURCES", v, indent=indent) dep_libs: Set[str] = set() for dep_full_name in set(flat["dep_libs"]): assert dep_full_name.startswith("//"), dep_name (_, dep_name) = dep_full_name.split(":") dep_libs.add(dep_name) dep_dirs = set(dep_libs) dep_dirs.discard("zlib") # Those directories are added by gfx/angle/moz.build. already_added_dirs = [ "angle_common", "translator", "libEGL", "libGLESv2", ] append_arr(lines, "USE_LIBS", dep_libs) append_arr( lines, "DIRS", ["../" + x for x in dep_dirs if x not in already_added_dirs] ) append_arr(lines, "OS_LIBS", os_libs) append_arr_commented(lines, "LDFLAGS", ldflags) for k, v in sorted(extras.items()): lines.append("{} = {}".format(k, v)) lib_type = desc["type"] if lib_type == "shared_library": lines.append(f'GeckoSharedLibrary("{name}")') elif lib_type == "static_library": lines.append(f'Library("{name}")') else: assert False, lib_type lines.append("") # EOF newline # Write it out mozbuild = target_dir / "moz.build" print(" ", " ", f"Writing {mozbuild}") data = b"\n".join((x.encode() for x in lines)) mozbuild.write_bytes(data) return required_files # - for target_name in libraries: reqs = export_target(target_name) required_files |= reqs # Copy all the files print("\nMigrate required files") i = 0 for x in required_files: i += 1 sys.stdout.write(f"\r Copying {i}/{len(required_files)}") sys.stdout.flush() assert x.startswith("//"), x x = x[2:] src = REPO_DIR / x dest = checkout_dir / x dest.parent.mkdir(parents=True, exist_ok=True) data = src.read_bytes() data = data.replace(b"\r\n", b"\n") dest.write_bytes(data) print("\n\nDone")