From bd7505b5260231c7a3d14240bd4da801338a4831 Mon Sep 17 00:00:00 2001 From: Mauricio Galindo Date: Mon, 19 Sep 2022 14:38:23 -0700 Subject: [PATCH 01/29] [SNAP FORK] avoid processing resources if none are explicitly specified --- rules/resources.bzl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rules/resources.bzl b/rules/resources.bzl index a92d69635..c787225a5 100644 --- a/rules/resources.bzl +++ b/rules/resources.bzl @@ -1341,8 +1341,7 @@ def _process_starlark( manifest or resource_files or defined_assets or - defined_assets_dir or - exports_manifest, + defined_assets_dir, ) # TODO(djwhang): Clean up the difference between neverlink the attribute used From ef0533ee7a901867291f522a14ea7d424802e811 Mon Sep 17 00:00:00 2001 From: Mauricio Galindo Date: Mon, 19 Sep 2022 14:40:49 -0700 Subject: [PATCH 02/29] [SNAP FORK] fix aar_import rule - acls dont work with external dependencies (i.e @foo//) - Propagate StarlarkAndroidResourcesInfo otherwise starlark rules wont be able to see those resources. - dont propagate r javas as deps. This will cause resource masking issues if two aars share the same package --- rules/aar_import/impl.bzl | 7 +++---- rules/aar_import/rule.bzl | 5 +++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/rules/aar_import/impl.bzl b/rules/aar_import/impl.bzl index c80d6182d..553ef1e51 100644 --- a/rules/aar_import/impl.bzl +++ b/rules/aar_import/impl.bzl @@ -154,7 +154,7 @@ def _process_resources( deps = ctx.attr.deps, exports = ctx.attr.exports, exports_manifest = getattr(ctx.attr, "exports_manifest", True), - propagate_resources = _acls.in_aar_propagate_resources(str(ctx.label)), + propagate_resources = True, # Tool and Processing related inputs aapt = _get_android_toolchain(ctx).aapt2.files_to_run, @@ -350,7 +350,7 @@ def _process_jars( ), source_jar = source_jar, neverlink = False, - deps = r_java_info + java_infos, # TODO(djwhang): Exports are not deps. + deps = java_infos, # TODO(djwhang): Exports are not deps. exports = (r_java_info if _acls.in_aar_import_exports_r_java(str(ctx.label)) else []) + java_infos, # TODO(djwhang): Deps are not exports. @@ -501,8 +501,7 @@ def impl(ctx): r_java = resources_ctx.r_java, exports = _utils.collect_providers(JavaInfo, ctx.attr.exports), enable_desugar_java8 = ctx.fragments.android.desugar_java8, - enable_imports_deps_check = - _acls.in_aar_import_deps_checker(str(ctx.label)), + enable_imports_deps_check = False, aar_embedded_jars_extractor_tool = _get_android_toolchain(ctx).aar_embedded_jars_extractor.files_to_run, bootclasspath = diff --git a/rules/aar_import/rule.bzl b/rules/aar_import/rule.bzl index df5c6ebcd..5e6302412 100644 --- a/rules/aar_import/rule.bzl +++ b/rules/aar_import/rule.bzl @@ -16,6 +16,10 @@ load(":attrs.bzl", _ATTRS = "ATTRS") load(":impl.bzl", _impl = "impl") +load( + "//rules:providers.bzl", + "StarlarkAndroidResourcesInfo", +) RULE_DOC = """ #### Examples @@ -49,6 +53,7 @@ aar_import = rule( AndroidLibraryResourceClassJarProvider, AndroidNativeLibsInfo, JavaInfo, + StarlarkAndroidResourcesInfo, ], toolchains = [ "//toolchains/android:toolchain_type", From b2057a446696644267afbc0fd145c4952643f2d6 Mon Sep 17 00:00:00 2001 From: Mauricio Galindo Date: Mon, 19 Sep 2022 14:49:32 -0700 Subject: [PATCH 03/29] [SNAP FORK] enable support multiplex workers in busybox actions --- rules/busybox.bzl | 51 ++++++++++++++++++++++++++++++++++++++++------- rules/java.bzl | 12 ++++++++++- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/rules/busybox.bzl b/rules/busybox.bzl index bcbb88053..d8156ca4a 100644 --- a/rules/busybox.bzl +++ b/rules/busybox.bzl @@ -283,7 +283,8 @@ def _package( transitive_input_files = [] args = ctx.actions.args() - args.use_param_file("@%s") + args.use_param_file("@%s", use_always=True) + args.set_param_file_format("multiline") args.add("--tool", "AAPT2_PACKAGE") args.add("--") args.add("--aapt2", aapt.executable) @@ -407,6 +408,8 @@ def _package( outputs = output_files, mnemonic = "PackageAndroidResources", progress_message = "Packaging Android Resources in %s" % ctx.label, + supports_workers = True, + supports_multiplex_workers = True, ) def _parse( @@ -427,7 +430,8 @@ def _parse( host_javabase: Target. The host javabase. """ args = ctx.actions.args() - args.use_param_file("@%s") + args.use_param_file("@%s", use_always=True) + args.set_param_file_format("multiline") args.add("--tool", "PARSE") args.add("--") args.add( @@ -448,6 +452,8 @@ def _parse( outputs = [out_symbols], mnemonic = "ParseAndroidResources", progress_message = "Parsing Android Resources in %s" % out_symbols.short_path, + supports_workers = True, + supports_multiplex_workers = True, ) def _make_merge_assets_flags(resources_node): @@ -493,7 +499,8 @@ def _merge_assets( host_javabase: Target. The host javabase. """ args = ctx.actions.args() - args.use_param_file("@%s") + args.use_param_file("@%s", use_always=True) + args.set_param_file_format("multiline") args.add("--tool", "MERGE_ASSETS") args.add("--") args.add("--assetsOutput", out_assets_zip) @@ -523,6 +530,7 @@ def _merge_assets( ctx = ctx, host_javabase = host_javabase, executable = busybox, + tools = [busybox], arguments = [args], inputs = depset( assets + [symbols], @@ -532,6 +540,8 @@ def _merge_assets( mnemonic = "MergeAndroidAssets", progress_message = "Merging Android Assets in %s" % out_assets_zip.short_path, + supports_workers = True, + supports_multiplex_workers = True, ) def _validate_and_link( @@ -573,7 +583,8 @@ def _validate_and_link( # Retrieves the list of files at runtime when a directory is passed. args = ctx.actions.args() - args.use_param_file("@%s") + args.use_param_file("@%s", use_always=True) + args.set_param_file_format("multiline") args.add("--tool", "LINK_STATIC_LIBRARY") args.add("--") args.add("--aapt2", aapt.executable) @@ -615,6 +626,8 @@ def _validate_and_link( mnemonic = "LinkAndroidResources", progress_message = "Linking Android Resources in " + out_file.short_path, + supports_workers = True, + supports_multiplex_workers = True, ) def _compile( @@ -644,7 +657,8 @@ def _compile( # Retrieves the list of files at runtime when a directory is passed. args = ctx.actions.args() - args.use_param_file("@%s") + args.use_param_file("@%s", use_always=True) + args.set_param_file_format("multiline") args.add("--tool", "COMPILE_LIBRARY_RESOURCES") args.add("--") args.add("--aapt2", aapt.executable) @@ -668,6 +682,8 @@ def _compile( outputs = [out_file], mnemonic = "CompileAndroidResources", progress_message = "Compiling Android Resources in %s" % out_file.short_path, + supports_workers = True, + supports_multiplex_workers = True, ) def _make_merge_compiled_flags(resources_node_info): @@ -720,7 +736,8 @@ def _merge_compiled( transitive_input_files = [] args = ctx.actions.args() - args.use_param_file("@%s") + args.use_param_file("@%s", use_always=True) + args.set_param_file_format("multiline") args.add("--tool", "MERGE_COMPILED") args.add("--") args.add("--classJarOutput", out_class_jar) @@ -763,12 +780,15 @@ def _merge_compiled( ctx = ctx, host_javabase = host_javabase, executable = busybox, + tools = [busybox], arguments = [args], inputs = depset(input_files, transitive = transitive_input_files), outputs = output_files, mnemonic = "StarlarkMergeCompiledAndroidResources", progress_message = "Merging compiled Android Resources in " + out_class_jar.short_path, + supports_workers = True, + supports_multiplex_workers = True, ) def _escape_mv(s): @@ -829,6 +849,7 @@ def _merge_manifests( # Args for busybox args = ctx.actions.args() args.use_param_file("@%s", use_always = True) + args.set_param_file_format("multiline") args.add("--tool", "MERGE_MANIFEST") args.add("--") if manifest: @@ -855,11 +876,14 @@ def _merge_manifests( ctx = ctx, host_javabase = host_javabase, executable = busybox, + tools = [busybox], arguments = [args], inputs = depset(directs, transitive = transitives), outputs = outputs, mnemonic = "MergeManifests", progress_message = "Merging Android Manifests in %s" % out_file.short_path, + supports_workers = True, + supports_multiplex_workers = True, ) def _process_databinding( @@ -894,6 +918,8 @@ def _process_databinding( res_dirs = _get_unique_res_dirs(resource_files) args = ctx.actions.args() + args.use_param_file("@%s", use_always=True) + args.set_param_file_format("multiline") args.add("--tool", "PROCESS_DATABINDING") args.add("--") args.add("--output_resource_directory", databinding_resources_dirname) @@ -910,6 +936,8 @@ def _process_databinding( outputs = [out_databinding_info] + out_databinding_processed_resources, mnemonic = "StarlarkProcessDatabinding", progress_message = "Processing data binding", + supports_workers = True, + supports_multiplex_workers = True, ) def _make_generate_binay_r_flags(resources_node): @@ -948,6 +976,8 @@ def _generate_binary_r( host_javabase: A Target. The host javabase. """ args = ctx.actions.args() + args.use_param_file("@%s", use_always=True) + args.set_param_file_format("multiline") args.add("--tool", "GENERATE_BINARY_R") args.add("--") args.add("--primaryRTxt", r_txt) @@ -967,17 +997,19 @@ def _generate_binary_r( # TODO(b/154003916): support transitive "--library transitive_r_txt_path,transitive_manifest_path" flags args.add("--classJarOutput", out_class_jar) args.add("--targetLabel", str(ctx.label)) - args.use_param_file("@%s") _java.run( ctx = ctx, host_javabase = host_javabase, executable = busybox, + tools = [busybox], arguments = [args], inputs = depset([r_txt, manifest], transitive = transitive_r_txts + transitive_manifests), outputs = [out_class_jar], mnemonic = "StarlarkRClassGenerator", progress_message = "Generating R classes", + supports_workers = True, + supports_multiplex_workers = True, ) def _make_aar( @@ -1012,6 +1044,8 @@ def _make_aar( when a resource conflict occurs. """ args = ctx.actions.args() + args.use_param_file("@%s", use_always=True) + args.set_param_file_format("multiline") args.add("--tool", "GENERATE_AAR") args.add("--") args.add( @@ -1035,6 +1069,7 @@ def _make_aar( ctx = ctx, host_javabase = host_javabase, executable = busybox, + tools = [busybox], arguments = [args], inputs = ( resource_files + @@ -1045,6 +1080,8 @@ def _make_aar( outputs = [out_aar], mnemonic = "StarlarkAARGenerator", progress_message = "Generating AAR package for %s" % ctx.label, + supports_workers = True, + supports_multiplex_workers = True, ) def _shrink( diff --git a/rules/java.bzl b/rules/java.bzl index 5eb74b0c1..64137c4d3 100644 --- a/rules/java.bzl +++ b/rules/java.bzl @@ -453,7 +453,17 @@ def _run( args["arguments"] = jvm_flags + [jar_args] + args.get("arguments", default = []) - ctx.actions.run(**args) + mnemonic = args.get("mnemonic") + supports_workers = args.pop("supports_workers", False) + supports_multiplex_workers = args.pop("supports_multiplex_workers", False) + execution_requirements = args.pop("execution_requirements", dict()) + if supports_workers: + execution_requirements["worker-key-mnemonic"] = mnemonic + execution_requirements["supports-workers"] = "1" + if supports_multiplex_workers: + execution_requirements["supports-multiplex-workers"] = "1" + + ctx.actions.run(execution_requirements = execution_requirements, **args) def _create_deploy_jar( ctx, From 80682853ae1d743f9d46e0b548a56921ed93c80d Mon Sep 17 00:00:00 2001 From: Mauricio Galindo Date: Mon, 19 Sep 2022 14:52:37 -0700 Subject: [PATCH 04/29] [SNAP FORK] remove missing dep --- rules/android_application/attrs.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/android_application/attrs.bzl b/rules/android_application/attrs.bzl index 55864c2a4..bf886e07e 100644 --- a/rules/android_application/attrs.bzl +++ b/rules/android_application/attrs.bzl @@ -48,7 +48,7 @@ ANDROID_APPLICATION_ATTRS = _attrs.add( ), _bundle_keystore_properties = attr.label( allow_single_file = True, - default = "//rules:bundle_keystore_properties.tmpl", + default = None, ), _feature_manifest_script = attr.label( allow_single_file = True, From 0e019238221595a8c878081b2d0742623d19023e Mon Sep 17 00:00:00 2001 From: Mauricio Galindo Date: Mon, 19 Sep 2022 15:07:58 -0700 Subject: [PATCH 05/29] [SNAP FORK] Always fix resources transitivity --- rules/android_library/impl.bzl | 2 +- rules/resources.bzl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rules/android_library/impl.bzl b/rules/android_library/impl.bzl index 6ccc5a5dd..aeaaf5d2e 100644 --- a/rules/android_library/impl.bzl +++ b/rules/android_library/impl.bzl @@ -175,7 +175,7 @@ def _process_resources(ctx, java_package, manifest_ctx, **unused_ctxs): enable_res_v3 = _flags.get(ctx).android_enable_res_v3, # TODO(b/144163743): remove fix_resource_transitivity, which was only added to emulate # misbehavior on the Java side. - fix_resource_transitivity = bool(ctx.attr.srcs), + fix_resource_transitivity = True, fix_export_exporting = acls.in_fix_export_exporting_rollout(str(ctx.label)), # Tool and Processing related inputs diff --git a/rules/resources.bzl b/rules/resources.bzl index c787225a5..0069ee4e0 100644 --- a/rules/resources.bzl +++ b/rules/resources.bzl @@ -1261,7 +1261,7 @@ def _process_starlark( neverlink = False, enable_data_binding = False, propagate_resources = True, - fix_resource_transitivity = False, + fix_resource_transitivity = True, aapt = None, android_jar = None, android_kit = None, @@ -1905,7 +1905,7 @@ def _process( enable_res_v3 = False, res_v3_dummy_manifest = None, res_v3_dummy_r_txt = None, - fix_resource_transitivity = False, + fix_resource_transitivity = True, fix_export_exporting = False, propagate_resources = True, zip_tool = None): From 6bbf6fb64912f267dc263c26fc32b9351674cfd3 Mon Sep 17 00:00:00 2001 From: Mauricio Galindo Date: Mon, 19 Sep 2022 16:05:37 -0700 Subject: [PATCH 06/29] [SNAP FORK] enable the resource processing pipeline for android_local_tests Robolectric tests need three main things: - java classes - the resource classes - an archive of compiled resources For the resources components this largely boils down to simply doing the same we do for android_binary, therefore the code borrows heavily for that implementation. --- .../android_local_test.bzl | 0 rules/android_local_test/resources.bzl | 72 +++++++++++++++++++ 2 files changed, 72 insertions(+) rename rules/{ => android_local_test}/android_local_test.bzl (100%) create mode 100644 rules/android_local_test/resources.bzl diff --git a/rules/android_local_test.bzl b/rules/android_local_test/android_local_test.bzl similarity index 100% rename from rules/android_local_test.bzl rename to rules/android_local_test/android_local_test.bzl diff --git a/rules/android_local_test/resources.bzl b/rules/android_local_test/resources.bzl new file mode 100644 index 000000000..ab0a44b03 --- /dev/null +++ b/rules/android_local_test/resources.bzl @@ -0,0 +1,72 @@ +# Copyright 2020 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation.""" + +load("//rules:acls.bzl", "acls") +load( + "//rules:processing_pipeline.bzl", + "ProviderInfo", + "processing_pipeline", +) +load("//rules:resources.bzl", _resources = "resources") +load("//rules:utils.bzl", "get_android_toolchain", "utils") + +def _process_manifest(ctx, **unused_ctxs): + manifest_ctx = _resources.bump_min_sdk( + ctx, + manifest = ctx.file.manifest, + floor = _resources.DEPOT_MIN_SDK_FLOOR if (_is_test_binary(ctx) and acls.in_enforce_min_sdk_floor_rollout(str(ctx.label))) else 0, + enforce_min_sdk_floor_tool = get_android_toolchain(ctx).enforce_min_sdk_floor_tool.files_to_run, + ) + + return ProviderInfo( + name = "manifest_ctx", + value = manifest_ctx, + ) + +def _process_resources_for_android_local_test(ctx, manifest_ctx, java_package, **unused_ctx): + packaged_resources_ctx = _resources.package( + ctx, + manifest = manifest_ctx.min_sdk_bumped_manifest, + manifest_values = utils.expand_make_vars(ctx, ctx.attr.manifest_values), + java_package = java_package, + use_legacy_manifest_merger = False, + should_throw_on_conflict = not acls.in_allow_resource_conflicts(str(ctx.label)), + deps = ctx.attr.deps + ctx.attr.associates, + aapt = get_android_toolchain(ctx).aapt2.files_to_run, + android_jar = ctx.attr._android_sdk[AndroidSdkInfo].android_jar, + busybox = get_android_toolchain(ctx).android_resources_busybox.files_to_run, + host_javabase = ctx.attr._host_javabase, + ) + return ProviderInfo( + name = "packaged_resources_ctx", + value = packaged_resources_ctx, + ) + +def _is_test_binary(ctx): + """Whether this android_binary target is a test binary. + + Args: + ctx: The context. + + Returns: + Boolean indicating whether the target is a test target. + """ + return ctx.attr.testonly or ctx.attr.instruments or str(ctx.label).find("/javatests/") >= 0 + +PROCESSORS_FOR_ANDROID_LOCAL_TEST = dict( + ManifestProcessor = _process_manifest, + ResourceProcessor = _process_resources_for_android_local_test, +) From fc4e2aeca24ec142b2d553b394d57e3242404575 Mon Sep 17 00:00:00 2001 From: Mauricio Galindo Date: Fri, 21 Oct 2022 17:05:43 -0700 Subject: [PATCH 07/29] Disable warnings for busybox actions Warnings go to stdout/stderr which makes it incompatible with workers --- rules/busybox.bzl | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/rules/busybox.bzl b/rules/busybox.bzl index d8156ca4a..4c3a1d257 100644 --- a/rules/busybox.bzl +++ b/rules/busybox.bzl @@ -118,6 +118,10 @@ def _make_resources_flag( ], ) +def _disable_warnings(args): + # Disable warnings - this are output to stdin/stderr which breaks worker mode + args.add("--logWarnings=false") + def _path(f): return f.path @@ -283,7 +287,7 @@ def _package( transitive_input_files = [] args = ctx.actions.args() - args.use_param_file("@%s", use_always=True) + args.use_param_file("@%s", use_always = True) args.set_param_file_format("multiline") args.add("--tool", "AAPT2_PACKAGE") args.add("--") @@ -391,6 +395,8 @@ def _package( if java_package: args.add("--packageForR", java_package) + _disable_warnings(args) + args.add_joined( "--resourceApks", resource_apks, @@ -402,7 +408,7 @@ def _package( ctx = ctx, host_javabase = host_javabase, executable = busybox, - tools = [aapt], + tools = [aapt, busybox], arguments = [args], inputs = depset(input_files, transitive = transitive_input_files), outputs = output_files, @@ -430,7 +436,7 @@ def _parse( host_javabase: Target. The host javabase. """ args = ctx.actions.args() - args.use_param_file("@%s", use_always=True) + args.use_param_file("@%s", use_always = True) args.set_param_file_format("multiline") args.add("--tool", "PARSE") args.add("--") @@ -443,6 +449,8 @@ def _parse( ) args.add("--output", out_symbols) + _disable_warnings(args) + _java.run( ctx = ctx, host_javabase = host_javabase, @@ -499,7 +507,7 @@ def _merge_assets( host_javabase: Target. The host javabase. """ args = ctx.actions.args() - args.use_param_file("@%s", use_always=True) + args.use_param_file("@%s", use_always = True) args.set_param_file_format("multiline") args.add("--tool", "MERGE_ASSETS") args.add("--") @@ -526,6 +534,8 @@ def _merge_assets( join_with = "&", ) + _disable_warnings(args) + _java.run( ctx = ctx, host_javabase = host_javabase, @@ -583,7 +593,7 @@ def _validate_and_link( # Retrieves the list of files at runtime when a directory is passed. args = ctx.actions.args() - args.use_param_file("@%s", use_always=True) + args.use_param_file("@%s", use_always = True) args.set_param_file_format("multiline") args.add("--tool", "LINK_STATIC_LIBRARY") args.add("--") @@ -615,6 +625,8 @@ def _validate_and_link( ) input_files.extend(resource_apks) + _disable_warnings(args) + _java.run( ctx = ctx, host_javabase = host_javabase, @@ -657,7 +669,7 @@ def _compile( # Retrieves the list of files at runtime when a directory is passed. args = ctx.actions.args() - args.use_param_file("@%s", use_always=True) + args.use_param_file("@%s", use_always = True) args.set_param_file_format("multiline") args.add("--tool", "COMPILE_LIBRARY_RESOURCES") args.add("--") @@ -672,6 +684,8 @@ def _compile( ) args.add("--output", out_file) + _disable_warnings(args) + _java.run( ctx = ctx, host_javabase = host_javabase, @@ -736,7 +750,7 @@ def _merge_compiled( transitive_input_files = [] args = ctx.actions.args() - args.use_param_file("@%s", use_always=True) + args.use_param_file("@%s", use_always = True) args.set_param_file_format("multiline") args.add("--tool", "MERGE_COMPILED") args.add("--") @@ -776,6 +790,8 @@ def _merge_compiled( ) transitive_input_files.append(transitive_compiled_resources) + _disable_warnings(args) + _java.run( ctx = ctx, host_javabase = host_javabase, @@ -872,6 +888,8 @@ def _merge_manifests( args.add("--log", out_log_file) outputs.append(out_log_file) + _disable_warnings(args) + _java.run( ctx = ctx, host_javabase = host_javabase, @@ -918,7 +936,7 @@ def _process_databinding( res_dirs = _get_unique_res_dirs(resource_files) args = ctx.actions.args() - args.use_param_file("@%s", use_always=True) + args.use_param_file("@%s", use_always = True) args.set_param_file_format("multiline") args.add("--tool", "PROCESS_DATABINDING") args.add("--") @@ -927,6 +945,8 @@ def _process_databinding( args.add("--dataBindingInfoOut", out_databinding_info) args.add("--appId", java_package) + _disable_warnings(args) + _java.run( ctx = ctx, host_javabase = host_javabase, @@ -976,7 +996,7 @@ def _generate_binary_r( host_javabase: A Target. The host javabase. """ args = ctx.actions.args() - args.use_param_file("@%s", use_always=True) + args.use_param_file("@%s", use_always = True) args.set_param_file_format("multiline") args.add("--tool", "GENERATE_BINARY_R") args.add("--") @@ -998,6 +1018,8 @@ def _generate_binary_r( args.add("--classJarOutput", out_class_jar) args.add("--targetLabel", str(ctx.label)) + _disable_warnings(args) + _java.run( ctx = ctx, host_javabase = host_javabase, @@ -1044,7 +1066,7 @@ def _make_aar( when a resource conflict occurs. """ args = ctx.actions.args() - args.use_param_file("@%s", use_always=True) + args.use_param_file("@%s", use_always = True) args.set_param_file_format("multiline") args.add("--tool", "GENERATE_AAR") args.add("--") @@ -1065,6 +1087,8 @@ def _make_aar( if should_throw_on_conflict: args.add("--throwOnResourceConflict") + _disable_warnings(args) + _java.run( ctx = ctx, host_javabase = host_javabase, From 43a25b3b199b18e80e1d84e49f8e5dea476c422b Mon Sep 17 00:00:00 2001 From: Mauricio Galindo Date: Mon, 24 Oct 2022 17:13:35 -0700 Subject: [PATCH 08/29] [SNAP FORK] disable min_sdk enforcement --- rules/acls/min_sdk_floors.bzl | 2 -- 1 file changed, 2 deletions(-) diff --git a/rules/acls/min_sdk_floors.bzl b/rules/acls/min_sdk_floors.bzl index 4dfe04412..27962f92b 100644 --- a/rules/acls/min_sdk_floors.bzl +++ b/rules/acls/min_sdk_floors.bzl @@ -20,9 +20,7 @@ cover the entire depot. MIN_SDK_FLOORS = { 19: [ - "//:__subpackages__", ], 14: [ - "//:__subpackages__", ], } From c1245390e4c8837a75eca2c7ada511ebd895cf76 Mon Sep 17 00:00:00 2001 From: Mauricio Galindo Date: Wed, 2 Nov 2022 14:02:54 -0700 Subject: [PATCH 09/29] Remove invalid attr --- rules/android_application/android_application_rule.bzl | 1 - rules/android_application/android_feature_module_rule.bzl | 5 ----- rules/android_application/gen_android_feature_manifest.sh | 2 -- 3 files changed, 8 deletions(-) diff --git a/rules/android_application/android_application_rule.bzl b/rules/android_application/android_application_rule.bzl index d7c15c2dd..b4f2664f3 100644 --- a/rules/android_application/android_application_rule.bzl +++ b/rules/android_application/android_application_rule.bzl @@ -164,7 +164,6 @@ def _create_feature_manifest( args.add(java_package) args.add(info.feature_name) args.add(info.title_id) - args.add(info.fused) args.add(aapt2.executable) ctx.actions.run( diff --git a/rules/android_application/android_feature_module_rule.bzl b/rules/android_application/android_feature_module_rule.bzl index 439548bce..9b702bb10 100644 --- a/rules/android_application/android_feature_module_rule.bzl +++ b/rules/android_application/android_feature_module_rule.bzl @@ -65,7 +65,6 @@ def _impl(ctx): title_id = ctx.attr.title_id, title_lib = ctx.attr.title_lib, feature_name = ctx.attr.feature_name, - fused = ctx.attr.fused, manifest = ctx.file.manifest, ), OutputGroupInfo(_validation = depset([validation])), @@ -115,9 +114,6 @@ def android_feature_module_macro(_android_binary, _android_library, **attrs): if not getattr(attrs, attr, None): fail("%s missing required attr <%s>" % (fqn, attr)) - if hasattr(attrs, "fused") and hasattr(attrs, "manifest"): - fail("%s cannot specify and . Prefer ") - targets = get_feature_module_paths(fqn) tags = getattr(attrs, "tags", []) @@ -193,7 +189,6 @@ EOF title_id = title_id, title_lib = str(targets.title_lib), feature_name = getattr(attrs, "feature_name", attrs.name), - fused = getattr(attrs, "fused", True), manifest = getattr(attrs, "manifest", None), tags = tags, transitive_configs = transitive_configs, diff --git a/rules/android_application/gen_android_feature_manifest.sh b/rules/android_application/gen_android_feature_manifest.sh index 5b5552e0e..23ac14739 100644 --- a/rules/android_application/gen_android_feature_manifest.sh +++ b/rules/android_application/gen_android_feature_manifest.sh @@ -18,7 +18,6 @@ base_apk="${2}" package="${3}" split="${4}" title_id="${5}" -fused="${6}" aapt="${7}" aapt_cmd="$aapt dump xmltree $base_apk --file AndroidManifest.xml" @@ -41,7 +40,6 @@ cat >$out_manifest < - From 63c6d68a96237e9d771d0952981aa5741ac0097d Mon Sep 17 00:00:00 2001 From: Mauricio Galindo Date: Wed, 16 Nov 2022 07:35:04 -0800 Subject: [PATCH 10/29] accept sources.jar as input for srcjars --- rules/aar_import/attrs.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/aar_import/attrs.bzl b/rules/aar_import/attrs.bzl index c8b265943..111b91a92 100644 --- a/rules/aar_import/attrs.bzl +++ b/rules/aar_import/attrs.bzl @@ -53,7 +53,7 @@ ATTRS = _attrs.add( "This needs to be the same value as the manifest's package.", ), srcjar = attr.label( - allow_single_file = [".srcjar"], + allow_single_file = ["sources.jar", ".srcjar"], doc = "A srcjar file that contains the source code for the JVM " + "artifacts stored within the AAR.", ), From a13352ff4da9d17ba372c29279f2714a8491a325 Mon Sep 17 00:00:00 2001 From: Mauricio Galindo Date: Sun, 20 Nov 2022 10:13:13 -0800 Subject: [PATCH 11/29] Implement namespaced R class The native version of the rules have this optimization (in our internal fork). This is that implementationm, but in starlark. See https://github.sc-corp.net/Snapchat/bazel/pull/126 for more details --- rules/aar_import/impl.bzl | 1 + rules/android_library/rule.bzl | 1 - rules/busybox.bzl | 26 ++++- rules/resources.bzl | 192 +++++++++++++++++++-------------- 4 files changed, 138 insertions(+), 82 deletions(-) diff --git a/rules/aar_import/impl.bzl b/rules/aar_import/impl.bzl index 553ef1e51..62baa6f3a 100644 --- a/rules/aar_import/impl.bzl +++ b/rules/aar_import/impl.bzl @@ -155,6 +155,7 @@ def _process_resources( exports = ctx.attr.exports, exports_manifest = getattr(ctx.attr, "exports_manifest", True), propagate_resources = True, + namespaced_r_class = False, # Tool and Processing related inputs aapt = _get_android_toolchain(ctx).aapt2.files_to_run, diff --git a/rules/android_library/rule.bzl b/rules/android_library/rule.bzl index e36bdd3b0..6d6585aa3 100644 --- a/rules/android_library/rule.bzl +++ b/rules/android_library/rule.bzl @@ -128,7 +128,6 @@ def _outputs(name, _package_name, _defined_local_resources): path_prefix = "" outputs.update( dict( - resources_src_jar = path_prefix + "%{name}.srcjar", resources_txt = path_prefix + "%{name}_symbols/R.txt", resources_jar = path_prefix + "%{name}_resources.jar", ), diff --git a/rules/busybox.bzl b/rules/busybox.bzl index 4c3a1d257..3ff084c83 100644 --- a/rules/busybox.bzl +++ b/rules/busybox.bzl @@ -645,11 +645,15 @@ def _validate_and_link( def _compile( ctx, out_file = None, + out_class_jar = None, + out_r_txt = None, + manifest = None, assets = [], assets_dir = None, resource_files = [], busybox = None, aapt = None, + android_jar = None, host_javabase = None): """Compile and store resources in a single archive. @@ -684,6 +688,23 @@ def _compile( ) args.add("--output", out_file) + optional_outputs = [] + if out_class_jar: + args.add("--classJarOutput", out_class_jar) + optional_outputs.append(out_class_jar) + if out_r_txt: + args.add("--rTxtOut", out_r_txt) + optional_outputs.append(out_r_txt) + + args.add("--targetLabel", ctx.label) + optional_inputs = [] + if manifest: + args.add("--manifest", manifest) + optional_inputs = [manifest] + if android_jar: + args.add("--androidJar", android_jar) + optional_inputs.append(android_jar) + _disable_warnings(args) _java.run( @@ -692,8 +713,8 @@ def _compile( executable = busybox, tools = [aapt], arguments = [args], - inputs = resource_files + assets, - outputs = [out_file], + inputs = resource_files + assets + optional_inputs, + outputs = [out_file] + optional_outputs, mnemonic = "CompileAndroidResources", progress_message = "Compiling Android Resources in %s" % out_file.short_path, supports_workers = True, @@ -1259,6 +1280,7 @@ busybox = struct( make_aar = _make_aar, shrink = _shrink, optimize = _optimize, + ANDROID_RESOURCES_STRICT_DEPS = _ANDROID_RESOURCES_STRICT_DEPS, # Exposed for testing mergee_manifests_flag = _mergee_manifests_flag, diff --git a/rules/resources.bzl b/rules/resources.bzl index 0069ee4e0..fe10d7124 100644 --- a/rules/resources.bzl +++ b/rules/resources.bzl @@ -1270,7 +1270,8 @@ def _process_starlark( host_javabase = None, instrument_xslt = None, xsltproc = None, - zip_tool = None): + zip_tool = None, + namespaced_r_class = True): """Processes Android Resources. Args: @@ -1526,6 +1527,7 @@ def _process_starlark( parsed_assets = ctx.actions.declare_file( "_migrated/" + ctx.label.name + "_symbols/assets.bin", ) + _busybox.parse( ctx, out_symbols = parsed_assets, @@ -1594,14 +1596,38 @@ def _process_starlark( compiled_resources = ctx.actions.declare_file( "_migrated/" + ctx.label.name + "_symbols/symbols.zip", ) - _busybox.compile( - ctx, - out_file = compiled_resources, - resource_files = processed_resources, - aapt = aapt, - busybox = busybox, - host_javabase = host_javabase, + + out_class_jar = ctx.actions.declare_file( + "_migrated/" + ctx.label.name + "_resources.jar", ) + processed_manifest = None + + # when using namespaced r classes the compile action generated both + # R.txt and the resources.jar without merging with outputs produced + # by transitive deps. Doing this means less changes that invalidate + # the action cache which in turn improves performance significantly. + if namespaced_r_class: + out_aapt2_r_txt = ctx.actions.declare_file( + "_migrated/" + ctx.label.name + "_symbols/R.txt", + ) + _busybox.compile( + ctx, + out_file = compiled_resources, + out_class_jar = out_class_jar, + out_r_txt = out_aapt2_r_txt, + manifest = manifest, + android_jar = android_jar, + resource_files = processed_resources, + aapt = aapt, + busybox = busybox, + host_javabase = host_javabase, + ) + r_txt = out_aapt2_r_txt + java_info = JavaInfo( + output_jar = out_class_jar, + compile_jar = out_class_jar, + ) + processed_manifest = manifest # TODO(b/160907203): Remove this fix once the native resource processing pipeline is turned off. if enable_data_binding: @@ -1616,77 +1642,86 @@ def _process_starlark( ) compiled_resources = fixed_compiled_resources - out_class_jar = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + "_resources.jar", - ) - processed_manifest = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + "_processed_manifest/AndroidManifest.xml", - ) - out_aapt2_r_txt = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + "_symbols/R.aapt2.txt", - ) - _busybox.merge_compiled( - ctx, - out_class_jar = out_class_jar, - out_manifest = processed_manifest, - out_aapt2_r_txt = out_aapt2_r_txt, - java_package = java_package, - manifest = manifest, - compiled_resources = compiled_resources, - direct_resources_nodes = - depset(transitive = direct_resources_nodes, order = "preorder"), - transitive_resources_nodes = depset( - transitive = transitive_resources_nodes, - order = "preorder", - ), - direct_compiled_resources = depset( - transitive = direct_compiled_resources, - order = "preorder", - ), - transitive_compiled_resources = depset( - transitive = transitive_compiled_resources, - order = "preorder", - ), - android_jar = android_jar, - busybox = busybox, - host_javabase = host_javabase, - ) - resources_ctx[_MERGED_MANIFEST] = processed_manifest + if not namespaced_r_class: + processed_manifest = ctx.actions.declare_file( + "_migrated/" + ctx.label.name + "_processed_manifest/AndroidManifest.xml", + ) + out_aapt2_r_txt = ctx.actions.declare_file( + "_migrated/" + ctx.label.name + "_symbols/R.aapt2.txt", + ) + _busybox.compile( + ctx, + out_file = compiled_resources, + manifest = manifest, + android_jar = android_jar, + resource_files = processed_resources, + aapt = aapt, + busybox = busybox, + host_javabase = host_javabase, + ) - apk = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + "_files/library.ap_", - ) - r_java = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + ".srcjar", - ) - r_txt = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + "_symbols/R.txt", - ) - _busybox.validate_and_link( - ctx, - out_r_src_jar = r_java, - out_r_txt = r_txt, - out_file = apk, - compiled_resources = compiled_resources, - transitive_compiled_resources = depset( - transitive = transitive_compiled_resources, - order = "preorder", - ), - java_package = java_package, - manifest = processed_manifest, - android_jar = android_jar, - aapt = aapt, - busybox = busybox, - host_javabase = host_javabase, - resource_apks = resource_apks, - ) - resources_ctx[_RESOURCES_APK] = apk + _busybox.merge_compiled( + ctx, + out_class_jar = out_class_jar, + out_manifest = processed_manifest, + out_aapt2_r_txt = out_aapt2_r_txt, + java_package = java_package, + manifest = manifest, + compiled_resources = compiled_resources, + direct_resources_nodes = + depset(transitive = direct_resources_nodes, order = "preorder"), + transitive_resources_nodes = depset( + transitive = transitive_resources_nodes, + order = "preorder", + ), + direct_compiled_resources = depset( + transitive = direct_compiled_resources, + order = "preorder", + ), + transitive_compiled_resources = depset( + transitive = transitive_compiled_resources, + order = "preorder", + ), + android_jar = android_jar, + busybox = busybox, + host_javabase = host_javabase, + ) - java_info = JavaInfo( - output_jar = out_class_jar, - compile_jar = out_class_jar, - source_jar = r_java, - ) + apk = ctx.actions.declare_file( + "_migrated/" + ctx.label.name + "_files/library.ap_", + ) + r_java = ctx.actions.declare_file( + "_migrated/" + ctx.label.name + ".srcjar", + ) + r_txt = ctx.actions.declare_file( + "_migrated/" + ctx.label.name + "_symbols/R.txt", + ) + _busybox.validate_and_link( + ctx, + out_r_src_jar = r_java, + out_r_txt = r_txt, + out_file = apk, + compiled_resources = compiled_resources, + transitive_compiled_resources = depset( + transitive = transitive_compiled_resources, + order = "preorder", + ), + java_package = java_package, + manifest = processed_manifest, + android_jar = android_jar, + aapt = aapt, + busybox = busybox, + host_javabase = host_javabase, + resource_apks = resource_apks, + ) + resources_ctx[_RESOURCES_APK] = apk + java_info = JavaInfo( + output_jar = out_class_jar, + compile_jar = out_class_jar, + source_jar = r_java, + ) + + resources_ctx[_MERGED_MANIFEST] = processed_manifest packages_to_r_txts_depset.setdefault(java_package, []).append(depset([out_aapt2_r_txt])) @@ -1877,7 +1912,6 @@ def _process_starlark( return resources_ctx - def _process( ctx, manifest = None, @@ -1940,9 +1974,9 @@ def _process( java_toolchain = java_toolchain, host_javabase = host_javabase, zip_tool = zip_tool, + namespaced_r_class = _busybox.ANDROID_RESOURCES_STRICT_DEPS not in ctx.disabled_features, ) - if _VALIDATION_OUTPUTS not in out_ctx: out_ctx[_VALIDATION_OUTPUTS] = [] From 7391c2d9d52763d06751f58209f0859dbe15ab92 Mon Sep 17 00:00:00 2001 From: Mauricio Galindo Date: Tue, 29 Nov 2022 10:06:44 -0800 Subject: [PATCH 12/29] Do Not infer package from path --- rules/android_binary_internal/impl.bzl | 2 +- rules/android_library/impl.bzl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rules/android_binary_internal/impl.bzl b/rules/android_binary_internal/impl.bzl index 2a005d8b5..c9274aa12 100644 --- a/rules/android_binary_internal/impl.bzl +++ b/rules/android_binary_internal/impl.bzl @@ -792,5 +792,5 @@ def impl(ctx): Returns: A list of providers. """ - java_package = java.resolve_package_from_label(ctx.label, ctx.attr.custom_package) + java_package = ctx.attr.custom_package if ctx.attr.custom_package else None return processing_pipeline.run(ctx, java_package, _PROCESSING_PIPELINE) diff --git a/rules/android_library/impl.bzl b/rules/android_library/impl.bzl index aeaaf5d2e..402c7d4df 100644 --- a/rules/android_library/impl.bzl +++ b/rules/android_library/impl.bzl @@ -584,5 +584,5 @@ def impl(ctx): Returns: A legacy struct provider. """ - java_package = _java.resolve_package_from_label(ctx.label, ctx.attr.custom_package) + java_package = ctx.attr.custom_package if ctx.attr.custom_package else None return processing_pipeline.run(ctx, java_package, _PROCESSING_PIPELINE) From 4997ae956ca6ab0c528f412a0577ca207dca37e7 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Fri, 5 May 2023 10:32:04 -0700 Subject: [PATCH 13/29] [Lyft] Hacks working around the custom JDK issues --- tools/jdk/BUILD | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/jdk/BUILD b/tools/jdk/BUILD index 08b8c862a..f904a43f1 100644 --- a/tools/jdk/BUILD +++ b/tools/jdk/BUILD @@ -1,5 +1,6 @@ -load("@bazel_tools//tools/jdk:default_java_toolchain.bzl", "default_java_toolchain") +load("@bazel_tools//tools/jdk:default_java_toolchain.bzl", "BASE_JDK9_JVM_OPTS", "default_java_toolchain") +# Setup default javacopts such as for errorprone without affecting external repos. default_java_toolchain( name = "toolchain_android_only", bootclasspath = [ @@ -7,6 +8,12 @@ default_java_toolchain( # TODO(b/175805830): Add this only when desugaring is enabled. "@bazel_tools//tools/android:desugar_java8_extra_bootclasspath", ], + # javac = ["//tools/error-prone:javac"], + java_runtime = "@lyft_jdk11//:jdk", + jvm_opts = BASE_JDK9_JVM_OPTS + ["-XX:-MaxFDLimit"], + package_configuration = ["@//:error_prone"], + source_version = "1.8", + target_version = "1.8", visibility = ["//visibility:public"], ) From da046c00ebb4646acb64e5b26b50f49da8947041 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Wed, 24 May 2023 17:10:36 -0700 Subject: [PATCH 14/29] Add flag for namespace resources --- rules/android_library/impl.bzl | 1 + rules/flags/flag_defs.bzl | 7 ++++++- rules/resources.bzl | 5 +++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/rules/android_library/impl.bzl b/rules/android_library/impl.bzl index 402c7d4df..ef7e023de 100644 --- a/rules/android_library/impl.bzl +++ b/rules/android_library/impl.bzl @@ -177,6 +177,7 @@ def _process_resources(ctx, java_package, manifest_ctx, **unused_ctxs): # misbehavior on the Java side. fix_resource_transitivity = True, fix_export_exporting = acls.in_fix_export_exporting_rollout(str(ctx.label)), + namespaced_r_class = _flags.get(ctx).namespaced_r_class, # Tool and Processing related inputs aapt = get_android_toolchain(ctx).aapt2.files_to_run, diff --git a/rules/flags/flag_defs.bzl b/rules/flags/flag_defs.bzl index 61f2cf7f0..9b5ff51a9 100644 --- a/rules/flags/flag_defs.bzl +++ b/rules/flags/flag_defs.bzl @@ -86,7 +86,6 @@ def define_flags(): description = "", ) - flags.EXPOSE_native_bool( name = "stamp", description = "Accesses the native --stamp CLI flag", @@ -97,3 +96,9 @@ def define_flags(): default = True, description = "Use Studio Deployer to install apks", ) + + flags.DEFINE_bool( + name = "namespaced_r_class", + default = False, + description = "Enables namespaced R class generation", + ) diff --git a/rules/resources.bzl b/rules/resources.bzl index fe10d7124..228a6c27a 100644 --- a/rules/resources.bzl +++ b/rules/resources.bzl @@ -1271,7 +1271,7 @@ def _process_starlark( instrument_xslt = None, xsltproc = None, zip_tool = None, - namespaced_r_class = True): + namespaced_r_class = False): """Processes Android Resources. Args: @@ -1942,6 +1942,7 @@ def _process( fix_resource_transitivity = True, fix_export_exporting = False, propagate_resources = True, + namespaced_r_class = False, zip_tool = None): out_ctx = _process_starlark( ctx, @@ -1974,7 +1975,7 @@ def _process( java_toolchain = java_toolchain, host_javabase = host_javabase, zip_tool = zip_tool, - namespaced_r_class = _busybox.ANDROID_RESOURCES_STRICT_DEPS not in ctx.disabled_features, + namespaced_r_class = namespaced_r_class, ) if _VALIDATION_OUTPUTS not in out_ctx: From 9691362bf9a00a108eb0286b4eccef59fc79395d Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Mon, 12 Jun 2023 08:29:07 -0700 Subject: [PATCH 15/29] Fix rules/android_local_test/resources.bzl to reference processed_manifest --- rules/android_local_test/resources.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rules/android_local_test/resources.bzl b/rules/android_local_test/resources.bzl index ab0a44b03..693591509 100644 --- a/rules/android_local_test/resources.bzl +++ b/rules/android_local_test/resources.bzl @@ -21,7 +21,7 @@ load( "processing_pipeline", ) load("//rules:resources.bzl", _resources = "resources") -load("//rules:utils.bzl", "get_android_toolchain", "utils") +load("//rules:utils.bzl", "get_android_toolchain", "utils") def _process_manifest(ctx, **unused_ctxs): manifest_ctx = _resources.bump_min_sdk( @@ -39,7 +39,7 @@ def _process_manifest(ctx, **unused_ctxs): def _process_resources_for_android_local_test(ctx, manifest_ctx, java_package, **unused_ctx): packaged_resources_ctx = _resources.package( ctx, - manifest = manifest_ctx.min_sdk_bumped_manifest, + manifest = manifest_ctx.processed_manifest, manifest_values = utils.expand_make_vars(ctx, ctx.attr.manifest_values), java_package = java_package, use_legacy_manifest_merger = False, From 60b23f406629104674e309702abf6910dae82b55 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Wed, 21 Jun 2023 10:54:31 -0700 Subject: [PATCH 16/29] Enable conditional merging of manifest permissions --- rules/busybox.bzl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rules/busybox.bzl b/rules/busybox.bzl index 3ff084c83..2621716da 100644 --- a/rules/busybox.bzl +++ b/rules/busybox.bzl @@ -713,7 +713,7 @@ def _compile( executable = busybox, tools = [aapt], arguments = [args], - inputs = resource_files + assets + optional_inputs, + inputs = resource_files + assets + optional_inputs, outputs = [out_file] + optional_outputs, mnemonic = "CompileAndroidResources", progress_message = "Compiling Android Resources in %s" % out_file.short_path, @@ -891,6 +891,7 @@ def _merge_manifests( args.add("--") if manifest: args.add("--manifest", manifest) + args.add("--mergeManifestPermissions") args.add_all( "--mergeeManifests", [mergee_manifests], From ca0b27c5898bb7fbcc4053d409a3e76ee33e93d2 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Wed, 21 Jun 2023 10:07:07 -0700 Subject: [PATCH 17/29] support-additional_providers-in-make_rule --- rules/android_binary_internal/rule.bzl | 6 ++++-- rules/android_library/rule.bzl | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/rules/android_binary_internal/rule.bzl b/rules/android_binary_internal/rule.bzl index 39f498c38..767a6efd0 100644 --- a/rules/android_binary_internal/rule.bzl +++ b/rules/android_binary_internal/rule.bzl @@ -29,7 +29,8 @@ def make_rule( attrs = ATTRS, implementation = impl, provides = _DEFAULT_PROVIDES, - additional_toolchains = []): + additional_toolchains = [], + additional_providers = []): """Makes the rule. Args: @@ -37,6 +38,7 @@ def make_rule( implementation: A function. The rule's implementation method. provides: A list. The providers that the rule must provide. additional_toolchains: A list. Additional toolchains passed to pass to rule(toolchains). + additional_providers: A list. Additional providers passed to pass to rule(providers). Returns: A rule. @@ -44,7 +46,7 @@ def make_rule( return rule( attrs = attrs, implementation = implementation, - provides = provides, + provides = provides + additional_providers, toolchains = [ "//toolchains/android:toolchain_type", "//toolchains/android_sdk:toolchain_type", diff --git a/rules/android_library/rule.bzl b/rules/android_library/rule.bzl index 6d6585aa3..789015a70 100644 --- a/rules/android_library/rule.bzl +++ b/rules/android_library/rule.bzl @@ -139,7 +139,8 @@ def make_rule( attrs = _ATTRS, implementation = _impl, outputs = _outputs, - additional_toolchains = []): + additional_toolchains = [], + additional_providers = []): """Makes the rule. Args: @@ -147,6 +148,7 @@ def make_rule( implementation: A function. The rule's implementation method. outputs: A dict, function, or None. The rule's outputs. additional_toolchains: A list. Additional toolchains passed to pass to rule(toolchains). + additional_providers: A list. Additional providers passed to pass to rule(providers). Returns: A rule. @@ -166,7 +168,7 @@ def make_rule( AndroidLibraryResourceClassJarProvider, AndroidNativeLibsInfo, JavaInfo, - ], + ] + additional_providers, outputs = outputs, toolchains = [ "//toolchains/android:toolchain_type", From 6d54e568a79abb2156dfa077c2ac2750952fb03c Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Thu, 6 Jul 2023 11:27:47 -0700 Subject: [PATCH 18/29] Don't collect transitive AndroidLibraryResourceClassJarProvider --- rules/resources.bzl | 8 -------- 1 file changed, 8 deletions(-) diff --git a/rules/resources.bzl b/rules/resources.bzl index 228a6c27a..a8d521473 100644 --- a/rules/resources.bzl +++ b/rules/resources.bzl @@ -1897,14 +1897,6 @@ def _process_starlark( AndroidLibraryResourceClassJarProvider( depset( (resources_ctx[_R_JAVA].runtime_output_jars if resources_ctx[_R_JAVA] else []), - transitive = [ - p.jars - for p in utils.collect_providers( - AndroidLibraryResourceClassJarProvider, - deps, - exports, - ) - ], order = "preorder", ), ), From 11280e00046a78699ebcfd8ea689646c1f462433 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Tue, 11 Jul 2023 17:11:44 -0700 Subject: [PATCH 19/29] Expand support for kt_android_local_test --- rules/android_local_test/resources.bzl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/rules/android_local_test/resources.bzl b/rules/android_local_test/resources.bzl index 693591509..e36f0d2d7 100644 --- a/rules/android_local_test/resources.bzl +++ b/rules/android_local_test/resources.bzl @@ -21,7 +21,8 @@ load( "processing_pipeline", ) load("//rules:resources.bzl", _resources = "resources") -load("//rules:utils.bzl", "get_android_toolchain", "utils") +load("//rules:utils.bzl", "compilation_mode", "get_android_toolchain", "utils") +load("//rules:attrs.bzl", _attrs = "attrs") def _process_manifest(ctx, **unused_ctxs): manifest_ctx = _resources.bump_min_sdk( @@ -39,6 +40,14 @@ def _process_manifest(ctx, **unused_ctxs): def _process_resources_for_android_local_test(ctx, manifest_ctx, java_package, **unused_ctx): packaged_resources_ctx = _resources.package( ctx, + resource_files = ctx.files.resource_files, + assets = ctx.files.assets, + assets_dir = ctx.attr.assets_dir, + resource_configs = ctx.attr.resource_configuration_filters, + densities = ctx.attr.densities, + nocompress_extensions = ctx.attr.nocompress_extensions, + compilation_mode = compilation_mode.get(ctx), + shrink_resources = _attrs.tristate.no, manifest = manifest_ctx.processed_manifest, manifest_values = utils.expand_make_vars(ctx, ctx.attr.manifest_values), java_package = java_package, @@ -66,7 +75,7 @@ def _is_test_binary(ctx): """ return ctx.attr.testonly or ctx.attr.instruments or str(ctx.label).find("/javatests/") >= 0 -PROCESSORS_FOR_ANDROID_LOCAL_TEST = dict( +PROCESSORS = dict( ManifestProcessor = _process_manifest, ResourceProcessor = _process_resources_for_android_local_test, ) From 918620e97d17f7591c2fd3a6dcff936d6368455c Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Tue, 11 Jul 2023 23:01:25 -0700 Subject: [PATCH 20/29] Add support for --output_library_merged_assets --- rules/aar_import/impl.bzl | 3 +- rules/android_library/impl.bzl | 1 + rules/flags/flag_defs.bzl | 6 ++++ rules/resources.bzl | 53 +++++++++++++++++++--------------- 4 files changed, 38 insertions(+), 25 deletions(-) diff --git a/rules/aar_import/impl.bzl b/rules/aar_import/impl.bzl index 62baa6f3a..29f708c0e 100644 --- a/rules/aar_import/impl.bzl +++ b/rules/aar_import/impl.bzl @@ -28,6 +28,7 @@ load( _java = "java", ) load("//rules:providers.bzl", "AndroidLintRulesInfo") +load("//rules/flags:flags.bzl", _flags = "flags") load( "//rules:resources.bzl", _resources = "resources", @@ -156,6 +157,7 @@ def _process_resources( exports_manifest = getattr(ctx.attr, "exports_manifest", True), propagate_resources = True, namespaced_r_class = False, + output_library_merged_assets = _flags.get(ctx).output_library_merged_assets, # Tool and Processing related inputs aapt = _get_android_toolchain(ctx).aapt2.files_to_run, @@ -177,7 +179,6 @@ def _process_resources( """ % package) - return struct(**resources_ctx) def _extract_jars( diff --git a/rules/android_library/impl.bzl b/rules/android_library/impl.bzl index ef7e023de..17ab53a56 100644 --- a/rules/android_library/impl.bzl +++ b/rules/android_library/impl.bzl @@ -178,6 +178,7 @@ def _process_resources(ctx, java_package, manifest_ctx, **unused_ctxs): fix_resource_transitivity = True, fix_export_exporting = acls.in_fix_export_exporting_rollout(str(ctx.label)), namespaced_r_class = _flags.get(ctx).namespaced_r_class, + output_library_merged_assets = _flags.get(ctx).output_library_merged_assets, # Tool and Processing related inputs aapt = get_android_toolchain(ctx).aapt2.files_to_run, diff --git a/rules/flags/flag_defs.bzl b/rules/flags/flag_defs.bzl index 9b5ff51a9..e764b8479 100644 --- a/rules/flags/flag_defs.bzl +++ b/rules/flags/flag_defs.bzl @@ -102,3 +102,9 @@ def define_flags(): default = False, description = "Enables namespaced R class generation", ) + + flags.DEFINE_bool( + name = "output_library_merged_assets", + default = True, + description = "If disabled, does not produce merged asset.zip outputs for library targets", + ) diff --git a/rules/resources.bzl b/rules/resources.bzl index a8d521473..afc79a1ab 100644 --- a/rules/resources.bzl +++ b/rules/resources.bzl @@ -1271,7 +1271,8 @@ def _process_starlark( instrument_xslt = None, xsltproc = None, zip_tool = None, - namespaced_r_class = False): + namespaced_r_class = False, + output_library_merged_assets = True): """Processes Android Resources. Args: @@ -1536,29 +1537,31 @@ def _process_starlark( busybox = busybox, host_javabase = host_javabase, ) - merged_assets = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + "_files/assets.zip", - ) - _busybox.merge_assets( - ctx, - out_assets_zip = merged_assets, - assets = assets, - assets_dir = assets_dir, - symbols = parsed_assets, - direct_resources_nodes = depset( - transitive = direct_resources_nodes, - order = "preorder", - ), - transitive_resources_nodes = depset( - transitive = transitive_resources_nodes, - order = "preorder", - ), - transitive_assets = transitive_assets, - transitive_assets_symbols = transitive_assets_symbols, - busybox = busybox, - host_javabase = host_javabase, - ) - resources_ctx[_VALIDATION_RESULTS].append(merged_assets) + + if output_library_merged_assets: + merged_assets = ctx.actions.declare_file( + "_migrated/" + ctx.label.name + "_files/assets.zip", + ) + _busybox.merge_assets( + ctx, + out_assets_zip = merged_assets, + assets = assets, + assets_dir = assets_dir, + symbols = parsed_assets, + direct_resources_nodes = depset( + transitive = direct_resources_nodes, + order = "preorder", + ), + transitive_resources_nodes = depset( + transitive = transitive_resources_nodes, + order = "preorder", + ), + transitive_assets = transitive_assets, + transitive_assets_symbols = transitive_assets_symbols, + busybox = busybox, + host_javabase = host_javabase, + ) + resources_ctx[_VALIDATION_RESULTS].append(merged_assets) if assets: compiled_assets = ctx.actions.declare_file( @@ -1935,6 +1938,7 @@ def _process( fix_export_exporting = False, propagate_resources = True, namespaced_r_class = False, + output_library_merged_assets = True, zip_tool = None): out_ctx = _process_starlark( ctx, @@ -1968,6 +1972,7 @@ def _process( host_javabase = host_javabase, zip_tool = zip_tool, namespaced_r_class = namespaced_r_class, + output_library_merged_assets = output_library_merged_assets, ) if _VALIDATION_OUTPUTS not in out_ctx: From 46e87de1f16c667169cc23335300c1c37f764163 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Tue, 11 Jul 2023 23:13:23 -0700 Subject: [PATCH 21/29] Support --merge_android_manifest_permissions --- rules/aar_import/impl.bzl | 1 + rules/android_library/impl.bzl | 1 + rules/busybox.bzl | 5 +++++ rules/flags/flag_defs.bzl | 6 ++++++ rules/resources.bzl | 10 ++++++++++ 5 files changed, 23 insertions(+) diff --git a/rules/aar_import/impl.bzl b/rules/aar_import/impl.bzl index 29f708c0e..d9ec38e36 100644 --- a/rules/aar_import/impl.bzl +++ b/rules/aar_import/impl.bzl @@ -158,6 +158,7 @@ def _process_resources( propagate_resources = True, namespaced_r_class = False, output_library_merged_assets = _flags.get(ctx).output_library_merged_assets, + merge_manifest_permissions = _flags.get(ctx).merge_manifest_permissions, # Tool and Processing related inputs aapt = _get_android_toolchain(ctx).aapt2.files_to_run, diff --git a/rules/android_library/impl.bzl b/rules/android_library/impl.bzl index 17ab53a56..f8456e2ee 100644 --- a/rules/android_library/impl.bzl +++ b/rules/android_library/impl.bzl @@ -179,6 +179,7 @@ def _process_resources(ctx, java_package, manifest_ctx, **unused_ctxs): fix_export_exporting = acls.in_fix_export_exporting_rollout(str(ctx.label)), namespaced_r_class = _flags.get(ctx).namespaced_r_class, output_library_merged_assets = _flags.get(ctx).output_library_merged_assets, + merge_manifest_permissions = _flags.get(ctx).merge_manifest_permissions, # Tool and Processing related inputs aapt = get_android_toolchain(ctx).aapt2.files_to_run, diff --git a/rules/busybox.bzl b/rules/busybox.bzl index 2621716da..a7c8e1c0a 100644 --- a/rules/busybox.bzl +++ b/rules/busybox.bzl @@ -856,6 +856,7 @@ def _merge_manifests( out_file = None, out_log_file = None, merge_type = "APPLICATION", + merge_manifest_permissions = False, manifest = None, mergee_manifests = depset(), manifest_values = None, @@ -869,6 +870,8 @@ def _merge_manifests( out_file: A File. The output merged manifest. out_log_file: A File. The output log from the merge tool. merge_type: A string, either APPLICATION or LIBRARY. Type of merging. + merge_manifest_permissions: A boolean, Determines of the manifest merger should + merge manifest permissions. manifest: A File. The primary AndroidManifest.xml. mergee_manifests: A depset of Files. All transitive manifests to be merged. manifest_values: A dictionary. Manifest values to substitute. @@ -909,6 +912,8 @@ def _merge_manifests( if out_log_file: args.add("--log", out_log_file) outputs.append(out_log_file) + if merge_manifest_permissions: + args.add("--mergeManifestPermissions") _disable_warnings(args) diff --git a/rules/flags/flag_defs.bzl b/rules/flags/flag_defs.bzl index e764b8479..98e3c8d20 100644 --- a/rules/flags/flag_defs.bzl +++ b/rules/flags/flag_defs.bzl @@ -108,3 +108,9 @@ def define_flags(): default = True, description = "If disabled, does not produce merged asset.zip outputs for library targets", ) + + flags.DEFINE_bool( + name = "merge_manifest_permissions", + default = False, + description = "If disabled, does not merge manifest permissions.", + ) diff --git a/rules/resources.bzl b/rules/resources.bzl index afc79a1ab..44c350b2b 100644 --- a/rules/resources.bzl +++ b/rules/resources.bzl @@ -494,6 +494,7 @@ def _package( enable_manifest_merging = True, should_compile_java_srcs = True, minsdk_proguard_config = None, + merge_manifest_permissions = False, aapt = None, has_local_proguard_specs = False, android_jar = None, @@ -553,6 +554,8 @@ def _package( should_compile_java_srcs: boolean. If native android_binary should perform java compilation. minsdk_proguard_config: Optional file. Proguard config for the minSdkVersion to include in the returned resource context. + merge_manifest_permissions: Whether to merge permissions when merging the + Android manifests. aapt: FilesToRunProvider. The aapt executable or FilesToRunProvider. has_local_proguard_specs: If the target has proguard specs. android_jar: File. The Android jar. @@ -668,6 +671,7 @@ def _package( mergee_manifests = mergee_manifests, manifest_values = manifest_values, merge_type = "APPLICATION", + merge_manifest_permissions = merge_manifest_permissions, java_package = java_package, busybox = busybox, host_javabase = host_javabase, @@ -1262,6 +1266,7 @@ def _process_starlark( enable_data_binding = False, propagate_resources = True, fix_resource_transitivity = True, + merge_manifest_permissions = False, aapt = None, android_jar = None, android_kit = None, @@ -1315,6 +1320,8 @@ def _process_starlark( continue to propagate others (AndroidLibraryResourceClassJarProvider). fix_resource_transitivity: Whether to ensure that transitive resources are correctly marked as transitive. + merge_manifest_permissions: Whether to merge permissions when merging the + Android manifests. aapt: FilesToRunProvider. The aapt executable or FilesToRunProvider. android_jar: File. The android Jar. android_kit: FilesToRunProvider. The android_kit executable or @@ -1506,6 +1513,7 @@ def _process_starlark( out_file = stamped_manifest, manifest = manifest, merge_type = "LIBRARY", + merge_manifest_permissions = merge_manifest_permissions, java_package = java_package, busybox = busybox, host_javabase = host_javabase, @@ -1935,6 +1943,7 @@ def _process( res_v3_dummy_manifest = None, res_v3_dummy_r_txt = None, fix_resource_transitivity = True, + merge_manifest_permissions = False, fix_export_exporting = False, propagate_resources = True, namespaced_r_class = False, @@ -1960,6 +1969,7 @@ def _process( resource_files = depset(transitive = [target.files for target in resource_files]).to_list(), enable_data_binding = enable_data_binding, fix_resource_transitivity = fix_resource_transitivity, + merge_manifest_permissions = merge_manifest_permissions, neverlink = neverlink, propagate_resources = propagate_resources, android_jar = android_jar, From 3c74dc1f3e12b4333f94d90d18ef445e6fa63531 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Wed, 12 Jul 2023 14:46:29 -0700 Subject: [PATCH 22/29] Disable g3itr --- rules/resources.bzl | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/rules/resources.bzl b/rules/resources.bzl index 44c350b2b..00071406c 100644 --- a/rules/resources.bzl +++ b/rules/resources.bzl @@ -582,18 +582,6 @@ def _package( g3itr_manifest = manifest - if xsltproc or instrument_xslt: - g3itr_manifest = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + "add_g3itr/AndroidManifest.xml", - ) - _add_g3itr( - ctx, - out_manifest = g3itr_manifest, - manifest = manifest, - xsltproc = xsltproc, - instrument_xslt = instrument_xslt, - ) - direct_resources_nodes = [] transitive_resources_nodes = [] transitive_assets = [] @@ -1520,19 +1508,6 @@ def _process_starlark( ) manifest = stamped_manifest - if instrument_xslt: - g3itr_manifest = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + "_g3itr_manifest/AndroidManifest.xml", - ) - _add_g3itr( - ctx, - out_manifest = g3itr_manifest, - manifest = manifest, - xsltproc = xsltproc, - instrument_xslt = instrument_xslt, - ) - manifest = g3itr_manifest - parsed_assets = ctx.actions.declare_file( "_migrated/" + ctx.label.name + "_symbols/assets.bin", ) From 18013f933861d3a8ad05d1d47364b31d94e22fe4 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Thu, 13 Jul 2023 19:22:47 -0700 Subject: [PATCH 23/29] Disable aar validation for now --- rules/aar_import/impl.bzl | 8 -------- 1 file changed, 8 deletions(-) diff --git a/rules/aar_import/impl.bzl b/rules/aar_import/impl.bzl index d9ec38e36..467e81792 100644 --- a/rules/aar_import/impl.bzl +++ b/rules/aar_import/impl.bzl @@ -561,14 +561,6 @@ def impl(ctx): ) providers.extend(lint_providers) - validation_outputs.append(_validate_rule( - ctx, - aar = aar, - package = package, - manifest = manifest_ctx.processed_manifest, - checks = _get_android_toolchain(ctx).aar_import_checks.files_to_run, - )) - providers.append( intellij.make_android_ide_info( ctx, From 74765909f89fd94981dcf14a52d92f4d93acbec5 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Fri, 14 Jul 2023 12:41:29 -0700 Subject: [PATCH 24/29] Rename android_application_macro generated target names to work better with the Lyft codebase --- rules/android_application/android_application_rule.bzl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rules/android_application/android_application_rule.bzl b/rules/android_application/android_application_rule.bzl index b4f2664f3..d9b4dd4bb 100644 --- a/rules/android_application/android_application_rule.bzl +++ b/rules/android_application/android_application_rule.bzl @@ -368,7 +368,6 @@ def android_application_macro(_android_binary, **attrs): # Create an android_binary base split, plus an android_application to produce the aab name = attrs.pop("name") - base_split_name = "%s_base" % name # default to [] if feature_modules = None is passed feature_modules = attrs.pop("feature_modules", []) or [] @@ -386,13 +385,13 @@ def android_application_macro(_android_binary, **attrs): attrs["deps"].append(str(module_targets.title_lib)) _android_binary( - name = base_split_name, + name = name, **attrs ) android_application( - name = name, - base_module = ":%s" % base_split_name, + name = "%s_aab" % name, + base_module = ":%s" % name, bundle_config_file = bundle_config_file, app_integrity_config = app_integrity_config, rotation_config = rotation_config, From 41d890dc697e6bab7fc559665f4433e03d9b919d Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Fri, 25 Aug 2023 08:18:24 -0700 Subject: [PATCH 25/29] Make the custom toolchain setup a bit easier --- tools/jdk/BUILD | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/tools/jdk/BUILD b/tools/jdk/BUILD index f904a43f1..b6e1e7aba 100644 --- a/tools/jdk/BUILD +++ b/tools/jdk/BUILD @@ -1,19 +1,6 @@ -load("@bazel_tools//tools/jdk:default_java_toolchain.bzl", "BASE_JDK9_JVM_OPTS", "default_java_toolchain") - -# Setup default javacopts such as for errorprone without affecting external repos. -default_java_toolchain( +alias( name = "toolchain_android_only", - bootclasspath = [ - "//tools/android:android_jar", - # TODO(b/175805830): Add this only when desugaring is enabled. - "@bazel_tools//tools/android:desugar_java8_extra_bootclasspath", - ], - # javac = ["//tools/error-prone:javac"], - java_runtime = "@lyft_jdk11//:jdk", - jvm_opts = BASE_JDK9_JVM_OPTS + ["-XX:-MaxFDLimit"], - package_configuration = ["@//:error_prone"], - source_version = "1.8", - target_version = "1.8", + actual = "@//:lyft_default_java_toolchain", visibility = ["//visibility:public"], ) From 6dd3c788b8cf80d7add0016a96148c9cb323add8 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Thu, 7 Sep 2023 21:48:52 -0700 Subject: [PATCH 26/29] Output proguard mappings from R8 --- rules/android_binary_internal/r8.bzl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rules/android_binary_internal/r8.bzl b/rules/android_binary_internal/r8.bzl index 0ace5492c..01d61ea39 100644 --- a/rules/android_binary_internal/r8.bzl +++ b/rules/android_binary_internal/r8.bzl @@ -74,11 +74,13 @@ def process_r8(ctx, jvm_ctx, packaged_resources_ctx, build_info_ctx, **_unused_c android_jar = get_android_sdk(ctx).android_jar proguard_specs = proguard.get_proguard_specs(ctx, packaged_resources_ctx.resource_proguard_config) min_sdk_version = getattr(ctx.attr, "min_sdk_version", None) + proguard_mappings_file = ctx.actions.declare_file(ctx.label.name + "_proguard.map") args = ctx.actions.args() args.add("--release") if min_sdk_version: args.add("--min-api", min_sdk_version) + args.add("--pg-map-output", proguard_mappings_file) args.add("--output", dexes_zip) args.add_all(proguard_specs, before_each = "--pg-conf") args.add("--lib", android_jar) @@ -90,7 +92,7 @@ def process_r8(ctx, jvm_ctx, packaged_resources_ctx, build_info_ctx, **_unused_c executable = get_android_toolchain(ctx).r8.files_to_run, arguments = [args], inputs = [android_jar, deploy_jar] + proguard_specs, - outputs = [dexes_zip], + outputs = [dexes_zip, proguard_mappings_file], mnemonic = "AndroidR8", jvm_flags = ["-Xmx8G"], progress_message = "R8 Optimizing, Desugaring, and Dexing %{label}", @@ -108,6 +110,7 @@ def process_r8(ctx, jvm_ctx, packaged_resources_ctx, build_info_ctx, **_unused_c name = "r8_ctx", value = struct( final_classes_dex_zip = dexes_zip, + proguard_mappings_file = proguard_mappings_file, providers = [android_dex_info], ), ) From 575b2f84b3c16c7e33cc0139b69023c11500a707 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Wed, 13 Sep 2023 14:24:03 -0700 Subject: [PATCH 27/29] Increase ThreadStackSize --- rules/android_binary_internal/r8.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/android_binary_internal/r8.bzl b/rules/android_binary_internal/r8.bzl index 01d61ea39..bae9bcfd6 100644 --- a/rules/android_binary_internal/r8.bzl +++ b/rules/android_binary_internal/r8.bzl @@ -94,7 +94,7 @@ def process_r8(ctx, jvm_ctx, packaged_resources_ctx, build_info_ctx, **_unused_c inputs = [android_jar, deploy_jar] + proguard_specs, outputs = [dexes_zip, proguard_mappings_file], mnemonic = "AndroidR8", - jvm_flags = ["-Xmx8G"], + jvm_flags = ["-Xmx8G", "-XX:ThreadStackSize=2048"], progress_message = "R8 Optimizing, Desugaring, and Dexing %{label}", ) From 275ab3ace6b0c7604b12dee50f81e3e868c2a1eb Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Wed, 20 Sep 2023 14:28:23 -0700 Subject: [PATCH 28/29] Increase R8 ram to 10g --- rules/android_binary_internal/r8.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/android_binary_internal/r8.bzl b/rules/android_binary_internal/r8.bzl index bae9bcfd6..a94580b07 100644 --- a/rules/android_binary_internal/r8.bzl +++ b/rules/android_binary_internal/r8.bzl @@ -94,7 +94,7 @@ def process_r8(ctx, jvm_ctx, packaged_resources_ctx, build_info_ctx, **_unused_c inputs = [android_jar, deploy_jar] + proguard_specs, outputs = [dexes_zip, proguard_mappings_file], mnemonic = "AndroidR8", - jvm_flags = ["-Xmx8G", "-XX:ThreadStackSize=2048"], + jvm_flags = ["-Xmx10G", "-XX:ThreadStackSize=2048"], progress_message = "R8 Optimizing, Desugaring, and Dexing %{label}", ) From cec1b126c71d3ca4ba840d8df708654455c51ff1 Mon Sep 17 00:00:00 2001 From: Oli Date: Mon, 9 Oct 2023 18:46:37 -0700 Subject: [PATCH 29/29] Add config to possibly skip unecessary linking for android_library targets --- rules/aar_import/impl.bzl | 1 + rules/android_library/impl.bzl | 1 + rules/flags/flag_defs.bzl | 6 +++ rules/resources.bzl | 74 ++++++++++++++++++++-------------- 4 files changed, 51 insertions(+), 31 deletions(-) diff --git a/rules/aar_import/impl.bzl b/rules/aar_import/impl.bzl index 467e81792..76426ba3e 100644 --- a/rules/aar_import/impl.bzl +++ b/rules/aar_import/impl.bzl @@ -169,6 +169,7 @@ def _process_resources( host_javabase = _common.get_host_javabase(ctx), instrument_xslt = _utils.only(_get_android_toolchain(ctx).add_g3itr_xslt.files.to_list()), xsltproc = _get_android_toolchain(ctx).xsltproc_tool.files_to_run, + link_library_resources = True, ) native_android_manifest = manifest diff --git a/rules/android_library/impl.bzl b/rules/android_library/impl.bzl index f8456e2ee..7ce8402ab 100644 --- a/rules/android_library/impl.bzl +++ b/rules/android_library/impl.bzl @@ -197,6 +197,7 @@ def _process_resources(ctx, java_package, manifest_ctx, **unused_ctxs): ), xsltproc = get_android_toolchain(ctx).xsltproc_tool.files_to_run, zip_tool = get_android_toolchain(ctx).zip_tool.files_to_run, + link_library_resources = _flags.get(ctx).link_library_resources, ) # TODO(b/139305816): Remove the ability for android_library to be added in diff --git a/rules/flags/flag_defs.bzl b/rules/flags/flag_defs.bzl index 98e3c8d20..14386b1e2 100644 --- a/rules/flags/flag_defs.bzl +++ b/rules/flags/flag_defs.bzl @@ -114,3 +114,9 @@ def define_flags(): default = False, description = "If disabled, does not merge manifest permissions.", ) + + flags.DEFINE_bool( + name = "link_library_resources", + default = True, + description = "If disabled, resource linking will be skipped on android libraries", + ) diff --git a/rules/resources.bzl b/rules/resources.bzl index 00071406c..f5026a0a2 100644 --- a/rules/resources.bzl +++ b/rules/resources.bzl @@ -1265,7 +1265,8 @@ def _process_starlark( xsltproc = None, zip_tool = None, namespaced_r_class = False, - output_library_merged_assets = True): + output_library_merged_assets = True, + link_library_resources = True): """Processes Android Resources. Args: @@ -1321,6 +1322,8 @@ def _process_starlark( instrument_xslt: File. The xslt transform to apply g3itr. xsltproc: FilesToRunProvider. The xsltproc executable or FilesToRunProvider. zip_tool: FilesToRunProvider. The zip tool executable or FilesToRunProvider. + link_library_resources: boolean. Whether to run or skip 'aapt link' on + android library resources. Returns: A dict containing _ResourcesProcessContextInfo provider fields. @@ -1632,9 +1635,11 @@ def _process_starlark( processed_manifest = ctx.actions.declare_file( "_migrated/" + ctx.label.name + "_processed_manifest/AndroidManifest.xml", ) + r_txt_filename = "R.aapt2.txt" if link_library_resources else "R.txt" out_aapt2_r_txt = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + "_symbols/R.aapt2.txt", + "_migrated/" + ctx.label.name + "_symbols/%s" % r_txt_filename, ) + _busybox.compile( ctx, out_file = compiled_resources, @@ -1673,34 +1678,39 @@ def _process_starlark( host_javabase = host_javabase, ) - apk = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + "_files/library.ap_", - ) - r_java = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + ".srcjar", - ) - r_txt = ctx.actions.declare_file( - "_migrated/" + ctx.label.name + "_symbols/R.txt", - ) - _busybox.validate_and_link( - ctx, - out_r_src_jar = r_java, - out_r_txt = r_txt, - out_file = apk, - compiled_resources = compiled_resources, - transitive_compiled_resources = depset( - transitive = transitive_compiled_resources, - order = "preorder", - ), - java_package = java_package, - manifest = processed_manifest, - android_jar = android_jar, - aapt = aapt, - busybox = busybox, - host_javabase = host_javabase, - resource_apks = resource_apks, - ) - resources_ctx[_RESOURCES_APK] = apk + if link_library_resources: + apk = ctx.actions.declare_file( + "_migrated/" + ctx.label.name + "_files/library.ap_", + ) + r_java = ctx.actions.declare_file( + "_migrated/" + ctx.label.name + ".srcjar", + ) + r_txt = ctx.actions.declare_file( + "_migrated/" + ctx.label.name + "_symbols/R.txt", + ) + _busybox.validate_and_link( + ctx, + out_r_src_jar = r_java, + out_r_txt = r_txt, + out_file = apk, + compiled_resources = compiled_resources, + transitive_compiled_resources = depset( + transitive = transitive_compiled_resources, + order = "preorder", + ), + java_package = java_package, + manifest = processed_manifest, + android_jar = android_jar, + aapt = aapt, + busybox = busybox, + host_javabase = host_javabase, + resource_apks = resource_apks, + ) + resources_ctx[_RESOURCES_APK] = apk + else: + r_txt = out_aapt2_r_txt + r_java = None + java_info = JavaInfo( output_jar = out_class_jar, compile_jar = out_class_jar, @@ -1923,7 +1933,8 @@ def _process( propagate_resources = True, namespaced_r_class = False, output_library_merged_assets = True, - zip_tool = None): + zip_tool = None, + link_library_resources = True): out_ctx = _process_starlark( ctx, java_package = java_package, @@ -1958,6 +1969,7 @@ def _process( zip_tool = zip_tool, namespaced_r_class = namespaced_r_class, output_library_merged_assets = output_library_merged_assets, + link_library_resources = link_library_resources, ) if _VALIDATION_OUTPUTS not in out_ctx: