From d0eaa54104d392a4067b4ab85513b58b5153c5f9 Mon Sep 17 00:00:00 2001 From: Jeremy Leconte Date: Fri, 2 Dec 2022 20:54:15 +0100 Subject: [PATCH] Reland "Add working Perf tests for Fuchsia" This is a reland of commit d8d86bd3329d05fdcc5402140917eb70b340c9c6 Original change's description: > Add working Perf tests for Fuchsia > > Utilize VM in perf pool for Fuchsia (separate CL) > > Bug: b/232740856 > Change-Id: Ia69f72bbf2046db3bc9b1e5edabda6cc4a339fe6 > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/285721 > Reviewed-by: Jeremy Leconte > Commit-Queue: Christoffer Jansson > Reviewed-by: Mirko Bonadei > Cr-Commit-Position: refs/heads/main@{#38787} Bug: b/232740856 Change-Id: I3ae8b17990157bde4a08fd8a98379cffd2c38148 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/285881 Reviewed-by: Jeremy Leconte Reviewed-by: Jeremy Leconte Commit-Queue: Jeremy Leconte Cr-Commit-Position: refs/heads/main@{#38802} --- BUILD.gn | 16 ++++++++ infra/specs/client.webrtc.perf.json | 61 ++++++++++++++++++++++++++++- infra/specs/gn_isolate_map.pyl | 4 ++ infra/specs/mixins.pyl | 21 +++++++++- infra/specs/mixins_webrtc.pyl | 19 +++++++++ infra/specs/test_suites.pyl | 5 +++ infra/specs/waterfalls.pyl | 9 ++--- 7 files changed, 128 insertions(+), 7 deletions(-) diff --git a/BUILD.gn b/BUILD.gn index ca60bdc24b..433e22e7b9 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -722,6 +722,22 @@ if (rtc_include_tests && !build_with_chromium) { } } + rtc_test("fuchsia_perf_tests") { + testonly = true + deps = [ + #TODO(fxbug.dev/115601) - Enable when fixed + #"call:call_perf_tests", + #"video:video_pc_full_stack_tests", + "modules/audio_coding:audio_coding_perf_tests", + "modules/audio_processing:audio_processing_perf_tests", + "pc:peerconnection_perf_tests", + "test:test_main", + "video:video_full_stack_tests", + ] + + data = webrtc_perf_tests_resources + } + rtc_test("webrtc_nonparallel_tests") { testonly = true deps = [ "rtc_base:rtc_base_nonparallel_tests" ] diff --git a/infra/specs/client.webrtc.perf.json b/infra/specs/client.webrtc.perf.json index 4f61d9ce6d..be4d952535 100644 --- a/infra/specs/client.webrtc.perf.json +++ b/infra/specs/client.webrtc.perf.json @@ -30,6 +30,7 @@ "android_devices": "1", "device_os": "M", "device_type": "shamu", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -65,6 +66,7 @@ "android_devices": "1", "device_os": "M", "device_type": "shamu", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -108,6 +110,7 @@ "android_devices": "1", "device_os": "M", "device_type": "hammerhead", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -143,6 +146,7 @@ "android_devices": "1", "device_os": "M", "device_type": "hammerhead", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -185,6 +189,7 @@ { "android_devices": "1", "device_type": "walleye", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -219,6 +224,7 @@ { "android_devices": "1", "device_type": "walleye", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -261,6 +267,7 @@ { "android_devices": "1", "device_type": "redfin", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -295,6 +302,7 @@ { "android_devices": "1", "device_type": "redfin", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -338,6 +346,7 @@ "android_devices": "1", "device_os": "MMB29Q", "device_type": "bullhead", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -373,6 +382,7 @@ "android_devices": "1", "device_os": "MMB29Q", "device_type": "bullhead", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -415,6 +425,7 @@ { "android_devices": "1", "device_type": "walleye", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -449,6 +460,7 @@ { "android_devices": "1", "device_type": "walleye", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -491,6 +503,7 @@ { "android_devices": "1", "device_type": "redfin", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -525,6 +538,7 @@ { "android_devices": "1", "device_type": "redfin", + "gce": "0", "os": "Android", "pool": "WebRTC-perf" } @@ -539,7 +553,45 @@ } ] }, - "Perf Fuchsia": {}, + "Perf Fuchsia": { + "isolated_scripts": [ + { + "args": [ + "--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb", + "--nologs" + ], + "isolate_name": "fuchsia_perf_tests", + "merge": { + "args": [ + "--test-suite", + "fuchsia_perf_tests" + ], + "script": "//tools_webrtc/perf/process_perf_results.py" + }, + "name": "fuchsia_perf_tests", + "resultdb": { + "enable": true, + "has_native_resultdb_integration": true + }, + "swarming": { + "can_use_on_swarming_builders": true, + "dimension_sets": [ + { + "cpu": "x86-64", + "gce": "1", + "os": "Ubuntu-18.04", + "pool": "WebRTC-perf" + } + ], + "hard_timeout": 10800, + "idempotent": false, + "io_timeout": 10800, + "service_account": "chromium-tester@chops-service-accounts.iam.gserviceaccount.com" + }, + "test_id_prefix": "ninja://:fuchsia_perf_tests/" + } + ] + }, "Perf Linux Bionic": { "isolated_scripts": [ { @@ -565,6 +617,7 @@ "can_use_on_swarming_builders": true, "dimension_sets": [ { + "gce": "0", "os": "Ubuntu-18.04", "pool": "WebRTC-perf" } @@ -599,6 +652,7 @@ "can_use_on_swarming_builders": true, "dimension_sets": [ { + "gce": "0", "os": "Ubuntu-18.04", "pool": "WebRTC-perf" } @@ -637,6 +691,7 @@ "dimension_sets": [ { "cpu": "x86-64", + "gce": "0", "os": "Mac-11", "pool": "WebRTC-perf" } @@ -672,6 +727,7 @@ "dimension_sets": [ { "cpu": "x86-64", + "gce": "0", "os": "Mac-11", "pool": "WebRTC-perf" } @@ -709,6 +765,7 @@ "dimension_sets": [ { "cpu": "arm64-64-Apple_M1", + "gce": "0", "os": "Mac-12", "pool": "WebRTC-perf" } @@ -746,6 +803,7 @@ "can_use_on_swarming_builders": true, "dimension_sets": [ { + "gce": "0", "os": "Windows-10", "pool": "WebRTC-perf" } @@ -780,6 +838,7 @@ "can_use_on_swarming_builders": true, "dimension_sets": [ { + "gce": "0", "os": "Windows-10", "pool": "WebRTC-perf" } diff --git a/infra/specs/gn_isolate_map.pyl b/infra/specs/gn_isolate_map.pyl index 25ab4f2e2d..7d1857225f 100644 --- a/infra/specs/gn_isolate_map.pyl +++ b/infra/specs/gn_isolate_map.pyl @@ -59,6 +59,10 @@ "label": "//sdk/android:android_instrumentation_test_apk", "type": "console_test_launcher", }, + "fuchsia_perf_tests": { + "label": "//:fuchsia_perf_tests", + "type": "raw", + }, "low_bandwidth_audio_test": { "label": "//audio:low_bandwidth_audio_test", "type": "console_test_launcher", diff --git a/infra/specs/mixins.pyl b/infra/specs/mixins.pyl index 86a5161d8e..20f342ac34 100644 --- a/infra/specs/mixins.pyl +++ b/infra/specs/mixins.pyl @@ -192,6 +192,15 @@ 'args': ['--out-dir', '${ISOLATED_OUTDIR}'] } }, + 'perf-fuchsia-perf-tests': { + 'merge': { + 'script': '//tools_webrtc/perf/process_perf_results.py', + 'args': ['--test-suite', 'fuchsia_perf_tests'] + }, + '$mixin_append': { + 'args': ['--nologs'] + } + }, 'perf-low-bandwidth-audio-perf-test': { 'merge': { 'script': '//tools_webrtc/perf/process_perf_results.py', @@ -209,7 +218,17 @@ 'swarming': { 'idempotent': False, 'dimensions': { - 'pool': 'WebRTC-perf' + 'pool': 'WebRTC-perf', + 'gce': '0' + } + } + }, + 'perf-pool-vm': { + 'swarming': { + 'idempotent': False, + 'dimensions': { + 'pool': 'WebRTC-perf', + 'gce': '1' } } }, diff --git a/infra/specs/mixins_webrtc.pyl b/infra/specs/mixins_webrtc.pyl index f026f7fa04..56de235da7 100644 --- a/infra/specs/mixins_webrtc.pyl +++ b/infra/specs/mixins_webrtc.pyl @@ -136,6 +136,15 @@ } } }, + 'perf-fuchsia-perf-tests': { + 'merge': { + 'script': '//tools_webrtc/perf/process_perf_results.py', + 'args': ['--test-suite', 'fuchsia_perf_tests'], + }, + '$mixin_append': { + 'args': ['--nologs'], + }, + }, 'perf-low-bandwidth-audio-perf-test': { 'merge': { 'script': '//tools_webrtc/perf/process_perf_results.py', @@ -157,6 +166,16 @@ 'idempotent': False, 'dimensions': { 'pool': 'WebRTC-perf', + 'gce': '0', + }, + }, + }, + 'perf-pool-vm': { + 'swarming': { + 'idempotent': False, + 'dimensions': { + 'pool': 'WebRTC-perf', + 'gce': '1', }, }, }, diff --git a/infra/specs/test_suites.pyl b/infra/specs/test_suites.pyl index 0769355409..ece0551f69 100644 --- a/infra/specs/test_suites.pyl +++ b/infra/specs/test_suites.pyl @@ -116,6 +116,11 @@ 'voip_unittests': {}, 'webrtc_nonparallel_tests': {}, }, + 'fuchsia_compatible_perf_tests': { + 'fuchsia_perf_tests': { + 'mixins': ['perf-fuchsia-perf-tests'], + }, + }, 'fuchsia_compatible_tests': { 'audio_decoder_unittests': {}, 'common_audio_unittests': {}, diff --git a/infra/specs/waterfalls.pyl b/infra/specs/waterfalls.pyl index f58f384e8a..6c9214b9d3 100644 --- a/infra/specs/waterfalls.pyl +++ b/infra/specs/waterfalls.pyl @@ -334,13 +334,12 @@ 'os_type': 'linux', 'mixins': [ - 'linux-bionic', 'x86-64', 'perf-pool', 'timeout-3h', 'perf-output', + 'linux-bionic', 'x86-64', 'perf-pool-vm', 'timeout-3h', 'perf-output', 'has_native_resultdb_integration', 'chromium-tester-service-account' ], - # TODO(b/232740856): Reenable when Fuchsia has it's own bot to run perf tests. - # 'test_suites': { - #  'isolated_scripts': 'desktop_perf_tests', - # } + 'test_suites': { + 'isolated_scripts': 'fuchsia_compatible_perf_tests', + } }, 'Perf Linux Bionic': { 'os_type':