Add ability to have multiple connected remote endpoints

Bug: webrtc:10138
Change-Id: Ic305c2f247588d75b6ced17052ba12d937d1a056
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/128864
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Sebastian Jansson <srte@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27460}

api/test/network_emulation_manager.h

@@ -86,18 +86,23 @@ class NetworkEmulationManager {
   // Creates a route between endpoints going through specified network nodes.
   // This route is single direction only and describe how traffic that was
   // sent by network interface |from| have to be delivered to the network
-  // interface |to|. Return object can be used to remove created route.
+  // interface |to|. Return object can be used to remove created route. The
+  // route must contains at least one network node inside it.
   //
-  // Assume there are endpoints E1, E2 and E3 and network nodes A, B, C and D.
+  // Assume that E{0-9} are endpoints and N{0-9} are network nodes, then
-  // Also assume, that there is a route constructed via A, B and C like this:
+  // creation of the route have to follow these rules:
-  // E1 -> A -> B -> C -> E2. In such case:
+  //   1. A route consists of a source endpoint, an ordered list of one or
-  //   * Caller mustn't use A, B and C in any route, that is leading to E2.
+  //      more network nodes, and a destination endpoint.
-  //   * If caller will then create a new route E1 -> D -> E3, then first
+  //   2. If (E1, ..., E2) is a route, then E1 != E2.
-  //     route will be corrupted, so if caller want to do this, first route
+  //      In other words, the source and the destination may not be the same.
-  //     should be deleted by ClearRoute(...) and then a new one should be
+  //   3. Given two simultaneously existing routes (E1, ..., E2) and
-  //     created.
+  //      (E3, ..., E4), either E1 != E3 or E2 != E4.
-  //   * Caller can use A, B or C for any other routes.
+  //      In other words, there may be at most one route from any given source
-  //   * Caller can create other routes leading to E2.
+  //      endpoint to any given destination endpoint.
+  //   4. Given two simultaneously existing routes (E1, ..., N1, ..., E2)
+  //      and (E3, ..., N2, ..., E4), either N1 != N2 or E2 != E4.
+  //      In other words, a network node may not belong to two routes that lead
+  //      to the same destination endpoint.
   virtual EmulatedRoute* CreateRoute(
       EmulatedEndpoint* from,
       const std::vector<EmulatedNetworkNode*>& via_nodes,

test/scenario/network/BUILD.gn

@@ -9,6 +9,13 @@
 import("../../../webrtc.gni")
 
 rtc_source_set("emulated_network") {
+  visibility = [
+    "../../../api:create_network_emulation_manager",
+    ":*",
+  ]
+  if (rtc_include_tests) {
+    visibility += [ "../:scenario" ]
+  }
   testonly = true
   sources = [
     "cross_traffic.cc",
@@ -57,8 +64,10 @@ rtc_source_set("network_emulation_unittest") {
     ":emulated_network",
     "../../../api:simulated_network_api",
     "../../../call:simulated_network",
+    "../../../rtc_base:gunit_helpers",
     "../../../rtc_base:logging",
     "../../../rtc_base:rtc_event",
+    "../../../system_wrappers:system_wrappers",
     "../../../test:test_support",
     "//third_party/abseil-cpp/absl/memory",
   ]

test/scenario/network/cross_traffic_unittest.cc

@@ -39,8 +39,9 @@ class CountingReceiver : public EmulatedNetworkReceiverInterface {
 struct TrafficCounterFixture {
   SimulatedClock clock{0};
   CountingReceiver counter;
-  EmulatedEndpoint endpoint{1 /*id */, rtc::IPAddress(), true /*is_enabled*/,
+  TaskQueueForTest task_queue_;
-                            &clock};
+  EmulatedEndpoint endpoint{/*id=*/1, rtc::IPAddress(), /*is_enabled=*/true,
+                            &task_queue_, &clock};
 };
 
 }  // namespace

test/scenario/network/network_emulation.cc

@@ -161,12 +161,14 @@ EmulatedNetworkNode::~EmulatedNetworkNode() = default;
 EmulatedEndpoint::EmulatedEndpoint(uint64_t id,
                                    const rtc::IPAddress& ip,
                                    bool is_enabled,
+                                   rtc::TaskQueue* task_queue,
                                    Clock* clock)
     : id_(id),
       peer_local_addr_(ip),
       is_enabled_(is_enabled),
-      send_node_(nullptr),
       clock_(clock),
+      task_queue_(task_queue),
+      router_(task_queue_),
       next_port_(kFirstEphemeralPort) {
   constexpr int kIPv4NetworkPrefixLength = 24;
   constexpr int kIPv6NetworkPrefixLength = 64;
@@ -191,18 +193,18 @@ uint64_t EmulatedEndpoint::GetId() const {
   return id_;
 }
 
-void EmulatedEndpoint::SetSendNode(EmulatedNetworkNode* send_node) {
-  send_node_ = send_node;
-}
-
 void EmulatedEndpoint::SendPacket(const rtc::SocketAddress& from,
                                   const rtc::SocketAddress& to,
                                   rtc::CopyOnWriteBuffer packet) {
   RTC_CHECK(from.ipaddr() == peer_local_addr_);
-  RTC_CHECK(send_node_);
+  struct Closure {
-  send_node_->OnPacketReceived(
+    void operator()() { endpoint->router_.OnPacketReceived(std::move(packet)); }
-      EmulatedIpPacket(from, to, std::move(packet),
+    EmulatedEndpoint* endpoint;
-                       Timestamp::us(clock_->TimeInMicroseconds())));
+    EmulatedIpPacket packet;
+  };
+  task_queue_->PostTask(Closure{
+      this, EmulatedIpPacket(from, to, std::move(packet),
+                             Timestamp::us(clock_->TimeInMicroseconds()))});
 }
 
 absl::optional<uint16_t> EmulatedEndpoint::BindReceiver(
@@ -293,10 +295,6 @@ bool EmulatedEndpoint::Enabled() const {
   return is_enabled_;
 }
 
-EmulatedNetworkNode* EmulatedEndpoint::GetSendNode() const {
-  return send_node_;
-}
-
 EndpointsContainer::EndpointsContainer(
     const std::vector<EmulatedEndpoint*>& endpoints)
     : endpoints_(endpoints) {}

test/scenario/network/network_emulation.h

@@ -30,13 +30,6 @@
 #include "system_wrappers/include/clock.h"
 
 namespace webrtc {
-namespace test {
-
-// Forward declare NetworkEmulationManagerImpl for friend access from
-// EmulatedEndpoint.
-class NetworkEmulationManagerImpl;
-
-}  // namespace test
 
 struct EmulatedIpPacket {
  public:
@@ -159,13 +152,13 @@ class EmulatedEndpoint : public EmulatedNetworkReceiverInterface {
   EmulatedEndpoint(uint64_t id,
                    const rtc::IPAddress& ip,
                    bool is_enabled,
+                   rtc::TaskQueue* task_queue,
                    Clock* clock);
   ~EmulatedEndpoint() override;
 
   uint64_t GetId() const;
 
-  // Set network node, that will be used to send packets to the network.
+  NetworkRouterNode* router() { return &router_; }
-  void SetSendNode(EmulatedNetworkNode* send_node);
   // Send packet into network.
   // |from| will be used to set source address for the packet in destination
   // socket.
@@ -200,11 +193,6 @@ class EmulatedEndpoint : public EmulatedNetworkReceiverInterface {
 
   const rtc::Network& network() const { return *network_.get(); }
 
- protected:
-  friend class test::NetworkEmulationManagerImpl;
-
-  EmulatedNetworkNode* GetSendNode() const;
-
  private:
   static constexpr uint16_t kFirstEphemeralPort = 49152;
   uint16_t NextPort() RTC_EXCLUSIVE_LOCKS_REQUIRED(receiver_lock_);
@@ -216,9 +204,10 @@ class EmulatedEndpoint : public EmulatedNetworkReceiverInterface {
   // Peer's local IP address for this endpoint network interface.
   const rtc::IPAddress peer_local_addr_;
   bool is_enabled_ RTC_GUARDED_BY(enabled_state_checker_);
-  EmulatedNetworkNode* send_node_;
   Clock* const clock_;
+  rtc::TaskQueue* const task_queue_;
   std::unique_ptr<rtc::Network> network_;
+  NetworkRouterNode router_;
 
   uint16_t next_port_ RTC_GUARDED_BY(receiver_lock_);
   std::map<uint16_t, EmulatedNetworkReceiverInterface*> port_to_receiver_

test/scenario/network/network_emulation_manager.cc

@@ -81,7 +81,7 @@ EmulatedEndpoint* NetworkEmulationManagerImpl::CreateEndpoint(
   bool res = used_ip_addresses_.insert(*ip).second;
   RTC_CHECK(res) << "IP=" << ip->ToString() << " already in use";
   auto node = absl::make_unique<EmulatedEndpoint>(
-      next_node_id_++, *ip, config.start_as_enabled, clock_);
+      next_node_id_++, *ip, config.start_as_enabled, &task_queue_, clock_);
   EmulatedEndpoint* out = node.get();
   endpoints_.push_back(std::move(node));
   return out;
@@ -109,7 +109,7 @@ EmulatedRoute* NetworkEmulationManagerImpl::CreateRoute(
   // provided here.
   RTC_CHECK(!via_nodes.empty());
 
-  from->SetSendNode(via_nodes[0]);
+  from->router()->SetReceiver(to->GetPeerLocalAddress(), via_nodes[0]);
   EmulatedNetworkNode* cur_node = via_nodes[0];
   for (size_t i = 1; i < via_nodes.size(); ++i) {
     cur_node->router()->SetReceiver(to->GetPeerLocalAddress(), via_nodes[i]);
@@ -131,12 +131,8 @@ void NetworkEmulationManagerImpl::ClearRoute(EmulatedRoute* route) {
     for (auto* node : route->via_nodes) {
       node->router()->RemoveReceiver(route->to->GetPeerLocalAddress());
     }
-    // Detach endpoint from current send node.
+    // Remove destination endpoint from source endpoint's router.
-    if (route->from->GetSendNode()) {
+    route->from->router()->RemoveReceiver(route->to->GetPeerLocalAddress());
-      route->from->GetSendNode()->router()->RemoveReceiver(
-          route->to->GetPeerLocalAddress());
-      route->from->SetSendNode(nullptr);
-    }
 
     route->active = false;
   });

test/scenario/network/network_emulation_unittest.cc

@@ -8,13 +8,16 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#include <atomic>
 #include <memory>
 
 #include "absl/memory/memory.h"
 #include "api/test/simulated_network.h"
 #include "call/simulated_network.h"
 #include "rtc_base/event.h"
+#include "rtc_base/gunit.h"
 #include "rtc_base/logging.h"
+#include "system_wrappers/include/sleep.h"
 #include "test/gmock.h"
 #include "test/gtest.h"
 #include "test/scenario/network/network_emulation.h"
@@ -22,6 +25,9 @@
 
 namespace webrtc {
 namespace test {
+namespace {
+
+constexpr int kNetworkPacketWaitTimeoutMs = 100;
 
 class SocketReader : public sigslot::has_slots<> {
  public:
@@ -57,6 +63,93 @@ class SocketReader : public sigslot::has_slots<> {
   int received_count_ RTC_GUARDED_BY(lock_) = 0;
 };
 
+class MockReceiver : public EmulatedNetworkReceiverInterface {
+ public:
+  MOCK_METHOD1(OnPacketReceived, void(EmulatedIpPacket packet));
+};
+
+class NetworkEmulationManagerThreeNodesRoutingTest : public ::testing::Test {
+ public:
+  NetworkEmulationManagerThreeNodesRoutingTest() {
+    e1_ = emulation_.CreateEndpoint(EmulatedEndpointConfig());
+    e2_ = emulation_.CreateEndpoint(EmulatedEndpointConfig());
+    e3_ = emulation_.CreateEndpoint(EmulatedEndpointConfig());
+  }
+
+  void SetupRouting(
+      std::function<void(EmulatedEndpoint*,
+                         EmulatedEndpoint*,
+                         EmulatedEndpoint*,
+                         NetworkEmulationManager*)> create_routing_func) {
+    create_routing_func(e1_, e2_, e3_, &emulation_);
+  }
+
+  void SendPacketsAndValidateDelivery() {
+    EXPECT_CALL(r_e1_e2_, OnPacketReceived(testing::_)).Times(1);
+    EXPECT_CALL(r_e2_e1_, OnPacketReceived(testing::_)).Times(1);
+    EXPECT_CALL(r_e1_e3_, OnPacketReceived(testing::_)).Times(1);
+    EXPECT_CALL(r_e3_e1_, OnPacketReceived(testing::_)).Times(1);
+
+    uint16_t common_send_port = 80;
+    uint16_t r_e1_e2_port = e2_->BindReceiver(0, &r_e1_e2_).value();
+    uint16_t r_e2_e1_port = e1_->BindReceiver(0, &r_e2_e1_).value();
+    uint16_t r_e1_e3_port = e3_->BindReceiver(0, &r_e1_e3_).value();
+    uint16_t r_e3_e1_port = e1_->BindReceiver(0, &r_e3_e1_).value();
+
+    // Next code is using API of EmulatedEndpoint, that is visible only for
+    // internals of network emulation layer. Don't use this API in other tests.
+    // Send packet from e1 to e2.
+    e1_->SendPacket(
+        rtc::SocketAddress(e1_->GetPeerLocalAddress(), common_send_port),
+        rtc::SocketAddress(e2_->GetPeerLocalAddress(), r_e1_e2_port),
+        rtc::CopyOnWriteBuffer(10));
+
+    // Send packet from e2 to e1.
+    e2_->SendPacket(
+        rtc::SocketAddress(e2_->GetPeerLocalAddress(), common_send_port),
+        rtc::SocketAddress(e1_->GetPeerLocalAddress(), r_e2_e1_port),
+        rtc::CopyOnWriteBuffer(10));
+
+    // Send packet from e1 to e3.
+    e1_->SendPacket(
+        rtc::SocketAddress(e1_->GetPeerLocalAddress(), common_send_port),
+        rtc::SocketAddress(e3_->GetPeerLocalAddress(), r_e1_e3_port),
+        rtc::CopyOnWriteBuffer(10));
+
+    // Send packet from e3 to e1.
+    e3_->SendPacket(
+        rtc::SocketAddress(e3_->GetPeerLocalAddress(), common_send_port),
+        rtc::SocketAddress(e1_->GetPeerLocalAddress(), r_e3_e1_port),
+        rtc::CopyOnWriteBuffer(10));
+
+    // Sleep at the end to wait for async packets delivery.
+    SleepMs(kNetworkPacketWaitTimeoutMs);
+  }
+
+ private:
+  // Receivers: r_<source endpoint>_<destination endpoint>
+  // They must be destroyed after emulation, so they should be declared before.
+  MockReceiver r_e1_e2_;
+  MockReceiver r_e2_e1_;
+  MockReceiver r_e1_e3_;
+  MockReceiver r_e3_e1_;
+
+  NetworkEmulationManagerImpl emulation_;
+  EmulatedEndpoint* e1_;
+  EmulatedEndpoint* e2_;
+  EmulatedEndpoint* e3_;
+};
+
+EmulatedNetworkNode* CreateEmulatedNodeWithDefaultBuiltInConfig(
+    NetworkEmulationManager* emulation) {
+  return emulation->CreateEmulatedNode(
+      absl::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig()));
+}
+
+}  // namespace
+
+using testing::_;
+
 TEST(NetworkEmulationManagerTest, GeneratedIpv4AddressDoesNotCollide) {
   NetworkEmulationManagerImpl network_manager;
   std::set<rtc::IPAddress> ips;
@@ -136,5 +229,52 @@ TEST(NetworkEmulationManagerTest, Run) {
   }
 }
 
+// Testing that packets are delivered via all routes using a routing scheme as
+// follows:
+//  * e1 -> n1 -> e2
+//  * e2 -> n2 -> e1
+//  * e1 -> n3 -> e3
+//  * e3 -> n4 -> e1
+TEST_F(NetworkEmulationManagerThreeNodesRoutingTest,
+       PacketsAreDeliveredInBothWaysWhenConnectedToTwoPeers) {
+  SetupRouting([](EmulatedEndpoint* e1, EmulatedEndpoint* e2,
+                  EmulatedEndpoint* e3, NetworkEmulationManager* emulation) {
+    auto* node1 = CreateEmulatedNodeWithDefaultBuiltInConfig(emulation);
+    auto* node2 = CreateEmulatedNodeWithDefaultBuiltInConfig(emulation);
+    auto* node3 = CreateEmulatedNodeWithDefaultBuiltInConfig(emulation);
+    auto* node4 = CreateEmulatedNodeWithDefaultBuiltInConfig(emulation);
+
+    emulation->CreateRoute(e1, {node1}, e2);
+    emulation->CreateRoute(e2, {node2}, e1);
+
+    emulation->CreateRoute(e1, {node3}, e3);
+    emulation->CreateRoute(e3, {node4}, e1);
+  });
+  SendPacketsAndValidateDelivery();
+}
+
+// Testing that packets are delivered via all routes using a routing scheme as
+// follows:
+//  * e1 -> n1 -> e2
+//  * e2 -> n2 -> e1
+//  * e1 -> n1 -> e3
+//  * e3 -> n4 -> e1
+TEST_F(NetworkEmulationManagerThreeNodesRoutingTest,
+       PacketsAreDeliveredInBothWaysWhenConnectedToTwoPeersOverSameSendLink) {
+  SetupRouting([](EmulatedEndpoint* e1, EmulatedEndpoint* e2,
+                  EmulatedEndpoint* e3, NetworkEmulationManager* emulation) {
+    auto* node1 = CreateEmulatedNodeWithDefaultBuiltInConfig(emulation);
+    auto* node2 = CreateEmulatedNodeWithDefaultBuiltInConfig(emulation);
+    auto* node3 = CreateEmulatedNodeWithDefaultBuiltInConfig(emulation);
+
+    emulation->CreateRoute(e1, {node1}, e2);
+    emulation->CreateRoute(e2, {node2}, e1);
+
+    emulation->CreateRoute(e1, {node1}, e3);
+    emulation->CreateRoute(e3, {node3}, e1);
+  });
+  SendPacketsAndValidateDelivery();
+}
+
 }  // namespace test
 }  // namespace webrtc