rfc9693xml2.original.xml   rfc9693.xml 
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="utf-8"?>
<!-- This template is for creating an Internet Draft using xml2rfc,
which is available here: http://xml.resource.org. -->
<!DOCTYPE rfc SYSTEM "rfc2629.dtd" [
<!-- One method to get references from the online citation libraries.
There has to be one entity for each item to be referenced.
An alternate method (rfc include) is described in the references. -->
<!ENTITY RFC1918 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC <!DOCTYPE rfc [
.1918.xml"> <!ENTITY nbsp "&#160;">
<!ENTITY RFC2119 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC <!ENTITY zwsp "&#8203;">
.2119.xml"> <!ENTITY nbhy "&#8209;">
<!ENTITY RFC2544 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC <!ENTITY wj "&#8288;">
.2544.xml">
<!ENTITY RFC3022 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.3022.xml">
<!ENTITY RFC4340 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.4340.xml">
<!ENTITY RFC4814 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.4814.xml">
<!ENTITY RFC5180 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.5180.xml">
<!ENTITY RFC6146 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.6146.xml">
<!ENTITY RFC7599 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.7599.xml">
<!ENTITY RFC8174 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.8174.xml">
<!ENTITY RFC8219 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.8219.xml">
<!ENTITY RFC9000 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.9000.xml">
<!ENTITY RFC9260 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.9260.xml">
<!ENTITY RFC9411 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC
.9411.xml">
]> ]>
<?xml-stylesheet type='text/xsl' href='rfc2629.xslt' ?>
<!-- used by XSLT processors -->
<!-- For a complete list and description of processing instructions (PIs),
please see http://xml.resource.org/authoring/README.html. -->
<!-- Below are generally applicable Processing Instructions (PIs) that most I-Ds
might want to use.
(Here they are set differently than their defaults in xml2rfc v1.32) -->
<?rfc strict="yes" ?>
<!-- give errors regarding ID-nits and DTD validation -->
<!-- control the table of contents (ToC) -->
<?rfc toc="yes"?>
<!-- generate a ToC -->
<?rfc tocdepth="4"?>
<!-- the number of levels of subsections in ToC. default: 3 -->
<!-- control references -->
<?rfc symrefs="yes"?>
<!-- use symbolic references tags, i.e, [RFC2119] instead of [1] -->
<?rfc sortrefs="yes" ?>
<!-- sort the reference entries alphabetically -->
<!-- control vertical white space
(using these PIs as follows is recommended by the RFC Editor) -->
<?rfc compact="yes" ?>
<!-- do not start each main section on a new page -->
<?rfc subcompact="no" ?>
<!-- keep one blank line between list items -->
<!-- end of list of popular I-D processing instructions -->
<rfc category="info" docName="draft-ietf-bmwg-benchmarking-stateful-09" ipr="tru
st200902">
<front>
<!-- The abbreviated title is used in the page header - it is only necessary
if the
full title is longer than 39 characters -->
<title abbrev="Benchmarking Stateful NATxy Gateways">Benchmarking Methodolog <rfc xmlns:xi="http://www.w3.org/2001/XInclude" category="info" docName="draft-i
y etf-bmwg-benchmarking-stateful-09" number="9693" consensus="true" ipr="trust2009
for Stateful NATxy Gateways using RFC 4814 Pseudorandom Port Numbers</title> 02" obsoletes="" updates="" submissionType="IETF" xml:lang="en" tocInclude="true
" tocDepth="4" symRefs="true" sortRefs="true" version="3">
<!-- add 'role="editor"' below for the editors if appropriate -->
<!-- Another author who claims to be an editor -->
<front>
<title abbrev="Benchmarking Stateful NATxy Gateways">Benchmarking Methodolog
y for Stateful NATxy Gateways</title>
<seriesInfo name="RFC" value="9693"/>
<author fullname="Gábor Lencse" initials="G." surname="Lencse"> <author fullname="Gábor Lencse" initials="G." surname="Lencse">
<organization>Széchenyi István University</organization> <organization>Széchenyi István University</organization>
<address> <address>
<postal> <postal>
<street>Egyetem tér 1.</street> <street>Egyetem tér 1.</street>
<!-- Reorder these if your country does things differently -->
<city>Győr</city> <city>Győr</city>
<region></region>
<code>H-9026</code> <code>H-9026</code>
<country>Hungary</country> <country>Hungary</country>
</postal> </postal>
<phone></phone>
<email>lencse@sze.hu</email> <email>lencse@sze.hu</email>
<uri></uri>
</address> </address>
</author> </author>
<author fullname="Keiichi Shima" initials="K." surname="Shima"> <author fullname="Keiichi Shima" initials="K." surname="Shima">
<organization>SoftBank Corp.</organization> <organization>SoftBank Corp.</organization>
<address> <address>
<postal> <postal>
<street>1-7-1 Kaigan</street> <street>1-7-1 Kaigan</street>
<city>Minato-ku</city> <region>Minato-ku, Tokyo</region>
<region>Tokyo</region>
<code>105-7529</code> <code>105-7529</code>
<country>Japan</country> <country>Japan</country>
</postal> </postal>
<phone></phone>
<email>shima@wide.ad.jp</email> <email>shima@wide.ad.jp</email>
<uri>https://softbank.co.jp/</uri> <uri>https://softbank.co.jp/</uri>
</address> </address>
</author> </author>
<date year="2024" month="December"/>
<date year="2024" /> <area>OPS</area>
<workgroup>bmwg</workgroup>
<!-- Meta-data Declarations -->
<area>Operations and Management Area</area>
<workgroup>Benchmarking Methodology Working Group</workgroup>
<!-- WG name at the upperleft corner of the doc,
IETF is fine for individual submissions.
If this element is not present, the default is "Network Working Group",
which is used by the RFC Editor as a nod to the history of the IETF. -
->
<keyword>Benchmarking, Stateful NATxy, Measurement Procedure, Throughput, Fr
ame Loss Rate, Latency, PDV</keyword>
<!-- Keywords will be incorporated into HTML output <keyword>Benchmarking</keyword>
files in a meta tag but they have no effect on text or nroff <keyword>Stateful NATxy</keyword>
output. If you submit your draft to the RFC Editor, the <keyword>Measurement Procedure</keyword>
keywords will be used for the search engine. --> <keyword>Throughput</keyword>
<keyword>Frame Loss Rate</keyword>
<keyword>Latency</keyword>
<keyword>PDV</keyword>
<abstract> <abstract>
<t>RFC 2544 has defined a benchmarking methodology for network <t>RFC 2544 defines a benchmarking methodology for network
interconnect devices. RFC 5180 addressed IPv6 specificities and it also interconnect devices. RFC 5180 addresses IPv6 specificities, and it also
provided a technology update but excluded IPv6 transition technologies. provides a technology update but excludes IPv6 transition technologies.
RFC 8219 addressed IPv6 transition technologies, including stateful NAT64. RFC 8219 addresses IPv6 transition technologies, including stateful
However, none of them discussed how to apply RFC 4814 pseudorandom port NAT64. However, none of them discuss how to apply pseudorandom port
numbers numbers from RFC 4814 to any stateful NATxy (such as NAT44, NAT64, and NAT
to any stateful NATxy (NAT44, NAT64, NAT66) technologies. 66)
This document discusses why using pseudorandom port numbers with statef technologies. This document discusses why using pseudorandom port
ul NATxy gateways is a numbers with stateful NATxy gateways is a difficult problem. It
difficult problem. It recommends a solution limiting the port number ra recommends a solution that limits the port number ranges and uses two
nges and using test phases (phase 1 and phase 2). This document shows how the classic
two test phases (phase 1 and phase 2). It is shown how the classic perf performance measurement procedures (e.g., throughput, frame loss rate,
ormance measurement latency, etc.) can be carried out. New performance metrics and
procedures (e.g. throughput, frame loss rate, latency, etc.) can be car measurement procedures are also defined for measuring the maximum
ried out. connection establishment rate, connection tear-down rate, and
New performance metrics and measurement procedures are also defined for connection tracking table capacity.
measuring </t>
maximum connection establishment rate, connection tear-down rate, and c
onnection
tracking table capacity.
</t>
</abstract> </abstract>
</front> </front>
<middle> <middle>
<section anchor="intro" title="Introduction"> <section anchor="intro" numbered="true" toc="default">
<t><xref target="RFC2544"/> has defined a comprehensive benchmarking <name>Introduction</name>
methodology for network interconnect devices, which is still in use. It
was
mainly IP version independent, but it used IPv4 in its examples.
<xref target="RFC5180"/> addressed IPv6 specificities
and also added technology updates, but declared IPv6 transition technologi
es
out of its scope. <xref target="RFC8219"/> addressed the IPv6 transition
technologies, including stateful NAT64. It has reused several benchmark
ing
procedures from <xref target="RFC2544"/> (e.g. throughput, frame loss rate
),
it has redefined the latency measurement and added further ones, e.g. t
he PDV
(packet delay variation) measurement.</t>
<t>However, none of them discussed, how to apply <xref target="RFC4814"
/>
pseudorandom port numbers, when benchmarking stateful NATxy (NAT44 (als
o called
NAPT) <xref target="RFC3022"/>, NAT64 <xref target="RFC6146"/>, and NAT
66) gateways.
(It should be noted that stateful NAT66 is not an IETF specification bu
t
refers to an IPv6 version of the stateful NAT44 specification.) The aut
hors are not
aware of any other RFCs that address this question.
</t>
<t>First, it is discussed why using pseudorandom port numbers with statefu
l NATxy
gateways is a difficult problem.
</t>
<t>Then a solution is recommended.
</t>
<section> <t><xref target="RFC2544" format="default"/> defines a comprehensive
benchmarking methodology for network interconnect devices that is still
in use. It is mainly independent of IP version, but it uses IPv4 in its
examples. <xref target="RFC5180" format="default"/> addresses IPv6
specificities and also adds technology updates but declares IPv6
transition technologies are out of its scope. <xref target="RFC8219"
format="default"/> addresses the IPv6 transition technologies, including
stateful NAT64. It reuses several benchmarking procedures from <xref
target="RFC2544" format="default"/> (e.g., throughput, frame loss rate),
and it redefines the latency measurement and adds further ones (e.g., the
Packet Delay Variation (PDV) measurement).</t>
<t>However, none of them discuss how to apply pseudorandom port
numbers from <xref target="RFC4814" format="default"/> when benchmarking
stateful NATxy gateways (such as NAT44 <xref
target="RFC3022" format="default"/>, NAT64 <xref target="RFC6146"
format="default"/>, and NAT66). (It should be noted that stateful NAT66
is not an IETF specification but refers to an IPv6 version of the
stateful NAT44 specification.) The authors are not aware of any other
RFCs that address this question.
</t>
<t>First, this document discusses why using pseudorandom port numbers with
stateful NATxy gateways is a difficult problem. Then, a solution is
recommended.</t>
<section numbered="true" toc="default">
<name>Requirements Language</name> <name>Requirements Language</name>
<t>The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", <t>The key words "<bcp14>MUST</bcp14>", "<bcp14>MUST NOT</bcp14>",
"SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT "<bcp14>REQUIRED</bcp14>", "<bcp14>SHALL</bcp14>", "<bcp14>SHALL
RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be NOT</bcp14>", "<bcp14>SHOULD</bcp14>", "<bcp14>SHOULD NOT</bcp14>",
interpreted as described in BCP 14 <xref target="RFC2119"/> "<bcp14>RECOMMENDED</bcp14>", "<bcp14>NOT RECOMMENDED</bcp14>",
<xref target="RFC8174"/> when, and only when, they appear in "<bcp14>MAY</bcp14>", and "<bcp14>OPTIONAL</bcp14>" in this document
all capitals, as shown here.</t> are to be interpreted as described in BCP 14 <xref target="RFC2119"
format="default"/> <xref target="RFC8174" format="default"/> when, and
only when, they appear in all capitals, as shown here.</t>
</section> </section>
<!-- [CHECK] The 'Requirements Language' section is optional -->
</section> </section>
<section anchor="problem" numbered="true" toc="default">
<name>Pseudorandom Port Numbers and Stateful Translation</name>
<section anchor="problem" title="Pseudorandom Port Numbers and Stateful Tran <t>In its appendix, <xref target="RFC2544" format="default"/>
slation"> defines a frame format for test frames, including specific source and
<t>In its appendix, <xref target="RFC2544"/> has defined a frame format destination port numbers. <xref target="RFC4814" format="default"/>
for test frames including specific source and destination port numbers. recommends using pseudorandom and uniformly distributed values for both
<xref target="RFC4814"/> recommends using pseudorandom and source and destination port numbers. However, stateful NATxy (such as NAT4
uniformly distributed values for both source and destination port 4,
numbers. However, stateful NATxy (NAT44, NAT64, NAT66) solutions use the NAT64, and NAT66) solutions use the port numbers to identify
port numbers to identify connections. The usage of pseudorandom port connections. The usage of pseudorandom port numbers causes different
numbers causes different problems depending on the direction. problems depending on the direction:
<list style="symbols"> </t>
<t> As for the client-to-server direction, pseudorandom source and <ul spacing="normal">
destination port numbers could be used, however, this approach would <li>
be a denial of service attack against the stateful NATxy gateway, <t>For the client-to-server direction, pseudorandom source and
destination port numbers could be used; however, this approach would
be a denial-of-service attack against the stateful NATxy gateway,
because it would exhaust its connection tracking table capacity. To tha t end, because it would exhaust its connection tracking table capacity. To tha t end,
let us see some calculations using the recommendations of RFC 4814: let us see some calculations using the recommendations of <xref target=
<list style="symbols"> "RFC4814" format="default"/>:
<t>The recommended source port range is: 1024-65535, thus its size is </t>
: 64512.</t> <ul spacing="normal">
<t>The recommended destination port range is: 1-49151, thus its size <li>
is: 49151.</t> <t>The recommended source port range is 1024-65535; thus, its size
<t>The number of source and destination port number combinations is: is 64512.</t>
3,170,829,312.</t> </li>
</list> <li>
<t>The recommended destination port range is 1-49151; thus, its si
ze is 49151.</t>
</li>
<li>
<t>The number of source and destination port number combinations i
s 3,170,829,312.</t>
</li>
</ul>
<t>
It should be noted that the usage of different source and destination IP a ddresses It should be noted that the usage of different source and destination IP a ddresses
further increases the number of connection tracking table entries.</t> further increases the number of connection tracking table entries.</t>
<t>As for the server-to-client direction, the stateful DUT (Device Unde </li>
r Test) would drop any <li>
packets that do not belong to an existing connection, therefore, the <t>For the server-to-client direction, the stateful Device Under Test
direct usage of pseudorandom port numbers from the above-mentioned rang (DUT) would drop any
es packets that do not belong to an existing connection; therefore, the
direct usage of pseudorandom port numbers from the ranges mentioned abo
ve
is not feasible.</t> is not feasible.</t>
</list> </li>
</t> </ul>
</section> </section>
<section anchor="setup_term" title="Test Setup and Terminology"> <section anchor="setup_term" numbered="true" toc="default">
<name>Test Setup and Terminology</name>
<t>Section 12 of <xref target="RFC2544"/> requires testing first using
a single protocol source and destination address pair an then also usin
g
multiple protocol addresses. The same approach is followed: first, a
single source and destination IP address pair is used, and then it is e
xplained how to
use multiple IP addresses.</t>
<section anchor="setup_term_single" title="When Testing with a Single IP A <t><xref target="RFC2544" sectionFormat="of" section="12"/> requires
ddress Pair"> testing using a single protocol source and destination address pair
first and then also using multiple protocol addresses. The same
approach is followed: first, a single source and destination IP address
pair is used, and then it is explained how to use multiple IP
addresses.</t>
<t>The methodology works with any IP versions to benchmark stateful N <section anchor="setup_term_single" numbered="true" toc="default">
ATxy <name>When Testing with a Single IP Address Pair</name>
gateways, where x and y are in {4, 6}. To facilitate an easy unde
rstanding,
two typical examples are used: stateful NAT44 and stateful NAT64.
</t>
<t>The Test Setup for the well-known stateful NAT44 (also called NAPT <t>The methodology works with any IP version to benchmark stateful
: NATxy gateways, where x and y are in {4, 6}. To facilitate an easy
Network Address and Port Translation) solution is shown in understanding, two typical examples are used: stateful NAT44 and
<xref target="test_setup_sfnat44"/>.</t> stateful NAT64.</t>
<t>Note that the <xref target="RFC1918"/> private IP addresses are <t>The test setup for the well-known stateful NAT44 (also called
used to facilitate an easy understanding of the example. And the Network Address and Port Translation (NAPT)) solution is shown in
usage of the IP addresses reserved for benchmarking is absolutely <xref target="test_setup_sfnat44" format="default"/>.</t>
legitimate.</t>
<figure anchor="test_setup_sfnat44" align="center" title="Test setup for <t>Note that the private IP addresses from <xref target="RFC1918"
benchmarking format="default"/> are used to facilitate an easy understanding of the
stateful NAT44 gateways"> example, and the usage of the IP addresses reserved for benchmarking
<preamble></preamble> is absolutely legitimate.</t>
<artwork align="left"><![CDATA[ <t keepWithNext="true"/>
<figure anchor="test_setup_sfnat44">
<name>Test Setup for Benchmarking Stateful NAT44 Gateways</name>
<artwork align="left" name="" type="" alt=""><![CDATA[
+--------------------------------------+ +--------------------------------------+
10.0.0.2 |Initiator Responder| 198.19.0.2 10.0.0.2 |Initiator Responder| 198.19.0.2
+-------------| Tester |<------------+ +-------------| Tester |<------------+
| private IPv4| [state table]| public IPv4 | | private IPv4| [state table]| public IPv4 |
| +--------------------------------------+ | | +--------------------------------------+ |
| | | |
| +--------------------------------------+ | | +--------------------------------------+ |
| 10.0.0.1 | DUT: | 198.19.0.1 | | 10.0.0.1 | DUT: | 198.19.0.1 |
+------------>| Stateful NAT44 gateway |-------------+ +------------>| Stateful NAT44 gateway |-------------+
private IPv4| [connection tracking table] | public IPv4 private IPv4| [connection tracking table] | public IPv4
+--------------------------------------+ +--------------------------------------+
]]></artwork>
]]></artwork>
<postamble></postamble>
</figure> </figure>
<t keepWithPrevious="true"/>
<t>The test setup for the stateful NAT64 solution <xref target="RFC6146"
format="default"/>, which is also widely used, is shown in
<xref target="test_setup_sfnat64" format="default"/>.</t>
<t> The Test Setup for the also widely used stateful NAT64 <xref targ <t keepWithNext="true"/>
et="RFC6146"/> <figure anchor="test_setup_sfnat64">
solution is shown in <xref target="test_setup_sfnat64"/>.</t> <name>Test Setup for Benchmarking Stateful NAT64 Gateways</name>
<artwork align="left" name="" type="" alt=""><![CDATA[
<figure anchor="test_setup_sfnat64" align="center" title="Test setup for
benchmarking
stateful NAT64 gateways">
<preamble></preamble>
<artwork align="left"><![CDATA[
+--------------------------------------+ +--------------------------------------+
2001:2::2 |Initiator Responder| 198.19.0.2 2001:2::2 |Initiator Responder| 198.19.0.2
+-------------| Tester |<------------+ +-------------| Tester |<------------+
| IPv6 address| [state table]| IPv4 address| | IPv6 address| [state table]| IPv4 address|
| +--------------------------------------+ | | +--------------------------------------+ |
| | | |
| +--------------------------------------+ | | +--------------------------------------+ |
| 2001:2::1 | DUT: | 198.19.0.1 | | 2001:2::1 | DUT: | 198.19.0.1 |
+------------>| Stateful NAT64 gateway |-------------+ +------------>| Stateful NAT64 gateway |-------------+
IPv6 address| [connection tracking table] | IPv4 address IPv6 address| [connection tracking table] | IPv4 address
+--------------------------------------+ +--------------------------------------+
]]></artwork> ]]></artwork>
<postamble></postamble>
</figure> </figure>
<t keepWithPrevious="true"/>
<t>As for the transport layer protocol, <xref target="RFC2544"
format="default"/> recommended testing with UDP, and it was also kept
in <xref target="RFC8219" format="default"/>. UDP is also kept for a
general recommendation; thus, the port numbers in the following text
are to be understood as UDP port numbers. The rationale and
limitations of this approach are discussed in <xref
target="udp_or_tcp" format="default"/>.</t>
<t>As for transport layer protocol, <xref target="RFC2544"/> recommen <t>The most important elements of the proposed benchmarking system are
ded defined as follows:</t>
testing with UDP, and it was kept also in <xref target="RFC8219"/
>. For
the general recommendation, UDP is also kept, thus the port numbe
rs in the
following text are to be understood as UDP port numbers. The rati
onale and limitations
of this approach are discussed in <xref target="udp_or_tcp"/>.</t
>
<t>The most important elements of the proposed benchmarking system ar <dl newline="false" spacing="normal">
e defined as follows. <dt>Connection:</dt>
<list style="symbols"> <dd>Although UDP itself is a connectionless protocol, stateful
<t>Connection: Although UDP itself is a connection-less protocol, NATxy gateways keep track of their translation mappings in the form
stateful NATxy of a "connection" as well as in the case of UDP using the same kind of
gateways keep track of their translation mappings in the form of entries as in TCP.</dd>
a "connection"
also in the case of UDP using the same kind of entries as in the
case of TCP.</t>
<t>Connection tracking table: The stateful NATxy gateway uses a conne
ction
tracking table to be able to perform the stateful translation in
the server to
client direction. Its size, policy, and content are unknown to th
e Tester.</t>
<t>Four tuple: The four numbers that identify a connection are so
urce IP address,
source port number, destination IP address, destination port numb
er.</t>
<t>State table: The Responder of the Tester extracts the four tup
le from each received
test frame and stores it in its state table. Recommendation is gi
ven for writing and
reading order of the state table in <xref target="st_wr_order"/>.
</t>
<t>Initiator: The port of the Tester that may initiate a connecti
on through the
stateful DUT in the client-to-server direction. Theoretically, it
can use
any source and destination port numbers from the ranges recommend
ed by
<xref target="RFC4814"/>: if the used four tuple does not belong
to an existing
connection, the DUT will register a new connection into its conne
ction tracking table.</t>
<t>Responder: The port of the Tester that may not initiate a conn
ection through
the stateful DUT in the server-to-client direction. It may send o
nly frames that
belong to an existing connection. To that end, it uses four tuple
s that have been
previously extracted from the received test frames and stored in
its state table.</t>
<t>Test phase 1: Test frames are sent only by the Initiator to th
e
Responder through the DUT to fill both the connection tracking ta
ble of the DUT
and the state table of the Responder. This is a newly introduced
operation phase
for stateful NATxy benchmarking. The necessity of this test phase
is explained in
<xref target="prelim"/>.</t>
<t>Test phase 2: The measurement procedures defined by <xref targ
et="RFC8219"/>
(e.g. throughput, latency, etc.) are performed in this test phase
after the completion
of test phase 1. Test frames are sent as required (e.g. bidirecti
onal
test or unidirectional test in any of the two directions).</t>
</list>
</t>
<t>
One further definition is used in the text of this document:
<list style="symbols">
<t> Black box testing: It is a testing approach when the Tester i
s not aware of the
details of the internal structure and operation of the DUT. It ca
n send input to the
DUT and observe the output of the DUT.</t>
</list>
</t>
</section>
<section anchor="setup_term_multiple" title="When Testing with Multiple IP <dt>Connection tracking table:</dt>
Addresses"> <dd>The stateful NATxy gateway uses a connection tracking table to
be able to perform the stateful translation in the server-to-client
direction. Its size, policy, and content are unknown to the
Tester.</dd>
<t>The number of the necessary and available IP addresses are considere <dt>Four tuple:</dt>
d.</t> <dd>The four numbers that identify a connection are source IP
address, source port number, destination IP address, and destination
port number.</dd>
<t>In <xref target="test_setup_sfnat44"/>, the single 198.19.0.1 IPv4 a <dt>State table:</dt>
ddress is used <dd>The Responder of the Tester extracts the four tuple from each
on the WAN side port of the stateful NAT44 gateway. However, in practic received test frame and stores it in its state table. A recommendation
e, not a single is given for the writing and reading order of the state table in <xref
IP address, but an IP address range is assigned to the WAN side port of target="st_wr_order" format="default"/>.</dd>
the stateful
NAT44 gateways. Its required size depends on the number of client nodes
and on the type
of the stateful NAT44 algorithm. (The traditional algorithm always repl
aces
the source port number, when a new connection is established. Thus it r
equires a larger
range than the extended algorithm, which replaces the source port numbe
r only when it
is necessary. Please refer to Table 1 and Table 2 of <xref target="LEN2
015"/>.)</t>
<t>When router testing is done, section 12 of <xref target="RFC2544"/> <dt>Initiator:</dt>
requires <dd>The port of the Tester that may initiate a connection through
testing first using a single source and destination IP address pair, an the stateful DUT in the client-to-server direction. Theoretically,
d then it can use any source and destination port numbers from the ranges
using destination IP addresses from 256 different networks. The 16-23 b recommended by <xref target="RFC4814" format="default"/>: if the
its of used four tuple does not belong to an existing connection, the DUT
the 198.18.0.0/24 and 198.19.0.0/24 addresses can be used to express th will register a new connection into its connection tracking
e 256 networks. table.</dd>
As this document does not deal with router testing, no multiple destina
tion networks are needed,
therefore, these bits are available for expressing multiple IP addresse
s that belong
to the same "/16" network. Moreover, both the 198.18.0.0/16 and the 198
.19.0.0/16
networks can be used on the right side of the test setup as private IP
addresses
from the 10.0.0.0/16 network are used on its left side.</t>
<figure anchor="test_setup_sfnat44_multi" align="center" title="Test setu <dt>Responder:</dt>
p for benchmarking <dd>The port of the Tester that may not initiate a connection
stateful NAT44 gateways using multiple IPv4 addresses"> through the stateful DUT in the server-to-client direction. It may
<preamble></preamble> only send frames that belong to an existing connection. To that end,
it uses four tuples that have been previously extracted from the
received test frames and stores in its state table.</dd>
<artwork align="left"><![CDATA[ <dt>Test phase 1:</dt>
10.0.0.2/16 – 10.0.255.254/16 198.19.0.0/15 - 198.19.255.254/15 <dd>The test frames are sent only by the Initiator to the Responder
through the DUT to fill both the connection tracking table of the
DUT and the state table of the Responder. This is a newly introduced
operation phase for stateful NATxy benchmarking. The necessity of
this test phase is explained in <xref target="prelim"
format="default"/>.</dd>
<dt>Test phase 2:</dt>
<dd>The measurement procedures defined by <xref target="RFC8219"
format="default"/> (e.g., throughput, latency, etc.) are performed in
this test phase after the completion of test phase 1. Test frames
are sent as required (e.g., a bidirectional test or a unidirectional te
st
in any of the two directions).</dd>
</dl>
<t>One further definition is used in the text of this document:</t>
<dl newline="false" spacing="normal">
<dt>Black box testing:</dt>
<dd>A testing approach when the Tester is not aware of the
details of the internal structure and operation of the DUT. It can
send input to the DUT and observe the output of the DUT.</dd>
</dl>
</section>
<section anchor="setup_term_multiple" numbered="true" toc="default">
<name>When Testing with Multiple IP Addresses</name>
<t>This section considers the number of the necessary and available IP
addresses.</t>
<t>In <xref target="test_setup_sfnat44" format="default"/>, the single
198.19.0.1 IPv4 address is used on the WAN side port of the stateful
NAT44 gateway. However, in practice, it is not a single IP address,
but rather an IP address range that is assigned to the WAN side port
of the stateful NAT44 gateways. Its required size depends on the
number of client nodes and on the type of the stateful NAT44
algorithm. (The traditional algorithm always replaces the source port
number when a new connection is established. Thus, it requires a
larger range than the extended algorithm, which replaces the source
port number only when it is necessary. Please refer to Tables 1 and
2 of <xref target="LEN2015" format="default"/>.)</t>
<t>When router testing is done, <xref target="RFC2544"
sectionFormat="of" section="12"/> requires testing using a
single source and destination IP address pair first and then using
destination IP addresses from 256 different networks. The 16-23 bits
of the 198.18.0.0/24 and 198.19.0.0/24 addresses can be used to
express the 256 networks. As this document does not deal with router
testing, no multiple destination networks are needed; therefore, these
bits are available for expressing multiple IP addresses that belong to
the same "/16" network. Moreover, both the 198.18.0.0/16 and the
198.19.0.0/16 networks can be used on the right side of the test setup,
as private IP addresses from the 10.0.0.0/16 network are used on its
left side.</t>
<t keepWithNext="true"/>
<figure anchor="test_setup_sfnat44_multi">
<name>Test Setup for Benchmarking Stateful NAT44 Gateways Using Multip
le IPv4 Addresses</name>
<artwork align="left" name="" type="" alt=""><![CDATA[
10.0.0.2/16 - 10.0.255.254/16 198.19.0.0/15 - 198.19.255.254/15
\ +--------------------------------------+ / \ +--------------------------------------+ /
\ |Initiator Responder| / \ |Initiator Responder| /
+-------------| Tester |<------------+ +-------------| Tester |<------------+
| private IPv4| [state table]| public IPv4 | | private IPv4| [state table]| public IPv4 |
| +--------------------------------------+ | | +--------------------------------------+ |
| | | |
| +--------------------------------------+ | | +--------------------------------------+ |
| 10.0.0.1/16 | DUT: | public IPv4 | | 10.0.0.1/16 | DUT: | public IPv4 |
+------------>| Stateful NAT44 gateway |-------------+ +------------>| Stateful NAT44 gateway |-------------+
private IPv4| [connection tracking table] | \ private IPv4| [connection tracking table] | \
+--------------------------------------+ \ +--------------------------------------+ \
198.18.0.1/15 - 198.18.255.255/15 198.18.0.1/15 - 198.18.255.255/15
]]></artwork> ]]></artwork>
<postamble></postamble>
</figure> </figure>
<t>A possible solution for assigning multiple IPv4 addresses is shown <t keepWithPrevious="true"/>
in <t>A possible solution for assigning multiple IPv4 addresses is shown
<xref target="test_setup_sfnat44_multi"/>. On the left side, the in <xref target="test_setup_sfnat44_multi" format="default"/>. On the
private IP left side, the private IP address range is abundantly large. (The
address range is abundantly large. (The 16-31 bits were used for 16-31 bits were used for generating nearly 64k potential different
generating nearly 64k source addresses, but the 8-15 bits are also available if needed.) On
potential different source addresses, but the 8-15 bits are also the right side, the 198.18.0.0./15 network is used, and it was cut
available into two equal parts. (Asymmetric division is also possible, if
if needed.) On the right side, the 198.18.0.0./15 network is used needed.)</t>
, and it was <t>It should be noted that these are the potential address ranges. The
cut into two equal parts. (Asymmetric division is also possible, actual address ranges to be used are discussed in <xref
if needed.)</t> target="restr_port_range" format="default"/>.</t>
<t>In the case of stateful NAT64, a single "/64" IPv6 prefix contains
<t>It should be noted that these are the potential address ranges. Th a high number of bits to express different IPv6 addresses. <xref
e actual target="test_setup_sfnat64_multi" format="default"/> shows an example
address ranges to be used are discussed in <xref target="restr_po where bits 96-111 are used for that purpose.
rt_range"/>.</t> </t>
<t keepWithNext="true"/>
<t>In the case of stateful NAT64, a single "/64" IPv6 prefix contains <figure anchor="test_setup_sfnat64_multi">
a high <name>Test Setup for Benchmarking Stateful NAT64 Gateways Using
number of bits to express different IPv6 addresses. <xref target= Multiple IPv6 and IPv4 Addresses</name>
"test_setup_sfnat64_multi"/> <artwork align="left" name="" type="" alt=""><![CDATA[
shows an example, where bits 96-111 are used for that purpose.
</t>
<figure anchor="test_setup_sfnat64_multi" align="center" title="Test Setu
p for benchmarking
stateful NAT64 gateways using multiple IPv6 and IPv4 addresses">
<preamble></preamble>
<artwork align="left"><![CDATA[
2001:2::[0000-ffff]:0002/64 198.19.0.0/15 - 198.19.255.254/15 2001:2::[0000-ffff]:0002/64 198.19.0.0/15 - 198.19.255.254/15
\ +--------------------------------------+ / \ +--------------------------------------+ /
IPv6 \ |Initiator Responder| / IPv6 \ |Initiator Responder| /
+-------------| Tester |<------------+ +-------------| Tester |<------------+
| addresses | [state table]| public IPv4 | | addresses | [state table]| public IPv4 |
| +--------------------------------------+ | | +--------------------------------------+ |
| | | |
| +--------------------------------------+ | | +--------------------------------------+ |
| 2001:2::1/64| DUT: | public IPv4 | | 2001:2::1/64| DUT: | public IPv4 |
+------------>| Stateful NAT64 gateway |-------------+ +------------>| Stateful NAT64 gateway |-------------+
IPv6 address | [connection tracking table] | \ IPv6 address | [connection tracking table] | \
+--------------------------------------+ \ +--------------------------------------+ \
198.18.0.1/15 - 198.18.255.255/15 198.18.0.1/15 - 198.18.255.255/15
]]></artwork> ]]></artwork>
<postamble></postamble>
</figure> </figure>
<t keepWithPrevious="true"/>
</section>
</section>
<section anchor="method" numbered="true" toc="default">
<name>Recommended Benchmarking Method</name>
<section anchor="restr_port_range" numbered="true" toc="default">
<name>Restricted Number of Network Flows</name>
<t>When a single IP address pair is used for testing, then the number
of network flows is determined by the number of source and
destination port number combinations. </t>
<t>The Initiator <bcp14>SHOULD</bcp14> use restricted ranges for
source and destination port numbers to avoid the exhaustion of the
connection tracking table capacity of the DUT as described in <xref
target="problem" format="default"/>. If it is possible, the size of
the source port number range <bcp14>SHOULD</bcp14> be larger (e.g., in
the order of a few tens of thousands), whereas the size of the
destination port number range <bcp14>SHOULD</bcp14> be smaller (e.g., it
may
vary from a few to several hundreds or thousands as needed). The
rationale is that source and destination port numbers that can be
observed in Internet traffic are not symmetrical. Whereas source
port numbers may be random, there are a few very popular destination
port numbers (e.g., 443 or 80; see <xref target="IIR2020"
format="default"/>), and others hardly occur. Additionally, it was found
that
their role is also asymmetric in the Linux kernel routing hash
function <xref target="LEN2020" format="default"/>.</t>
<t>However, in some special cases, the size of the source port range
is limited. For example, when benchmarking the Customer Edge (CE) and
Border Relay (BR) of a Mapping of Address and Port using Translation
(MAP-T) system <xref target="RFC7599" format="default"/> together (as
a compound system performing stateful NAT44), the source port
range is limited to the number of source port numbers assigned to each
subscriber. (It could be as low as 2048 ports.)</t>
<t>When multiple IP addresses are used, then the port number ranges
should be even more restricted, as the number of potential network
flows is the product of the size of:</t>
<ul>
<li>the source IP address range,</li>
<li>the source port number range,</li>
<li>the destination IP address range, and</li>
<li>the destination port number range.</li>
</ul>
<t>In addition, the recommended method requires the enumeration of all
their possible combinations in test phase 1 as described in <xref
target="ctrl_conntrack" format="default"/>.</t>
<t>The number of network flows can be used as a parameter. The
performance of the stateful NATxy gateway <bcp14>MAY</bcp14> be
examined as a function of this parameter as described in <xref
target="sc_net_flows" format="default"/>.</t>
</section> </section>
</section> <section anchor="prelim" numbered="true" toc="default">
<name>Test Phase 1</name>
<t>Test phase 1 serves two purposes:</t>
<section anchor="method" title="Recommended Benchmarking Method"> <ol spacing="normal" type="1">
<li>
<t>The connection tracking table of the DUT is filled. This is
important because its maximum connection establishment rate may
be lower than its maximum frame forwarding rate (that is,
its throughput).</t>
</li>
<li>
<t>The state table of the Responder is filled with valid four
tuples. It is a precondition for the Responder to be able to
transmit frames that belong to connections that exist in the
connection tracking table of the DUT.</t>
</li>
</ol>
<section anchor="restr_port_range" title="Restricted Number of Network <t>Whereas the above two things are always necessary before test phase
Flows"> 2, test phase 1 can be used without test phase 2. This is done when
the maximum connection establishment rate is measured (as described in
<xref target="meas_max_conn_est_rate" format="default"/>).</t>
<t>When a single IP address pair is used for testing then the number of <t>Test phase 1 <bcp14>MUST</bcp14> be performed before all tests are
network performed in test phase 2. The following things happen in test phase
flows is determined by the number of source port number destination por 1:</t>
t number
combinations. </t>
<t>The Initiator SHOULD use restricted ranges for source and destinatio <ol spacing="normal" type="1">
n <li>
port numbers to avoid the exhaustion of the connection tracking table <t>The Initiator sends test frames to the Responder through the
capacity of the DUT as described in <xref target="problem"/>. DUT at a specific frame rate.</t>
If it is possible, the size of the source port number range SHOULD be l </li>
arger (e.g. in the order <li>
of a few times ten thousand), whereas the size of the destination port <t>The DUT performs the stateful translation of the test frames,
number and it also stores the new connections in its connection tracking
range SHOULD be smaller (may vary from a few to several hundreds or tho table.</t>
usands </li>
as needed). <li>
The rationale is that source and destination port numbers that can be o <t>The Responder receives the translated test frames and updates
bserved in its state table with the received four tuples. The Responder
the Internet traffic are not symmetrical. Whereas source port numbers m transmits no test frames during test phase 1.</t>
ay be random, </li>
there are a few very popular destination port numbers (e.g. 443, 80, et </ol>
c., <t>When test phase 1 is performed in preparation for test phase 2, the
see <xref target="IIR2020"/>), and others hardly occur. And it was applied frame rate <bcp14>SHOULD</bcp14> be safely lower than the
found that their role is also asymmetric in the Linux kernel routing ha maximum connection establishment rate. (It implies that maximum
sh connection establishment rate measurement <bcp14>MUST</bcp14> be
function <xref target="LEN2020"/>.</t> performed first.) Please refer to <xref target="ctrl_conntrack"
<t>However, in some special cases, the size of the source port range is format="default"/> for further conditions regarding timeout and the
limited. E.g. enumeration of all possible four tuples.</t>
when benchmarking the CE and BR of a MAP-T <xref target="RFC7599"/> sys </section>
tem together (as a compound system
performing stateful NAT44), then the source port range is limited to th
e number of
source port numbers assigned to each subscriber. (It could be as low as
2048 ports.) </t>
<t>When multiple IP addresses are used, then the port number ranges sho <section anchor="consider_stateful" numbered="true" toc="default">
uld be even <name>Consideration of the Cases of Stateful Operation</name>
more restricted, as the number of potential network flows is the produc <t>The authors consider the most important events that may happen
t of the size during the operation of a stateful NATxy gateway and the Actions of
of the source IP address range, the size of the source port number rang the gateway as follows.</t>
e, the size of the
destination IP address range, and the size of the destination port numb
er range.
And the recommended method requires the enumeration of all their possib
le combinations in test
phase 1 as described in <xref target="ctrl_conntrack"/>.</t>
<t>The number of network flows can be used as a parameter. The performa <ol>
nce of the <li>
stateful NATxy gateway MAY be examined as a function of this parameter <t>EVENT: A packet not belonging to an existing connection arrives
as described in the client-to-server direction.</t>
in <xref target="sc_net_flows"/>.</t> <t>ACTION: A new connection is registered into the connection
</section> tracking table, and the packet is translated and forwarded.</t>
</li>
<li>
<t>EVENT: A packet not belonging to an existing connection
arrives in the server-to-client direction.</t>
<t>ACTION: The packet is discarded.</t>
</li>
<li>
<t>EVENT: A packet belonging to an existing connection arrives
(in any direction).</t>
<t>ACTION: The packet is translated and forwarded, and the
timeout counter of the corresponding connection tracking table
entry is reset.</t>
</li>
<li>
<t>EVENT: A connection tracking table entry times out.</t>
<t>ACTION: The entry is deleted from the connection tracking
table.</t>
</li>
</ol>
<section anchor="prelim" title="Test Phase 1"> <t>Due to "black box" testing, the Tester is not able to directly
<t>Test phase 1 serves two purposes: examine (or delete) the entries of the connection tracking
<list style="numbers"> table. However, the entries can and <bcp14>MUST</bcp14> be
<t>The connection tracking table of the DUT is filled. It is im controlled by setting an appropriate timeout value and carefully
portant, selecting the port numbers of the packets (as described in <xref
because its maximum connection establishment rate may be lower target="ctrl_conntrack" format="default"/>) to be able to produce
than its maximum meaningful and repeatable measurement results.</t>
frame forwarding rate (that is throughput).</t> <t>This document aims to support the measurement of the following
<t>The state table of the Responder is filled with valid four t performance characteristics of a stateful NATxy gateway:</t>
uples. It is <ul spacing="normal">
a precondition for the Responder to be able to transmit frames <li>
that belong to <t>maximum connection establishment rate</t>
connections that exist in the connection tracking table of the </li>
DUT.</t> <li>
</list> <t>all "classic" performance metrics like throughput, frame loss rat
Whereas the above two things are always necessary before test pha e, latency, etc.</t>
se 2, </li>
test phase 1 can be used without test phase 2. It is done so <li>
when the maximum connection establishment rate is measured (as de <t>connection tear-down rate</t>
scribed in </li>
<xref target="meas_max_conn_est_rate"/>). <li>
</t> <t>connection tracking table capacity</t>
<t>Test phase 1 MUST be performed before all tests performed in </li>
test phase 2. The following things happen in test phase 1: </ul>
<list style="numbers"> </section>
<t>The Initiator sends test frames to the Responder through the
DUT at a
specific frame rate.</t>
<t>The DUT performs the stateful translation of the test frames
and it also
stores the new connections in its connection tracking table.</t
>
<t>The Responder receives the translated test frames and update
s its state
table with the received four tuples. The responder transmits no
test frames
during test phase 1.</t>
</list>
</t>
<t>When test phase 1 is performed in preparation for
test phase 2, the applied frame rate SHOULD be safely lower than
the maximum connection establishment rate. (It implies that maxim
um connection
establishment rate measurement MUST be performed first.)
Please refer to <xref target="ctrl_conntrack"/> for further condi
tions regarding
timeout and the enumeration of all possible four tuples.</t>
</section>
<section anchor="consider_stateful" title="Consideration of the Cases o <section anchor="ctrl_conntrack" numbered="true" toc="default">
f Stateful Operation"> <name>Control of the Connection Tracking Table Entries</name>
<t>The authors consider the most important events that may happen <t>It is necessary to control the connection tracking table entries of
during the operation the DUT to achieve clear conditions for the measurements. One can
of a stateful NATxy gateway, and the Actions of the gateway as fo simply achieve the following two extreme situations:</t>
llows.
<list style="numbers">
<t>EVENT: A packet not belonging to an existing connection arri
ves in the client-to-server
direction. ACTION: A new connection is registered into the conn
ection tracking
table and the packet is translated and forwarded.</t>
<t>EVENT: A packet not belonging to an existing connection arri
ves in the server-to-client
direction. ACTION: The packet is discarded.</t>
<t>EVENT: A packet belonging to an existing connection arrives
(in any direction).
ACTION: The packet is translated and forwarded and the timeout
counter of the corresponding
connection tracking table entry is reset.</t>
<t>EVENT: A connection tracking table entry times out. ACTION:
The entry is deleted from
the connection tracking table.</t>
</list>
</t>
<t>Due to "black box" testing, the Tester is not able to directly
examine (or delete) the entries
of the connection tracking table. But the entries can be and MUST
be controlled by setting
an appropriate timeout value and carefully selecting the port num
bers of the packets
(as described in <xref target="ctrl_conntrack"/>) to be able to p
roduce meaningful and
repeatable measurement results.
</t>
<t>This document aims to support the measurement of the following
performance characteristics
of a stateful NATxy gateway:
<list style="numbers">
<t>maximum connection establishment rate</t>
<t>all "classic" performance metrics like throughput, frame los
s rate, latency, etc.</t>
<t>connection tear-down rate</t>
<t>connection tracking table capacity</t>
</list>
</t>
</section>
<section anchor="ctrl_conntrack" title="Control of the Connection Track <ol spacing="normal">
ing Table Entries"> <li>
<t>It is necessary to control the connection tracking table entri All frames create a new entry in the connection tracking table
es of the DUT, and no old entries are deleted during the test. This is
of the DUT to achieve clear conditions for the measurements. One required for measuring the maximum connection establishment
can simply rate.
achieve the following two extreme situations: </li>
<list style="numbers"> <li>
<t>All frames create a new entry in the connection tracking tab No new entries are created in the connection tracking table of
le of the DUT and no the DUT, and no old ones are deleted during the test. This is ideal
old entries are deleted during the test. This is required for m for the measurements to be executed in phase 2, like throughput,
easuring the maximum latency, etc.
connection establishment rate.</t> </li>
<t>No new entries are created in the connection tracking table </ol>
of the DUT and no old
ones are deleted during the test. This is ideal for the measure
ments to be executed in phase 2,
like throughput, latency, etc.</t>
</list>
</t>
<t>From this point, the following two assumptions are used: <t>From this point, the following two assumptions are used:</t>
<list style="numbers">
<t>The connection tracking table of the stateful NATxy is large
enough to store all
connections defined by the different four tuples.</t>
<t>Each experiment is started with an empty connection tracking
table. (It can be ensured
by deleting its content before the experiment.)</t>
</list>
</t>
<t>The first extreme situation can be achieved by <ol spacing="normal" type="1">
<list style="symbols"> <li anchor="assumption1">
<t>using different four tuples for every single test frame in t The connection tracking table of the stateful NATxy is large
est phase 1 and</t> enough to store all connections defined by the different four
<t> setting the UDP timeout of the NATxy gateway to a value hig tuples.
her than the length of </li>
test phase 1.</t> <li anchor="assumption2">
</list> Each experiment is started with an empty connection tracking
</t> table. (This can be ensured by deleting its content before the
experiment.)
</li>
</ol>
<t>The second extreme situation can be achieved by <t>The first extreme situation can be achieved by:</t>
<list style="symbols"> <ul spacing="normal">
<t>enumerating all possible four tuples in test phase 1 and</t> <li>
<t>setting the UDP timeout of the NATxy gateway to a value high <t>using different four tuples for every single test frame in test p
er than the length of hase 1 and</t>
test phase 1 plus the gap between the two phases plus the lengt </li>
h of test phase 2.</t> <li>
</list> <t>setting the UDP timeout of the NATxy gateway to a value higher
</t> than the length of test phase 1.</t>
</li>
</ul>
<t>The second extreme situation can be achieved by:</t>
<t> <ul spacing="normal">
<xref target="RFC4814"/> REQUIRES pseudorandom port numbers, whic <li>
h the authors believe is a good <t>enumerating all possible four tuples in test phase 1 and</t>
approximation of the distribution of the source port numbers a NA </li>
Txy gateway on the <li>
Internet may face with. <t>setting the UDP timeout of the NATxy gateway to a value higher
</t> than the length of test phase 1 plus the gap between the two
phases plus the length of test phase 2.</t>
</li>
</ul>
<t> <t>As described in <xref target="RFC4814" format="default"/>, pseudorand
It should be noted that although the enumeration of all possible om
four tuples port numbers are <bcp14>REQUIRED</bcp14>, which the authors believe is a
is not a requirement for the first extreme situation and the usag good approximation of the
e of distribution of the source port numbers a NATxy gateway on the
different four tuples in test phase 1 is not a Internet may be faced with.
requirement for the second extreme situation, pseudorandom </t>
enumeration of all possible four tuples in test phase 1
is a good solution in both cases. It may be computing efficiently
generated by preparing a random permutation of the previously
enumerated all possible four tuples using Dustenfeld's random shu
ffle
algorithm <xref target="DUST1964"/>.
</t>
<t>The enumeration of the four tuples in increasing or decreasing <!-- [rfced] For clarity, how may we rephrase "it may be computing efficiently
order generated by preparing" in the text below?
(or in any other specific order) MAY be used as an additional mea
surement.
</t>
</section>
<section anchor="meas_max_conn_est_rate" title="Measurement of the Maxi Original:
mum Connection Establishment Rate">
<t>The maximum connection establishment rate is an important characte
ristic of
the stateful NATxy gateway and its determination is necessary for
the safe
execution of test phase 1 (without frame loss) before test phase
2.
</t>
<t>The measurement procedure of the maximum connection establishm
ent rate is
very similar to the throughput measurement procedure defined in
<xref target="RFC2544"/>.
</t>
<t>Procedure: The Initiator sends a specific number of test frame
s using all
different four tuples at a specific rate through the DUT.
The Responder counts the frames that are successfully translated
by the DUT.
If the count of offered frames is equal to the count of received
frames, the rate of the offered stream is raised and the test is
rerun.
If fewer frames are received than were transmitted, the rate of t
he offered
stream is reduced and the test is rerun.
</t>
<t>The maximum connection establishment rate is the fastest rate
at which
the count of test frames successfully translated by the DUT is eq
ual to the number
of test frames sent to it by the Initiator.
</t>
<t>Note: In practice, the usage of binary search is RECOMMENDED.<
/t>
</section>
<section anchor="validation_of_conn" title="Validation of Connection Es It may be computing efficiently generated by preparing a
tablishment"> random permutation of the previously enumerated all possible four
<t>Due to "black box" testing, the entries of the connection tracking tuples using Durstenfeld's random shuffle algorithm [DUST1964].
table of
the DUT may not be directly examined, but the presence of the con
nections can be
checked easily by sending frames from the Responder to the Initia
tor in
test phase 2 using all four tuples stored in the state table of t
he Tester
(at a low enough frame rate). The arrival of all test frames indi
cates that the
connections are indeed present.
</t>
<t>Procedure: When all the desired N number of test frames were s Perhaps:
ent by the Initiator
to the Receiver at frame rate R in test phase 1 for the maximum c
onnection
establishment rate measurement, and the Receiver has successfully
received all
the N frames, the establishment of the connections is checked in
test
phase 2 as follows:
<list style="symbols">
<t>The Responder sends test frames to the Initiator at frame ra
te r=R*alpha,
for the duration of N/r using a different four tuple from its s
tate table for
each test frame.</t>
<t>The Initiator counts the received frames, and if all N frame
s are arrived
then the R frame rate of the maximum connection establishment r
ate measurement
(performed in test phase 1) is raised for the next iteration,
otherwise lowered (as well as in the case if test frames were m
issing
in the preliminary test phase).</t>
</list>
</t>
<t>Notes:
<list style="symbols">
<t>The alpha is a kind of "safety factor", it aims to make su
re that
the frame rate used for the validation is not too high, a
nd test may fail only
in the case if at least one connection is not present in
the connection
tracking table of the DUT. (So alpha should be typically
less than 1, e.g.
0.8 or 0.5.)
</t>
<t>The duration of N/r and the frame rate of r means that
N frames are sent for validation.</t>
<t>The order of four tuple selection is arbitrary provide
d that all four tuples MUST be used.</t>
<t>Please refer to <xref target="meas_contr_capacity"/> f
or a short analysis
of the operation of the measurement and what problems may
occur.</t>
</list>
</t>
</section>
<section anchor="real_test" title="Test Phase 2"> Efficient computing may be generated by preparing a
<t>As for the traffic direction, there are three possible cases durin random permutation of the previously enumerated all possible four
g tuples using Durstenfeld's random shuffle algorithm [DUST1964].
test phase 2:
<list style="symbols">
<t>bidirectional traffic: The Initiator sends test frames to th
e Responder and
the Responder sends test frames to the Initiator.</t>
<t>unidirectional traffic from the Initiator to the Responder:
The Initiator
sends test frames to the Responder but the Responder does not s
end test frames to
the Initiator.</t>
<t>unidirectional traffic from the Responder to the Initiator:
The Responder
sends test frames to the Initiator but the Initiator does not s
end test frames to
the Responder.</t>
</list>
</t>
<t>If the Initiator sends test frames, then it uses pseudorandom
source port numbers and
destination port numbers from the restricted port number ranges.
(If it uses multiple
source and/or destination IP addresses, then their ranges are als
o limited.)
The responder receives the test frames, updates its state table,
and processes the test
frames as required by the given measurement procedure (e.g. only
counts them for the
throughput test, handles timestamps for latency or PDV tests, etc
.).
</t>
<t>If the Responder sends test frames, then it uses the four tupl
es from its state
table. The reading order of the state table may follow different
policies (discussed
in <xref target="st_wr_order"/>). The Initiator receives the test
frames and
processes them as required by the given measurement procedure.
</t>
<t>
As for the actual measurement procedures, the usage of the update
d ones
from Section 7 of <xref target="RFC8219"/> is RECOMMENDED.
</t>
</section>
<section anchor="meas_conn_tear_down_rate" title="Measurement of the Co -->
nnection Tear-down Rate">
<t>Connection tear-down can cause significant load for the NATxy <t>Although the enumeration of all possible four tuples is not a
gateway. requirement for the first extreme situation and the usage of
The connection tear-down performance can be measured as follows: different four tuples in test phase 1 is not a requirement for the
<list style="numbers"> second extreme situation, pseudorandom
<t>Load a certain number of connections (N) into the connection enumeration of all possible four tuples in test phase 1 is a good
tracking table of the DUT (in the same way as done to measure t solution in both cases. Pseudorandom enumeration of all possible four tu
he ples may be computing efficiently generated by preparing a
maximum connection establishment rate).</t> random permutation of the previously enumerated all possible four
<t>Record TimestampA.</t> tuples using Durstenfeld's random shuffle algorithm <xref
<t>Delete the content of the connection tracking table of the D target="DUST1964" format="default"/>.</t>
UT.</t>
<t>Record TimestampB.</t> <t>The enumeration of the four tuples in increasing or decreasing
</list> order (or in any other specific order) <bcp14>MAY</bcp14> be used as
The connection tear-down rate can be computed as: an additional measurement.</t>
</t>
<t>connection tear-down rate = N / ( TimestampB - TimestampA) </section>
<section anchor="meas_max_conn_est_rate" numbered="true" toc="default">
<name>Measurement of the Maximum Connection Establishment Rate</name>
<t>The maximum connection establishment rate is an important
characteristic of the stateful NATxy gateway, and its determination is
necessary for the safe execution of test phase 1 (without frame loss)
before test phase 2.
</t>
<t>The measurement procedure of the maximum connection establishment
rate is very similar to the throughput measurement procedure defined
in <xref target="RFC2544" format="default"/>.
</t> </t>
<t>The connection tear-down rate SHOULD be measured for various v
alues of N.
</t>
<t>It is assumed that the content of the connection tracking table may b
e deleted
by an out-of-band control mechanism specific to the given NATxy g
ateway implementation.
(E.g. by removing the appropriate kernel module under Linux.)
</t>
<t>It is noted that the performance of removing the entire content of th
e connection
tracking table at one time may be different from removing all the
entries one by one.
</t>
</section> <t>The procedure is as follows:</t>
<ul>
<li>The Initiator sends a specific number of test frames using all
different four tuples at a specific rate through the DUT.</li>
<li>The Responder counts the frames that are successfully translated
by the DUT.</li>
<li>If the count of offered frames is equal to the count of received
frames, the rate of the offered stream is raised and the test is
rerun.</li>
<li>If fewer frames are received than were transmitted, the rate of
the offered stream is reduced and the test is rerun.</li>
</ul>
<section anchor="meas_contr_capacity" title="Measurement of the Connect <t>The maximum connection establishment rate is the fastest rate at
ion Tracking Table Capacity"> which the count of test frames successfully translated by the DUT is
<t>The connection tracking table capacity is an important metric equal to the number of test frames sent to it by the Initiator.
of stateful </t>
NATxy gateways. Its measurement is not easy, because an elementar
y
step of a validated maximum connection establishment rate measure
ment (defined in
<xref target="validation_of_conn"/>) may have only a few distinct
observable outcomes,
but some of them may have different root causes:
<list style="numbers">
<t>During test phase 1, the number of test frames received by t
he
Responder is less than the number of test frames sent by the In
itiator.
It may have different root causes, including:
<list style="numbers">
<t>The R frame sending rate was higher than the maximum conne
ction
establishment rate. (Note that now the maximum connection
establishment rate is considered unknown because one can
not measure the
maximum connection establishment without assumption 1 in
<xref target="ctrl_conntrack"/>!)
This root cause may be eliminated by lowering the R rate
and re-executing
the test. (This step may be performed multiple times, whi
le R>0.)</t>
<t>The capacity of the connection tracking table of the D
UT has been
exhausted. (And either the DUT does not want to delete
connections
or the deletion of the connections makes it slower. Thi
s case is not
investigated further in test phase 1.)</t>
</list>
</t>
<t>During test phase 1, the number of test frames received by t
he
Responder equals the number of test frames sent by the Initiato
r.
In this case, the connections are validated in test phase 1.
The validation may have two kinds of observable results:
<list style="numbers">
<t>The number of validation frames received by the Initiator
equals the number of validation frames sent by the Respon
der.
(It proves that the capacity of the connection tracking t
able of
the DUT is enough and both R and r were chosen properly.)
</t>
<t>The number of validation frames received by the Initia
tor
is less than the number of validation frames sent by the
Responder.
This phenomenon may have various root causes:
<list style="numbers">
<t>The capacity of the connection tracking table of the
DUT has been
exhausted. (It does not matter, whether some existing c
onnections are
discarded and new ones are stored, or the new connectio
ns are discarded.
Some connections are lost anyway, and it makes validati
on fail.)</t>
<t>The R frame sending rate used by the Initiator was t
oo high in
test phase 1 and thus some connections were not establi
shed,
even though all test frames arrived at the Responder. T
his root cause
may be eliminated by lowering the R rate and re-executi
ng the test.
(This step may be performed multiple times, while R>0.)
</t>
<t>The r frame sending rate used by the Responder was t
oo high in
test phase 2 and thus some test frames did not arrive a
t the Initiator, even
though all connections were present in the connection t
racking table of the DUT.
This root cause may be eliminated by lowering the r rat
e and re-executing the test.
(This step may be performed multiple times, while r>0.)
</t>
</list>
And here is the problem: as the above three root causes a
re indistinguishable,
it is not easy to decide, whether R or r should be decrea
sed.
</t>
</list>
</t>
</list>
</t>
<t>Experience shows that the DUT may collapse if its memory is ex <t>Note: In practice, the usage of binary search is
hausted. <bcp14>RECOMMENDED</bcp14>.</t>
Such a situation may make the connection </section>
tracking table capacity measurements rather inconvenient. This po <section anchor="validation_of_conn" numbered="true" toc="default">
ssibility is included <name>Validation of Connection Establishment</name>
in the recommended measurement procedure, but the detection and e <t>Due to "black box" testing, the entries of the connection tracking
limination of such table of the DUT may not be directly examined. However, the presence of
a situation is not addressed. (E.g. how the algorithm can reset t the
he DUT.) connections can be checked easily by sending frames from the Responder
</t> to the Initiator in test phase 2 using all four tuples stored in the
state table of the Tester (at a low enough frame rate). The arrival of
all test frames indicates that the connections are indeed present.
</t>
<t>For the connection tracking table size measurement, first one <t>The procedure is as follows:</t>
needs a safe <t>When all the desired N number of test frames are sent by the
number: C0. It is a precondition, that C0 number of connections c Initiator to the Receiver at frame rate R in test phase 1 for the
an surely be maximum connection establishment rate measurement and the Receiver
stored in the connection tracking table of the DUT. Using C0, one has successfully received all the N frames, the establishment
can determine of the connections is checked in test phase 2 as follows:</t>
the maximum connection establishment rate using C0 number of conn <ul>
ections. <li>
It is done with a binary search using validation. The result is R The Responder sends test frames to the Initiator at frame rate
0. The values r=R*alpha for the duration of N/r, using a different four tuple
C0 and R0 will serve as "safe" starting values for the following from its state table for each test frame.
two searches. </li>
</t>
<t>First, an exponential search is performed to find the order of <li>
magnitude of the The Initiator counts the received frames, and if all N frames
connection tracking table capacity. The search stops if the DUT c have arrived, then the R frame rate of the maximum connection
ollapses OR establishment rate measurement (performed in test phase 1) is
the maximum connection establishment rate severely drops (e.g. to raised for the next iteration; otherwise, it is lowered (as well a
its one tenth) s in
due to doubling the number of connections. the case that test frames were missing in the preliminary test
</t> phase, as well).
</li>
</ul>
<t>Then, the result of the exponential search gives the order of <t>Notes:</t>
magnitude of <ul spacing="normal">
the size of the connection tracking table. Before disclosing the <li>
possible algorithms to The alpha is a kind of "safety factor"; it aims to make sure
determine the exact size of the connection tracking table, three that the frame rate used for the validation is not too high, and t
possible he
replacement policies for the NATxy gateway are considered: test may fail only in the case of if at least one connection is no
<list style="numbers"> t
<t>The gateway does not delete any live connections until their present in the connection tracking table of the DUT. (Therefore, a
timeout expires.</t> lpha
<t>The gateway replaces the live connections according to LRU ( should be typically less than 1, e.g., 0.8 or 0.5.)
least recently used) policy.</t> </li>
<t>The gateway does a garbage collection when its connection tr <li>
acking table is full The duration of N/r and the frame rate of r means that N frames
and a frame with a new four tuple arrives. During the garbage c are sent for validation.
ollection, it deletes the K </li>
least recently used connections, where K is greater than 1.</t> <li>
</list> The order of four tuple selection is arbitrary, provided that
Now, it is examined what happens and how many validation frames a all four tuples <bcp14>MUST</bcp14> be used.
rrive in the there cases. </li>
Let the size of the connection tracking table be S, and the numbe <li>
r of preliminary Please refer to <xref target="meas_contr_capacity"
frames be N, where S is less than N. format="default"/> for a short analysis of the operation of the
<list style="numbers"> measurement and what problems may occur.
<t>The connections defined by the first S test frames are regis </li>
tered into </ul>
the connection tracking table of the DUT, and the last N-S conn
ections are lost.
(It is another question if the last N-S test frames are transla
ted and
forwarded in test phase 1 or simply dropped.) During validation
, the validation
frames with four tuples corresponding to the first S test frame
s will arrive at the
Initiator and the other N-S validation frames will be lost.</t>
<t>All connections are registered into the connection tracking
table of the DUT,
but the first N-S connections are replaced (and thus lost). Dur
ing validation,
the validation frames with four tuples corresponding to the las
t S test frames
will arrive to the Initiator, and the other N-S validation fram
es will be lost. </t>
<t>Depending on the values of K, S, and N, maybe less than S co
nnections will survive.
In the worst case, only S-K+1 validation frames arrive, even th
ough, the size of
the connection tracking table is S.</t>
</list>
If one knows that the stateful NATxy gateway uses the first or se
cond replacement
policy and one also knows that both R and r rates are low enough,
then the final
step of determining the size of the connection tracking table is
simple. If the Responder
sent N validation frames and the Initiator received N' of them, t
hen the size of the
connection tracking table is N'.
</t>
<t>In the general case, a binary search is performed to find the </section>
exact value of the connection
tracking table capacity within E error. The search chooses the lo
wer half of
the interval if the DUT collapses OR the maximum connection estab
lishment
rate severely drops (e.g. to its half) otherwise it chooses the h
igher half.
The search stops if the size of the interval is less than the E e
rror.
</t>
<t>The algorithms for the general case are defined using C like p <section anchor="real_test" numbered="true" toc="default">
seudocode in <name>Test Phase 2</name>
<xref target="meas_contr_capacity_algo"/>. In practice, this algo
rithm may
be made more efficient in a way that the binary search for the ma
ximum
connection establishment rate stops, if an elementary test fails
at a rate
under RS*beta or RS*gamma during the external search or during th
e final
binary search for the capacity of the connection tracking table,
respectively.
(This saves a high amount of execution time by eliminating the lo
ng-lasting tests at
low rates.)
</t>
<figure anchor="meas_contr_capacity_algo" align="center" title="Measurem <t>As for the traffic direction, there are three possible cases
ent of the Connection Tracking Table Capacity"> during test phase 2:</t>
<sourcecode type="pseudocode"><![CDATA[
<ol spacing="normal" type="1">
<li>
<t>Bidirectional traffic: The Initiator sends test frames to the
Responder, and the Responder sends test frames to the
Initiator.</t>
</li>
<li>
<t>Unidirectional traffic from the Initiator to the Responder: The
Initiator sends test frames to the Responder, but the Responder
does not send test frames to the Initiator.</t>
</li>
<li>
<t>Unidirectional traffic from the Responder to the Initiator: The
Responder sends test frames to the Initiator, but the Initiator
does not send test frames to the Responder.</t>
</li>
</ol>
<t>If the Initiator sends test frames, then it uses pseudorandom
source port numbers and destination port numbers from the restricted
port number ranges. (If it uses multiple source and/or destination IP
addresses, then their ranges are also limited.) The Responder
receives the test frames, updates its state table, and processes the
test frames as required by the given measurement procedure (e.g., only
counts them for the throughput test, handles timestamps for latency or
PDV tests, etc.).</t>
<t>If the Responder sends test frames, then it uses the four tuples
from its state table. The reading order of the state table may follow
different policies (discussed in <xref target="st_wr_order"
format="default"/>). The Initiator receives the test frames and
processes them as required by the given measurement procedure.</t>
<t>As for the actual measurement procedures, the usage of the updated
ones from <xref target="RFC8219" sectionFormat="of" section="7"/> is
<bcp14>RECOMMENDED</bcp14>.</t>
</section>
<section anchor="meas_conn_tear_down_rate" numbered="true" toc="default">
<name>Measurement of the Connection Tear-Down Rate</name>
<t>Connection tear-down can cause significant load for the NATxy
gateway. The connection tear-down performance can be measured as
follows:</t>
<ol spacing="normal" type="1">
<li>Load a certain number of connections (N) into the connection
tracking table of the DUT (in the same way as done to measure the
maximum connection establishment rate).</li>
<li>Record TimestampA.</li>
<li>Delete the content of the connection tracking table of the DUT.</l
i>
<li>Record TimestampB.</li>
</ol>
<t>The connection tear-down rate can be computed as:</t>
<t indent="5">connection tear-down rate = N / ( TimestampB - TimestampA)
</t>
<t>The connection tear-down rate <bcp14>SHOULD</bcp14> be measured for
various values of N.</t>
<t>It is assumed that the content of the connection tracking table may
be deleted by an out-of-band control mechanism specific to the given
NATxy gateway implementation (e.g., by removing the appropriate kernel
module under Linux).</t>
<t>It is noted that the performance of removing the entire content of
the connection tracking table at one time may be different from
removing all the entries one by one.</t>
</section>
<section anchor="meas_contr_capacity" numbered="true" toc="default">
<name>Measurement of the Connection Tracking Table Capacity</name>
<t>The connection tracking table capacity is an important metric of
stateful NATxy gateways. Its measurement is not easy, because an
elementary step of a validated maximum connection establishment rate
measurement (defined in <xref target="validation_of_conn"
format="default"/>) may have only a few distinct observable outcomes,
but some of them may have different root causes:</t>
<ul spacing="normal">
<li>
<t>During test phase 1, the number of test frames received by the
Responder is less than the number of test frames sent by the
Initiator. It may have different root causes, including:</t>
<ul spacing="normal">
<li>
<t>The R frame sending rate was higher than the maximum
connection establishment rate. (Note that now the maximum
connection establishment rate is considered unknown because
one cannot measure the maximum connection establishment
without <xref target="assumption1" format="none">assumption 1</x
ref> in <xref target="ctrl_conntrack"
format="default"/>.) This root cause may be eliminated by
lowering the R rate and re-executing the test. (This step may
be performed multiple times while R&gt;0.)</t>
</li>
<li>
<t>The capacity of the connection tracking table of the DUT
has been exhausted (and either the DUT does not want to
delete connections or the deletion of the connections makes it
slower; this case is not investigated further in test phase
1).</t>
</li>
</ul>
</li>
<li>
<t>During test phase 1, the number of test frames received by the
Responder equals the number of test frames sent by the Initiator.
In this case, the connections are validated in test phase 2. The
validation may have two kinds of observable results:</t>
<ol spacing="normal" type="1">
<li>
<t>The number of validation frames received by the Initiator
equals the number of validation frames sent by the Responder.
(It proves that the capacity of the connection tracking table
of the DUT is enough and both R and r were chosen
properly.)</t>
</li>
<li>
<t>The number of validation frames received by the Initiator
is less than the number of validation frames sent by the
Responder. This phenomenon may have various root causes:</t>
<ul spacing="normal">
<li>
<t>The capacity of the connection tracking table of the
DUT has been exhausted. (It does not matter whether some
existing connections are discarded and new ones are
stored or if the new connections are discarded. Some
connections are lost anyway, and it makes validation
fail.)</t>
</li>
<li>
<t>The R frame sending rate used by the Initiator was too
high in test phase 1; thus, some connections were not
established even though all test frames arrived at the
Responder. This root cause may be eliminated by lowering
the R rate and re-executing the test. (This step may be
performed multiple times while R&gt;0.)</t>
</li>
<li>
<t>The r frame sending rate used by the Responder was too
high in test phase 2; thus, some test frames did not
arrive at the Initiator even though all connections were
present in the connection tracking table of the DUT. This
root cause may be eliminated by lowering the r rate and
re-executing the test. (This step may be performed
multiple times while r&gt;0.)</t>
</li>
</ul>
<t>This is the problem: As the above three root causes are
indistinguishable, it is not easy to decide whether R or r
should be decreased.</t>
</li>
</ol>
</li>
</ul>
<t>Experience shows that the DUT may collapse if its memory is
exhausted. Such a situation may make the connection tracking table
capacity measurements rather inconvenient. This possibility is
included in the recommended measurement procedure, but the detection
and elimination of such a situation is not addressed (e.g., how the
algorithm can reset the DUT).</t>
<t>For the connection tracking table size measurement, first, one needs
a safe number: C0. It is a precondition that C0 number of connections
can surely be stored in the connection tracking table of the
DUT. Using C0, one can determine the maximum connection establishment
rate using C0 number of connections. It is done with a binary search
using validation. The result is R0. The values C0 and R0 will serve as
"safe" starting values for the following two searches.</t>
<t>First, an exponential search is performed to find the order of
magnitude of the connection tracking table capacity. The search stops
if the DUT collapses OR the maximum connection establishment rate
severely drops (e.g., to its one tenth) due to doubling the number of
connections.</t>
<t>Then, the result of the exponential search gives the order of
magnitude of the size of the connection tracking table. Before
disclosing the possible algorithms to determine the exact size of the
connection tracking table, three possible replacement policies for the
NATxy gateway are considered:</t>
<ol spacing="normal" type="1">
<li>
<t>The gateway does not delete any live connections until their time
out expires.</t>
</li>
<li>
<t>The gateway replaces the live connections according to the Least
Recently Used (LRU) policy.</t>
</li>
<li>
<t>The gateway does a garbage collection when its connection
tracking table is full and a frame with a new four tuple
arrives. During the garbage collection, it deletes the K LRU connect
ions, where K is greater than 1.</t>
</li>
</ol>
<t>Now, it is examined what happens and how many validation frames
arrive in the three cases. Let the size of the connection tracking
table be S and the number of preliminary frames be N, where S is less
than N.</t>
<ol spacing="normal" type="1">
<li>
<t>The connections defined by the first S test frames are
registered into the connection tracking table of the DUT, and
the last N-S connections are lost. (It is another question if the
last N-S test frames are translated and forwarded in test phase 1
or simply dropped.) During validation, the validation frames with
four tuples corresponding to the first S test frames will arrive
at the Initiator and the other N-S validation frames will be
lost.</t>
</li>
<li>
<t>All connections are registered into the connection tracking
table of the DUT, but the first N-S connections are replaced (and
thus lost). During validation, the validation frames with four
tuples corresponding to the last S test frames will arrive to the
Initiator, and the other N-S validation frames will be lost.</t>
</li>
<li>
<t>Depending on the values of K, S, and N, maybe less than S
connections will survive. In the worst case, only S-K+1
validation frames arrive, even though the size of the connection
tracking table is S.</t>
</li>
</ol>
<t>If one knows that the stateful NATxy gateway uses the first or
second replacement policy and one also knows that both R and r rates
are low enough, then the final step of determining the size of the
connection tracking table is simple. If the Responder sent N
validation frames and the Initiator received N' of them, then the size
of the connection tracking table is N'.</t>
<t>In the general case, a binary search is performed to find the exact
value of the connection tracking table capacity within E error. The
search chooses the lower half of the interval if the DUT collapses OR
the maximum connection establishment rate severely drops (e.g., to its
half); otherwise, it chooses the higher half. The search stops if the
size of the interval is less than the E error.</t>
<t>The algorithms for the general case are defined using C-like
pseudocode in <xref target="meas_contr_capacity_algo"
format="default"/>. In practice, this algorithm may be made more
efficient in the way that the binary search for the maximum connection
establishment rate stops if an elementary test fails at a rate under
RS*beta or RS*gamma during the external search or during the final
binary search for the capacity of the connection tracking table,
respectively. (This saves a high amount of execution time by
eliminating the long-lasting tests at low rates.)
</t>
<figure anchor="meas_contr_capacity_algo">
<name>Measurement of the Connection Tracking Table Capacity</name>
<sourcecode type="pseudocode"><![CDATA[
// The binarySearchForMaximumConnectionCstablishmentRate(c,r) // The binarySearchForMaximumConnectionCstablishmentRate(c,r)
// function performs a binary search for the maximum connection // function performs a binary search for the maximum connection
// establishment rate in the [0, r] interval using c number of // establishment rate in the [0, r] interval using c number of
// connections. // connections.
// This is an exponential search for finding the order of magnitude // This is an exponential search for finding the order of magnitude
// of the connection tracking table capacity // of the connection tracking table capacity
// Variables: // Variables:
// C0 and R0 are beginning safe values for the connection // C0 and R0 are beginning safe values for the connection
// tracking table size and connection establishment rate, // tracking table size and connection establishment rate,
// respectively // respectively
// CS and RS are their currently used safe values // CS and RS are their currently used safe values
// CT and RT are their values for the current examination // CT and RT are their values for the current examination
// beta is a factor expressing an unacceptable drop in R (e.g. // beta is a factor expressing an unacceptable drop in R (e.g.,
// beta=0.1) // beta=0.1)
// maxrate is the maximum frame rate for the media // maxrate is the maximum frame rate for the media
R0=binarySearchForMaximumConnectionCstablishmentRate(C0,maxrate); R0=binarySearchForMaximumConnectionCstablishmentRate(C0,maxrate);
for ( CS=C0, RS=R0; 1; CS=CT, RS=RT ) for ( CS=C0, RS=R0; 1; CS=CT, RS=RT )
{ {
CT=2*CS; CT=2*CS;
RT=binarySearchForMaximumConnectionCstablishmentRate(CT,RS); RT=binarySearchForMaximumConnectionCstablishmentRate(CT,RS);
if ( DUT_collapsed || RT < RS*beta ) if ( DUT_collapsed || RT < RS*beta )
break; break;
} }
// At this point, the size of the connection tracking table is // At this point, the size of the connection tracking table is
// between CS and CT. // between CS and CT.
// This is the final binary search for finding the connection // This is the final binary search for finding the connection
// tracking table capacity within E error // tracking table capacity within E error
// Variables: // Variables:
// CS and RS are the safe values for connection tracking table size // CS and RS are the safe values for connection tracking table size
// and connection establishment rate, respectively // and connection establishment rate, respectively
// C and R are the values for the current examination // C and R are the values for the current examination
// gamma is a factor expressing an unacceptable drop in R // gamma is a factor expressing an unacceptable drop in R
// (e.g. gamma=0.5) // (e.g., gamma=0.5)
for ( D=CT-CS; D>E; D=CT-CS ) for ( D=CT-CS; D>E; D=CT-CS )
{ {
C=(CS+CT)/2; C=(CS+CT)/2;
R=binarySearchForMaximumConnectionCstablishmentRate(C,RS); R=binarySearchForMaximumConnectionCstablishmentRate(C,RS);
if ( DUT_collapsed || R < RS*gamma ) if ( DUT_collapsed || R < RS*gamma )
CT=C; // take the lower half of the interval CT=C; // take the lower half of the interval
else else
CS=C,RS=R; // take the upper half of the interval CS=C,RS=R; // take the upper half of the interval
} }
// At this point, the size of the connection tracking table is // At this point, the size of the connection tracking table is
// CS within E error. // CS within E error.
]]></sourcecode> ]]></sourcecode>
<postamble></postamble>
</figure> </figure>
<t keepWithPrevious="true"/>
</section>
</section> <section anchor="st_wr_order" numbered="true" toc="default">
<name>Writing and Reading Order of the State Table</name>
<section anchor="st_wr_order" title="Writing and Reading Order of the S <t>As for the writing policy of the state table of the Responder,
tate Table"> round robin is <bcp14>RECOMMENDED</bcp14>, because it ensures that its
<t>As for the writing policy of the state table of the Responder, entries are automatically kept fresh and consistent with that of the
round robin is RECOMMENDED, connection tracking table of the DUT.
because it ensures that its entries are automatically kept fresh </t>
and consistent with <t>The Responder can read its state table in various orders, for
that of the connection tracking table of the DUT. example:
</t> </t>
<t>The Responder can read its state table in various orders, for <ul spacing="normal">
example: <li>
<list style="symbols"> <t>pseudorandom</t>
<t>pseudorandom</t> </li>
<t>round-robin</t> <li>
</list> <t>round robin</t>
</t> </li>
<t> </ul>
Pseudorandom is RECOMMENDED to follow the approach of <xref targe <t>Pseudorandom is <bcp14>RECOMMENDED</bcp14> to follow the approach
t="RFC4814"/>. of <xref target="RFC4814" format="default"/>. Round robin may be used
Round-robin may be used as a computationally cheaper alternative. as a computationally cheaper alternative.
</t> </t>
</section> </section>
</section> </section>
<section anchor="meas_scalability" numbered="true" toc="default">
<name>Scalability Measurements</name>
<section anchor="meas_scalability" title="Scalability Measurements"> <t>As for scalability measurements, no new types of performance metrics
<t>As for scalability measurements, no new types of performance metrics are defined, but it is <bcp14>RECOMMENDED</bcp14> to perform measurement
are defined, series through which the value of one or more parameter(s) are
but it is RECOMMENDED to perform measurement series through which the v changed to discover how the various values of the given parameter(s)
alue of one or more parameter(s) influence the performance of the DUT.
is/are changed to discover how the various values of the given paramete </t>
r(s) influence <section anchor="sc_net_flows" numbered="true" toc="default">
the performance of the DUT. <name>Scalability Against the Number of Network Flows</name>
</t> <t>The scalability measurements aim to quantify how the performance of
the stateful NATxy gateways degrades with the increase of the number
<section anchor="sc_net_flows" title="Scalability Against the Number of of network flows.</t>
Network Flows"> <t>As for the actual values for the number of network flows to be used
during the measurement series, it is <bcp14>RECOMMENDED</bcp14> to use
<t>The scalability measurements aim to quantify how the performance some representative values from the range of the potential number of
of the stateful NATxy gateways degrades with the increase of the network flows the DUT may be faced with during its intended usage.</t>
number of <t>It is important how the given number of network flows are
network flows.</t> generated. The sizes of the ranges of the source and destination IP
addresses and port numbers are essential parameters to be reported
<t>As for the actual values for the number of network flows to be together with the results. Please also see <xref
used during target="reporting_format" format="default"/> about the reporting
the measurement series, it is RECOMMENDED to use some representat format.</t>
ive values from <t>If a single IP address pair is used, then it is <bcp14>RECOMMENDED</b
the range of the potential number of network flows the DUT may be cp14> to use:
faced with
during its intended usage.</t>
<t>It is important, how the given number of network flows are gen
erated. The sizes
of the ranges of the source and destination IP addresses and port
numbers are
essential parameters to be reported together with the results. Pl
ease see also
<xref target="reporting_format"/> about the reporting format.</t>
<t>If a single IP address pair is used, then it is RECOMMENDED to
use
<list style="symbols">
<t>a fixed, larger source port number range (e.g., a few times
10,000)</t>
<t>a variable size destination port number range (e.g. 10; 100;
1,000; etc.),
where its expedient granularity depends on the purpose.</t>
</list>
</t> </t>
<ul spacing="normal">
<li>
<t>a fixed, larger source port number range (e.g., a few times
10,000) and</t>
</li>
<li>
<t>a variable-size destination port number range (e.g., 10, 100,
1,000, etc.), where its expedient granularity depends on the
purpose.</t>
</li>
</ul>
</section> </section>
<section anchor="sc_cpu_cores" numbered="true" toc="default">
<section anchor="sc_cpu_cores" title="Scalability Against the Number of <name>Scalability Against the Number of CPU Cores</name>
CPU Cores"> <t>Stateful NATxy gateways are often implemented in software that is
not bound to a specific hardware but can be executed by commodity
<t>Stateful NATxy gateways are often implemented in software that are servers. To facilitate the comparison of their performance, it can be
not bound useful to determine:
to a specific hardware but can be executed by commodity servers. </t>
To facilitate <ul spacing="normal">
the comparison of their performance, it can be useful to determin <li>
e <t>the performance of the various implementations using a single
<list style="symbols"> core of a well-known CPU and</t>
<t>the performance of the various implementations using a single co </li>
re of a well-known CPU</t> <li>
<t>the scale-up of the performance of the various implementatio <t>the scale-up of the performance of the various implementations
ns with the number of CPU cores.</t> with the number of CPU cores.</t>
</list> </li>
</t> </ul>
<t>If the number of the available CPU cores is a power of two, then it
<t>If the number of the available CPU cores is a power of two, then i is <bcp14>RECOMMENDED</bcp14> to perform the tests with 1, 2, 4, 8,
t is RECOMMENDED 16, etc. number of active CPU cores of the DUT.</t>
to perform the tests with 1, 2, 4, 8, 16, etc. number of active C
PU cores of the DUT.</t>
</section> </section>
</section> </section>
<section anchor="reporting_format" title="Reporting Format"> <section anchor="reporting_format" numbered="true" toc="default">
<name>Reporting Format</name>
<t>Measurements MUST be executed multiple times. The necessary number o <t>Measurements <bcp14>MUST</bcp14> be executed multiple times. The
f repetitions necessary number of repetitions to achieve statistically reliable
to achieve statistically reliable results may depend on the consistent results may depend on the consistent or scattered nature of the results.
or scattered nature of the results. The report of the results <bcp14>MUST</bcp14> contain the number of
The report of the results MUST contain the number of repetitions of the repetitions of the measurements. The median is <bcp14>RECOMMENDED</bcp14>
measurements. as the summarizing function of the results complemented with the first
Median is RECOMMENDED as the summarizing function of the results comple percentile and the 99th percentile as indices of the dispersion of the
mented with the results. The average and standard deviation <bcp14>MAY</bcp14> also be
first percentile and the 99th percentile as indices of the dispersion o reported.
f the results. </t>
Average and standard deviation MAY also be reported. <t>All parameters and settings that may influence the performance of the
</t> DUT <bcp14>MUST</bcp14> be reported. Some of them may be specific to the
given NATxy gateway implementation, like the "hashsize" (hash table
<t>All parameters and settings that may influence the performance of th size) and "nf_conntrack_max" (number of connection tracking table
e DUT MUST be entries) values for iptables or the limit of the number of states for
reported. Some of them may be specific to the given NATxy gateway imple OpenBSD PF (set by the "set limit states number" command in the pf.conf
mentation, like the file).
"hashsize" (hash table size) and "nf_conntrack_max" (number of connecti </t>
on tracking <t keepWithNext="true"/>
table entries) values for iptables or the limit of the number of states
for OpenBSD PF
(set by the "set limit states number" command in the pf.conf file).
</t>
<figure anchor="iptables-conn-scale" align="center" title="Example table:
Maximum connection establishment rate of iptables against the number of s
essions">
<preamble></preamble>
<artwork align="left"><![CDATA[
number of sessions (req.) 0.4M 4M 40M 400M
source port numbers (req.) 40,000 40,000 40,000 40,000
destination port numbers (req.) 10 100 1,000 10,000
"hashsize" (i.s.) 2^17 2^20 2^23 2^27
"nf_conntrack_max" (i.s.) 2^20 2^23 2^26 2^30
num. sessions / "hashsize" (i.s.) 3.05 3.81 4.77 2.98
number of experiments (req.) 10 10 10 10
error of binary search (req.) 1,000 1,000 1,000 1,000
connections/s median (req.)
connections/s 1st perc. (req.)
connections/s 99th perc. (req.)
]]></artwork>
<postamble></postamble>
</figure>
<t><xref target="iptables-conn-scale"/> shows an example of table headi
ngs for
reporting the measurement results for the scalability of the iptables s
tateful NAT44
implementation against the number of sessions. The table indicates the
always required fields
(req.) and the implementation-specific ones (i.s.).
A computed value was also added in row 6; it is the number of sessions
per hashsize ratio, which
helps the reader to interpret the achieved maximum connection establish
ment rate.
(A lower value results in shorter linked lists hanging on the entries o
f the hash
table thus facilitating higher performance. The ratio is varying, becau
se the number of
sessions is always a power of 10, whereas the hash table size is a powe
r of 2.)
To reflect the accuracy of the results, the table contains the value of
the "error" of the
binary search, which expresses the stopping criterion for the binary se
arch. The binary
search stops, when the difference between the "higher limit" and "lower
limit" of the
binary search is less than or equal to "error".
</t>
<t> The table MUST be complemented with reporting the relevant paramete
rs of the
DUT. If the DUT is a general-purpose computer and some software NATxy g
ateway implementation is tested,
then the hardware description SHOULD include: computer type, CPU type,
and number of active CPU cores,
memory type, size and speed, network interface card type (reflecting al
so the speed),
the fact that direct cable connections were used or the type of the swi
tch used for
interconnecting the Tester and the DUT. Operating system type and versi
on, kernel version, and the version
of the NATxy gateway implementation (including last commit date and num
ber if applicable) SHOULD also be given.
</t>
</section>
<section anchor="impl_exp" title="Implementation and Experience"> <table anchor="iptables-conn-scale" align="left">
<t>The stateful extension of siitperf <xref target="SIITPERF"/> is an i <name>Example Table of the Maximum Connection Establishment Rate of
mplementation of this concept. Iptables Against the Number of Sessions</name>
Its first version only supporting multiple port numbers is documented i <tbody>
n this (open access) paper <xref target="LEN2022"/>. <tr>
Its extended version also supporting multiple IP addresses is documente <td align="left">number of sessions (req.)</td>
d in this (open access) paper <xref target="LEN2024b"/>. <td align="right">0.4M</td>
</t> <td align="right">4M</td>
<t> The proposed benchmarking methodology has been validated by perform <td align="right">40M</td>
ing <td align="right">400M</td>
benchmarking measurements with three radically different stateful NAT64 </tr>
implementations (Jool, tayga+iptables, OpenBSD PF) in (open access) pap <tr>
er <td align="left">source port numbers (req.)</td>
<xref target="LEN2023"/>. <td align="right">40,000</td>
</t> <td align="right">40,000</td>
<t>Further experience with this methodology using siitperf for measurin <td align="right">40,000</td>
g the <td align="right">40,000</td>
scalability of the iptables stateful NAT44 and Jool stateful NAT64 </tr>
implementations are described in <tr>
<xref target="I-D.lencse-v6ops-transition-scalability"/>. <td align="left">destination port numbers (req.)</td>
</t> <td align="right">10</td>
<t>This methodology was successfully applied for the benchmarking of va <td align="right">100</td>
rious <td align="right">1,000</td>
IPv4aas (IPv4-as-a-Service) technologies without the usage of technolog <td align="right">10,000</td>
y-specific </tr>
Testers by reducing the aggregate of their CE (Customer Edge) and PE (P <tr>
rovider Edge) <td align="left">"hashsize" (i.s.)</td>
devices to a stateful NAT44 gateway documented in (open access) paper < <td align="right">2<sup>17</sup></td>
xref target="LEN2024a"/>. <td align="right">2<sup>20</sup></td>
</t> <td align="right">2<sup>23</sup></td>
</section> <td align="right">2<sup>27</sup></td>
</tr>
<tr>
<td align="left">"nf_conntrack_max" (i.s.)</td>
<td align="right">2<sup>20</sup></td>
<td align="right">2<sup>23</sup></td>
<td align="right">2<sup>26</sup></td>
<td align="right">2<sup>30</sup></td>
</tr>
<tr>
<td align="left">num. sessions / "hashsize" (i.s.)</td>
<td align="right">3.05</td>
<td align="right">3.81</td>
<td align="right">4.77</td>
<td align="right">2.98</td>
</tr>
<tr>
<td align="left">number of experiments (req.)</td>
<td align="right">10</td>
<td align="right">10</td>
<td align="right">10</td>
<td align="right">10</td>
</tr>
<tr>
<td align="left">error of binary search (req.)</td>
<td align="right">1,000</td>
<td align="right">1,000</td>
<td align="right">1,000</td>
<td align="right">1,000</td>
</tr>
<tr>
<td align="left">connections/s median (req.)</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
<tr>
<td align="left">connections/s 1st perc. (req.)</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
<tr>
<td align="left">connections/s 99th perc. (req.)</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
</tbody>
</table>
<section anchor="udp_or_tcp" title="Limitations of using UDP as Transport La <t keepWithPrevious="true"/>
yer Protocol">
<t>The test frame format defined in RFC 2544 exclusively uses UDP (and
not TCP) as a
transport layer protocol. Testing with UDP was kept in both RFC 5180 an
d RFC 8219 regarding
the standard benchmarking procedures (throughput, latency, frame loss r
ate, etc.).
The benchmarking methodology proposed in this document follows this lon
g established
benchmarking tradition using UDP as a transport layer protocol, too. Th
e rationale for this
is that the standard benchmarking procedures require sending frames at
arbitrary constant
frame rates, which would violate the flow control and congestion contro
l algorithms of the
TCP protocol. TCP connection setup (using the three-way handshake) woul
d further complicate testing.
</t>
<t>Further potential transport layer protocols e.g., DCCP <xref target= <t><xref target="iptables-conn-scale" format="default"/> shows an
"RFC4340"/> and SCTP <xref target="RFC9260"/> example of table headings for reporting the measurement results regarding
are outside of the scope of this document, as the widely-used the
stateful NAT44 and stateful NAT64 implementations do not support them. scalability of the iptables stateful NAT44 implementation against the
Although QUIC <xref target="RFC9000"/> number of sessions. The table indicates the required fields
is also considered a transport layer protocol, but QUIC packets are car (req.) and the implementation-specific ones (i.s.). A computed value
ried in UDP was also added in row 6; it is the number of sessions per hashsize
datagrams thus QUIC does not need a special handling. ratio, which helps the reader to interpret the achieved maximum
</t> connection establishment rate. (A lower value results in shorter linked
lists hanging on the entries of the hash table, thus facilitating higher
performance. The ratio is varying, because the number of sessions is
always a power of 10, whereas the hash table size is a power of 2.) To
reflect the accuracy of the results, the table contains the value of the
"error" of the binary search, which expresses the stopping criterion for
the binary search. The binary search stops when the difference between
the "higher limit" and "lower limit" of the binary search is less than
or equal to the "error".
<t>Some stateful NATxy solutions handle TCP and UDP differently, e.g. i </t>
ptables uses 30s <t>The table <bcp14>MUST</bcp14> be complemented with reporting the
timeout for UDP and 60s timeout for TCP. Thus benchmarking results prod relevant parameters of the DUT. If the DUT is a general-purpose computer
uced using UDP do not and some software NATxy gateway implementation is tested, then the
necessarily characterize the performance of a NATxy gateway well enough wh hardware description <bcp14>SHOULD</bcp14> include the following:</t>
en they <ul>
are used for forwarding Internet traffic. As for the given example, tim <li>computer type</li>
eout values of the DUT may <li>CPU type</li>
be adjusted, but it requires extra consideration. <li>number of active CPU cores</li>
</t> <li>memory type</li>
<t>Other differences in handling UDP or TCP are also possible. Thus, th <li>size and speed</li>
e authors recommend that <li>network interface card type (also reflecting the speed)</li>
further investigations should be performed in this field. <li>direct cable connections or the type of switch used for
</t> interconnecting the Tester and the DUT</li>
<t>As a mitigation of this problem, this document recommends that testi </ul>
ng with protocols using TCP <t>The operating system type and
(like HTTP and HTTPS up to version 2) can be performed as described in version, kernel version, and version of the NATxy gateway
<xref target="RFC9411"/>. implementation (including the last commit date and number if applicable)
This approach also solves the potential problem of protocol helpers may <bcp14>SHOULD</bcp14> also be given.
be present in the stateful DUT. </t>
</t>
<t>As for HTTP/3, it uses QUIC, which uses UDP as stated above. It shou
ld be noted that QUIC is treated
as any other UDP payload. The proposed measurement method does not aim
to measure the performance
of QUIC, rather it aims to measure the performance of the stateful NATx
y gateway.
</t>
</section> </section>
<section anchor="Acknowledgements" title="Acknowledgements"> <section anchor="impl_exp" numbered="true" toc="default">
<t>The authors would like to thank Al Morton, Sarah Banks, Edwin Cordeiro, <name>Implementation and Experience</name>
Lukasz Bromirski,
Sándor Répás, Tamás Hetényi, Timothy Winters, Eduard Vasilenko, Minh Ng
oc Tran, Paolo Volpato,
Zeqi Lai, and Bertalan Kovács for their comments.</t>
<t>The authors thank Warren Kumari, Michael Scharf, Alexey Melnikov, Ro
bert Sparks, David Dong,
Roman Danyliw, Erik Kline, Murray Kucherawy, Zaheduzzaman Sarker, and É
ric Vyncke
for their reviews and comments.</t>
<t>This work was supported by the Japan Trust International Research Coo
peration Program
of the National Institute of Information and Communications Technology (
NICT), Japan.</t>
</section>
<!-- Possibly a 'Contributors' section ... -->
<section anchor="IANA" title="IANA Considerations">
<t>This document does not make any request to IANA.</t>
</section>
<section anchor="Security" title="Security Considerations">
<t>This document has no further security considerations beyond that of <xre
f target="RFC8219"/>.
They should be cited here so that they be applied not only for the
benchmarking of IPv6 transition technologies but also for the benchmarki
ng of any
stateful NATxy gateways (allowing for x=y, too).</t>
</section>
</middle>
<!-- *****BACK MATTER ***** -->
<back> <t>The stateful extension of siitperf <xref target="SIITPERF"
<!-- References split into informative and normative --> format="default"/> is an implementation of this concept. Its first
version that only supports multiple port numbers is documented in this
(open access) paper: <xref target="LEN2022" format="default"/>. Its
extended version that also supports multiple IP addresses is documented in
this (open access) paper: <xref target="LEN2024b" format="default"/>.
</t>
<!-- There are 2 ways to insert reference entries from the citation libraries <t>The proposed benchmarking methodology has been validated by
: performing benchmarking measurements with three radically different
1. define an ENTITY at the top, and use "ampersand character"RFC2629; here ( stateful NAT64 implementations (Jool, tayga+iptables, and OpenBSD PF) in t
as shown) his
2. simply use a PI "less than character"?rfc include="reference.RFC.2119.xml (open access) paper: <xref target="LEN2023" format="default"/>.</t>
"?> here
(for I-Ds: include="reference.I-D.narten-iana-considerations-rfc2434bis.x
ml")
Both are cited textually in the same manner: by using xref elements. <t>Further experience with this methodology of using siitperf for measurin
If you use the PI option, xml2rfc will, by default, try to find included fil g
es in the same the scalability of the iptables stateful NAT44 and Jool stateful NAT64
directory as the including file. You can also define the XML_LIBRARY environ implementations are described in <xref
ment variable target="I-D.lencse-v6ops-transition-scalability" format="default"/>.</t>
with a value containing a set of directories to search. These can be either
in the local
filing system or remote ones accessed by http (http://domain/dir/... ).-->
<references title="Normative References"> <t>This methodology was successfully applied for the benchmarking of
<!--?rfc include="http://xml.resource.org/public/rfc/bibxml/reference.RFC.21 various IPv4-as-a-Service (IPv4aas) technologies without the usage of
19.xml"?--> technology-specific Testers by reducing the aggregate of their Customer
Edge (CE) and Provider Edge (PE) devices to a stateful NAT44 gateway
documented in this (open access) paper: <xref target="LEN2024a"
format="default"/>.</t>
</section>
&RFC2119; <section anchor="udp_or_tcp" numbered="true" toc="default">
&RFC1918; <name>Limitations of Using UDP as a Transport Layer Protocol</name>
&RFC2544;
&RFC3022;
&RFC4340;
&RFC4814;
&RFC5180;
&RFC6146;
&RFC7599;
&RFC8174;
&RFC8219;
&RFC9000;
&RFC9260;
&RFC9411;
</references> <t>The test frame format defined in <xref target="RFC2544"/> exclusively u
ses UDP (and
not TCP) as a transport layer protocol. Testing with UDP was kept in
both <xref target="RFC5180"/> and <xref target="RFC8219"/> regarding the s
tandard benchmarking
procedures (throughput, latency, frame loss rate, etc.). The
benchmarking methodology proposed in this document follows this long-estab
lished benchmarking tradition using UDP as a transport layer
protocol, too. The rationale for this is that the standard benchmarking
procedures require sending frames at arbitrary constant frame rates,
which would violate the flow control and congestion control algorithms
of the TCP protocol. TCP connection setup (using the three-way
handshake) would further complicate testing.</t>
<references title="Informative References"> <t>Further potential transport layer protocols, e.g., the Datagram Congest
<!-- Here we use entities that we defined at the beginning. --> ion Control Protocol (DCCP) <xref
target="RFC4340" format="default"/> and the Stream Control Transmission Pr
otocol (SCTP) <xref target="RFC9260"
format="default"/>, are outside of the scope of this document, as the
widely used stateful NAT44 and stateful NAT64 implementations do not
support them. Although QUIC <xref target="RFC9000" format="default"/> is
also considered a transport layer protocol, QUIC packets are carried
in UDP datagrams; thus, QUIC does not need a special handling.</t>
<?rfc include='reference.I-D.lencse-v6ops-transition-scalability'?> <t>Some stateful NATxy solutions handle TCP and UDP differently,
e.g., iptables use a 30s timeout for UDP and a 60s timeout for TCP. Thus,
benchmarking results produced using UDP do not necessarily characterize
the performance of a NATxy gateway well enough when they are used for
forwarding Internet traffic. As for the given example, timeout values of
the DUT may be adjusted, but it requires extra consideration.</t>
<reference anchor="DUST1964" <t>Other differences in handling UDP or TCP are also possible. Thus, the
target="https://dl.acm.org/doi/10.1145/364520.364540"> authors recommend that further investigations should be performed in
<front> this field.</t>
<title>Algorithm 235: Random permutation
</title>
<author initials="R." surname="Durstenfeld">
<organization></organization>
</author>
<date day="" month="July" year="1964"/>
</front>
<seriesInfo name="" value="Communications of the ACM, vol. 7, no. 7, p.420
."/>
<seriesInfo name="DOI" value="10.1145/364520.364540"/>
</reference>
<reference anchor="IIR2020" <t>As a mitigation of this problem, this document recommends that
target="https://www.iij.ad.jp/en/dev/iir/pdf/iir_vol49_report_EN.pdf"> testing with protocols using TCP (like HTTP and HTTPS up to version 2)
<front> can be performed as described in <xref target="RFC9411"
<title>Periodic observation report: Internet trends as seen from IIJ inf format="default"/>. This approach also solves the potential problem of
rastructure - 2020 protocol helpers that may be present in the stateful DUT.</t>
</title>
<author initials="T." surname="Kurahashi"> <t>As for HTTP/3, it uses QUIC, which uses UDP as stated above. It
<organization></organization> should be noted that QUIC is treated as any other UDP payload. The
</author> proposed measurement method does not aim to measure the performance of
<author initials="Y." surname="Matsuzaki"> QUIC, rather, it aims to measure the performance of the stateful NATxy
<organization></organization> gateway.</t>
</author> </section>
<author initials="T." surname="Sasaki">
<organization></organization>
</author>
<author initials="T." surname="Saito">
<organization></organization>
</author>
<author initials="F." surname="Tsutsuji">
<organization></organization>
</author>
<date day="" month="Dec" year="2020"/>
</front>
<seriesInfo name="" value="Internet Infrastructure Review, vol. 49"/>
</reference>
<reference anchor="LEN2015" <section anchor="IANA" numbered="true" toc="default">
target="http://www.hit.bme.hu/~lencse/publications/e98-b_8_1580.pdf"> <name>IANA Considerations</name>
<front> <t>This document has no IANA actions.</t>
<title>Estimation of the Port Number Consumption of Web Browsing </section>
</title>
<author initials="G." surname="Lencse"> <section anchor="Security" numbered="true" toc="default">
<organization></organization> <name>Security Considerations</name>
</author> <t>This document has no further security considerations beyond that of
<xref target="RFC8219" format="default"/>. They should be cited here so
that they can be applied not only for the benchmarking of IPv6 transition
technologies but also for the benchmarking of any stateful NATxy
gateways (allowing for x=y, too).</t>
</section>
</middle>
<back>
<date day="1" month="8" year="2015"/> <displayreference target="I-D.lencse-v6ops-transition-scalability" to="SCALAB
</front> ILITY"/>
<seriesInfo name="" value="IEICE Transactions on Communications, vol. E98- <references>
B, no. 8. pp. 1580-1588"/> <name>References</name>
<seriesInfo name="DOI" value="DOI: 10.1587/transcom.E98.B.1580"/> <references>
</reference> <name>Normative References</name>
<reference anchor="LEN2020" <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.21
target="http://ijates.org/index.php/ijates/article/view/291"> 19.xml"/>
<front> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.1
<title>Adding RFC 4814 Random Port Feature to Siitperf: Design, Implemen 918.xml"/>
tation and Performance Estimation <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.2
</title> 544.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.3
022.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.4
340.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.4
814.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.5
180.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.6
146.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.7
599.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8
174.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8
219.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9
000.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9
260.xml"/>
<xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9
411.xml"/>
</references>
<references>
<name>Informative References</name>
<author initials="G." surname="Lencse"> <!-- [I-D.lencse-v6ops-transition-scalability] IESG state: Expired as of 06/19/2
<organization></organization> 4-->
</author> <xi:include href="https://datatracker.ietf.org/doc/bibxml3/draft-lencse-v
<date day="" month="" year="2020"/> 6ops-transition-scalability.xml"/>
</front>
<seriesInfo name="" value="International Journal of Advances in Telecommun
ications, Electrotechnics, Signals and Systems, vol 9, no 3, pp. 18-26."/>
<seriesInfo name="DOI" value="10.11601/ijates.v9i3.291"/>
</reference>
<reference anchor="LEN2022" <reference anchor="DUST1964" target="https://dl.acm.org/doi/pdf/10.1145/
target="https://www.sciencedirect.com/science/article/pii/S0140366422001803" 364520.364540">
> <front>
<front> <title>Algorithm 235: Random permutation
<title>Design and Implementation of a Software Tester for Benchmarking S </title>
tateful NAT64xy Gateways: <author initials="R." surname="Durstenfeld">
Theory and Practice of Extending Siitperf for Stateful Tests <organization/>
</title> </author>
<date month="July" year="1964"/>
</front>
<refcontent>Communications of the ACM, vol. 7, no. 7, p. 420</refconte
nt>
<seriesInfo name="DOI" value="10.1145/364520.364540"/>
</reference>
<author initials="G." surname="Lencse"> <reference anchor="IIR2020" target="https://www.iij.ad.jp/en/dev/iir/pdf
<organization></organization> /iir_vol49_report_EN.pdf">
</author> <front>
<title>Periodic Observation Report: Internet Trends as Seen from IIJ
Infrastructure - 2020
</title>
<author initials="T." surname="Kurahashi">
<organization/>
</author>
<author initials="Y." surname="Matsuzaki">
<organization/>
</author>
<author initials="T." surname="Sasaki">
<organization/>
</author>
<author initials="T." surname="Saito">
<organization/>
</author>
<author initials="F." surname="Tsutsuji">
<organization/>
</author>
<date month="December" year="2020"/>
</front>
<refcontent>Internet Initiative Japan Inc.</refcontent>
<refcontent>Internet Infrastructure Review, vol. 49</refcontent>
</reference>
<date day="1" month="August" year="2022"/> <reference anchor="LEN2015" target="https://www.hit.bme.hu/~lencse/publi
</front> cations/e98-b_8_1580.pdf">
<seriesInfo name="" value="Computer Communications, vol. 172, no. 1, pp. 7 <front>
5-88"/> <title>Estimation of the Port Number Consumption of Web Browsing
<seriesInfo name="DOI" value="10.1016/j.comcom.2022.05.028"/> </title>
</reference> <author initials="G." surname="Lencse">
<organization/>
</author>
<date month="August" year="2015"/>
</front>
<refcontent>IEICE Transactions on Communications, vol. E98-B, no. 8. p
p. 1580-1588</refcontent>
<seriesInfo name="DOI" value="10.1587/transcom.E98.B.1580"/>
</reference>
<reference anchor="LEN2023" <reference anchor="LEN2020" target="http://ijates.org/index.php/ijates/a
target="https://www.sciencedirect.com/science/article/pii/S0140366423002931" rticle/view/291">
> <front>
<front> <title>Adding RFC 4814 Random Port Feature to Siitperf: Design, Impl
<title>Benchmarking methodology for stateful NAT64 gateways ementation and Performance Estimation
</title> </title>
<author initials="G." surname="Lencse">
<organization/>
</author>
<date month="November" year="2020"/>
</front>
<refcontent>International Journal of Advances in Telecommunications, E
lectrotechnics, Signals and Systems, vol 9, no 3, pp. 18-26.</refcontent>
<seriesInfo name="DOI" value="10.11601/ijates.v9i3.291"/>
</reference>
<author initials="G." surname="Lencse"> <reference anchor="LEN2022" target="https://www.sciencedirect.com/scienc
<organization></organization> e/article/pii/S0140366422001803">
</author> <front>
<author initials="K." surname="Shima"> <title>Design and Implementation of a Software Tester for Benchmarki
<organization></organization> ng Stateful NAT64xy Gateways: Theory and Practice of Extending Siitperf for Stat
</author> eful Tests
<author initials="K." surname="Cho"> </title>
<organization></organization> <author initials="G." surname="Lencse">
</author> <organization/>
</author>
<date month="August" year="2022"/>
</front>
<refcontent>Computer Communications, vol. 192, pp. 75-88</refcontent>
<seriesInfo name="DOI" value="10.1016/j.comcom.2022.05.028"/>
</reference>
<date day="1" month="October" year="2023"/> <reference anchor="LEN2023" target="https://www.sciencedirect.com/scienc
</front> e/article/pii/S0140366423002931">
<seriesInfo name="" value="Computer Communications, vol. 210, no. 1, pp. 2 <front>
56-272"/> <title>Benchmarking methodology for stateful NAT64 gateways
<seriesInfo name="DOI" value="10.1016/j.comcom.2023.08.009"/> </title>
</reference> <author initials="G." surname="Lencse">
<organization/>
</author>
<author initials="K." surname="Shima">
<organization/>
</author>
<author initials="K." surname="Cho">
<organization/>
</author>
<date month="October" year="2023"/>
</front>
<refcontent>Computer Communications, vol. 210, pp. 256-272</refcontent
>
<seriesInfo name="DOI" value="10.1016/j.comcom.2023.08.009"/>
</reference>
<reference anchor="LEN2024a" <reference anchor="LEN2024a" target="https://www.sciencedirect.com/scien
target="https://www.sciencedirect.com/science/article/pii/S0140366424000999" ce/article/pii/S0140366424000999">
> <front>
<front> <title>Benchmarking methodology for IPv4aaS technologies:
<title>Benchmarking methodology for IPv4aaS technologies:
Comparison of the scalability of the Jool implementation of 464XL AT and MAP-T Comparison of the scalability of the Jool implementation of 464XL AT and MAP-T
</title> </title>
<author initials="G." surname="Lencse">
<author initials="G." surname="Lencse"> <organization/>
<organization></organization> </author>
</author> <author initials="Á." surname="Bazsó">
<author initials="Á." surname="Bazsó"> <organization/>
<organization></organization> </author>
</author> <date month="April" year="2024"/>
</front>
<refcontent>Computer Communications, vol. 219, pp. 243-258</refcontent
>
<seriesInfo name="DOI" value="10.1016/j.comcom.2024.03.007"/>
</reference>
<date day="1" month="April" year="2024"/> <reference anchor="LEN2024b" target="https://www.sciencedirect.com/scien
</front> ce/article/abs/pii/S0140366424001993">
<seriesInfo name="" value="Computer Communications, vol. 219, no. 1, pp. 2 <front>
43-258"/> <title>Making stateless and stateful network performance measurement
<seriesInfo name="DOI" value="10.1016/j.comcom.2024.03.007"/> s unbiased
</reference> </title>
<author initials="G." surname="Lencse">
<organization/>
</author>
<date month="September" year="2024"/>
</front>
<refcontent>Computer Communications, vol. 225, pp. 141-155</refcontent
>
<seriesInfo name="DOI" value="10.1016/j.comcom.2024.05.018"/>
</reference>
<reference anchor="LEN2024b" <reference anchor="SIITPERF" target="https://github.com/lencsegabor/siit
target="https://www.sciencedirect.com/science/article/abs/pii/S0140366424001 perf">
993"> <front>
<front> <title>Siitperf: An RFC 8219 compliant SIIT and stateful NAT64/NAT44
<title>Making stateless and stateful network performance measurements un tester
biased </title>
</title> <author>
<organization/>
</author>
<date month="September" year="2023"/>
</front>
<refcontent>commit 165cb7f</refcontent>
</reference>
</references>
</references>
<author initials="G." surname="Lencse"> <section anchor="Acknowledgements" numbered="false" toc="default">
<organization></organization> <name>Acknowledgements</name>
</author>
<!-- <date day="1" month="April" year="2024"/> --> <t>The authors would like to thank <contact fullname="Al Morton"/>,
</front> <contact fullname="Sarah Banks"/>, <contact fullname="Edwin Cordeiro"/>,
<seriesInfo name="" value="Computer Communications"/> <contact fullname="Lukasz Bromirski"/>, <contact fullname="Sándor
<seriesInfo name="DOI" value="10.1016/j.comcom.2024.05.018"/> Répás"/>, <contact fullname="Tamás Hetényi"/>, <contact
</reference> fullname="Timothy Winters"/>, <contact fullname="Eduard Vasilenko"/>,
<contact fullname="Minh Ngoc Tran"/>, <contact fullname="Paolo
Volpato"/>, <contact fullname="Zeqi Lai"/>, and <contact
fullname="Bertalan Kovács"/> for their comments.</t>
<t>The authors thank <contact fullname="Warren Kumari"/>, <contact
fullname="Michael Scharf"/>, <contact fullname="Alexey Melnikov"/>,
<contact fullname="Robert Sparks"/>, <contact fullname="David Dong"/>,
<contact fullname="Roman Danyliw"/>, <contact fullname="Erik Kline"/>,
<contact fullname="Murray Kucherawy"/>, <contact fullname="Zaheduzzaman
Sarker"/>, and <contact fullname="Éric Vyncke"/> for their reviews and
comments.</t>
<t>This work was supported by the Japan Trust International Research
Cooperation Program of the National Institute of Information and
Communications Technology (NICT), Japan.</t>
</section>
</back>
<reference anchor="SIITPERF" <!-- [rfced] Please review the "Inclusive Language" portion of the online
target="https://github.com/lencsegabor/siitperf"> Style Guide <https://www.rfc-editor.org/styleguide/part2/#inclusive_language>
<front> and let us know if any changes are needed. Updates of this nature typically
<title>Siitperf: An RFC 8219 compliant SIIT and stateful NAT64/NAT44 tes result in more precise language, which is helpful for readers.
ter written in C++ using DPDK
</title>
<author initials="G." surname="Lencse"> a. For example, please consider whether "black" should be updated.
<organization></organization>
</author>
<date day="" month="" year="2019-2023" /> b. In addition, please consider whether "tradition" and "traditional" should
</front> be updated for clarity. While the NIST website
<seriesInfo name="" value="source code"/> <https://www.nist.gov/nist-research-library/nist-technical-series-publications-a
<seriesInfo name="" value="available from GitHub"/> uthor-instructions#table1>
</reference> indicates that this term is potentially biased, it is also ambiguous.
<!-- --> "Tradition" is a subjective term, as it is not the same for everyone.
</references> -->
<section anchor="change_log" title="Change Log">
<section title="00">
<t>Initial version.
</t>
</section>
<section title="01">
<t>Updates based on the comments received on the BMWG mailing list and min
or corrections.
</t>
</section>
<section title="02">
<t><xref target="ctrl_conntrack"/> was completely re-written. As a consequ
ence,
the occurrences of the now undefined "mostly different" source port num
ber destination
port number combinations were deleted from <xref target="meas_max_conn_
est_rate"/>,
too.
</t>
</section>
<section title="03">
<t>Added <xref target="consider_stateful"/> about the consideration of the
cases of stateful operation.
</t>
<t>Consistency checking. Removal of some parts obsoleted by the previous r
e-writing
of <xref target="ctrl_conntrack"/>.
</t>
<t>Added <xref target="meas_conn_tear_down_rate"/> about the method for me
asuring connection tear-down rate.
</t>
<t>Updates for <xref target="impl_exp"/> about the implementation and expe
rience.
</t>
</section>
<section title="04">
<t>Update of the abstract.
</t>
<t>Added <xref target="validation_of_conn"/> about validation of connectio
n establishment.
</t>
<t>Added <xref target="meas_contr_capacity"/> about the method for measuri
ng connection tracking table capacity.
</t>
<t>Consistency checking and corrections.
</t>
</section>
<section title="00 - WG item">
<t>Added measurement setup for Stateful NAT64 gateways.
</t>
<t>Consistency checking and corrections.
</t>
</section>
<section title="01">
<t>Added Section 4.5.1 about typical types of measurement series and repor
ting format.
</t>
</section>
<section title="02">
<t>Added the usage of multiple IP addresses.</t>
<t>Section 4.5.1 was removed and split into two Sections:
<xref target="meas_scalability"/> about scalability measurements and
<xref target="reporting_format"/> about reporting format.
</t>
</section>
<section title="03">
<t>Updated the usage of multiple IP addresses.</t>
<t>Test phases were renamed as follows:
<list style="symbols">
<t>preliminary test phase --> test phase 1</t>
<t>real test phase --> test phase 2.</t>
</list>
</t>
</section>
<section title="04">
<t>Minor updates to <xref target="setup_term_multiple"/> and <xref target=
"impl_exp"/>.</t>
</section>
<section title="05">
<t>Minor updates addressing WGLC nits (adding the definition of "black box
", and
performing a high amount of grammatical corrections).</t>
</section>
<section title="06">
<t>Language editing addressing preliminary AD review comments by eliminati
ng the occurrences
of first person singular ("we", "our").</t>
</section>
<section title="07">
<t>Updates addressing IESG Last Call comments.</t>
</section>
</section>
</back>
</rfc> </rfc>
 End of changes. 131 change blocks. 
1616 lines changed or deleted 1412 lines changed or added

This html diff was produced by rfcdiff 1.48.