test patch D49047
Contents
tri-point topology testbed config over emulated 1Gbps x 20ms WAN
Two virtual machines (VMs) as traffic senders are hosted by Bhyve in two separate physical boxs (Beelink SER5 AMD Mini PC). The Bhyve hosts are running in FreeBSD 14.2 release OS.
The 1st VM(n1fbsd) generates TCP traffic while the 2nd VM(n2fbsd) generates UDP traffic.
A single traffic receiver is running in Ubuntu Linux 24.04 LTS. The traffic receiver is a physical box (same Beelink box) for the simplicity of a tri-point topology.
The three physical boxes are connected through a 5-Port Gigabit Ethernet Switch (TP-Link TL-SG105). The swich has a shared 1MB Packet Buffer Memory, which is just 40% of the 2.5 Mbytes BDP (1000Mbps x 20ms).
testbed: 
additional 20ms latency is added/emulated in the receiver: Bandwidth Delay Product(BDP) is around 2.5 Mbytes == 1000Mbps x 20ms.
cc@Linux:~ % tc qdisc show dev enp1s0 qdisc fq_codel 0: root refcnt 2 limit 10240p flows 1024 quantum 1514 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64 cc@Linux:~ % sudo tc qdisc add dev enp1s0 root netem delay 20ms cc@Linux:~ % tc qdisc show dev enp1s0 qdisc netem 8001: root refcnt 2 limit 1000 delay 20ms cc@Linux:~ % root@n1fbsd:~ # ping -c 5 -S n1fbsd Linux PING Linux (192.168.50.46) from n1fbsdvm: 56 data bytes 64 bytes from 192.168.50.46: icmp_seq=0 ttl=64 time=21.296 ms 64 bytes from 192.168.50.46: icmp_seq=1 ttl=64 time=21.411 ms 64 bytes from 192.168.50.46: icmp_seq=2 ttl=64 time=21.285 ms 64 bytes from 192.168.50.46: icmp_seq=3 ttl=64 time=21.355 ms 64 bytes from 192.168.50.46: icmp_seq=4 ttl=64 time=21.390 ms --- Linux ping statistics --- 5 packets transmitted, 5 packets received, 0.0% packet loss round-trip min/avg/max/stddev = 21.285/21.347/21.411/0.050 ms root@n1fbsd:~ # root@n2fbsd:~ # ping -c 5 -S n2fbsd Linux PING Linux (192.168.50.46) from n2fbsdvm: 56 data bytes 64 bytes from 192.168.50.46: icmp_seq=0 ttl=64 time=21.197 ms 64 bytes from 192.168.50.46: icmp_seq=1 ttl=64 time=21.372 ms 64 bytes from 192.168.50.46: icmp_seq=2 ttl=64 time=21.520 ms 64 bytes from 192.168.50.46: icmp_seq=3 ttl=64 time=21.685 ms 64 bytes from 192.168.50.46: icmp_seq=4 ttl=64 time=21.345 ms --- Linux ping statistics --- 5 packets transmitted, 5 packets received, 0.0% packet loss round-trip min/avg/max/stddev = 21.197/21.424/21.685/0.166 ms root@n2fbsd:~ #
- sender/receiver sysctl tuning
n1fbsd and n2fbsd: cat /etc/sysctl.conf ... # Don't cache ssthresh from previous connection net.inet.tcp.hostcache.enable=0 # In crease FreeBSD maximum socket buffer size up to 5MB kern.ipc.maxsockbuf=5242880 # Increase FreeBSD Max size of automatic send/receive buffer up to 5MB net.inet.tcp.sendbuf_max=5242880 net.inet.tcp.recvbuf_max=5242880 root@n1fbsd:~ # cc@Linux:~$ cat /etc/sysctl.conf ... # allow testing with 5MB buffers net.core.rmem_max = 5242880 net.core.wmem_max = 5242880 # increase Linux autotuning TCP buffer limit to 5MB net.ipv4.tcp_rmem = 4096 131072 5242880 net.ipv4.tcp_wmem = 4096 16384 5242880 # don't cache ssthresh from previous connection net.ipv4.tcp_no_metrics_save = 1 cc@Linux:~$
- iperf3 command (sender2 generates UDP traffic)
iperf3 -B n1fbsd --cport 54321 -c Linux -p 5201 -l 1M -t 40 -i 1 -f m -VC cubic iperf3 -B n2fbsd --cport 54321 -c Linux -p 5202 -t 50 -i 1 -f m --udp -b 200m
test analysis method
I use my siftr2 for pipe study code kernel module to capture TCP stats, and my log tool review_siftr2_log to analyze the data.
The key data to look at are these three values in siftr2.
sent_inflight_bytes = tp->snd_max - tp->snd_una; // siftr's old code on inflight data (pipe) calculation def_pipe = tcp_compute_pipe(tp); // FreeBSD default stack pipe calculation if V_tcp_do_newsack == 1 old_pipe = tp->snd_max - tp->t_ccv.curack; // the old FreeBSD default stack pipe calculation if V_tcp_do_newsack == 0 in xx_cc_post_recovery()
test result before patch D49047
- code of pipe calculation before patch D49047
1 int
2 tcp_compute_pipe(struct tcpcb *tp)
3 {
4 if (tp->t_fb->tfb_compute_pipe == NULL) {
5 return (tp->snd_max - tp->snd_una +
6 tp->sackhint.sack_bytes_rexmit -
7 tp->sackhint.sacked_bytes -
8 tp->sackhint.lost_bytes);
9 } else {
10 return((*tp->t_fb->tfb_compute_pipe)(tp));
11 }
12 }
13
14 static void
15 cubic_post_recovery(struct cc_var *ccv)
16 {
17 struct cubic *cubic_data;
18 int pipe;
19 uint32_t mss = tcp_fixed_maxseg(ccv->tp);
20
21 cubic_data = ccv->cc_data;
22 pipe = 0;
23
24 if (IN_FASTRECOVERY(CCV(ccv, t_flags))) {
25 /*
26 * If inflight data is less than ssthresh, set cwnd
27 * conservatively to avoid a burst of data, as suggested in
28 * the NewReno RFC. Otherwise, use the CUBIC method.
29 *
30 * XXXLAS: Find a way to do this without needing curack
31 */
32 if (V_tcp_do_newsack)
33 pipe = tcp_compute_pipe(ccv->tp); << def_pipe
34 else
35 pipe = CCV(ccv, snd_max) - ccv->curack; << old_pipe
36
37 if (pipe < CCV(ccv, snd_ssthresh))
38 /*
39 * Ensure that cwnd does not collapse to 1 MSS under
40 * adverse conditions. Implements RFC6582
41 */
42 CCV(ccv, snd_cwnd) = max(pipe, mss) + mss;
43 else
44 /* Update cwnd based on beta and adjusted W_max. */
45 CCV(ccv, snd_cwnd) = max(((uint64_t)cubic_data->W_max *
46 CUBIC_BETA) >> CUBIC_SHIFT,
47 2 * mss);
48 }
49
50 /* Calculate the average RTT between congestion epochs. */
51 if (cubic_data->epoch_ack_count > 0 &&
52 cubic_data->sum_rtt_usecs >= cubic_data->epoch_ack_count) {
53 cubic_data->mean_rtt_usecs = (int)(cubic_data->sum_rtt_usecs /
54 cubic_data->epoch_ack_count);
55 }
56
57 cubic_data->epoch_ack_count = 0;
58 cubic_data->sum_rtt_usecs = 0;
59 }
- with V_tcp_do_newsack == 1, the def_pipe value is conservative to let cwnd be less burst
##direction relative_timestamp cwnd ssthresh data_size inflight_bytes def_pipe old_pipe t_flags o 0.000000 14480 1073725440 0 0 0 425266961 N/A o 0.000125 14480 1073725440 37 0 0 425266961 N/A i 0.020397 14480 1073725440 0 37 37 425266998 N/A << curack not assigned, so ignore the first 3 lines as the flow is in slow-start phase o 0.028410 14517 1073725440 1448 0 0 0 N/A o 0.028479 14517 1073725440 1448 1448 1448 1448 N/A o 0.028496 14517 1073725440 1448 2896 2896 2896 N/A o 0.028511 14517 1073725440 1448 4344 4344 4344 N/A o 0.028553 14517 1073725440 1448 5792 5792 5792 N/A o 0.028580 14517 1073725440 1448 7240 7240 7240 N/A o 0.028604 14517 1073725440 1448 8688 8688 8688 N/A o 0.028629 14517 1073725440 1448 10136 10136 10136 N/A o 0.028652 14517 1073725440 1448 11584 11584 11584 N/A o 0.028672 14517 1073725440 1448 13032 13032 13032 N/A i 0.049146 14517 1073725440 0 14480 14480 14480 N/A << in slow-start phase, all three pipe values are same ... o 0.494398 530005 1073725440 1448 528520 528520 528520 N/A i 0.510055 530005 1073725440 0 529968 529968 529968 N/A i 0.510090 530005 1073725440 0 529968 528520 529968 N/A o 0.510113 496664 265002 1448 529968 496664 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | << for the first loss recovery, def_pipe has a decreasing trend while others don't i 0.510148 265002 265002 0 529968 496664 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.510167 479288 265002 0 529968 485080 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.510179 489424 265002 1448 529968 485080 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.510204 489424 265002 1448 529968 486528 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.510224 489424 265002 1448 529968 487976 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.510242 489424 265002 1448 529968 489424 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.511261 489424 265002 0 529968 489424 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.512392 370688 265002 0 529968 409784 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.513875 309872 265002 0 529968 315664 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.515239 107152 265002 0 529968 157832 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.515284 139008 265002 1448 529968 33304 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.515320 139008 265002 1448 529968 34752 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.515342 139008 265002 1448 529968 36200 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.515360 139008 265002 1448 529968 37648 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.516467 139008 265002 0 529968 37648 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.530711 240368 265002 0 529968 13032 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.530762 263536 265002 0 529968 13032 529968 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.530860 263536 265002 0 528520 11584 528520 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.530920 259192 265002 0 528520 11584 528520 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.535941 259192 265002 0 522728 5792 522728 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.536411 2896 265002 1448 0 0 0 N/A o 0.536460 2896 265002 1448 1448 1448 1448 N/A ... i 6.387183 402278 150592 0 401096 401096 401096 N/A o 6.387206 330144 199824 1448 401096 330144 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | i 6.387455 199824 199824 0 401096 330144 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | i 6.404650 304080 199824 0 401096 312768 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | i 6.405692 291048 199824 0 401096 292496 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | i 6.406971 102808 199824 0 401096 195480 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 6.406997 72400 199824 1448 401096 69504 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 6.407016 72400 199824 1448 401096 70952 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 6.407022 72400 199824 1448 401096 72400 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | i 6.407563 72400 199824 0 401096 72400 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 6.407578 131768 199824 1448 401096 7240 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | i 6.427665 131768 199824 0 399648 5792 399648 TF_FASTRECOVERY | TF_CONGRECOVERY | o 6.427774 195480 199824 0 399648 5792 399648 TF_FASTRECOVERY | TF_CONGRECOVERY | i 6.428163 195480 199824 0 395304 1448 395304 TF_FASTRECOVERY | TF_CONGRECOVERY | << same here in later loss recovery ...
Raw data files: iperf3 output siftr2 log siftr2 log analysis result
TCP CUBIC congestion window chart: 
with V_tcp_do_newsack == 0, the tcp cubic pipe value is the old_pipe from cubic_post_recovery(), and the cwnd is applied by that old_pipe.
##direction relative_timestamp cwnd ssthresh data_size inflight_bytes def_pipe old_pipe t_flags ... i 0.383445 392445 1073725440 0 401096 373584 401096 N/A o 0.383477 367792 196222 1448 401096 367792 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | << first loss recovery after slow-start i 0.383533 196222 196222 0 401096 367792 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.384274 359104 196222 0 401096 362000 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.385659 266432 196222 0 401096 296840 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.387024 88328 196222 0 401096 189688 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.387116 53576 196222 1448 401096 49232 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.387170 53576 196222 1448 401096 50680 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.387187 53576 196222 1448 401096 52128 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.387201 53576 196222 1448 401096 53576 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.388538 53576 196222 0 401096 53576 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.388557 149144 196222 1448 401096 8688 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.388575 149144 196222 1448 401096 10136 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.388581 149144 196222 1448 401096 11584 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.388585 149144 196222 1448 401096 13032 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.404117 149144 196222 0 401096 13032 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.404157 194032 196222 0 401096 13032 401096 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.407996 194032 196222 0 399648 11584 399648 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.408037 189688 196222 0 399648 11584 399648 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.409044 189688 196222 0 393856 5792 393856 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.409101 191136 196222 0 393856 5792 393856 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.409226 191136 196222 0 389512 1448 389512 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.409654 2896 196222 1448 0 0 0 N/A o 0.409667 2896 196222 1448 1448 1448 1448 N/A ... o 8.172252 709520 196222 1448 725448 705176 725448 N/A i 8.172537 709520 196222 0 726896 706624 726896 N/A o 8.172546 679112 354760 1448 726896 679112 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | i 8.173112 354760 354760 0 726896 679112 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | i 8.174432 626984 354760 0 726896 644360 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | i 8.176589 586440 354760 0 726896 595128 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | o 8.176601 603816 354760 1448 726896 574856 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | i 8.178586 603816 354760 0 726896 574856 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | i 8.179820 580648 354760 0 726896 545896 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | i 8.181222 474944 354760 0 726896 456120 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | i 8.182614 501008 354760 0 726896 414128 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | i 8.183763 266432 354760 0 726896 325800 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | i 8.191412 333040 354760 0 726896 304080 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | i 8.192842 205616 354760 0 726896 154936 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | o 8.192854 201272 354760 0 726896 2896 726896 TF_FASTRECOVERY | TF_CONGRECOVERY | i 8.197127 201272 354760 0 725448 1448 725448 TF_FASTRECOVERY | TF_CONGRECOVERY | << but I don't see much impact of the old_pipe to cwnd here because the old_pipe > ssthresh o 8.197670 2896 354760 1448 0 0 0 N/A o 8.197720 2896 354760 1448 1448 1448 1448 N/A i 8.218422 2896 354760 0 2896 2896 2896 N/A ... o 27.155659 343176 348968 1448 340280 317112 340280 TF_FASTRECOVERY | TF_CONGRECOVERY | o 27.155667 343176 348968 1448 341728 318560 341728 TF_FASTRECOVERY | TF_CONGRECOVERY | i 27.169991 343176 348968 0 343176 320008 343176 TF_FASTRECOVERY | TF_CONGRECOVERY | o 27.170070 317112 348968 1448 315664 315664 315664 N/A << when old_pipe < ssthresh, cwnd is old_pipe+mss i 27.174263 317112 348968 0 317112 317112 317112 N/A
Raw data files: base_newsackoff iperf3 output base_newsackoff siftr2 log base_newsackoff siftr2 log analysis result
TCP CUBIC congestion window chart: 
test result after patch D49047
- code of pipe calculation after patch D49047
1 int
2 tcp_compute_pipe(struct tcpcb *tp)
3 {
4 int pipe;
5
6 if (tp->t_fb->tfb_compute_pipe != NULL) {
7 pipe = (*tp->t_fb->tfb_compute_pipe)(tp);
8 } else if (V_tcp_do_newsack) {
9 pipe = tp->snd_max - tp->snd_una + << def_pipe with V_tcp_do_newsack == 1
10 tp->sackhint.sack_bytes_rexmit -
11 tp->sackhint.sacked_bytes -
12 tp->sackhint.lost_bytes;
13 } else {
14 pipe = tp->snd_nxt - tp->snd_fack + tp->sackhint.sack_bytes_rexmit;
15 }
16 return (imax(pipe, 0));
17 }
- with V_tcp_do_newsack == 1, the def_pipe value is conservative to let cwnd be less burst
##direction relative_timestamp cwnd ssthresh data_size inflight_bytes def_pipe old_pipe t_flags ... o 0.558271 425749 1073725440 1448 424264 424264 424264 N/A i 0.558275 425749 1073725440 0 425712 425712 425712 N/A o 0.558302 405440 212874 1448 425712 405440 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | << first loss recovery after slow-start i 0.558313 212874 212874 0 425712 405440 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.575032 399648 212874 0 425712 401096 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.575802 344624 212874 0 425712 362000 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.575819 356208 212874 1448 425712 344624 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.575836 356208 212874 1448 425712 346072 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.575841 356208 212874 1448 425712 347520 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.575845 356208 212874 1448 425712 348968 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.577197 356208 212874 0 425712 348968 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.579360 75296 212874 0 425712 211408 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.579407 47784 212874 0 425712 46336 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.579429 178104 212874 1448 425712 8688 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.579470 178104 212874 1448 425712 10136 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.579481 178104 212874 1448 425712 11584 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.579492 178104 212874 1448 425712 13032 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.596516 178104 212874 0 424264 11584 424264 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.596569 587888 212874 1448 424264 402544 424264 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.596641 587888 212874 1448 425712 403992 425712 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.596656 587888 212874 1448 427160 405440 427160 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.596669 587888 212874 1448 428608 406888 428608 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.596704 587888 212874 1448 430056 408336 430056 TF_FASTRECOVERY | TF_CONGRECOVERY | ... o 3.242194 301313 107152 1448 299736 299736 299736 N/A i 3.258996 301313 107152 0 301184 301184 301184 N/A o 3.259050 288152 150592 1448 301184 288152 301184 TF_FASTRECOVERY | TF_CONGRECOVERY | i 3.259128 150592 150592 0 301184 288152 301184 TF_FASTRECOVERY | TF_CONGRECOVERY | i 3.259157 266432 150592 0 301184 273672 301184 TF_FASTRECOVERY | TF_CONGRECOVERY | o 3.259182 276568 150592 1448 301184 272224 301184 TF_FASTRECOVERY | TF_CONGRECOVERY | o 3.259220 276568 150592 1448 301184 273672 301184 TF_FASTRECOVERY | TF_CONGRECOVERY | o 3.259235 276568 150592 1448 301184 275120 301184 TF_FASTRECOVERY | TF_CONGRECOVERY | o 3.259247 276568 150592 1448 301184 276568 301184 TF_FASTRECOVERY | TF_CONGRECOVERY | i 3.260378 276568 150592 0 301184 276568 301184 TF_FASTRECOVERY | TF_CONGRECOVERY | i 3.262178 250504 150592 0 301184 257744 301184 TF_FASTRECOVERY | TF_CONGRECOVERY | i 3.263948 1448 150592 0 301184 99912 301184 TF_FASTRECOVERY | TF_CONGRECOVERY | i 3.280018 57920 150592 0 301184 7240 301184 TF_FASTRECOVERY | TF_CONGRECOVERY | o 3.280058 149144 150592 0 301184 7240 301184 TF_FASTRECOVERY | TF_CONGRECOVERY | i 3.280158 149144 150592 0 299736 5792 299736 TF_FASTRECOVERY | TF_CONGRECOVERY | o 3.281027 2896 150592 1448 0 0 0 N/A o 3.281041 2896 150592 1448 1448 1448 1448 N/A i 3.301704 2896 150592 0 2896 2896 2896 N/A ... i 5.096083 317241 150592 0 317112 315664 317112 N/A i 5.096107 317241 150592 0 317112 314216 317112 N/A o 5.096130 312768 157832 1448 317112 312768 317112 TF_FASTRECOVERY | TF_CONGRECOVERY | i 5.096784 157832 157832 0 317112 312768 317112 TF_FASTRECOVERY | TF_CONGRECOVERY | i 5.097917 221544 157832 0 317112 251952 317112 TF_FASTRECOVERY | TF_CONGRECOVERY | i 5.099790 20272 157832 0 317112 114392 317112 TF_FASTRECOVERY | TF_CONGRECOVERY | i 5.116968 44888 157832 0 317112 1448 317112 TF_FASTRECOVERY | TF_CONGRECOVERY | o 5.117695 2896 157832 1448 0 0 0 N/A o 5.117722 2896 157832 1448 1448 1448 1448 N/A i 5.138282 2896 157832 0 2896 2896 2896 N/A ...
Raw data files: patched_newsackon iperf3 output patched_newsackon siftr2 log patched_newsackon siftr2 log analysis result
TCP CUBIC congestion window chart: 
with V_tcp_do_newsack == 0, the tcp cubic pipe value is the old_pipe from cubic_post_recovery().
old_pipe = pipe = CCV(ccv, snd_max) - ccv->curack;
##direction relative_timestamp cwnd ssthresh data_size inflight_bytes def_pipe old_pipe t_flags ... o 0.473233 393893 1073725440 1448 390960 0 390960 N/A o 0.473241 393893 1073725440 1448 392408 0 392408 N/A i 0.473250 393893 1073725440 0 393856 0 393856 N/A o 0.473259 363448 196946 1448 393856 363448 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | << before the fist loss recovery, def_pipe is 0 o 0.473269 351864 196946 0 393856 363448 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.473354 351864 196946 0 392408 363448 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.473371 360552 196946 1448 392408 354760 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.473387 360552 196946 1448 392408 356208 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.473398 360552 196946 1448 392408 357656 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.473407 360552 196946 1448 392408 359104 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.490885 360552 196946 0 392408 359104 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.490923 359104 196946 0 392408 351864 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.491602 356208 196946 0 392408 344624 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.493306 263536 196946 0 392408 278016 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.494132 63712 196946 0 392408 144800 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.494157 57920 196946 0 392408 5792 392408 TF_FASTRECOVERY | TF_CONGRECOVERY | i 0.494413 57920 196946 0 390960 5792 390960 TF_FASTRECOVERY | TF_CONGRECOVERY | o 0.494646 2896 196946 1448 0 0 0 N/A o 0.494681 2896 196946 1448 1448 1448 1448 N/A i 0.515166 2896 196946 0 2896 2896 2896 N/A ... i 4.253925 309872 165072 0 309872 289600 309872 N/A o 4.253977 254848 154936 1448 309872 254848 309872 TF_FASTRECOVERY | TF_CONGRECOVERY | << with V_tcp_do_newsack == 0, the def_pipe is also conservative i 4.255817 154936 154936 0 309872 254848 309872 TF_FASTRECOVERY | TF_CONGRECOVERY | i 4.256935 13032 154936 0 309872 112944 309872 TF_FASTRECOVERY | TF_CONGRECOVERY | o 4.256978 43440 154936 1448 309872 2896 309872 TF_FASTRECOVERY | TF_CONGRECOVERY | o 4.257023 43440 154936 1448 309872 4344 309872 TF_FASTRECOVERY | TF_CONGRECOVERY | o 4.257038 43440 154936 1448 309872 5792 309872 TF_FASTRECOVERY | TF_CONGRECOVERY | o 4.257048 43440 154936 1448 309872 7240 309872 TF_FASTRECOVERY | TF_CONGRECOVERY | i 4.274566 43440 154936 0 309872 7240 309872 TF_FASTRECOVERY | TF_CONGRECOVERY | o 4.274604 153488 154936 0 309872 5792 309872 TF_FASTRECOVERY | TF_CONGRECOVERY | i 4.277550 153488 154936 0 308424 5792 308424 TF_FASTRECOVERY | TF_CONGRECOVERY | o 4.277807 2896 154936 1448 0 0 0 N/A o 4.277857 2896 154936 1448 1448 1448 1448 N/A ... o 7.950245 340280 207064 1448 438744 340280 438744 TF_FASTRECOVERY | TF_CONGRECOVERY | i 7.969163 207064 207064 0 438744 340280 438744 TF_FASTRECOVERY | TF_CONGRECOVERY | i 7.969772 152040 207064 0 438744 217200 438744 TF_FASTRECOVERY | TF_CONGRECOVERY | i 7.970823 94120 207064 0 438744 104256 438744 TF_FASTRECOVERY | TF_CONGRECOVERY | o 7.970837 102808 207064 1448 438744 1448 438744 TF_FASTRECOVERY | TF_CONGRECOVERY | o 7.970855 102808 207064 1448 438744 2896 438744 TF_FASTRECOVERY | TF_CONGRECOVERY | o 7.970865 102808 207064 1448 438744 4344 438744 TF_FASTRECOVERY | TF_CONGRECOVERY | i 7.991343 102808 207064 0 437296 4344 437296 TF_FASTRECOVERY | TF_CONGRECOVERY | o 7.991729 2896 207064 1448 0 0 0 N/A o 7.991782 2896 207064 1448 1448 1448 1448 N/A i 8.012280 2896 207064 0 2896 2896 2896 N/A ...
Raw data files: patched_newsackoff iperf3 output patched_newsackoff siftr2 log patched_newsackoff siftr2 log analysis result
TCP CUBIC congestion window chart: 