TCP Connection Analysis Why the Socket Remains in the FIN_WAIT_1 State Post Killing the Process

Symptoms

Analysis

Root Cause

# 记录下new state的报文的日志
iptables -A INPUT -p tcp -m state --state NEW -j LOG --log-prefix "[iptables] INPUT NEW: "
# iptables -A OUTPUT -m state --state INVALID -j DROP
  • Does the problem always occur? What are the triggering conditions?
  • Why is the FIN packet considered INVALID?

When Does the Problem Trigger

Why FIN Packet is Considered INVALID

Test: Iptables Rule Setting

#!/bin/sh
iptables -P INPUT ACCEPT
iptables -F
iptables -X
iptables -Z
# 在日志里记录INPUT chain里过来的每个报文的状态
iptables -A INPUT -p tcp -m state --state NEW -j LOG --log-prefix "[iptables] INPUT NEW: "
iptables -A INPUT -p TCP -m state --state ESTABLISHED -j LOG --log-prefix "[iptables] INPUT ESTABLISHED: "
iptables -A INPUT -p TCP -m state --state RELATED -j LOG --log-prefix "[iptables] INPUT RELATED: "
iptables -A INPUT -p TCP -m state --state INVALID -j LOG --log-prefix "[iptables] INPUT INVALID: "
iptables -A INPUT -i lo -j ACCEPT
iptables -A INPUT -p tcp --dport 22 -j ACCEPT
iptables -A INPUT -p tcp --dport 21 -j ACCEPT
iptables -A INPUT -p tcp --dport 80 -j ACCEPT
iptables -A INPUT -p tcp --dport 443 -j ACCEPT
iptables -A INPUT -p tcp --dport 8088 -m state --state NEW -j ACCEPT
iptables -A INPUT -p icmp --icmp-type 8 -j ACCEPT
iptables -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
# 在日志里记录OUTPUT chain里过来的每个报文的状态
iptables -A OUTPUT -p tcp -m state --state NEW -j LOG --log-prefix "[iptables] OUTPUT NEW: "
iptables -A OUTPUT -p TCP -m state --state ESTABLISHED -j LOG --log-prefix "[iptables] OUTPUT ESTABLISHED: "
iptables -A OUTPUT -p TCP -m state --state RELATED -j LOG --log-prefix "[iptables] OUTPUT RELATED: "
iptables -A OUTPUT -p TCP -m state --state INVALID -j LOG --log-prefix "[iptables] OUTPUT INVALID: "
# iptables -A OUTPUT -m state --state INVALID -j DROP
iptables -P INPUT DROP
iptables -P OUTPUT ACCEPT
iptables -P FORWARD DROP
service iptables save
systemctl restart iptables.service
sysctl-w net. netfilter. nf_conntrack_tcp_timeout_established = 20

Code Logic

nf_conntrack_in @net/netfilter/nf_conntrack_core.c
|--> resolve_normal_ct @net/netfilter/nf_conntrack_core.c // 利用__nf_conntrack_find_get查找对应的连接跟踪表项,没找到则init新的conntrack表项
|--> init_conntrack @net/netfilter/nf_conntrack_core.c // 初始化conntrack表项
|--> tcp_new @net/netfilter/nf_conntrack_proto_tcp.c // 到TCP协议的处理逻辑,called when a new connection for this protocol found。在这里根据tcp_conntracks数组决定状态。

reslove_normal_ct

/* look for tuple match */
hash = hash_conntrack_raw(&tuple, zone);
h = __nf_conntrack_find_get(net, zone, &tuple, hash);
if (!h) {
h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
skb, dataoff, hash);
if (!h)
return NULL;
if (IS_ERR(h))
return (void *)h;
}

init_conntrack

if (!l4proto->new(ct, skb, dataoff, timeouts)) {
nf_conntrack_free(ct);
pr_debug("init conntrack: can't track with proto module\n");
return NULL;
}

tcp_new

/* Called when a new connection for this protocol found. */
static bool tcp_new(struct nf_conn *ct, const struct sk_buff *skb,
unsigned int dataoff, unsigned int *timeouts)
{
enum tcp_conntrack new_state;
const struct tcphdr *th;
struct tcphdr _tcph;
struct net *net = nf_ct_net(ct);
struct nf_tcp_net *tn = tcp_pernet(net);
const struct ip_ct_tcp_state *sender = &ct->proto.tcp.seen[0];
const struct ip_ct_tcp_state *receiver = &ct->proto.tcp.seen[1];
th = skb_header_pointer(skb, dataoff, sizeof(_tcph), &_tcph);
BUG_ON(th == NULL);
/* Don't need lock here: this conntrack not in circulation yet */
// 这里get_conntrack_index拿到的是TCP_FIN_SET,是枚举类型tcp_bit_set的值
new_state = tcp_conntracks[0][get_conntrack_index(th)][TCP_CONNTRACK_NONE];
/* Invalid: delete conntrack */
if (new_state >= TCP_CONNTRACK_MAX) {
pr_debug("nf_ct_tcp: invalid new deleting.\n");
return false;
}
......
}
  • The subscript of the outermost layer of the tcp_conntrack array is 0, indicating ORIGINAL, or the packet sending end.
  • In the scenario described in this article, the outer mark of the middle layer is determined by get_conntrack_index. Get_conntrack_index(th) obtains the value TCP_FIN_SET of enum tcp_bit_set (defined as follows) based on the FIN flag in the packet. tcp_bit_set is in one-to-one correspondence with the middle layer subscript of the tcp_conntracks array to be introduced below.
/* What TCP flags are set from RST/SYN/FIN/ACK. */
enum tcp_bit_set {
TCP_SYN_SET,
TCP_SYNACK_SET,
TCP_FIN_SET,
TCP_ACK_SET,
TCP_RST_SET,
TCP_NON
  • The subscript of the inner layer is TCP_CONNTRACK_NONE, which corresponds to 0 in enum tcp_conntrack.

tcp_conntracks Array

static const u8 tcp_conntracks[2][6][TCP_CONNTRACK_MAX] = {
{
/* ORIGINAL */
/*syn*/ { sSS, sSS, sIG, sIG, sIG, sIG, sIG, sSS, sSS, sS2 },
/*synack*/ { sIV, sIV, sSR, sIV, sIV, sIV, sIV, sIV, sIV, sSR },
/*fin*/ { sIV, sIV, sFW, sFW, sLA, sLA, sLA, sTW, sCL, sIV },
/*ack*/ { sES, sIV, sES, sES, sCW, sCW, sTW, sTW, sCL, sIV },
/*rst*/ { sIV, sCL, sCL, sCL, sCL, sCL, sCL, sCL, sCL, sCL },
/*none*/ { sIV, sIV, sIV, sIV, sIV, sIV, sIV, sIV, sIV, sIV }
},
{
/* REPLY */
/*syn*/ { sIV, sS2, sIV, sIV, sIV, sIV, sIV, sIV, sIV, sS2 },
/*synack*/ { sIV, sSR, sIG, sIG, sIG, sIG, sIG, sIG, sIG, sSR },
/*fin*/ { sIV, sIV, sFW, sFW, sLA, sLA, sLA, sTW, sCL, sIV },
/*ack*/ { sIV, sIG, sSR, sES, sCW, sCW, sTW, sTW, sCL, sIG },
/*rst*/ { sIV, sCL, sCL, sCL, sCL, sCL, sCL, sCL, sCL, sCL },
/*none*/ { sIV, sIV, sIV, sIV, sIV, sIV, sIV, sIV, sIV, sIV }
}
};
tcp_conntracks[0][get_conntrack_index(th)][TCP_CONNTRACK_NONE] =>tcp_conntracks[0][get_conntrack_index(th)][0]
  • When the packet carries FIN: tcp_conntracks0[0] = tcp_conntracks0[0] => INVALID state // case described in this article
  • When the packet carries RESET: tcp_conntracks0[0] = tcp_conntracks0[0] => INVALID state
  • When the packet carries SYNACK: tcp_conntracks0[0] = tcp_conntracks0[0] => INVALID state
  • When the packet carries both SYN and ACK, the packet is considered NEW by the conntrack module.

Conclusion

Original Source:

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Alibaba Cloud

Alibaba Cloud

Follow me to keep abreast with the latest technology news, industry insights, and developer trends. Alibaba Cloud website:https://www.alibabacloud.com