[MME] Crash due to a race condition

A race condition can occur in the following situations.
In conclusion, we can use this situation to determine
whether or not the UE Context has been removed and avoiding a crash.

For example, suppose a UE Context is removed in the followings.

1. Attach Request
2. Authentication-Information-Request
3. Authentication-Information-Answer
4. Authentication Request
5. Authentication Response(MAC Failed)
6. Authentication Reject
7. UEContextReleaseCommand
8. UEContextReleaseComplete

The MME then sends a Purge-UE-request to the HSS and deletes
the UE context as soon as it receives a Purge-UE-Answer.

Suppose an Attach Request is received from the same UE
between Purge-UE-Request/Answer, then the MME and HSS start
the Authentication-Information-Request/Answer process.

This can lead to the following situations.

1. Purge-UE-Request
2. Attach Request
3. Authentication-Information-Request
4. Purge-UE-Answer
5. [UE Context Removed]
6. Authentication-Information-Answer

Since the UE Context has already been deleted
when the Authentication-Information-Answer is received,
it cannot be processed properly.

Therefore, mme_ue_cycle() is used to check
whether the UE Context has been deleted and
decide whether to process or
ignore the Authentication-Information-Answer as shown below.

lib/core/ogs-timer.c

@@ -146,7 +146,6 @@ void ogs_timer_stop_debug(ogs_timer_t *timer, const char *file_line)
     manager = timer->manager;
     ogs_assert(manager);
     timer = ogs_timer_cycle(manager, timer);
-    ogs_assert(timer);
     if (!timer) {
         ogs_fatal("ogs_timer_stop() failed in %s", file_line);
         ogs_assert_if_reached();

lib/diameter/gx/message.c

@@ -1,4 +1,4 @@
-/*
+/* Gx Interface, 3GPP TS 29.212 section 4
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  *
  * This file is part of Open5GS.

lib/diameter/gx/message.h

@@ -1,4 +1,4 @@
-/*
+/* Gx Interface, 3GPP TS 29.212 section 4
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  *
  * This file is part of Open5GS.

lib/diameter/gx/ogs-diameter-gx.h

@@ -1,4 +1,4 @@
-/*
+/* Gx Interface, 3GPP TS 29.212 section 4
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  *
  * This file is part of Open5GS.

lib/diameter/gy/message.c

@@ -1,4 +1,4 @@
-/*
+/* Gy Interface, 3GPP TS 32.299
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  * Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
  *

lib/diameter/gy/message.h

@@ -1,4 +1,4 @@
-/*
+/* Gy Interface, 3GPP TS 32.299
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  * Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
  *

lib/diameter/gy/ogs-diameter-gy.h

@@ -1,4 +1,4 @@
-/*
+/* Gy Interface, 3GPP TS 32.299
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  * Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
  *

src/mme/mme-sm.c

@@ -407,8 +407,53 @@ void mme_state_operational(ogs_fsm_t *s, mme_event_t *e)
         break;
 
     case MME_EVENT_S6A_MESSAGE:
-        mme_ue = e->mme_ue;
-        ogs_assert(mme_ue);
+        /*
+         * A race condition can occur in the following situations.
+         * In conclusion, we can use this situation to determine
+         * whether or not the UE Context has been removed and avoiding a crash.
+         *
+         * For example, suppose a UE Context is removed in the followings.
+         *
+         * 1. Attach Request
+         * 2. Authentication-Information-Request
+         * 3. Authentication-Information-Answer
+         * 4. Authentication Request
+         * 5. Authentication Response(MAC Failed)
+         * 6. Authentication Reject
+         * 7. UEContextReleaseCommand
+         * 8. UEContextReleaseComplete
+         *
+         * The MME then sends a Purge-UE-request to the HSS and deletes
+         * the UE context as soon as it receives a Purge-UE-Answer.
+         *
+         * Suppose an Attach Request is received from the same UE
+         * between Purge-UE-Request/Answer, then the MME and HSS start
+         * the Authentication-Information-Request/Answer process.
+         *
+         * This can lead to the following situations.
+         *
+         * 1. Purge-UE-Request
+         * 2. Attach Request
+         * 3. Authentication-Information-Request
+         * 4. Purge-UE-Answer
+         * 5. [UE Context Removed]
+         * 6. Authentication-Information-Answer
+         *
+         * Since the UE Context has already been deleted
+         * when the Authentication-Information-Answer is received,
+         * it cannot be processed properly.
+         *
+         * Therefore, mme_ue_cycle() is used to check
+         * whether the UE Context has been deleted and
+         * decide whether to process or
+         * ignore the Authentication-Information-Answer as shown below.
+         */
+        mme_ue = mme_ue_cycle(e->mme_ue);
+        if (!mme_ue) {
+            ogs_error("UE(mme-ue) context has already been removed");
+            goto cleanup;
+        }
+
         s6a_message = e->s6a_message;
         ogs_assert(s6a_message);
 
@@ -473,6 +518,8 @@ void mme_state_operational(ogs_fsm_t *s, mme_event_t *e)
             ogs_error("Invalid Type[%d]", s6a_message->cmd_code);
             break;
         }
+
+cleanup:
         ogs_subscription_data_free(&s6a_message->idr_message.subscription_data);
         ogs_subscription_data_free(&s6a_message->ula_message.subscription_data);
         ogs_free(s6a_message);

src/mme/s1ap-build.c

@@ -574,7 +574,28 @@ ogs_pkbuf_t *s1ap_build_initial_context_setup_request(
         emmbuf = NULL;
     }
 
-    ogs_assert(E_RABToBeSetupListCtxtSUReq->list.count);
+    if (!E_RABToBeSetupListCtxtSUReq->list.count) {
+        ogs_error("    IMSI[%s] NAS-EPS Type[%d] "
+                "ENB_UE_S1AP_ID[%d] MME_UE_S1AP_ID[%d]",
+                mme_ue->imsi_bcd, mme_ue->nas_eps.type,
+                enb_ue->enb_ue_s1ap_id, enb_ue->mme_ue_s1ap_id);
+        ogs_list_for_each(&mme_ue->sess_list, sess) {
+            ogs_error("    APN[%s]",
+                    sess->session ? sess->session->name : "Unknown");
+            ogs_list_for_each(&sess->bearer_list, bearer) {
+                if (OGS_FSM_CHECK(&bearer->sm, esm_state_inactive))
+                    ogs_error("    IN-ACTIVE");
+                else if (OGS_FSM_CHECK(&bearer->sm, esm_state_active))
+                    ogs_error("    ACTIVE");
+                else
+                    ogs_error("    OTHER STATE");
+
+                ogs_error("    EBI[%d] QCI[%d] SGW-S1U-TEID[%d]",
+                        bearer->ebi, bearer->qos.index, bearer->sgw_s1u_teid);
+            }
+        }
+        return NULL;
+    }
 
     ie = CALLOC(1, sizeof(S1AP_InitialContextSetupRequestIEs_t));
     ASN_SEQUENCE_ADD(&InitialContextSetupRequest->protocolIEs, ie);

src/pcrf/pcrf-gx-path.c

@@ -1,4 +1,4 @@
-/*
+/* Gx Interface, 3GPP TS 29.212 section 4
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  *
  * This file is part of Open5GS.
@@ -55,8 +55,8 @@ static OGS_POOL(rx_sess_state_pool, struct rx_sess_state);
 static ogs_thread_mutex_t sess_state_mutex;
 
 static struct session_handler *pcrf_gx_reg = NULL;
-static struct disp_hdl *hdl_gx_fb = NULL; 
-static struct disp_hdl *hdl_gx_ccr = NULL; 
+static struct disp_hdl *hdl_gx_fb = NULL;
+static struct disp_hdl *hdl_gx_ccr = NULL;
 
 static void pcrf_gx_raa_cb(void *data, struct msg **msg);
 
@@ -169,7 +169,7 @@ static struct rx_sess_state *find_rx_state(struct sess_state *gx, os0_t sid)
 
     ogs_assert(gx);
     ogs_assert(sid);
-    
+
     ogs_list_for_each(&gx->rx_list, rx_sess_data) {
         if (!strcmp((char *)rx_sess_data->sid, (char *)sid))
             return rx_sess_data;
@@ -199,13 +199,13 @@ static void state_cleanup(struct sess_state *sess_data, os0_t sid, void *opaque)
         ogs_free(sess_data->sid);
 
     remove_rx_state_all(sess_data);
-    
+
     ogs_thread_mutex_lock(&sess_state_mutex);
     ogs_pool_free(&sess_state_pool, sess_data);
     ogs_thread_mutex_unlock(&sess_state_mutex);
 }
 
-static int pcrf_gx_fb_cb(struct msg **msg, struct avp *avp, 
+static int pcrf_gx_fb_cb(struct msg **msg, struct avp *avp,
         struct session *sess, void *opaque, enum disp_action *act)
 {
     /* This CB should never be called */
@@ -214,7 +214,7 @@ static int pcrf_gx_fb_cb(struct msg **msg, struct avp *avp,
     return ENOTSUP;
 }
 
-static int pcrf_gx_ccr_cb( struct msg **msg, struct avp *avp, 
+static int pcrf_gx_ccr_cb( struct msg **msg, struct avp *avp,
         struct session *sess, void *opaque, enum disp_action *act)
 {
     int rv;
@@ -486,7 +486,7 @@ static int pcrf_gx_ccr_cb( struct msg **msg, struct avp *avp,
                 ret = fd_msg_avp_add (avp, MSG_BRW_LAST_CHILD, avpch1);
                 ogs_assert(ret == 0);
             }
-            
+
             if (gx_message.session_data.session.ambr.downlink) {
                 ret = fd_msg_avp_new(
                         ogs_diam_gx_apn_aggregate_max_bitrate_dl, 0, &avpch1);
@@ -779,7 +779,7 @@ int pcrf_gx_send_rar(
                 rx_message->result_code = OGS_DIAM_INVALID_AVP_VALUE;
                 goto out;
             }
-            
+
             for (j = 0; j < gx_message.session_data.num_of_pcc_rule; j++) {
                 if (gx_message.session_data.pcc_rule[j].qos.index ==
                         qos_index) {
@@ -814,7 +814,7 @@ int pcrf_gx_send_rar(
                 ogs_error("CHECK WEBUI : No PCC Rule in DB [QoS Index:%d]",
                         qos_index);
                 ogs_error("Please add PCC Rule using WEBUI");
-                rx_message->result_code = 
+                rx_message->result_code =
                     OGS_DIAM_RX_DIAMETER_REQUESTED_SERVICE_NOT_AUTHORIZED;
                 goto out;
             }
@@ -827,7 +827,7 @@ int pcrf_gx_send_rar(
             }
 
             if (!pcc_rule) {
-                pcc_rule = 
+                pcc_rule =
                     &rx_sess_data->pcc_rule[rx_sess_data->num_of_pcc_rule];
 
                 /* Device PCC Rule Info from DB Profile */
@@ -871,7 +871,7 @@ int pcrf_gx_send_rar(
                     rv = ogs_pcc_rule_install_flow_from_media(
                             pcc_rule, media_component);
                     if (rv != OGS_OK) {
-                        rx_message->result_code = 
+                        rx_message->result_code =
                             OGS_DIAM_RX_DIAMETER_FILTER_RESTRICTIONS;
                         ogs_error("install_flow() failed");
                         goto out;
@@ -995,16 +995,16 @@ int pcrf_gx_send_rar(
 
     ret = clock_gettime(CLOCK_REALTIME, &sess_data->ts);
     ogs_assert(ret == 0);
-    
-    /* Keep a pointer to the session data for debug purpose, 
+
+    /* Keep a pointer to the session data for debug purpose,
      * in real life we would not need it */
     svg = sess_data;
-    
+
     /* Store this value in the session */
     ret = fd_sess_state_store(pcrf_gx_reg, session, &sess_data);
     ogs_assert(ret == 0);
     ogs_assert(sess_data == NULL);
-    
+
     /* Send the request */
     ret = fd_msg_send(&req, pcrf_gx_raa_cb, svg);
     ogs_assert(ret == 0);
@@ -1043,7 +1043,7 @@ static void pcrf_gx_raa_cb(void *data, struct msg **msg)
     unsigned long dur;
     int error = 0;
     int new;
-    
+
     uint32_t result_code;
 
     ogs_debug("[PCRF] Re-Auth-Answer");
@@ -1055,7 +1055,7 @@ static void pcrf_gx_raa_cb(void *data, struct msg **msg)
     ret = fd_msg_sess_get(fd_g_config->cnf_dict, *msg, &session, &new);
     ogs_assert(ret == 0);
     ogs_assert(new == 0);
-    
+
     ret = fd_sess_state_retrieve(pcrf_gx_reg, session, &sess_data);
     ogs_assert(ret == 0);
     ogs_assert(sess_data);
@@ -1116,11 +1116,11 @@ static void pcrf_gx_raa_cb(void *data, struct msg **msg)
 
     /* Free the message */
     ogs_assert(pthread_mutex_lock(&ogs_diam_logger_self()->stats_lock) == 0);
-    dur = ((ts.tv_sec - sess_data->ts.tv_sec) * 1000000) + 
+    dur = ((ts.tv_sec - sess_data->ts.tv_sec) * 1000000) +
         ((ts.tv_nsec - sess_data->ts.tv_nsec) / 1000);
     if (ogs_diam_logger_self()->stats.nb_recv) {
         /* Ponderate in the avg */
-        ogs_diam_logger_self()->stats.avg = (ogs_diam_logger_self()->stats.avg * 
+        ogs_diam_logger_self()->stats.avg = (ogs_diam_logger_self()->stats.avg *
             ogs_diam_logger_self()->stats.nb_recv + dur) /
             (ogs_diam_logger_self()->stats.nb_recv + 1);
         /* Min, max */
@@ -1135,18 +1135,18 @@ static void pcrf_gx_raa_cb(void *data, struct msg **msg)
     }
     if (error)
         ogs_diam_logger_self()->stats.nb_errs++;
-    else 
+    else
         ogs_diam_logger_self()->stats.nb_recv++;
 
     ogs_assert(pthread_mutex_unlock(&ogs_diam_logger_self()->stats_lock) == 0);
-    
+
     /* Display how long it took */
     if (ts.tv_nsec > sess_data->ts.tv_nsec)
-        ogs_trace("in %d.%06ld sec", 
+        ogs_trace("in %d.%06ld sec",
                 (int)(ts.tv_sec - sess_data->ts.tv_sec),
                 (long)(ts.tv_nsec - sess_data->ts.tv_nsec) / 1000);
     else
-        ogs_trace("in %d.%06ld sec", 
+        ogs_trace("in %d.%06ld sec",
                 (int)(ts.tv_sec + 1 - sess_data->ts.tv_sec),
                 (long)(1000000000 + ts.tv_nsec - sess_data->ts.tv_nsec) / 1000);
 
@@ -1157,7 +1157,7 @@ static void pcrf_gx_raa_cb(void *data, struct msg **msg)
     ret = fd_msg_free(*msg);
     ogs_assert(ret == 0);
     *msg = NULL;
-    
+
     return;
 }
 
@@ -1202,7 +1202,7 @@ void pcrf_gx_final(void)
     int ret;
 
     ret = fd_sess_handler_destroy(&pcrf_gx_reg, NULL);
-    ogs_assert(ret == 0); 
+    ogs_assert(ret == 0);
 
     if (hdl_gx_fb)
         (void) fd_disp_unregister(&hdl_gx_fb, NULL);
@@ -1242,7 +1242,7 @@ static int encode_pcc_rule_definition(
             ret = fd_msg_avp_new(ogs_diam_gx_flow_information, 0, &avpch2);
             ogs_assert(ret == 0);
 
-            ret = fd_msg_avp_new(ogs_diam_gx_flow_direction, 0, &avpch3); 
+            ret = fd_msg_avp_new(ogs_diam_gx_flow_direction, 0, &avpch3);
             ogs_assert(ret == 0);
             val.i32 = flow->direction;
             ret = fd_msg_avp_setvalue(avpch3, &val);
@@ -1250,7 +1250,7 @@ static int encode_pcc_rule_definition(
             ret = fd_msg_avp_add(avpch2, MSG_BRW_LAST_CHILD, avpch3);
             ogs_assert(ret == 0);
 
-            ret = fd_msg_avp_new(ogs_diam_gx_flow_description, 0, &avpch3); 
+            ret = fd_msg_avp_new(ogs_diam_gx_flow_description, 0, &avpch3);
             ogs_assert(ret == 0);
             val.os.data = (uint8_t *)flow->description;
             val.os.len = strlen(flow->description);
@@ -1373,6 +1373,6 @@ static int encode_pcc_rule_definition(
 
     ret = fd_msg_avp_add(avp, MSG_BRW_LAST_CHILD, avpch1);
     ogs_assert(ret == 0);
-    
+
     return OGS_OK;
 }

src/smf/gx-handler.c

@@ -1,4 +1,4 @@
-/*
+/* Gx Interface, 3GPP TS 29.212 section 4
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  * Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
  *

src/smf/gx-handler.h

@@ -1,4 +1,4 @@
-/*
+/* Gx Interface, 3GPP TS 29.212 section 4
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  *
  * This file is part of Open5GS.

src/smf/gx-path.c

@@ -1,4 +1,4 @@
-/*
+/* Gx Interface, 3GPP TS 29.212 section 4
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  *
  * This file is part of Open5GS.
@@ -87,6 +87,7 @@ static void state_cleanup(struct sess_state *sess_data, os0_t sid, void *opaque)
     ogs_thread_mutex_unlock(&sess_state_mutex);
 }
 
+/* 3GPP TS 29.212 5.6.2 Credit-Control-Request */
 void smf_gx_send_ccr(smf_sess_t *sess, ogs_gtp_xact_t *xact,
         uint32_t cc_request_type)
 {
@@ -701,6 +702,7 @@ void smf_gx_send_ccr(smf_sess_t *sess, ogs_gtp_xact_t *xact,
     ogs_assert(pthread_mutex_unlock(&ogs_diam_logger_self()->stats_lock) == 0);
 }
 
+/* 3GPP TS 29.212 5b.6.5 Credit-Control-Answer */
 static void smf_gx_cca_cb(void *data, struct msg **msg)
 {
     int rv;

src/smf/gy-handler.c

@@ -1,4 +1,4 @@
-/*
+/* Gy Interface, 3GPP TS 32.299
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  * Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
  *

src/smf/gy-handler.h

@@ -1,4 +1,4 @@
-/*
+/* Gy Interface, 3GPP TS 32.299
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  * Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
  *

src/smf/gy-path.c

@@ -1,4 +1,4 @@
-/*
+/* Gy Interface, 3GPP TS 32.299
  * Copyright (C) 2019 by Sukchan Lee <acetcom@gmail.com>
  * Copyright (C) 2022 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
  *