MME: select SGW by RR,TAC,ENB_ID || SMF: select PFCP associated UPF by RR,TAC,APN,ENB_ID (re-worked) (#470)

* MME: select SGW by RR,TAC,ENB_ID
and enable round robin inside each option

* SMF: select PFCP associated UPF by RR,TAC,APN,ENB_ID
and enable round robin inside each option

configs/open5gs/mme.yaml.in

@@ -263,7 +263,7 @@ mme:
 #     addr: 127.0.2.2
 #     addr: 127.0.4.2
 #
-# o SGW selection by eNodeB TAC
+# o SGW selection by eNodeB TAC (either single TAC or multiple TACs)
 #
 #   selection_mode: tac
 #   gtpc:
@@ -272,6 +272,14 @@ mme:
 #     - addr: 127.0.2.2
 #       tac: [25000, 27000, 28000]
 #
+# o SGW selection by eNodeB ID (either single enb_id or multiple enb_ids, decimal or hex representation)
+#
+#   selection_mode: enb_id
+#   gtpc:
+#     - addr: 127.0.0.2
+#       enb_id: 463
+#     - addr: 127.0.2.2
+#       enb_id: [0x12345, 9413, 0x98765]
 #
 sgw:
     gtpc:

configs/open5gs/smf.yaml.in

@@ -69,6 +69,23 @@ logger:
 #      - addr: 127.0.0.3
 #      - addr: ::1
 #
+#  o UPF (PFCP) selection can be based on three options. Round Robin is default.
+#      Round Robin (rr)
+#        OR
+#      UE's serving eNB/gNB TAC (tac)
+#        OR
+#      UE's APN (apn)
+#        OR
+#      UE's serving eNB/gNB_id (enb_id)
+#
+#    smf:
+#      pfcp:
+#        - addr: 127.0.0.3
+#        - upf_selection_mode: rr   < one of these
+#        - upf_selection_mode: tac
+#        - upf_selection_mode: apn
+#        - upf_selection_mode: enb_id
+#
 #  <GTP-C Server>
 #
 #  o GTP-C Server(127.0.0.3:2123, [fe80::3%@loopback_devname@]:2123)
@@ -243,6 +260,43 @@ nrf:
 #    pfcp:
 #      addr: 127.0.0.4
 #
+#  <UPF_SELECTION_MODE>
+#
+# o Round-Robin
+#   (If `upf_selection_mode` is omitted in smf/pfcp, the default mode is Round-Robin)
+#
+#  upf:
+#    pfcp:
+#      - addr: 127.0.0.4
+#      - addr: 127.0.0.12
+#
+# o UPF selection by eNodeB TAC (either single TAC or multiple TACs)
+#
+#  upf:
+#    pfcp:
+#      - addr: 127.0.0.4
+#        tac: 1                   < either single TAC or multiple TACs
+#      - addr: 127.0.0.12
+#        tac: [3,5,8]
+#
+# o SGW selection by UE's APN (either single APN or multiple APNs)
+#
+#  upf:
+#    pfcp:
+#      - addr: 127.0.0.4
+#        apn: ims                 < either single APN or multiple APNs
+#      - addr: 127.0.0.12
+#        apn: [internet, web]
+#
+# o SGW selection by eNodeB ID (either single enb_id or multiple enb_ids, decimal or hex representation)
+#
+#  upf:
+#    pfcp:
+#      - addr: 127.0.0.4
+#        enb_id: 463              < either single enb_id or multiple enb_id,
+#      - addr: 127.0.0.12           can be entered in decimal or hex format
+#        enb_id: [0x12345, 9413]
+#
 upf:
     pfcp:
       - addr: 127.0.0.4

lib/core/ogs-3gpp-types.h

@@ -47,6 +47,8 @@ extern "C" {
 #define OGS_MAX_IMEISV_LEN              \
     OGS_BCD_TO_BUFFER_LEN(OGS_MAX_IMEISV_BCD_LEN)
 
+#define OGS_MAX_NUM_OF_ENB_ID           16
+#define OGS_MAX_NUM_OF_APN              16
 #define OGS_MAX_NUM_OF_HOSTNAME         16
 #define OGS_MAX_DNN_LEN                 100
 #define OGS_MAX_APN_LEN                 OGS_MAX_DNN_LEN

lib/pfcp/context.c

@@ -242,6 +242,22 @@ int ogs_pfcp_context_parse_config(const char *local, const char *remote)
                                 dev = ogs_yaml_iter_value(&pfcp_iter);
                             } else if (!strcmp(pfcp_key, "apn")) {
                                 /* Skip */
+                            } else if (!strcmp(pfcp_key, "upf_selection_mode")) {
+                                ogs_assert(ogs_yaml_iter_type(&pfcp_iter) !=
+                                        YAML_SCALAR_NODE);
+                                const char *upf_selection_mode = ogs_yaml_iter_value(&pfcp_iter);
+
+                                if (!strcmp(upf_selection_mode, "rr"))
+                                    self.upf_selection_mode = UPF_SELECT_RR;
+                                else if (!strcmp(upf_selection_mode, "tac"))
+                                    self.upf_selection_mode = UPF_SELECT_TAC;
+                                else if (!strcmp(upf_selection_mode, "apn"))
+                                    self.upf_selection_mode = UPF_SELECT_APN;
+                                else if (!strcmp(upf_selection_mode, "enb_id"))
+                                    self.upf_selection_mode = UPF_SELECT_ENB_ID;
+                                else
+                                    ogs_warn("unknown upf_selection_mode `%s`",
+                                            upf_selection_mode);
                             } else
                                 ogs_warn("unknown key `%s`", pfcp_key);
                         }
@@ -437,6 +453,10 @@ int ogs_pfcp_context_parse_config(const char *local, const char *remote)
                         uint16_t port = self.pfcp_port;
                         uint16_t tac[OGS_MAX_NUM_OF_TAI] = {0,};
                         uint8_t num_of_tac = 0;
+                        const char * apn[OGS_MAX_NUM_OF_APN];
+                        uint8_t num_of_apn = 0;
+                        uint32_t enb_id[OGS_MAX_NUM_OF_ENB_ID] = {0,};
+                        uint8_t num_of_enb_id = 0;
 
                         if (ogs_yaml_iter_type(&pfcp_array) ==
                                 YAML_MAPPING_NODE) {
@@ -517,6 +537,63 @@ int ogs_pfcp_context_parse_config(const char *local, const char *remote)
                                 } while (
                                     ogs_yaml_iter_type(&tac_iter) ==
                                         YAML_SEQUENCE_NODE);
+                            } else if (!strcmp(pfcp_key, "apn")) {
+                                ogs_yaml_iter_t apn_iter;
+                                ogs_yaml_iter_recurse(&pfcp_iter, &apn_iter);
+                                ogs_assert(ogs_yaml_iter_type(&apn_iter) !=
+                                    YAML_MAPPING_NODE);
+
+                                do {
+                                    const char *v = NULL;
+
+                                    ogs_assert(num_of_apn <=
+                                            OGS_MAX_NUM_OF_APN);
+                                    if (ogs_yaml_iter_type(&apn_iter) ==
+                                            YAML_SEQUENCE_NODE) {
+                                        if (!ogs_yaml_iter_next(&apn_iter))
+                                            break;
+                                    }
+
+                                    v = ogs_yaml_iter_value(&apn_iter);
+                                    if (v) {
+                                        apn[num_of_apn] = v;
+                                        num_of_apn++;
+                                    }
+                                } while (
+                                    ogs_yaml_iter_type(&apn_iter) ==
+                                        YAML_SEQUENCE_NODE);
+                            } else if (!strcmp(pfcp_key, "enb_id")) {
+                                ogs_yaml_iter_t enb_id_iter;
+                                ogs_yaml_iter_recurse(&pfcp_iter, &enb_id_iter);
+                                ogs_assert(ogs_yaml_iter_type(&enb_id_iter) !=
+                                    YAML_MAPPING_NODE);
+
+                                do {
+                                    const char *v = NULL;
+
+                                    ogs_assert(num_of_enb_id <= 
+                                            OGS_MAX_NUM_OF_ENB_ID);
+                                    if (ogs_yaml_iter_type(&enb_id_iter) ==
+                                            YAML_SEQUENCE_NODE) {
+                                        if (!ogs_yaml_iter_next(&enb_id_iter))
+                                            break;
+                                    }
+
+                                    v = ogs_yaml_iter_value(&enb_id_iter);
+                                    if (v) {
+                                        if (strncmp(v,"0x",2)) {
+                                            // integer enb_id
+                                            enb_id[num_of_enb_id] = atoi(v);
+                                        } else {
+                                            // hex enb_id
+                                            enb_id[num_of_enb_id] = strtol(v, NULL, 16);
+                                        }                                        
+                                        num_of_enb_id++;
+                                    }
+                                } while (
+                                    ogs_yaml_iter_type(&enb_id_iter) ==
+                                        YAML_SEQUENCE_NODE);
+                            
                             } else
                                 ogs_warn("unknown key `%s`", pfcp_key);
                         }
@@ -541,6 +618,14 @@ int ogs_pfcp_context_parse_config(const char *local, const char *remote)
                         if (num_of_tac != 0)
                             memcpy(node->tac, tac, sizeof(node->tac));
 
+                        node->num_of_apn = num_of_apn;
+                        if (num_of_apn != 0)
+                            memcpy(node->apn, apn, sizeof(node->apn));
+                        
+                        node->num_of_enb_id = num_of_enb_id;
+                        if (num_of_enb_id != 0)
+                            memcpy(node->enb_id, enb_id, sizeof(node->enb_id));
+
                     } while (ogs_yaml_iter_type(&pfcp_array) ==
                             YAML_SEQUENCE_NODE);
                 }

lib/pfcp/context.h

@@ -33,6 +33,13 @@ extern "C" {
 
 typedef struct ogs_pfcp_node_s ogs_pfcp_node_t;
 
+typedef enum {
+    UPF_SELECT_RR = 0,  /* Default UPF Selection Method */
+    UPF_SELECT_TAC,     /* Select UPF by enb_tac */
+    UPF_SELECT_APN,     /* Select UPF by UE's APN */
+    UPF_SELECT_ENB_ID,  /* Select UPF by enb_id */
+} upf_select_e;
+
 typedef struct ogs_pfcp_context_s {
     uint32_t        pfcp_port;      /* PFCP local port */
     const char      *tun_ifname;    /* PFCP TUN Interface Name */
@@ -47,6 +54,7 @@ typedef struct ogs_pfcp_context_s {
     uint32_t        pfcp_started;   /* UTC time when the PFCP entity started */
 
     ogs_list_t      n4_list;        /* PFCP Node List */
+    upf_select_e    upf_selection_mode;  /* Selection algorithm for selecting UPF */
     ogs_pfcp_node_t *node;    /* Iterator for Peer round-robin */
 
     ogs_list_t      dev_list;       /* Tun Device List */
@@ -79,6 +87,10 @@ typedef struct ogs_pfcp_node_s {
 
     uint16_t        tac[OGS_MAX_NUM_OF_TAI];
     uint8_t         num_of_tac;
+    const char*     apn[OGS_MAX_APN_LEN];
+    uint8_t         num_of_apn;
+    uint32_t        enb_id[OGS_MAX_NUM_OF_ENB_ID];
+    uint8_t         num_of_enb_id;
 
     ogs_list_t      gtpu_resource_list; /* User Plane IP Resource Information */
 } ogs_pfcp_node_t;

src/mme/mme-context.c

@@ -1408,6 +1408,8 @@ int mme_context_parse_config()
                         uint16_t port = self.gtpc_port;
                         uint16_t tac[OGS_MAX_NUM_OF_TAI] = {0,};
                         uint8_t num_of_tac = 0;
+                        uint32_t enb_id[OGS_MAX_NUM_OF_ENB_ID] = {0,};
+                        uint8_t num_of_enb_id = 0;                        
 
                         if (ogs_yaml_iter_type(&gtpc_array) ==
                                 YAML_MAPPING_NODE) {
@@ -1488,6 +1490,36 @@ int mme_context_parse_config()
                                 } while (
                                     ogs_yaml_iter_type(&tac_iter) ==
                                         YAML_SEQUENCE_NODE);
+                            } else if (!strcmp(gtpc_key, "enb_id")) {
+                                ogs_yaml_iter_t enb_id_iter;
+                                ogs_yaml_iter_recurse(&gtpc_iter, &enb_id_iter);
+                                ogs_assert(ogs_yaml_iter_type(&enb_id_iter) !=
+                                    YAML_MAPPING_NODE);
+
+                                do {
+                                    const char *v = NULL;
+
+                                    ogs_assert(num_of_enb_id <=
+                                            OGS_MAX_NUM_OF_ENB_ID);
+                                    if (ogs_yaml_iter_type(&enb_id_iter) ==
+                                            YAML_SEQUENCE_NODE) {
+                                        if (!ogs_yaml_iter_next(&enb_id_iter))
+                                            break;
+                                    }
+                                    v = ogs_yaml_iter_value(&enb_id_iter);
+                                    if (v) {
+                                        if (strncmp(v,"0x",2)) {
+                                            // integer enb_id
+                                            enb_id[num_of_enb_id] = atoi(v);
+                                        } else {
+                                            // hex enb_id
+                                            enb_id[num_of_enb_id] = strtol(v, NULL, 16);
+                                        }                                        
+                                        num_of_enb_id++;
+                                    }
+                                } while (
+                                    ogs_yaml_iter_type(&enb_id_iter) ==
+                                        YAML_SEQUENCE_NODE);
                             } else
                                 ogs_warn("unknown key `%s`", gtpc_key);
                         }
@@ -1511,6 +1543,10 @@ int mme_context_parse_config()
                         if (num_of_tac != 0)
                             memcpy(sgw->tac, tac, sizeof(sgw->tac));
 
+                        sgw->num_of_enb_id = num_of_enb_id;
+                        if (num_of_enb_id != 0)
+                            memcpy(sgw->enb_id, enb_id, sizeof(sgw->enb_id));
+
                     } while (ogs_yaml_iter_type(&gtpc_array) ==
                             YAML_SEQUENCE_NODE);
                 } else if (!strcmp(mme_key, "selection_mode")) {
@@ -1521,6 +1557,8 @@ int mme_context_parse_config()
                         self.sgw_selection = SGW_SELECT_RR;
                     else if (!strcmp(selection_mode, "tac"))
                         self.sgw_selection = SGW_SELECT_TAC;
+                    else if (!strcmp(selection_mode, "enb_id"))
+                        self.sgw_selection = SGW_SELECT_ENB_ID;
                     else
                         ogs_warn("unknown sgw_selection mode `%s`",
                                 selection_mode);
@@ -2162,6 +2200,8 @@ mme_ue_t *mme_ue_add(enb_ue_t *enb_ue)
     mme_ue_t *mme_ue = NULL;
     mme_event_t e;
 
+    char buf[OGS_ADDRSTRLEN];
+
     ogs_assert(enb_ue);
     enb = enb_ue->enb;
     ogs_assert(enb);
@@ -2179,37 +2219,157 @@ mme_ue_t *mme_ue_add(enb_ue_t *enb_ue)
     /* Create New GUTI */
     mme_ue_new_guti(mme_ue);
 
-    if (mme_self()->sgw_selection == SGW_SELECT_RR) {
-        /* Setup SGW with round-robin manner */
-        if (mme_self()->sgw == NULL)
-            mme_self()->sgw = ogs_list_first(&mme_self()->sgw_list);
+    // on first UE connection, initialise at top of SGW list
+    // for subsequent UE connections, begin from current position 
+    if (mme_self()->sgw == NULL)
+        mme_self()->sgw = ogs_list_first(&mme_self()->sgw_list);
 
+    if (mme_self()->sgw_selection == SGW_SELECT_RR) {
+        /* Select SGW with round-robin manner */
+        ogs_debug("Select SGW by RR");
+
+        // list SGW used
+        OGS_ADDR(&mme_self()->sgw->gnode->addr, buf);
+        ogs_debug("UE using SGW on IP[%s]", 
+            buf);
+        // setup GTP path with selected SGW
         ogs_assert(mme_self()->sgw);
         OGS_SETUP_GTP_NODE(mme_ue, mme_self()->sgw->gnode);
-
+        // iterate to next SGW in list for next UE attach (RR)
         mme_self()->sgw = ogs_list_next(mme_self()->sgw);
+
     } else if (mme_self()->sgw_selection == SGW_SELECT_TAC) {
         /* Select SGW by eNB TAC */
-        int i, found = 0;
+        ogs_debug("Select SGW by enb_tac,(RR)");
+        /* 
+        - starting from list position of last used SGW, search down list for a SGW that serves the enb_tac
+            - if suitable SGW found
+                - use it
+            - if no suitable SGW found, keep searching
+            - if bottom of list reached, reset search to top of list
+            - if completed full cyclic search of list and still not found a suitable SGW, use default (first)
+        */
 
-        mme_self()->sgw = ogs_list_first(&mme_self()->sgw_list);
-        while (mme_self()->sgw && !found) {
-            for (i = 0; i < mme_self()->sgw->num_of_tac && !found; i++)
-                found = mme_self()->sgw->tac[i] == enb_ue->saved.tai.tac ? 1: 0;
+        int i, found=0, numreset=0;;
+        char startSGWIP[OGS_ADDRSTRLEN];
+        OGS_ADDR(&mme_self()->sgw->gnode->addr, startSGWIP);
 
-            if (!found)
-                mme_self()->sgw = ogs_list_next(mme_self()->sgw);
-        }
-
-        if (!found) {
-            ogs_warn("No corresponding SGW found for eNB TAC[%d]",
+        // search SGW list, find next SGW that serves the TAC || use default (first) if not found
+        while (!found) {
+            // (when end of SGW list reached, reset search to top of SGW list)
+            if(mme_self()->sgw == NULL){
+                mme_self()->sgw = ogs_list_first(&mme_self()->sgw_list);
+                numreset = 1;
+            }
+            // search SGW list, find next SGW that serves the TAC
+            while(mme_self()->sgw && !found) {
+                // if cyclic list check complete and still not found a SGW, break
+                OGS_ADDR(&mme_self()->sgw->gnode->addr, buf);
+                if (numreset == 1 && !strcmp(buf,startSGWIP)) {
+                    break;
+                }
+                // search from current list position
+                for (i = 0; i < mme_self()->sgw->num_of_tac && !found; i++){
+                    found = mme_self()->sgw->tac[i] == enb_ue->saved.tai.tac ? 1: 0;
+                }
+                if(found){
+                    // found a SGW that serves the TAC                   
+                    OGS_ADDR(&mme_self()->sgw->gnode->addr, buf);
+                    ogs_debug("SGW on IP[%s] serves enb_tac[%d] - use it", 
+                            buf, enb_ue->saved.tai.tac);
+                } else {
+                    // not found, check next SGW in list
+                    mme_self()->sgw = ogs_list_next(mme_self()->sgw);
+                }
+            }
+            // after checking from the top of the list and not finding a suitable SGW
+            if (!found && numreset == 1 ){
+                // default to first in list
+                ogs_warn("No SGW found that serves enb_tac[%d], defaulting to first SGW in mme.yaml list",
                     enb_ue->saved.tai.tac);
-            ogs_warn("Defaulting to first SGW in mme.yaml list");
-            mme_self()->sgw = ogs_list_first(&mme_self()->sgw_list);
-        }
+                mme_self()->sgw = ogs_list_first(&mme_self()->sgw_list);
+                break;
+            }
+        } 
 
+        // list SGW used
+        OGS_ADDR(&mme_self()->sgw->gnode->addr, buf);
+        ogs_debug("UE using SGW on IP[%s]", 
+            buf);
+
+        // setup GTP path with selected SGW
         ogs_assert(mme_self()->sgw);
         OGS_SETUP_GTP_NODE(mme_ue, mme_self()->sgw->gnode);
+
+        // iterate to next SGW in list for next UE attach (RR)
+        mme_self()->sgw = ogs_list_next(mme_self()->sgw);
+
+    } else if (mme_self()->sgw_selection == SGW_SELECT_ENB_ID) {
+        /* Select SGW by eNB_ID */
+        ogs_debug("Select SGW by enb_id,(RR)");
+        /* 
+        - starting from list position of last used SGW, search down list for a SGW that serves the enb_id
+            - if suitable SGW found
+                - use it
+            - if no suitable SGW found, keep searching
+            - if bottom of list reached, reset search to top of list
+            - if completed full cyclic search of list and still not found a suitable SGW, use default (first)
+        */
+
+        int i, found=0, numreset=0;
+        char startSGWIP[OGS_ADDRSTRLEN];
+        OGS_ADDR(&mme_self()->sgw->gnode->addr, startSGWIP);
+
+        // search SGW list, find next SGW that serves the enb_id || use default (first) if not found
+        while (!found) {
+            // (when end of SGW list reached, reset search to top of SGW list)
+            if(mme_self()->sgw == NULL){
+                mme_self()->sgw = ogs_list_first(&mme_self()->sgw_list);
+                numreset = 1;
+            }
+            // search SGW list, find next SGW that serves the enb_id
+            while(mme_self()->sgw && !found) {
+                // if cyclic list check complete and still not found a SGW, break
+                OGS_ADDR(&mme_self()->sgw->gnode->addr, buf);
+                if (numreset == 1 && !strcmp(buf,startSGWIP)) {
+                    break;
+                }
+                // search from current list position
+                for (i = 0; i < mme_self()->sgw->num_of_enb_id && !found; i++){
+                    found = mme_self()->sgw->enb_id[i] == enb->enb_id ? 1: 0;
+                }
+                if(found){
+                    // found a SGW that serves the enb_id                   
+                    OGS_ADDR(&mme_self()->sgw->gnode->addr, buf);
+                    ogs_debug("SGW on IP[%s] serves enb_id[%d]/[0x%02x] - use it", 
+                        buf, enb->enb_id, enb->enb_id);
+                } else {
+                    // not found, check next SGW in list
+                    mme_self()->sgw = ogs_list_next(mme_self()->sgw);
+                }
+            }
+            // after checking from the top of the list and not finding a suitable SGW
+            if (!found && numreset == 1 ){
+                // default to first in list
+                ogs_warn("No SGW found that serves enb_id[%d]/[0x%02x], defaulting to first SGW in mme.yaml list",
+                    enb->enb_id, enb->enb_id);
+                mme_self()->sgw = ogs_list_first(&mme_self()->sgw_list);
+                break;
+            }
+        } 
+
+        // list SGW used
+        OGS_ADDR(&mme_self()->sgw->gnode->addr, buf);
+        ogs_debug("UE using SGW on IP[%s]", 
+            buf);
+
+        // setup GTP path with selected SGW
+        ogs_assert(mme_self()->sgw);
+        OGS_SETUP_GTP_NODE(mme_ue, mme_self()->sgw->gnode);
+
+        // iterate to next SGW in list for next UE attach (RR)
+        mme_self()->sgw = ogs_list_next(mme_self()->sgw);
+
     } else
         ogs_assert_if_reached();
         

src/mme/mme-context.h

@@ -63,6 +63,7 @@ typedef uint32_t mme_p_tmsi_t;
 typedef enum {
     SGW_SELECT_RR = 0,  /* Default SGW Selection Method */
     SGW_SELECT_TAC,
+    SGW_SELECT_ENB_ID,
 } sgw_select_e;
 
 typedef struct served_gummei_s {
@@ -170,6 +171,8 @@ typedef struct mme_sgw_s {
 
     uint16_t        tac[OGS_MAX_NUM_OF_TAI];
     uint8_t         num_of_tac;
+    uint32_t        enb_id[OGS_MAX_NUM_OF_ENB_ID];
+    uint8_t         num_of_enb_id;
 
     ogs_gtp_node_t  *gnode;
 } mme_sgw_t;

src/smf/context.c

@@ -599,10 +599,12 @@ smf_ue_t *smf_ue_find_by_imsi(uint8_t *imsi, int imsi_len)
 }
 
 smf_sess_t *smf_sess_add_by_apn(smf_ue_t *smf_ue, char *apn,
-        uint8_t pdn_type, uint8_t ebi, ogs_paa_t *paa)
+        uint8_t pdn_type, uint8_t ebi, ogs_paa_t *paa, ogs_gtp_create_session_request_t *req)
 {
+    ogs_debug("smf_sess_add_by_apn");
     smf_event_t e;
 
+    char buf[OGS_ADDRSTRLEN];
     char buf1[OGS_ADDRSTRLEN];
     char buf2[OGS_ADDRSTRLEN];
     smf_sess_t *sess = NULL;
@@ -677,18 +679,337 @@ smf_sess_t *smf_sess_add_by_apn(smf_ue_t *smf_ue, char *apn,
         sess->ipv4 ? OGS_INET_NTOP(&sess->ipv4->addr, buf1) : "",
         sess->ipv6 ? OGS_INET6_NTOP(&sess->ipv6->addr, buf2) : "");
 
-    /* Select UPF with round-robin manner */
+    // on first UE connection, initialise at top of UPF (PFCP) list
+    // for subsequent UE connections, begin from current position 
     if (ogs_pfcp_self()->node == NULL)
         ogs_pfcp_self()->node = ogs_list_first(&ogs_pfcp_self()->n4_list);
 
-    for (; ogs_pfcp_self()->node;
-        ogs_pfcp_self()->node = ogs_list_next(ogs_pfcp_self()->node)) {
-        if (OGS_FSM_CHECK(
-                &ogs_pfcp_self()->node->sm, smf_pfcp_state_associated)) {
-            OGS_SETUP_PFCP_NODE(sess, ogs_pfcp_self()->node);
-            break;
+    if (ogs_pfcp_self()->upf_selection_mode == UPF_SELECT_RR) {
+        /* Select UPF (PFCP) with round-robin manner */
+        ogs_debug("Select UPF by RR");
+        /* 
+        - starting from list position of last used UPF, search down list for next PFCP associated UPF
+            - if PFCP associated UPF found
+                - use it
+            - if no associated UPF found, keep searching
+            - if bottom of list reached, reset search to top of list
+            - if completed full cyclic search of list and still not found a UPF, use default (first)
+        */
+
+        int connect=0, numreset=0;
+        char startUPFIP[OGS_ADDRSTRLEN];
+        OGS_ADDR(&ogs_pfcp_self()->node->addr, startUPFIP);
+
+        //search UPF list, find next UPF that is associated
+        while (!connect)
+        {
+            // (when end of UPF (PFCP) list reached, reset search to top of UPF list)
+            if(ogs_pfcp_self()->node == NULL){
+                ogs_pfcp_self()->node = ogs_list_first(&ogs_pfcp_self()->n4_list);
+                numreset = 1;
+            }
+            // search UPF (PFCP) list, find next UPF that is associated
+            while(ogs_pfcp_self()->node && !connect) {
+                // if cyclic list check complete and still not found a UPF, break
+                OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+                if (numreset == 1 && !strcmp(buf,startUPFIP)) {
+                    break;
+                }
+                // has UPF <x> associated over PFCP?
+                if (OGS_FSM_CHECK( &ogs_pfcp_self()->node->sm, smf_pfcp_state_associated) ){
+                    // then use it
+                    connect = 1;
+                } else {
+                    // else check next UPF in list
+                    OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+                    ogs_debug("UPF on IP[%s] is not PFCP associated",
+                        buf);
+                    ogs_pfcp_self()->node = ogs_list_next(ogs_pfcp_self()->node);
+                }
+            }            
+            // after checking from the top of the list and not finding a suitable UPF
+            if (!connect && numreset == 1 ){
+                // default to first in list
+                ogs_error("No UPF (PFCP) is currently PFCP associated");
+                ogs_error("Defaulting to first UPF (PFCP) in smf.yaml list");
+                ogs_pfcp_self()->node = ogs_list_first(&ogs_pfcp_self()->n4_list);
+                break;
+            }
         }
-    }
+
+        // list UPF used
+        OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+        ogs_debug("UE using UPF on IP[%s]", 
+            buf);
+
+        // setup GTP session with selected UPF
+        ogs_assert(ogs_pfcp_self()->node);
+        OGS_SETUP_PFCP_NODE(sess, ogs_pfcp_self()->node);
+
+        // iterate to next UPF in list for next UE attach (RR)
+        ogs_pfcp_self()->node = ogs_list_next(ogs_pfcp_self()->node);
+
+
+    } else if (ogs_pfcp_self()->upf_selection_mode == UPF_SELECT_TAC) {
+        /* Select UPF by eNB/gNB TAC */
+        ogs_debug("Select UPF by UE's enb_tac,(RR)");
+        /* 
+        - starting from list position of last used UPF, search down list for a UPF that serves the enb_tac **AND** is PFCP associated
+            - if suitable UPF found
+                - if PFCP associated
+                    - use it
+                - else keep searching
+            - if no suitable UPF found, keep searching
+            - if bottom of list reached, reset search to top of list
+            - if completed full cyclic search of list and still not found a UPF, use default (first)
+        */
+
+        int i, found=0, numreset=0;
+        char startUPFIP[OGS_ADDRSTRLEN];
+        OGS_ADDR(&ogs_pfcp_self()->node->addr, startUPFIP);
+        
+        // fetch the user location information from the incoming S5c request
+        ogs_gtp_uli_t uli;
+        ogs_gtp_parse_uli(&uli, &req->user_location_information);
+
+        //ogs_info("UE enb_tac: [%d]", uli.tai.tac); 
+      
+        //search UPF list, find next UPF that serves the enb_tac **AND** is associated
+        while (!found)
+        {
+            // (when end of UPF (PFCP) list reached, reset search to top of UPF list)
+            if(ogs_pfcp_self()->node == NULL){
+                ogs_pfcp_self()->node = ogs_list_first(&ogs_pfcp_self()->n4_list);
+                numreset = 1;
+            }
+            // search UPF list, find next UPF that serves the enb_tac **AND** is associated
+            while(ogs_pfcp_self()->node && !found) {
+                // if cyclic list check complete and still not found a UPF, break
+                OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+                if (numreset == 1 && !strcmp(buf,startUPFIP)) {
+                    break;
+                }
+                // search from current list position
+                for (i = 0; i < ogs_pfcp_self()->node->num_of_tac && !found; i++){
+                    found = ogs_pfcp_self()->node->tac[i] == uli.tai.tac ? 1: 0;
+                }
+                if(found){
+                    // has UPF <x> associated over PFCP?
+                    if (!OGS_FSM_CHECK( &ogs_pfcp_self()->node->sm, smf_pfcp_state_associated )){
+                        // no - keep searching
+                        OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+                        ogs_warn("UPF on IP[%s] serves enb_tac[%d] - but is **NOT** associated", 
+                            buf, uli.tai.tac);
+                        found = 0;
+                        ogs_pfcp_self()->node = ogs_list_next(ogs_pfcp_self()->node);
+                    } else {
+                        // found a UPF that serves the enb_tac **AND** is associated
+                        OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+                        ogs_debug("UPF on IP[%s] serves enb_tac[%d] - and is associated, use it", 
+                            buf, uli.tai.tac);
+                    }
+                } else {
+                    // not found, check next UPF in list
+                    ogs_pfcp_self()->node = ogs_list_next(ogs_pfcp_self()->node);
+                }
+            }
+            // after checking from the top of the list and not finding a suitable UPF
+            if (!found && numreset == 1 ){
+                // default to first in list
+                ogs_error("No UPF that serves enb_tac[%d] is currently PFCP associated", 
+                    uli.tai.tac);
+                ogs_error("Defaulting to first UPF (PFCP) in smf.yaml list");
+                ogs_pfcp_self()->node = ogs_list_first(&ogs_pfcp_self()->n4_list);
+                break;
+            }
+        }
+
+        // list UPF used
+        OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+        ogs_debug("UE using UPF on IP[%s]", 
+            buf);
+
+        // setup GTP session with selected UPF
+        ogs_assert(ogs_pfcp_self()->node);
+        OGS_SETUP_PFCP_NODE(sess, ogs_pfcp_self()->node);
+
+        // iterate to next UPF in list for next UE attach (RR)
+        ogs_pfcp_self()->node = ogs_list_next(ogs_pfcp_self()->node);
+
+    } else if (ogs_pfcp_self()->upf_selection_mode == UPF_SELECT_APN) {
+        /* Select UPF by UE APN */
+        ogs_debug("Select UPF by UE's apn,(RR)");
+        /* 
+        - starting from list position of last used UPF, search down list for a UPF that serves the UE's APN **AND** is PFCP associated
+            - if suitable UPF found
+                - if PFCP associated
+                    - use it
+                - else keep searching
+            - if no suitable UPF found, keep searching
+            - if bottom of list reached, reset search to top of list
+            - if completed full cyclic search of list and still not found a UPF, use default (first)
+        */
+
+        int i, found=0, numreset=0;
+        char startUPFIP[OGS_ADDRSTRLEN];
+        OGS_ADDR(&ogs_pfcp_self()->node->addr, startUPFIP);
+
+        //ogs_info("UE apn: [%s]", apn); 
+        
+        // search UPF list, find next UPF that serves the ue_apn **AND** is associated
+        while (!found)
+        {
+            // (when end of UPF (PFCP) list reached, reset search to top of UPF list)
+            if(ogs_pfcp_self()->node == NULL){
+                ogs_pfcp_self()->node = ogs_list_first(&ogs_pfcp_self()->n4_list);
+                numreset = 1;
+            }
+            // search UPF list, find next UPF that serves the ue_apn **AND** is associated
+            while(ogs_pfcp_self()->node && !found) {
+                // if cyclic list check complete and still not found a UPF, break
+                OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+                if (numreset == 1 && !strcmp(buf,startUPFIP)) {
+                    break;
+                }
+                // search from current list position
+                for (i = 0; i < ogs_pfcp_self()->node->num_of_apn && !found; i++){
+                    found = strncmp( ogs_pfcp_self()->node->apn[i], apn, OGS_MAX_APN_LEN ) ? 0: 1;
+                }
+                if(found){
+                    // has UPF <x> associated over PFCP?
+                    if (!OGS_FSM_CHECK( &ogs_pfcp_self()->node->sm, smf_pfcp_state_associated )){
+                        // no - keep searching
+                        OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+                        ogs_warn("UPF on IP[%s] serves ue_apn[%s] - but is **NOT** associated", 
+                            buf, apn);
+                        found = 0;
+                        ogs_pfcp_self()->node = ogs_list_next(ogs_pfcp_self()->node);
+                    } else {
+                        // found a UPF that serves the ue_apn **AND** is associated
+                        OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+                        ogs_debug("UPF on IP[%s] serves ue_apn[%s] - and is associated, use it", 
+                            buf, apn);
+                    }
+                } else {
+                    // not found, check next UPF in list
+                    ogs_pfcp_self()->node = ogs_list_next(ogs_pfcp_self()->node);
+                }
+            }            
+            // after checking from the top of the list and not finding a suitable UPF
+            if (!found && numreset == 1 ){
+                // default to first in list
+                ogs_error("No UPF that serves ue_apn[%s] is currently PFCP associated", 
+                    apn);
+                ogs_error("Defaulting to first UPF (PFCP) in smf.yaml list");
+                ogs_pfcp_self()->node = ogs_list_first(&ogs_pfcp_self()->n4_list);
+                break;
+            }
+        }
+
+        // list UPF used
+        OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+        ogs_debug("UE using UPF on IP[%s]", 
+            buf);
+
+        // setup GTP session with selected UPF
+        ogs_assert(ogs_pfcp_self()->node);
+        OGS_SETUP_PFCP_NODE(sess, ogs_pfcp_self()->node);
+
+        // iterate to next UPF in list for next UE attach (RR)
+        ogs_pfcp_self()->node = ogs_list_next(ogs_pfcp_self()->node);
+
+    } else if (ogs_pfcp_self()->upf_selection_mode == UPF_SELECT_ENB_ID) {
+        /* Select UPF by eNB/gNB ID */
+        ogs_debug("Select UPF by UE's enb_id,(RR)");
+        /* 
+        - starting from list position of last used UPF, search down list for a UPF that serves the enb_id **AND** is PFCP associated
+            - if suitable UPF found
+                - if PFCP associated
+                    - use it
+                - else keep searching
+            - if no suitable UPF found, keep searching
+            - if bottom of list reached, reset search to top of list
+            - if completed full cyclic search of list and still not found a UPF, use default (first)
+        */
+
+        int i, found=0, numreset=0;
+        char startUPFIP[OGS_ADDRSTRLEN];
+        OGS_ADDR(&ogs_pfcp_self()->node->addr, startUPFIP);
+        
+        // fetch the user location information from the incoming S5c request
+        ogs_gtp_uli_t uli;
+        ogs_gtp_parse_uli(&uli, &req->user_location_information);
+
+        int UE_enb_id = ((uli.e_cgi.cell_id & 0xfffff00) >> 8); //shift 2 bytes right
+        //int UE_cell_id = (uli.e_cgi.cell_id & 0x00000ff);
+        // e_cgi.cell_id = [enb_id, cell_id] [20bit, 8bit]
+
+        //ogs_info("UE enb_id:  [0x%02x]", UE_enb_id); 
+
+        //search UPF list, find next UPF that serves the enb_id **AND** is associated
+        while (!found)
+        {
+            // (when end of UPF (PFCP) list reached, reset search to top of UPF list)
+            if(ogs_pfcp_self()->node == NULL){
+                ogs_pfcp_self()->node = ogs_list_first(&ogs_pfcp_self()->n4_list);
+                numreset = 1;
+            }
+            // search UPF list, find next UPF that serves the enb_id **AND** is associated
+            while(ogs_pfcp_self()->node && !found) {
+                // if cyclic list check complete and still not found a UPF, break
+                OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+                if (numreset == 1 && !strcmp(buf,startUPFIP)) {
+                    break;
+                }
+                // search from current list position
+                for (i = 0; i < ogs_pfcp_self()->node->num_of_enb_id && !found; i++){
+                    found = ogs_pfcp_self()->node->enb_id[i] == UE_enb_id ? 1: 0;
+                }
+                if(found){
+                    // has UPF <x> associated over PFCP?
+                    if (!OGS_FSM_CHECK( &ogs_pfcp_self()->node->sm, smf_pfcp_state_associated )){
+                        // no - keep searching
+                        OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+                        ogs_warn("UPF on IP[%s] serves enb_id[%d]/[0x%02x] - but is **NOT** associated", 
+                            buf, UE_enb_id, UE_enb_id);
+                        found = 0;
+                        ogs_pfcp_self()->node = ogs_list_next(ogs_pfcp_self()->node);
+                    } else {
+                        // found a UPF that serves the enb_id **AND** is associated
+                        OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+                        ogs_debug("UPF on IP[%s] serves enb_id[%d]/[0x%02x] - and is associated, use it", 
+                            buf, UE_enb_id, UE_enb_id);
+                    }
+                } else {
+                    // not found, check next UPF in list
+                    ogs_pfcp_self()->node = ogs_list_next(ogs_pfcp_self()->node);
+                }
+            }
+            // after checking from the top of the list and not finding a suitable UPF
+            if (!found && numreset == 1 ){
+                // default to first in list
+                ogs_error("No UPF that serves enb_id[%d]/[0x%02x] is currently PFCP associated", 
+                    UE_enb_id, UE_enb_id);
+                ogs_error("Defaulting to first UPF (PFCP) in smf.yaml list");
+                ogs_pfcp_self()->node = ogs_list_first(&ogs_pfcp_self()->n4_list);
+                break;
+            }
+        }
+
+        // list UPF used
+        OGS_ADDR(&ogs_pfcp_self()->node->addr, buf);
+        ogs_debug("UE using UPF on IP[%s]", 
+            buf);
+
+        // setup GTP session with selected UPF
+        ogs_assert(ogs_pfcp_self()->node);
+        OGS_SETUP_PFCP_NODE(sess, ogs_pfcp_self()->node);
+
+        // iterate to next UPF in list for next UE attach (RR)
+        ogs_pfcp_self()->node = ogs_list_next(ogs_pfcp_self()->node);
+
+    } else
+        ogs_assert_if_reached();
 
     /* Set Default Bearer */
     ogs_list_init(&sess->bearer_list);
@@ -794,7 +1115,7 @@ smf_sess_t *smf_sess_add_by_gtp_message(ogs_gtp_message_t *message)
     }
 
     sess = smf_sess_add_by_apn(smf_ue, apn, req->pdn_type.u8,
-                    req->bearer_contexts_to_be_created.eps_bearer_id.u8, paa);
+                    req->bearer_contexts_to_be_created.eps_bearer_id.u8, paa, req);
     return sess;
 }
 
@@ -856,6 +1177,8 @@ void smf_sess_set_ue_ip(smf_sess_t *sess)
 
 smf_sess_t *smf_sess_add_by_psi(smf_ue_t *smf_ue, uint8_t psi)
 {
+    ogs_debug("smf_sess_add_by_psi");
+
     smf_event_t e;
 
     smf_sess_t *sess = NULL;

src/smf/context.h

@@ -275,7 +275,7 @@ smf_ue_t *smf_ue_find_by_imsi(uint8_t *imsi, int imsi_len);
 
 smf_sess_t *smf_sess_add_by_gtp_message(ogs_gtp_message_t *message);
 smf_sess_t *smf_sess_add_by_apn(smf_ue_t *smf_ue, char *apn,
-        uint8_t pdn_type, uint8_t ebi, ogs_paa_t *addr);
+        uint8_t pdn_type, uint8_t ebi, ogs_paa_t *addr,  ogs_gtp_create_session_request_t *message);
 
 smf_sess_t *smf_sess_add_by_sbi_message(ogs_sbi_message_t *message);
 smf_sess_t *smf_sess_add_by_psi(smf_ue_t *smf_ue, uint8_t psi);