FS-2762

conf/sip_profiles/internal.xml

@@ -42,6 +42,26 @@
 	<!-- If you want FreeSWITCH to shutdown if this profile fails to load, uncomment the next line. -->
 	<!-- <param name="shutdown-on-fail" value="true"/> -->
     <param name="sip-trace" value="no"/>
+
+    <!-- 
+        Sometimes, in extremely rare edge cases, the Sofia SIP stack may stop
+        responding. These options allow you to enable and control a watchdog
+        on the Sofia SIP stack so that if it stops responding for the
+        specified number of milliseconds, it will cause FreeSWITCH to shut
+        down immediately. This is useful if you run in an HA environment and
+        need to ensure automated recovery from such a condition. Note that if
+        your server is idle a lot, the watchdog may fire due to not receiving
+        any SIP messages. Thus, if you expect your system to be idle, you
+        should leave the watchdog disabled. It can be toggled on and off
+        through the FreeSWITCH CLI either on an individual profile basis or
+        globally for all profiles. So, if you run in an HA environment with a
+        master and slave, you should use the CLI to make sure the watchdog is
+        only enabled on the master. 
+    -->
+    <param name="watchdog-enabled" value="no"/>
+    <param name="watchdog-step-timeout" value="30000"/>
+    <param name="watchdog-event-timeout" value="30000"/>
+
     <param name="log-auth-failures" value="true"/>
     <param name="forward-unsolicited-mwi-notify" value="false"/>
 

src/mod/endpoints/mod_sofia/mod_sofia.c

@@ -3174,6 +3174,17 @@ static switch_status_t cmd_profile(char **argv, int argc, switch_stream_handle_t
 		goto done;
 	}
 
+	if (!strcasecmp(argv[1], "watchdog")) {
+		if (argc > 2) {
+			int value = switch_true(argv[2]);
+			profile->watchdog_enabled = value;
+			stream->write_function(stream, "%s sip debugging on %s", value ? "Enabled" : "Disabled", profile->name);
+		} else {
+			stream->write_function(stream, "Usage: sofia profile <name> watchdog <on/off>\n");
+		}
+		goto done;
+	}
+
 
 	if (!strcasecmp(argv[1], "gwlist")) {
 		int up = 1;
@@ -3510,12 +3521,14 @@ SWITCH_STANDARD_API(sofia_function)
 		"[register|unregister] [<gateway name>|all]|"
 		"killgw <gateway name>|"
 		"[stun-auto-disable|stun-enabled] [true|false]]|"
-		"siptrace [on|off]\n"
+		"siptrace <on|off>|"
+		"watchdog <on|off>\n"
 		"sofia status|xmlstatus profile <name> [ reg <contact str> ] | [ pres <pres str> ] | [ user <user@domain> ]\n"
 		"sofia status|xmlstatus gateway <name>\n"
 		"sofia loglevel <all|default|tport|iptsec|nea|nta|nth_client|nth_server|nua|soa|sresolv|stun> [0-9]\n"
 		"sofia tracelevel <console|alert|crit|err|warning|notice|info|debug>\n"
-		"sofa global siptrace [on|off]\n"
+		"sofia global siptrace <on|off>|"
+		"watchdog <on|off>\n"
 		"--------------------------------------------------------------------------------\n";
 
 	if (session) {
@@ -3572,21 +3585,30 @@ SWITCH_STANDARD_API(sofia_function)
 		stream->write_function(stream, "%s", usage_string);
 		goto done;
 	} else if (!strcasecmp(argv[0], "global")) {
-		int on = -1;
+		int ston = -1;
+		int wdon = -1;
 
 		if (argc > 1) {
 			if (!strcasecmp(argv[1], "siptrace")) {
 				if (argc > 2) {
-					on = switch_true(argv[2]);
+					ston = switch_true(argv[2]);
+				}
+			}
+			if (!strcasecmp(argv[1], "watchdog")) {
+				if (argc > 2) {
+					wdon = switch_true(argv[2]);
 				}
 			}
 		}
 
-		if (on != -1) {
-			sofia_glue_global_siptrace(on);
-			stream->write_function(stream, "+OK Global siptrace %s", on ? "on" : "off");
+		if (ston != -1) {
+			sofia_glue_global_siptrace(ston);
+			stream->write_function(stream, "+OK Global siptrace %s", ston ? "on" : "off");
+		} else if (wdon != -1) {
+			sofia_glue_global_watchdog(wdon);
+			stream->write_function(stream, "+OK Global watchdog %s", wdon ? "on" : "off");
 		} else {
-			stream->write_function(stream, "-ERR Usage: siptrace on|off");
+			stream->write_function(stream, "-ERR Usage: siptrace <on|off>|watchdog <on|off>");
 		}
 		
 		goto done;
@@ -4668,6 +4690,7 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_sofia_load)
 	switch_console_set_complete("add sofia tracelevel ::[console:alert:crit:err:warning:notice:info:debug");
 
 	switch_console_set_complete("add sofia global siptrace ::[on:off");
+	switch_console_set_complete("add sofia global watchdog ::[on:off");
 
 	switch_console_set_complete("add sofia profile");
 	switch_console_set_complete("add sofia profile restart all");
@@ -4683,6 +4706,8 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_sofia_load)
 	switch_console_set_complete("add sofia profile ::sofia::list_profiles killgw ::sofia::list_profile_gateway");
 	switch_console_set_complete("add sofia profile ::sofia::list_profiles siptrace on");
 	switch_console_set_complete("add sofia profile ::sofia::list_profiles siptrace off");
+	switch_console_set_complete("add sofia profile ::sofia::list_profiles watchdog on");
+	switch_console_set_complete("add sofia profile ::sofia::list_profiles watchdog off");
 
 	switch_console_set_complete("add sofia profile ::sofia::list_profiles gwlist up");
 	switch_console_set_complete("add sofia profile ::sofia::list_profiles gwlist down");

src/mod/endpoints/mod_sofia/mod_sofia.h

@@ -566,6 +566,7 @@ struct sofia_profile {
 	switch_time_t last_root_step;
 	uint32_t step_timeout;
 	uint32_t event_timeout;
+	int watchdog_enabled;
 };
 
 struct private_object {
@@ -1020,6 +1021,7 @@ void sofia_glue_tech_simplify(private_object_t *tech_pvt);
 switch_console_callback_match_t *sofia_reg_find_reg_url_multi(sofia_profile_t *profile, const char *user, const char *host);
 switch_bool_t sofia_glue_profile_exists(const char *key);
 void sofia_glue_global_siptrace(switch_bool_t on);
+void sofia_glue_global_watchdog(switch_bool_t on);
 void sofia_glue_proxy_codec(switch_core_session_t *session, const char *r_sdp);
 switch_status_t sofia_glue_sdp_map(const char *r_sdp, switch_event_t **fmtp, switch_event_t **pt);
 void sofia_glue_build_vid_refresh_message(switch_core_session_t *session, const char *pl);

src/mod/endpoints/mod_sofia/sofia.c

@@ -1326,7 +1326,7 @@ void *SWITCH_THREAD_FUNC sofia_profile_worker_thread_run(switch_thread_t *thread
 				}
 			}
 
-			if (fail) {
+			if (profile->watchdog_enabled && fail) {
 				int arg = 1;
 				switch_session_ctl_t command = SCSC_SHUTDOWN_NOW;
 
@@ -2327,9 +2327,11 @@ switch_status_t reconfig_sofia(sofia_profile_t *profile)
 						} else {
 							sofia_clear_pflag(profile, PFLAG_DEL_SUBS_ON_REG);
 						}
-					} else if (!strcasecmp(var, "watchdog_step_timeout")) {
+					} else if (!strcasecmp(var, "watchdog-enabled")) {
+						profile->watchdog_enabled = switch_true(val);
+					} else if (!strcasecmp(var, "watchdog-step-timeout")) {
 						profile->step_timeout = (unsigned long) atol(val);
-					} else if (!strcasecmp(var, "watchdog_event_timeout")) {
+					} else if (!strcasecmp(var, "watchdog-event-timeout")) {
 						profile->event_timeout = (unsigned long) atol(val);
 					} else if (!strcasecmp(var, "in-dialog-chat")) {
 						if (switch_true(val)) {
@@ -3011,9 +3013,11 @@ switch_status_t config_sofia(int reload, char *profile_name)
 						} else {
 							sofia_clear_pflag(profile, PFLAG_LOG_AUTH_FAIL);
 						}
-					} else if (!strcasecmp(var, "watchdog_step_timeout")) {
+					} else if (!strcasecmp(var, "watchdog-enabled")) {
+						profile->watchdog_enabled = switch_true(val);
+					} else if (!strcasecmp(var, "watchdog-step-timeout")) {
 						profile->step_timeout = atoi(val);
-					} else if (!strcasecmp(var, "watchdog_event_timeout")) {
+					} else if (!strcasecmp(var, "watchdog-event-timeout")) {
 						profile->event_timeout = atoi(val);
 
 					} else if (!strcasecmp(var, "in-dialog-chat")) {

src/mod/endpoints/mod_sofia/sofia_glue.c

@@ -4752,6 +4752,26 @@ void sofia_glue_global_siptrace(switch_bool_t on)
 
 }
 
+void sofia_glue_global_watchdog(switch_bool_t on)
+{
+	switch_hash_index_t *hi;
+	const void *var;
+	void *val;
+	sofia_profile_t *pptr;
+
+	switch_mutex_lock(mod_sofia_globals.hash_mutex);
+	if (mod_sofia_globals.profile_hash) {
+		for (hi = switch_hash_first(NULL, mod_sofia_globals.profile_hash); hi; hi = switch_hash_next(hi)) {
+			switch_hash_this(hi, &var, NULL, &val);
+			if ((pptr = (sofia_profile_t *) val)) {
+				pptr->watchdog_enabled = (on ? 1 : 0);
+			}
+		}
+	}
+	switch_mutex_unlock(mod_sofia_globals.hash_mutex);
+
+}
+
 void sofia_glue_del_profile(sofia_profile_t *profile)
 {
 	sofia_gateway_t *gp;