Index: pjproject/trunk/pjlib/src/pj/ioqueue_epoll.c
===================================================================
--- pjproject/trunk/pjlib/src/pj/ioqueue_epoll.c	(revision 2039)
+++ pjproject/trunk/pjlib/src/pj/ioqueue_epoll.c	(revision 2295)
@@ -176,4 +176,11 @@
     struct epoll_event *events;
     struct queue       *queue;
+
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+    pj_mutex_t	       *ref_cnt_mutex;
+    pj_ioqueue_key_t	active_list;
+    pj_ioqueue_key_t	closing_list;
+    pj_ioqueue_key_t	free_list;
+#endif
 };
 
@@ -182,4 +189,9 @@
  */
 #include "ioqueue_common_abs.c"
+
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+/* Scan closing keys to be put to free list again */
+static void scan_closing_keys(pj_ioqueue_t *ioqueue);
+#endif
 
 /*
@@ -207,4 +219,5 @@
     pj_status_t rc;
     pj_lock_t *lock;
+    int i;
 
     /* Check that arguments are valid. */
@@ -223,4 +236,44 @@
     ioqueue->count = 0;
     pj_list_init(&ioqueue->hlist);
+
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+    /* When safe unregistration is used (the default), we pre-create
+     * all keys and put them in the free list.
+     */
+
+    /* Mutex to protect key's reference counter 
+     * We don't want to use key's mutex or ioqueue's mutex because
+     * that would create deadlock situation in some cases.
+     */
+    rc = pj_mutex_create_simple(pool, NULL, &ioqueue->ref_cnt_mutex);
+    if (rc != PJ_SUCCESS)
+	return rc;
+
+
+    /* Init key list */
+    pj_list_init(&ioqueue->free_list);
+    pj_list_init(&ioqueue->closing_list);
+
+
+    /* Pre-create all keys according to max_fd */
+    for ( i=0; i<max_fd; ++i) {
+	pj_ioqueue_key_t *key;
+
+	key = PJ_POOL_ALLOC_T(pool, pj_ioqueue_key_t);
+	key->ref_count = 0;
+	rc = pj_mutex_create_recursive(pool, NULL, &key->mutex);
+	if (rc != PJ_SUCCESS) {
+	    key = ioqueue->free_list.next;
+	    while (key != &ioqueue->free_list) {
+		pj_mutex_destroy(key->mutex);
+		key = key->next;
+	    }
+	    pj_mutex_destroy(ioqueue->ref_cnt_mutex);
+	    return rc;
+	}
+
+	pj_list_push_back(&ioqueue->free_list, key);
+    }
+#endif
 
     rc = pj_lock_create_simple_mutex(pool, "ioq%p", &lock);
@@ -257,4 +310,6 @@
 PJ_DEF(pj_status_t) pj_ioqueue_destroy(pj_ioqueue_t *ioqueue)
 {
+    pj_ioqueue_key_t *key;
+
     PJ_ASSERT_RETURN(ioqueue, PJ_EINVAL);
     PJ_ASSERT_RETURN(ioqueue->epfd > 0, PJ_EINVALIDOP);
@@ -263,4 +318,27 @@
     os_close(ioqueue->epfd);
     ioqueue->epfd = 0;
+
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+    /* Destroy reference counters */
+    key = ioqueue->active_list.next;
+    while (key != &ioqueue->active_list) {
+	pj_mutex_destroy(key->mutex);
+	key = key->next;
+    }
+
+    key = ioqueue->closing_list.next;
+    while (key != &ioqueue->closing_list) {
+	pj_mutex_destroy(key->mutex);
+	key = key->next;
+    }
+
+    key = ioqueue->free_list.next;
+    while (key != &ioqueue->free_list) {
+	pj_mutex_destroy(key->mutex);
+	key = key->next;
+    }
+
+    pj_mutex_destroy(ioqueue->ref_cnt_mutex);
+#endif
     return ioqueue_destroy(ioqueue);
 }
@@ -304,6 +382,25 @@
     }
 
+    /* If safe unregistration (PJ_IOQUEUE_HAS_SAFE_UNREG) is used, get
+     * the key from the free list. Otherwise allocate a new one. 
+     */
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+
+    /* Scan closing_keys first to let them come back to free_list */
+    scan_closing_keys(ioqueue);
+
+    pj_assert(!pj_list_empty(&ioqueue->free_list));
+    if (pj_list_empty(&ioqueue->free_list)) {
+	rc = PJ_ETOOMANY;
+	goto on_return;
+    }
+
+    key = ioqueue->free_list.next;
+    pj_list_erase(key);
+#else
     /* Create key. */
     key = (pj_ioqueue_key_t*)pj_pool_zalloc(pool, sizeof(pj_ioqueue_key_t));
+#endif
+
     rc = ioqueue_init_key(pool, ioqueue, key, sock, user_data, cb);
     if (rc != PJ_SUCCESS) {
@@ -313,10 +410,10 @@
 
     /* Create key's mutex */
-    rc = pj_mutex_create_recursive(pool, NULL, &key->mutex);
+ /*   rc = pj_mutex_create_recursive(pool, NULL, &key->mutex);
     if (rc != PJ_SUCCESS) {
 	key = NULL;
 	goto on_return;
     }
-
+*/
     /* os_epoll_ctl. */
     ev.events = EPOLLIN | EPOLLERR;
@@ -346,4 +443,39 @@
 }
 
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+/* Increment key's reference counter */
+static void increment_counter(pj_ioqueue_key_t *key)
+{
+    pj_mutex_lock(key->ioqueue->ref_cnt_mutex);
+    ++key->ref_count;
+    pj_mutex_unlock(key->ioqueue->ref_cnt_mutex);
+}
+
+/* Decrement the key's reference counter, and when the counter reach zero,
+ * destroy the key.
+ *
+ * Note: MUST NOT CALL THIS FUNCTION WHILE HOLDING ioqueue's LOCK.
+ */
+static void decrement_counter(pj_ioqueue_key_t *key)
+{
+    pj_lock_acquire(key->ioqueue->lock);
+    pj_mutex_lock(key->ioqueue->ref_cnt_mutex);
+    --key->ref_count;
+    if (key->ref_count == 0) {
+
+	pj_assert(key->closing == 1);
+	pj_gettimeofday(&key->free_time);
+	key->free_time.msec += PJ_IOQUEUE_KEY_FREE_DELAY;
+	pj_time_val_normalize(&key->free_time);
+
+	pj_list_erase(key);
+	pj_list_push_back(&key->ioqueue->closing_list, key);
+
+    }
+    pj_mutex_unlock(key->ioqueue->ref_cnt_mutex);
+    pj_lock_release(key->ioqueue->lock);
+}
+#endif
+
 /*
  * pj_ioqueue_unregister()
@@ -364,5 +496,7 @@
     pj_assert(ioqueue->count > 0);
     --ioqueue->count;
+#if !PJ_IOQUEUE_HAS_SAFE_UNREG
     pj_list_erase(key);
+#endif
 
     ev.events = 0;
@@ -375,9 +509,22 @@
     }
 
-    pj_lock_release(ioqueue->lock);
-
     /* Destroy the key. */
     pj_sock_close(key->fd);
+
+    pj_lock_release(ioqueue->lock);
+
+
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+    /* Mark key is closing. */
+    key->closing = 1;
+
+    /* Decrement counter. */
+    decrement_counter(key);
+
+    /* Done. */
+    pj_mutex_unlock(key->mutex);
+#else
     pj_mutex_destroy(key->mutex);
+#endif
 
     return PJ_SUCCESS;
@@ -420,4 +567,27 @@
     }	
 }
+
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+/* Scan closing keys to be put to free list again */
+static void scan_closing_keys(pj_ioqueue_t *ioqueue)
+{
+    pj_time_val now;
+    pj_ioqueue_key_t *h;
+
+    pj_gettimeofday(&now);
+    h = ioqueue->closing_list.next;
+    while (h != &ioqueue->closing_list) {
+	pj_ioqueue_key_t *next = h->next;
+
+	pj_assert(h->closing != 0);
+
+	if (PJ_TIME_VAL_GTE(now, h->free_time)) {
+	    pj_list_erase(h);
+	    pj_list_push_back(&ioqueue->free_list, h);
+	}
+	h = next;
+    }
+}
+#endif
 
 /*
@@ -442,4 +612,14 @@
     count = os_epoll_wait( ioqueue->epfd, events, ioqueue->max, msec);
     if (count == 0) {
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+    /* Check the closing keys only when there's no activity and when there are
+     * pending closing keys.
+     */
+    if (count == 0 && !pj_list_empty(&ioqueue->closing_list)) {
+	pj_lock_acquire(ioqueue->lock);
+	scan_closing_keys(ioqueue);
+	pj_lock_release(ioqueue->lock);
+    }
+#endif
 	TRACE_((THIS_FILE, "os_epoll_wait timed out"));
 	return count;
@@ -468,4 +648,8 @@
 	if ((events[i].events & EPOLLIN) && 
 	    (key_has_pending_read(h) || key_has_pending_accept(h))) {
+
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+	    increment_counter(h);
+#endif
 	    queue[processed].key = h;
 	    queue[processed].event_type = READABLE_EVENT;
@@ -477,4 +661,8 @@
 	 */
 	if ((events[i].events & EPOLLOUT) && key_has_pending_write(h)) {
+
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+	    increment_counter(h);
+#endif
 	    queue[processed].key = h;
 	    queue[processed].event_type = WRITEABLE_EVENT;
@@ -487,4 +675,8 @@
 	 */
 	if ((events[i].events & EPOLLOUT) && (h->connecting)) {
+
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+	    increment_counter(h);
+#endif
 	    queue[processed].key = h;
 	    queue[processed].event_type = WRITEABLE_EVENT;
@@ -497,4 +689,8 @@
 	 */
 	if (events[i].events & EPOLLERR && (h->connecting)) {
+		
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+	    increment_counter(h);
+#endif		
 	    queue[processed].key = h;
 	    queue[processed].event_type = EXCEPTION_EVENT;
@@ -520,4 +716,8 @@
             break;
         }
+
+#if PJ_IOQUEUE_HAS_SAFE_UNREG
+	decrement_counter(queue[i].key);
+#endif
     }
 
