Changeset 4247 for pjproject/trunk/pjlib/src/pj/ssl_sock_ossl.c
- Timestamp:
- Sep 7, 2012 8:58:48 AM (12 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
pjproject/trunk/pjlib/src/pj/ssl_sock_ossl.c
r4146 r4247 106 106 107 107 /* 108 * Structure of SSL socket write buffer.108 * Structure of SSL socket write data. 109 109 */ 110 110 typedef struct write_data_t { 111 PJ_DECL_LIST_MEMBER(struct write_data_t); 111 112 pj_ioqueue_op_key_t key; 112 113 pj_size_t record_len; … … 122 123 123 124 /* 124 * Structure of SSL socket write state.125 */ 126 typedef struct write_state_t {125 * Structure of SSL socket write buffer (circular buffer). 126 */ 127 typedef struct send_buf_t { 127 128 char *buf; 128 129 pj_size_t max_len; 129 130 char *start; 130 131 pj_size_t len; 131 write_data_t *last_data; 132 } write_state_t; 133 134 /* 135 * Structure of write data pending. 136 */ 137 typedef struct write_pending_t { 138 PJ_DECL_LIST_MEMBER(struct write_pending_t); 139 write_data_t data; 140 } write_pending_t; 132 } send_buf_t; 141 133 142 134 /* … … 174 166 read_data_t *ssock_rbuf; 175 167 176 write_state_t write_state; 177 write_pending_t write_pending; 178 write_pending_t write_pending_empty; 179 pj_lock_t *write_mutex; /* protect write BIO and write_state */ 168 write_data_t write_pending;/* list of pending write to OpenSSL */ 169 write_data_t write_pending_empty; /* cache for write_pending */ 170 pj_bool_t flushing_write_pend; /* flag of flushing is ongoing*/ 171 send_buf_t send_buf; 172 write_data_t send_pending; /* list of pending write to network */ 173 pj_lock_t *write_mutex; /* protect write BIO and send_buf */ 180 174 181 175 SSL_CTX *ossl_ctx; … … 198 192 199 193 194 static write_data_t* alloc_send_data(pj_ssl_sock_t *ssock, pj_size_t len); 195 static void free_send_data(pj_ssl_sock_t *ssock, write_data_t *wdata); 200 196 static pj_status_t flush_delayed_send(pj_ssl_sock_t *ssock); 201 197 … … 1055 1051 } 1056 1052 1053 static write_data_t* alloc_send_data(pj_ssl_sock_t *ssock, pj_size_t len) 1054 { 1055 send_buf_t *send_buf = &ssock->send_buf; 1056 pj_size_t avail_len, skipped_len = 0; 1057 char *reg1, *reg2; 1058 pj_size_t reg1_len, reg2_len; 1059 write_data_t *p; 1060 1061 /* Check buffer availability */ 1062 avail_len = send_buf->max_len - send_buf->len; 1063 if (avail_len < len) 1064 return NULL; 1065 1066 /* If buffer empty, reset start pointer and return it */ 1067 if (send_buf->len == 0) { 1068 send_buf->start = send_buf->buf; 1069 send_buf->len = len; 1070 p = (write_data_t*)send_buf->start; 1071 goto init_send_data; 1072 } 1073 1074 /* Free space may be wrapped/splitted into two regions, so let's 1075 * analyze them if any region can hold the write data. 1076 */ 1077 reg1 = send_buf->start + send_buf->len; 1078 if (reg1 >= send_buf->buf + send_buf->max_len) 1079 reg1 -= send_buf->max_len; 1080 reg1_len = send_buf->max_len - send_buf->len; 1081 if (reg1 + reg1_len > send_buf->buf + send_buf->max_len) { 1082 reg1_len = send_buf->buf + send_buf->max_len - reg1; 1083 reg2 = send_buf->buf; 1084 reg2_len = send_buf->start - send_buf->buf; 1085 } else { 1086 reg2 = NULL; 1087 reg2_len = 0; 1088 } 1089 1090 /* More buffer availability check, note that the write data must be in 1091 * a contigue buffer. 1092 */ 1093 avail_len = PJ_MAX(reg1_len, reg2_len); 1094 if (avail_len < len) 1095 return NULL; 1096 1097 /* Get the data slot */ 1098 if (reg1_len >= len) { 1099 p = (write_data_t*)reg1; 1100 } else { 1101 p = (write_data_t*)reg2; 1102 skipped_len = reg1_len; 1103 } 1104 1105 /* Update buffer length */ 1106 send_buf->len += len + skipped_len; 1107 1108 init_send_data: 1109 /* Init the new send data */ 1110 pj_bzero(p, sizeof(*p)); 1111 pj_list_init(p); 1112 pj_list_push_back(&ssock->send_pending, p); 1113 1114 return p; 1115 } 1116 1117 static void free_send_data(pj_ssl_sock_t *ssock, write_data_t *wdata) 1118 { 1119 send_buf_t *buf = &ssock->send_buf; 1120 write_data_t *spl = &ssock->send_pending; 1121 1122 pj_assert(!pj_list_empty(&ssock->send_pending)); 1123 1124 /* Free slot from the buffer */ 1125 if (spl->next == wdata && spl->prev == wdata) { 1126 /* This is the only data, reset the buffer */ 1127 buf->start = buf->buf; 1128 buf->len = 0; 1129 } else if (spl->next == wdata) { 1130 /* This is the first data, shift start pointer of the buffer and 1131 * adjust the buffer length. 1132 */ 1133 buf->start = (char*)wdata->next; 1134 if (wdata->next > wdata) { 1135 buf->len -= ((char*)wdata->next - buf->start); 1136 } else { 1137 /* Overlapped */ 1138 unsigned right_len, left_len; 1139 right_len = buf->buf + buf->max_len - (char*)wdata; 1140 left_len = (char*)wdata->next - buf->buf; 1141 buf->len -= (right_len + left_len); 1142 } 1143 } else if (spl->prev == wdata) { 1144 /* This is the last data, just adjust the buffer length */ 1145 if (wdata->prev < wdata) { 1146 unsigned jump_len; 1147 jump_len = (char*)wdata - 1148 ((char*)wdata->prev + wdata->prev->record_len); 1149 buf->len -= (wdata->record_len + jump_len); 1150 } else { 1151 /* Overlapped */ 1152 unsigned right_len, left_len; 1153 right_len = buf->buf + buf->max_len - 1154 ((char*)wdata->prev + wdata->prev->record_len); 1155 left_len = (char*)wdata + wdata->record_len - buf->buf; 1156 buf->len -= (right_len + left_len); 1157 } 1158 } 1159 /* For data in the middle buffer, just do nothing on the buffer. The slot 1160 * will be freed when freeing other data that is in the first/last. 1161 */ 1162 1163 /* Remove the data from send pending list */ 1164 pj_list_erase(wdata); 1165 } 1166 1167 #if 0 1168 /* Just for testing send buffer alloc/free */ 1169 #include <pj/rand.h> 1170 pj_status_t pj_ssl_sock_ossl_test_send_buf(pj_pool_t *pool) 1171 { 1172 enum { MAX_CHUNK_NUM = 20 }; 1173 unsigned chunk_size, chunk_cnt, i; 1174 write_data_t *wdata[MAX_CHUNK_NUM] = {0}; 1175 pj_time_val now; 1176 pj_ssl_sock_t *ssock = NULL; 1177 pj_ssl_sock_param param; 1178 pj_status_t status; 1179 1180 pj_gettimeofday(&now); 1181 pj_srand((unsigned)now.sec); 1182 1183 pj_ssl_sock_param_default(¶m); 1184 status = pj_ssl_sock_create(pool, ¶m, &ssock); 1185 if (status != PJ_SUCCESS) { 1186 return status; 1187 } 1188 1189 if (ssock->send_buf.max_len == 0) { 1190 ssock->send_buf.buf = (char*) 1191 pj_pool_alloc(ssock->pool, 1192 ssock->param.send_buffer_size); 1193 ssock->send_buf.max_len = ssock->param.send_buffer_size; 1194 ssock->send_buf.start = ssock->send_buf.buf; 1195 ssock->send_buf.len = 0; 1196 } 1197 1198 chunk_size = ssock->param.send_buffer_size / MAX_CHUNK_NUM / 2; 1199 chunk_cnt = 0; 1200 for (i = 0; i < MAX_CHUNK_NUM; i++) { 1201 wdata[i] = alloc_send_data(ssock, pj_rand() % chunk_size + 321); 1202 if (wdata[i]) 1203 chunk_cnt++; 1204 else 1205 break; 1206 } 1207 1208 while (chunk_cnt) { 1209 i = pj_rand() % MAX_CHUNK_NUM; 1210 if (wdata[i]) { 1211 free_send_data(ssock, wdata[i]); 1212 wdata[i] = NULL; 1213 chunk_cnt--; 1214 } 1215 } 1216 1217 if (ssock->send_buf.len != 0) 1218 status = PJ_EBUG; 1219 1220 pj_ssl_sock_close(ssock); 1221 return status; 1222 } 1223 #endif 1224 1225 1057 1226 /* Flush write BIO to network socket. Note that any access to write BIO 1058 1227 * MUST be serialized, so mutex protection must cover any call to OpenSSL … … 1068 1237 char *data; 1069 1238 pj_ssize_t len; 1070 1071 write_state_t *write_st = &ssock->write_state;1072 1239 write_data_t *wdata; 1073 pj_size_t avail_len, needed_len, skipped_len = 0;1240 pj_size_t needed_len; 1074 1241 pj_status_t status; 1075 1242 1243 pj_lock_acquire(ssock->write_mutex); 1244 1076 1245 /* Check if there is data in write BIO, flush it if any */ 1077 if (!BIO_pending(ssock->ossl_wbio)) 1246 if (!BIO_pending(ssock->ossl_wbio)) { 1247 pj_lock_release(ssock->write_mutex); 1078 1248 return PJ_SUCCESS; 1249 } 1079 1250 1080 1251 /* Get data and its length */ 1081 1252 len = BIO_get_mem_data(ssock->ossl_wbio, &data); 1082 if (len == 0) 1253 if (len == 0) { 1254 pj_lock_release(ssock->write_mutex); 1083 1255 return PJ_SUCCESS; 1256 } 1084 1257 1085 1258 /* Calculate buffer size needed, and align it to 8 */ … … 1087 1260 needed_len = ((needed_len + 7) >> 3) << 3; 1088 1261 1089 /* Check buffer availability */ 1090 avail_len = write_st->max_len - write_st->len; 1091 if (avail_len < needed_len) 1262 /* Allocate buffer for send data */ 1263 wdata = alloc_send_data(ssock, needed_len); 1264 if (wdata == NULL) { 1265 pj_lock_release(ssock->write_mutex); 1092 1266 return PJ_ENOMEM; 1093 1094 /* More buffer availability check, note that the write data must be in 1095 * a contigue buffer. 1096 */ 1097 if (write_st->len == 0) { 1098 1099 write_st->start = write_st->buf; 1100 wdata = (write_data_t*)write_st->start; 1101 1102 } else { 1103 1104 char *reg1, *reg2; 1105 pj_size_t reg1_len, reg2_len; 1106 1107 /* Unused slots may be wrapped/splitted into two regions, so let's 1108 * analyze them if any region can hold the write data. 1109 */ 1110 reg1 = write_st->start + write_st->len; 1111 if (reg1 >= write_st->buf + write_st->max_len) 1112 reg1 -= write_st->max_len; 1113 reg1_len = write_st->max_len - write_st->len; 1114 if (reg1 + reg1_len > write_st->buf + write_st->max_len) { 1115 reg1_len = write_st->buf + write_st->max_len - reg1; 1116 reg2 = write_st->buf; 1117 reg2_len = write_st->start - write_st->buf; 1118 } else { 1119 reg2 = NULL; 1120 reg2_len = 0; 1121 } 1122 avail_len = PJ_MAX(reg1_len, reg2_len); 1123 if (avail_len < needed_len) 1124 return PJ_ENOMEM; 1125 1126 /* Get write data pointer and update buffer length */ 1127 if (reg1_len >= needed_len) { 1128 wdata = (write_data_t*)reg1; 1129 } else { 1130 wdata = (write_data_t*)reg2; 1131 /* Unused slot in region 1 is skipped as current write data 1132 * doesn't fit it. 1133 */ 1134 skipped_len = reg1_len; 1135 } 1136 } 1137 1138 /* Copy the data and set its properties into the buffer */ 1139 pj_bzero(wdata, sizeof(write_data_t)); 1267 } 1268 1269 /* Copy the data and set its properties into the send data */ 1140 1270 wdata->app_key = send_key; 1141 1271 wdata->record_len = needed_len; … … 1144 1274 wdata->flags = flags; 1145 1275 pj_memcpy(&wdata->data, data, len); 1276 1277 /* Reset write BIO */ 1278 BIO_reset(ssock->ossl_wbio); 1279 1280 /* Ticket #1573: Don't hold mutex while calling PJLIB socket send(). */ 1281 pj_lock_release(ssock->write_mutex); 1146 1282 1147 1283 /* Send it */ … … 1158 1294 } 1159 1295 1160 /* Oh no, EWOULDBLOCK! */ 1161 if (status == PJ_STATUS_FROM_OS(OSERR_EWOULDBLOCK)) { 1162 /* Just return PJ_SUCCESS here, the pending data will be sent in next 1163 * call of this function since the data is still stored in write BIO. 1296 if (status != PJ_EPENDING) { 1297 /* When the sending is not pending, remove the wdata from send 1298 * pending list. 1164 1299 */ 1165 return PJ_SUCCESS; 1166 } 1167 1168 /* Reset write BIO after flushed */ 1169 BIO_reset(ssock->ossl_wbio); 1170 1171 if (status == PJ_EPENDING) { 1172 /* Update write state */ 1173 pj_assert(skipped_len==0 || write_st->last_data); 1174 write_st->len += needed_len + skipped_len; 1175 if (write_st->last_data) 1176 write_st->last_data->record_len += skipped_len; 1177 write_st->last_data = wdata; 1300 pj_lock_acquire(ssock->write_mutex); 1301 free_send_data(ssock, wdata); 1302 pj_lock_release(ssock->write_mutex); 1178 1303 } 1179 1304 … … 1214 1339 int err; 1215 1340 1341 /* Perform SSL handshake */ 1216 1342 pj_lock_acquire(ssock->write_mutex); 1217 1218 /* Perform SSL handshake */1219 1343 err = SSL_do_handshake(ssock->ossl_ssl); 1344 pj_lock_release(ssock->write_mutex); 1220 1345 1221 1346 /* SSL_do_handshake() may put some pending data into SSL write BIO, … … 1227 1352 return status; 1228 1353 } 1229 1230 pj_lock_release(ssock->write_mutex);1231 1354 1232 1355 if (err < 0) { … … 1357 1480 update_certs_info(ssock); 1358 1481 1359 pj_lock_acquire(ssock->write_mutex); 1482 // Ticket #1573: Don't hold mutex while calling 1483 // PJLIB socket send(). 1484 //pj_lock_acquire(ssock->write_mutex); 1360 1485 status = flush_delayed_send(ssock); 1361 pj_lock_release(ssock->write_mutex); 1486 //pj_lock_release(ssock->write_mutex); 1487 1488 /* If flushing is ongoing, treat it as success */ 1489 if (status == PJ_EBUSY) 1490 status = PJ_SUCCESS; 1362 1491 1363 1492 if (status != PJ_SUCCESS && status != PJ_EPENDING) { … … 1432 1561 /* Update write buffer state */ 1433 1562 pj_lock_acquire(ssock->write_mutex); 1434 ssock->write_state.start += wdata->record_len; 1435 ssock->write_state.len -= wdata->record_len; 1436 if (ssock->write_state.last_data == wdata) { 1437 pj_assert(ssock->write_state.len == 0); 1438 ssock->write_state.last_data = NULL; 1439 } 1563 free_send_data(ssock, wdata); 1440 1564 pj_lock_release(ssock->write_mutex); 1441 1565 … … 1548 1672 1549 1673 /* Prepare write/send state */ 1550 pj_assert(ssock-> write_state.max_len == 0);1551 ssock-> write_state.buf = (char*)1552 1553 1554 ssock-> write_state.max_len = ssock->param.send_buffer_size;1555 ssock-> write_state.start = ssock->write_state.buf;1556 ssock-> write_state.len = 0;1674 pj_assert(ssock->send_buf.max_len == 0); 1675 ssock->send_buf.buf = (char*) 1676 pj_pool_alloc(ssock->pool, 1677 ssock->param.send_buffer_size); 1678 ssock->send_buf.max_len = ssock->param.send_buffer_size; 1679 ssock->send_buf.start = ssock->send_buf.buf; 1680 ssock->send_buf.len = 0; 1557 1681 1558 1682 /* Start handshake timer */ … … 1627 1751 1628 1752 /* Prepare write/send state */ 1629 pj_assert(ssock-> write_state.max_len == 0);1630 ssock-> write_state.buf = (char*)1753 pj_assert(ssock->send_buf.max_len == 0); 1754 ssock->send_buf.buf = (char*) 1631 1755 pj_pool_alloc(ssock->pool, 1632 1756 ssock->param.send_buffer_size); 1633 ssock-> write_state.max_len = ssock->param.send_buffer_size;1634 ssock-> write_state.start = ssock->write_state.buf;1635 ssock-> write_state.len = 0;1757 ssock->send_buf.max_len = ssock->param.send_buffer_size; 1758 ssock->send_buf.start = ssock->send_buf.buf; 1759 ssock->send_buf.len = 0; 1636 1760 1637 1761 #ifdef SSL_set_tlsext_host_name … … 1806 1930 pj_list_init(&ssock->write_pending); 1807 1931 pj_list_init(&ssock->write_pending_empty); 1932 pj_list_init(&ssock->send_pending); 1808 1933 pj_timer_entry_init(&ssock->timer, 0, ssock, &on_timer); 1809 1934 … … 2039 2164 } 2040 2165 2041 /* Write plain data to SSL and flush write BIO. Note that accessing 2042 * write BIO must be serialized, so a call to this function must be 2043 * protected by write mutex of SSL socket. 2044 */ 2166 /* Write plain data to SSL and flush write BIO. */ 2045 2167 static pj_status_t ssl_write(pj_ssl_sock_t *ssock, 2046 2168 pj_ioqueue_op_key_t *send_key, … … 2057 2179 * until re-negotiation is completed. 2058 2180 */ 2181 pj_lock_acquire(ssock->write_mutex); 2059 2182 nwritten = SSL_write(ssock->ossl_ssl, data, size); 2183 pj_lock_release(ssock->write_mutex); 2060 2184 2061 2185 if (nwritten == size) { … … 2088 2212 } 2089 2213 2090 /* Flush delayed data sending in the write pending list. Note that accessing 2091 * write pending list must be serialized, so a call to this function must be 2092 * protected by write mutex of SSL socket. 2093 */ 2214 /* Flush delayed data sending in the write pending list. */ 2094 2215 static pj_status_t flush_delayed_send(pj_ssl_sock_t *ssock) 2095 2216 { 2217 /* Check for another ongoing flush */ 2218 if (ssock->flushing_write_pend) 2219 return PJ_EBUSY; 2220 2221 pj_lock_acquire(ssock->write_mutex); 2222 2223 /* Again, check for another ongoing flush */ 2224 if (ssock->flushing_write_pend) { 2225 pj_lock_release(ssock->write_mutex); 2226 return PJ_EBUSY; 2227 } 2228 2229 /* Set ongoing flush flag */ 2230 ssock->flushing_write_pend = PJ_TRUE; 2231 2096 2232 while (!pj_list_empty(&ssock->write_pending)) { 2097 write_ pending_t *wp;2233 write_data_t *wp; 2098 2234 pj_status_t status; 2099 2235 2100 2236 wp = ssock->write_pending.next; 2101 2237 2102 status = ssl_write(ssock, &wp->data.key, wp->data.data.ptr, 2103 wp->data.plain_data_len, wp->data.flags); 2104 if (status != PJ_SUCCESS) 2238 /* Ticket #1573: Don't hold mutex while calling socket send. */ 2239 pj_lock_release(ssock->write_mutex); 2240 2241 status = ssl_write(ssock, &wp->key, wp->data.ptr, 2242 wp->plain_data_len, wp->flags); 2243 if (status != PJ_SUCCESS) { 2244 /* Reset ongoing flush flag first. */ 2245 ssock->flushing_write_pend = PJ_FALSE; 2105 2246 return status; 2106 2247 } 2248 2249 pj_lock_acquire(ssock->write_mutex); 2107 2250 pj_list_erase(wp); 2108 2251 pj_list_push_back(&ssock->write_pending_empty, wp); 2109 2252 } 2110 2253 2254 /* Reset ongoing flush flag */ 2255 ssock->flushing_write_pend = PJ_FALSE; 2256 2257 pj_lock_release(ssock->write_mutex); 2258 2111 2259 return PJ_SUCCESS; 2112 2260 } 2113 2261 2114 /* Sending is delayed, push back the sending data into pending list. Note that 2115 * accessing write pending list must be serialized, so a call to this function 2116 * must be protected by write mutex of SSL socket. 2117 */ 2262 /* Sending is delayed, push back the sending data into pending list. */ 2118 2263 static pj_status_t delay_send (pj_ssl_sock_t *ssock, 2119 2264 pj_ioqueue_op_key_t *send_key, … … 2122 2267 unsigned flags) 2123 2268 { 2124 write_pending_t *wp; 2269 write_data_t *wp; 2270 2271 pj_lock_acquire(ssock->write_mutex); 2125 2272 2126 2273 /* Init write pending instance */ … … 2129 2276 pj_list_erase(wp); 2130 2277 } else { 2131 wp = PJ_POOL_ZALLOC_T(ssock->pool, write_ pending_t);2132 } 2133 2134 wp-> data.app_key = send_key;2135 wp-> data.plain_data_len = size;2136 wp->data. data.ptr = data;2137 wp-> data.flags = flags;2278 wp = PJ_POOL_ZALLOC_T(ssock->pool, write_data_t); 2279 } 2280 2281 wp->app_key = send_key; 2282 wp->plain_data_len = size; 2283 wp->data.ptr = data; 2284 wp->flags = flags; 2138 2285 2139 2286 pj_list_push_back(&ssock->write_pending, wp); 2287 2288 pj_lock_release(ssock->write_mutex); 2140 2289 2141 2290 /* Must return PJ_EPENDING */ … … 2157 2306 PJ_ASSERT_RETURN(ssock->ssl_state==SSL_STATE_ESTABLISHED, PJ_EINVALIDOP); 2158 2307 2159 pj_lock_acquire(ssock->write_mutex); 2308 // Ticket #1573: Don't hold mutex while calling PJLIB socket send(). 2309 //pj_lock_acquire(ssock->write_mutex); 2160 2310 2161 2311 /* Flush delayed send first. Sending data might be delayed when … … 2164 2314 status = flush_delayed_send(ssock); 2165 2315 if (status == PJ_EBUSY) { 2166 /* Re-negotiation is on progress, delay sending */2316 /* Re-negotiation or flushing is on progress, delay sending */ 2167 2317 status = delay_send(ssock, send_key, data, *size, flags); 2168 2318 goto on_return; … … 2179 2329 2180 2330 on_return: 2181 pj_lock_release(ssock->write_mutex);2331 //pj_lock_release(ssock->write_mutex); 2182 2332 return status; 2183 2333 }
Note: See TracChangeset
for help on using the changeset viewer.