10 #include <sys/param.h> 13 #include <sys/types.h> 27 #define PCMK_IPC_VERSION 1 30 #define PCMK_IPC_DEFAULT_QUEUE_MAX 500 32 struct crm_ipc_response_header {
33 struct qb_ipc_response_header qb;
40 static int hdr_offset = 0;
41 static unsigned int ipc_buffer_max = 0;
42 static unsigned int pick_ipc_buffer(
unsigned int max);
47 if (hdr_offset == 0) {
48 hdr_offset =
sizeof(
struct crm_ipc_response_header);
50 if (ipc_buffer_max == 0) {
51 ipc_buffer_max = pick_ipc_buffer(0);
58 return pick_ipc_buffer(0);
62 generateReference(
const char *custom1,
const char *custom2)
64 static uint ref_counter = 0;
67 (custom1? custom1 :
"_empty_"),
68 (custom2? custom2 :
"_empty_"),
69 (
unsigned long)time(NULL), ref_counter++);
74 const char *host_to,
const char *sys_to,
75 const char *sys_from,
const char *uuid_from,
const char *origin)
77 char *true_from = NULL;
78 xmlNode *request = NULL;
79 char *reference = generateReference(task, sys_from);
81 if (uuid_from != NULL) {
83 }
else if (sys_from != NULL) {
84 true_from = strdup(sys_from);
86 crm_err(
"No sys from specified");
101 if (host_to != NULL && strlen(host_to) > 0) {
105 if (msg_data != NULL) {
118 create_reply_adv(xmlNode * original_request, xmlNode * xml_response_data,
const char *origin)
120 xmlNode *reply = NULL;
130 crm_err(
"Cannot create new_message, no message type in original message");
135 crm_err(
"Cannot create new_message, original message was not a request");
141 crm_err(
"Cannot create new_message, malloc failed");
157 if (host_from != NULL && strlen(host_from) > 0) {
161 if (xml_response_data != NULL) {
194 while (g_hash_table_iter_next(&iter, &key, (gpointer *) & client)) {
195 if (strcmp(client->id,
id) == 0) {
201 crm_trace(
"No client found with id=%s",
id);
210 }
else if (c->
name == NULL && c->
id == NULL) {
212 }
else if (c->
name == NULL) {
222 switch (client_type) {
227 #ifdef HAVE_GNUTLS_GNUTLS_H 252 crm_err(
"Exiting with %d active connections", active);
261 qb_ipcs_connection_t *c = NULL;
263 if (service == NULL) {
267 c = qb_ipcs_connection_first_get(service);
270 qb_ipcs_connection_t *last = c;
272 c = qb_ipcs_connection_next_get(service, last);
276 qb_ipcs_disconnect(last);
277 qb_ipcs_connection_unref(last);
292 client_from_connection(qb_ipcs_connection_t *c,
void *key, uid_t uid_client)
296 if (client == NULL) {
304 if (client->
user == NULL) {
305 client->
user = strdup(
"#unprivileged");
307 crm_err(
"Unable to enforce ACLs for user ID %d, assuming unprivileged",
320 if (client->
id == NULL) {
321 crm_err(
"Could not generate UUID for client");
343 crm_client_t *client = client_from_connection(NULL, key, 0);
352 static gid_t uid_cluster = 0;
353 static gid_t gid_cluster = 0;
359 if (uid_cluster == 0) {
361 static bool need_log = TRUE;
364 crm_warn(
"Could not find user and group IDs for user %s",
371 if (uid_client != 0) {
372 crm_trace(
"Giving access to group %u", gid_cluster);
374 qb_ipcs_connection_auth_set(c, -1, gid_cluster, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
380 client = client_from_connection(c, NULL, uid_client);
381 if (client == NULL) {
385 if ((uid_client == 0) || (uid_client == uid_cluster)) {
390 crm_debug(
"Connecting %p for uid=%d gid=%d pid=%u id=%s", c, uid_client, gid_client, client->
pid, client->
id);
395 static struct iovec *
396 pcmk__new_ipc_event()
398 struct iovec *iov = calloc(2,
sizeof(
struct iovec));
413 free(event[0].iov_base);
414 free(event[1].iov_base);
420 free_event(gpointer
data)
443 crm_trace(
"Destroying %p/%p (%d remaining)",
448 crm_trace(
"Destroying remote connection %p (%d remaining)",
490 if ((errno == 0) && (qmax_int > 0)) {
501 struct qb_ipcs_connection_stats stats;
503 stats.client_pid = 0;
504 qb_ipcs_connection_stats_get(c, &stats, 0);
505 return stats.client_pid;
512 char *uncompressed = NULL;
513 char *text = ((
char *)data) +
sizeof(
struct crm_ipc_response_header);
514 struct crm_ipc_response_header *header =
data;
517 *
id = ((
struct qb_ipc_response_header *)data)->id;
520 *flags = header->flags;
532 crm_err(
"Filtering incompatible v%d IPC message, we only support versions <= %d",
537 if (header->size_compressed) {
539 unsigned int size_u = 1 + header->size_uncompressed;
540 uncompressed = calloc(1, size_u);
542 crm_trace(
"Decompressing message data %u bytes into %u bytes",
543 header->size_compressed, size_u);
545 rc = BZ2_bzBuffToBuffDecompress(uncompressed, &size_u, text, header->size_compressed, 1, 0);
556 CRM_ASSERT(text[header->size_uncompressed - 1] == 0);
568 crm_ipcs_flush_events_cb(gpointer
data)
585 delay_next_flush(
crm_client_t *c,
unsigned int queue_len)
588 guint delay = (queue_len < 5)? (1000 + 100 * queue_len) : 1500;
590 c->
event_timer = g_timeout_add(delay, crm_ipcs_flush_events_cb, c);
597 unsigned int sent = 0;
598 unsigned int queue_len = 0;
613 struct crm_ipc_response_header *header = NULL;
614 struct iovec *
event = NULL;
624 rc = qb_ipcs_event_sendv(c->
ipcs, event, 2);
631 header =
event[0].iov_base;
632 if (header->size_compressed) {
633 crm_trace(
"Event %d to %p[%d] (%lld compressed bytes) sent",
634 header->qb.id, c->
ipcs, c->
pid, (
long long) rc);
636 crm_trace(
"Event %d to %p[%d] (%lld bytes) sent: %.120s",
637 header->qb.id, c->
ipcs, c->
pid, (
long long) rc,
638 (
char *) (event[1].iov_base));
644 if (sent > 0 || queue_len) {
645 crm_trace(
"Sent %d events (%d remaining) for %p[%d]: %s (%lld)",
646 sent, queue_len, c->
ipcs, c->
pid,
657 if ((c->
queue_backlog <= 1) || (queue_len < c->queue_backlog)) {
659 crm_warn(
"Client with process ID %u has a backlog of %u messages " 662 crm_err(
"Evicting client with process ID %u due to backlog of %u messages " 665 qb_ipcs_disconnect(c->
ipcs);
671 delay_next_flush(c, queue_len);
684 static unsigned int biggest = 0;
686 unsigned int total = 0;
687 char *compressed = NULL;
689 struct crm_ipc_response_header *header = calloc(1,
sizeof(
struct crm_ipc_response_header));
695 if (max_send_size == 0) {
696 max_send_size = ipc_buffer_max;
702 iov = pcmk__new_ipc_event();
703 iov[0].iov_len = hdr_offset;
704 iov[0].iov_base = header;
707 header->size_uncompressed = 1 + strlen(buffer);
708 total = iov[0].iov_len + header->size_uncompressed;
710 if (total < max_send_size) {
711 iov[1].iov_base = buffer;
712 iov[1].iov_len = header->size_uncompressed;
715 unsigned int new_size = 0;
718 (buffer, header->size_uncompressed, max_send_size, &compressed, &new_size)) {
721 header->size_compressed = new_size;
723 iov[1].iov_len = header->size_compressed;
724 iov[1].iov_base = compressed;
728 biggest = QB_MAX(header->size_compressed, biggest);
731 ssize_t rc = -EMSGSIZE;
734 biggest = QB_MAX(header->size_uncompressed, biggest);
737 (
"Could not compress the message (%u bytes) into less than the configured ipc limit (%u bytes). " 738 "Set PCMK_ipc_buffer to a higher value (%u bytes suggested)",
739 header->size_uncompressed, max_send_size, 4 * biggest);
747 header->qb.size = iov[0].iov_len + iov[1].iov_len;
748 header->qb.id = (
int32_t)request;
752 return header->qb.size;
760 struct crm_ipc_response_header *header = iov[0].iov_base;
772 header->flags |=
flags;
774 header->qb.id =
id++;
781 struct iovec *iov_copy = pcmk__new_ipc_event();
784 iov_copy[0].iov_len = iov[0].iov_len;
785 iov_copy[0].iov_base = malloc(iov[0].iov_len);
786 memcpy(iov_copy[0].iov_base, iov[0].iov_base, iov[0].iov_len);
788 iov_copy[1].iov_len = iov[1].iov_len;
789 iov_copy[1].iov_base = malloc(iov[1].iov_len);
790 memcpy(iov_copy[1].iov_base, iov[1].iov_base, iov[1].iov_len);
792 add_event(c, iov_copy);
798 rc = qb_ipcs_response_sendv(c->
ipcs, iov, 2);
799 if (rc < header->qb.size) {
800 crm_notice(
"Response %d to pid %d failed: %s " 801 CRM_XS " bytes=%u rc=%lld ipcs=%p",
803 header->qb.size, (
long long) rc, c->
ipcs);
806 crm_trace(
"Response %d sent, %lld bytes to %p[%d]",
807 header->qb.id, (
long long) rc, c->
ipcs, c->
pid);
815 if (flags & crm_ipc_server_event) {
821 if (rc == -EPIPE || rc == -ENOTCONN) {
832 struct iovec *iov = NULL;
836 return -EDESTADDRREQ;
869 #define MIN_MSG_SIZE 12336 870 #define MAX_MSG_SIZE 128*1024 876 unsigned int max_buf_size;
878 unsigned int buf_size;
884 qb_ipcc_connection_t *ipc;
889 pick_ipc_buffer(
unsigned int max)
891 static unsigned int global_max = 0;
893 if (global_max == 0) {
894 const char *env = getenv(
"PCMK_ipc_buffer");
906 return QB_MAX(max, global_max);
916 client->name = strdup(name);
917 client->buf_size = pick_ipc_buffer(max_size);
918 client->buffer = malloc(client->buf_size);
921 client->max_buf_size = client->buf_size;
924 client->pfd.events = POLLIN;
925 client->pfd.revents = 0;
940 client->need_reply = FALSE;
941 client->ipc = qb_ipcc_connect(client->name, client->buf_size);
943 if (client->ipc == NULL) {
949 if (client->pfd.fd < 0) {
950 crm_debug(
"Could not obtain file descriptor for %s connection: %s (%d)", client->name,
pcmk_strerror(errno), errno);
954 qb_ipcc_context_set(client->ipc, client);
956 #ifdef HAVE_IPCS_GET_BUFFER_SIZE 957 client->max_buf_size = qb_ipcc_get_buffer_size(client->ipc);
958 if (client->max_buf_size > client->buf_size) {
959 free(client->buffer);
960 client->buffer = calloc(1, client->max_buf_size);
961 client->buf_size = client->max_buf_size;
972 crm_trace(
"Disconnecting %s IPC connection %p (%p)", client->name, client, client->ipc);
975 qb_ipcc_connection_t *ipc = client->ipc;
978 qb_ipcc_disconnect(ipc);
987 if (client->ipc && qb_ipcc_is_connected(client->ipc)) {
988 crm_notice(
"Destroying an active IPC connection to %s", client->name);
999 crm_trace(
"Destroying IPC connection to %s: %p", client->name, client);
1000 free(client->buffer);
1011 if (client && client->ipc && (qb_ipcc_fd_get(client->ipc, &fd) == 0)) {
1015 crm_perror(LOG_ERR,
"Could not obtain file IPC descriptor for %s",
1016 (client? client->name :
"unspecified client"));
1025 if (client == NULL) {
1029 }
else if (client->ipc == NULL) {
1033 }
else if (client->pfd.fd < 0) {
1038 rc = qb_ipcc_is_connected(client->ipc);
1040 client->pfd.fd = -EINVAL;
1063 client->pfd.revents = 0;
1064 rc = poll(&(client->pfd), 1, 0);
1065 return (rc < 0)? -errno : rc;
1071 struct crm_ipc_response_header *header = (
struct crm_ipc_response_header *)(
void*)client->buffer;
1073 if (header->size_compressed) {
1075 unsigned int size_u = 1 + header->size_uncompressed;
1077 unsigned int new_buf_size = QB_MAX((hdr_offset + size_u), client->max_buf_size);
1078 char *uncompressed = calloc(1, new_buf_size);
1080 crm_trace(
"Decompressing message data %u bytes into %u bytes",
1081 header->size_compressed, size_u);
1083 rc = BZ2_bzBuffToBuffDecompress(uncompressed + hdr_offset, &size_u,
1084 client->buffer + hdr_offset, header->size_compressed, 1, 0);
1101 CRM_ASSERT(size_u == header->size_uncompressed);
1103 memcpy(uncompressed, client->buffer, hdr_offset);
1104 header = (
struct crm_ipc_response_header *)(
void*)uncompressed;
1106 free(client->buffer);
1107 client->buf_size = new_buf_size;
1108 client->buffer = uncompressed;
1111 CRM_ASSERT(client->buffer[hdr_offset + header->size_uncompressed - 1] == 0);
1118 struct crm_ipc_response_header *header = NULL;
1126 client->buffer[0] = 0;
1127 client->msg_size = qb_ipcc_event_recv(client->ipc, client->buffer,
1128 client->buf_size, 0);
1129 if (client->msg_size >= 0) {
1130 int rc = crm_ipc_decompress(client);
1136 header = (
struct crm_ipc_response_header *)(
void*)client->buffer;
1138 crm_err(
"Filtering incompatible v%d IPC message, we only support versions <= %d",
1143 crm_trace(
"Received %s event %d, size=%u, rc=%d, text: %.100s",
1144 client->name, header->qb.id, header->qb.size, client->msg_size,
1145 client->buffer + hdr_offset);
1152 crm_err(
"Connection to %s failed", client->name);
1157 return header->size_uncompressed;
1166 return client->buffer +
sizeof(
struct crm_ipc_response_header);
1172 struct crm_ipc_response_header *header = NULL;
1175 if (client->buffer == NULL) {
1179 header = (
struct crm_ipc_response_header *)(
void*)client->buffer;
1180 return header->flags;
1187 return client->name;
1191 internal_ipc_send_recv(
crm_ipc_t * client,
const void *iov)
1196 rc = qb_ipcc_sendv_recv(client->ipc, iov, 2, client->buffer, client->buf_size, -1);
1203 internal_ipc_send_request(
crm_ipc_t * client,
const void *iov,
int ms_timeout)
1206 time_t timeout = time(NULL) + 1 + (ms_timeout / 1000);
1209 rc = qb_ipcc_sendv(client->ipc, iov, 2);
1216 internal_ipc_get_reply(
crm_ipc_t * client,
int request_id,
int ms_timeout)
1218 time_t timeout = time(NULL) + 1 + (ms_timeout / 1000);
1224 crm_trace(
"client %s waiting on reply to msg id %d", client->name, request_id);
1227 rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, 1000);
1229 struct crm_ipc_response_header *hdr = NULL;
1231 int rc = crm_ipc_decompress(client);
1237 hdr = (
struct crm_ipc_response_header *)(
void*)client->buffer;
1238 if (hdr->qb.id == request_id) {
1241 }
else if (hdr->qb.id < request_id) {
1244 crm_err(
"Discarding old reply %d (need %d)", hdr->qb.id, request_id);
1250 crm_err(
"Discarding newer reply %d (need %d)", hdr->qb.id, request_id);
1255 crm_err(
"Server disconnected client %s while waiting for msg id %d", client->name,
1260 }
while (time(NULL) < timeout);
1272 static int factor = 8;
1273 struct crm_ipc_response_header *header;
1277 if (client == NULL) {
1283 crm_notice(
"Connection to %s closed", client->name);
1287 if (ms_timeout == 0) {
1291 if (client->need_reply) {
1292 crm_trace(
"Trying again to obtain pending reply from %s", client->name);
1293 rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, ms_timeout);
1295 crm_warn(
"Sending to %s (%p) is disabled until pending reply is received", client->name,
1300 crm_notice(
"Lost reply from %s (%p) finally arrived, sending re-enabled", client->name,
1302 client->need_reply = FALSE;
1313 header = iov[0].iov_base;
1314 header->flags |=
flags;
1321 if(header->size_compressed) {
1322 if(factor < 10 && (client->max_buf_size / 10) < (rc / factor)) {
1323 crm_notice(
"Compressed message exceeds %d0%% of the configured ipc limit (%u bytes), " 1324 "consider setting PCMK_ipc_buffer to %u or higher",
1325 factor, client->max_buf_size, 2 * client->max_buf_size);
1330 crm_trace(
"Sending from client: %s request id: %d bytes: %u timeout:%d msg...",
1331 client->name, header->qb.id, header->qb.size, ms_timeout);
1335 rc = internal_ipc_send_request(client, iov, ms_timeout);
1338 crm_trace(
"Failed to send from client %s request %d with %u bytes...",
1339 client->name, header->qb.id, header->qb.size);
1343 crm_trace(
"Message sent, not waiting for reply to %d from %s to %u bytes...",
1344 header->qb.id, client->name, header->qb.size);
1349 rc = internal_ipc_get_reply(client, header->qb.id, ms_timeout);
1358 client->need_reply = TRUE;
1362 rc = internal_ipc_send_recv(client, iov);
1366 struct crm_ipc_response_header *hdr = (
struct crm_ipc_response_header *)(
void*)client->buffer;
1368 crm_trace(
"Received response %d, size=%u, rc=%ld, text: %.200s", hdr->qb.id, hdr->qb.size,
1376 crm_trace(
"Response not received: rc=%ld, errno=%d", rc, errno);
1383 }
else if (rc == -ETIMEDOUT) {
1384 crm_warn(
"Request %d to %s (%p) failed: %s (%ld) after %dms",
1385 header->qb.id, client->name, client->ipc,
pcmk_strerror(rc), rc, ms_timeout);
1388 }
else if (rc <= 0) {
1389 crm_warn(
"Request %d to %s (%p) failed: %s (%ld)",
1390 header->qb.id, client->name, client->ipc,
pcmk_strerror(rc), rc);
1401 const char *client_name,
const char *major_version,
const char *minor_version)
1403 xmlNode *hello_node = NULL;
1404 xmlNode *hello = NULL;
1406 if (uuid == NULL || strlen(uuid) == 0
1407 || client_name == NULL || strlen(client_name) == 0
1408 || major_version == NULL || strlen(major_version) == 0
1409 || minor_version == NULL || strlen(minor_version) == 0) {
1410 crm_err(
"Missing fields, Hello message will not be valid.");
1415 crm_xml_add(hello_node,
"major_version", major_version);
1416 crm_xml_add(hello_node,
"minor_version", minor_version);
1417 crm_xml_add(hello_node,
"client_name", client_name);
#define CRM_CHECK(expr, failure_action)
const char * crm_ipc_buffer(crm_ipc_t *client)
void pcmk_free_ipc_event(struct iovec *event)
Free an I/O vector created by crm_ipc_prepare()
void crm_write_blackbox(int nsig, struct qb_log_callsite *callsite)
void crm_ipc_close(crm_ipc_t *client)
#define crm_notice(fmt, args...)
const char * pcmk_strerror(int rc)
const char * bz2_strerror(int rc)
char * crm_generate_uuid(void)
uint32_t crm_ipc_buffer_flags(crm_ipc_t *client)
int crm_ipc_send(crm_ipc_t *client, xmlNode *message, enum crm_ipc_flags flags, int32_t ms_timeout, xmlNode **reply)
crm_client_t * crm_client_get(qb_ipcs_connection_t *c)
qb_ipcs_connection_t * ipcs
const char * crm_xml_add_int(xmlNode *node, const char *name, int value)
Create an XML attribute with specified name and integer value.
long long crm_int_helper(const char *text, char **end_text)
bool crm_set_client_queue_max(crm_client_t *client, const char *qmax)
Raise IPC eviction threshold for a client, if allowed.
crm_client_t * crm_client_get_by_id(const char *id)
xmlNode * create_reply_adv(xmlNode *original_request, xmlNode *xml_response_data, const char *origin)
const char * crm_xml_add(xmlNode *node, const char *name, const char *value)
Create an XML attribute with specified name and value.
bool crm_ipc_connect(crm_ipc_t *client)
Establish an IPC connection to a Pacemaker component.
int crm_parse_int(const char *text, const char *default_text)
Parse an integer value from a string.
struct crm_remote_s * remote
int crm_user_lookup(const char *name, uid_t *uid, gid_t *gid)
#define CRM_LOG_ASSERT(expr)
void crm_client_destroy(crm_client_t *c)
int crm_ipc_get_fd(crm_ipc_t *client)
void crm_client_init(void)
#define clear_bit(word, bit)
void crm_client_disconnect_all(qb_ipcs_service_t *service)
ssize_t crm_ipc_prepare(uint32_t request, xmlNode *message, struct iovec **result, uint32_t max_send_size)
ssize_t crm_ipcs_flush_events(crm_client_t *c)
xmlNode * string2xml(const char *input)
ssize_t crm_ipcs_sendv(crm_client_t *c, struct iovec *iov, enum crm_ipc_flags flags)
const char * crm_client_type_text(enum client_type client_type)
#define crm_warn(fmt, args...)
crm_client_t * crm_client_new(qb_ipcs_connection_t *c, uid_t uid_client, gid_t gid_client)
#define set_bit(word, bit)
#define crm_debug(fmt, args...)
struct crm_ipc_s crm_ipc_t
const char * crm_element_value(const xmlNode *data, const char *name)
Retrieve the value of an XML attribute.
const char * crm_ipc_name(crm_ipc_t *client)
GHashTable * client_connections
unsigned int crm_ipc_default_buffer_size(void)
#define crm_trace(fmt, args...)
void crm_ipc_destroy(crm_ipc_t *client)
xmlNode * create_xml_node(xmlNode *parent, const char *name)
gboolean add_message_xml(xmlNode *msg, const char *field, xmlNode *xml)
void free_xml(xmlNode *child)
void crm_ipcs_send_ack(crm_client_t *c, uint32_t request, uint32_t flags, const char *tag, const char *function, int line)
unsigned int queue_backlog
xmlNode * crm_ipcs_recv(crm_client_t *c, void *data, size_t size, uint32_t *id, uint32_t *flags)
crm_client_t * crm_client_alloc(void *key)
Allocate a new crm_client_t object and generate its ID.
bool crm_compress_string(const char *data, int length, int max, char **result, unsigned int *result_len)
ssize_t crm_ipcs_send(crm_client_t *c, uint32_t request, xmlNode *message, enum crm_ipc_flags flags)
#define PCMK_IPC_DEFAULT_QUEUE_MAX
#define crm_perror(level, fmt, args...)
Log a system error message.
xmlNode * create_hello_message(const char *uuid, const char *client_name, const char *major_version, const char *minor_version)
#define crm_err(fmt, args...)
#define crm_log_xml_notice(xml, text)
char * dump_xml_unformatted(xmlNode *msg)
int crm_ipc_ready(crm_ipc_t *client)
Check whether an IPC connection is ready to be read.
#define XML_ATTR_RESPONSE
long crm_ipc_read(crm_ipc_t *client)
Wrappers for and extensions to libqb IPC.
char * generate_hash_key(const char *crm_msg_reference, const char *sys)
int crm_ipcs_client_pid(qb_ipcs_connection_t *c)
#define crm_log_xml_trace(xml, text)
void crm_client_cleanup(void)
const char * crm_client_name(crm_client_t *c)
bool crm_ipc_connected(crm_ipc_t *client)
crm_ipc_t * crm_ipc_new(const char *name, size_t max_size)
xmlNode * create_request_adv(const char *task, xmlNode *msg_data, const char *host_to, const char *sys_to, const char *sys_from, const char *uuid_from, const char *origin)
char * crm_strdup_printf(char const *format,...) __attribute__((__format__(__printf__
#define create_request(task, xml_data, host_to, sys_to, sys_from, uuid_from)
char * uid2username(uid_t uid)
enum crm_ais_msg_types type