/* * Copyright (c) 2012-2013 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include #include #include #include #include #include #include #include #include #include #include #include #define MAX_ATM_VALUES (2 * 4096) #define MAX_TRACE_BUFFER_SIZE (0x40000000) /* Restrict to 1GB per task */ #define MAX_MAILBOX_SIZE (8 * 4096) #define ATM_VALUE_TO_HANDLE(x) (CAST_DOWN(atm_voucher_id_t, (x))) #define HANDLE_TO_ATM_VALUE(x) (CAST_DOWN(atm_value_t, (x))) #define ATM_MAX_HASH_TABLE_SIZE (256) #define AID_HASH_MASK (0xFF) #define AID_TO_HASH(x) ((x) & (AID_HASH_MASK)) #define ATM_LIST_DEAD_MAX 15 #define AID_ARRAY_COUNT_MAX (256) struct atm_value_hash atm_value_hash_table[ATM_MAX_HASH_TABLE_SIZE]; extern int maxproc; /* Global flag to disable ATM. ATM get value and memory registration will return error. */ boolean_t disable_atm = FALSE; #if DEVELOPMENT || DEBUG queue_head_t atm_descriptors_list; queue_head_t atm_values_list; #endif ipc_voucher_attr_control_t voucher_attr_control; /* communication channel from ATM to voucher system */ static zone_t atm_value_zone, atm_descriptors_zone, atm_link_objects_zone; static aid_t get_aid(); static atm_value_t atm_value_alloc_init(); static void atm_value_dealloc(atm_value_t atm_value); static void atm_hash_table_init(); static void atm_value_hash_table_insert(atm_value_t new_atm_value); static void atm_value_hash_table_delete(atm_value_t atm_value); static atm_value_t get_atm_value_from_aid(aid_t aid); static void atm_value_get_ref(atm_value_t atm_value); static kern_return_t atm_listener_insert(atm_value_t atm_value, atm_task_descriptor_t task_descriptor, mailbox_offset_t mailbox_offset); static void atm_listener_delete_all(atm_value_t atm_value); static atm_task_descriptor_t atm_task_descriptor_alloc_init(mach_port_t trace_buffer,uint64_t buffer_size, void *mailbox_addr, uint64_t mailbox_array_size, __assert_only task_t task); static void atm_descriptor_get_reference(atm_task_descriptor_t task_descriptor); static void atm_task_descriptor_dealloc(atm_task_descriptor_t task_descriptor); static mach_atm_subaid_t atm_get_min_sub_aid(atm_value_t atm_value); static void atm_get_min_sub_aid_array(aid_t *aid_array, mach_atm_subaid_t *subaid_array, uint32_t count) __unused; static kern_return_t atm_value_unregister(atm_value_t atm_value, atm_task_descriptor_t task_descriptor, mailbox_offset_t mailbox_offset); static kern_return_t atm_listener_delete(atm_value_t atm_value, atm_task_descriptor_t task_descriptor, mailbox_offset_t mailbox_offset); static void atm_link_get_reference(atm_link_object_t link_object); static void atm_link_dealloc(atm_link_object_t link_object); kern_return_t atm_invoke_collection(atm_value_t atm_value, uint64_t sub_activity_id, uint32_t flags); kern_return_t atm_send_user_notification(aid_t aid, uint64_t subaid, mach_port_t *buffers_array, uint64_t *sizes_array, mach_msg_type_number_t count, uint32_t flags); kern_return_t atm_release_value( ipc_voucher_attr_manager_t __assert_only manager, mach_voucher_attr_key_t __assert_only key, mach_voucher_attr_value_handle_t value, mach_voucher_attr_value_reference_t sync); kern_return_t atm_get_value( ipc_voucher_attr_manager_t __assert_only manager, mach_voucher_attr_key_t __assert_only key, mach_voucher_attr_recipe_command_t command, mach_voucher_attr_value_handle_array_t prev_values, mach_msg_type_number_t __assert_only prev_value_count, mach_voucher_attr_content_t recipe, mach_voucher_attr_content_size_t recipe_size, mach_voucher_attr_value_handle_t *out_value, ipc_voucher_t *out_value_voucher); kern_return_t atm_extract_content( ipc_voucher_attr_manager_t __assert_only manager, mach_voucher_attr_key_t __assert_only key, mach_voucher_attr_value_handle_array_t values, mach_msg_type_number_t value_count, mach_voucher_attr_recipe_command_t *out_command, mach_voucher_attr_content_t out_recipe, mach_voucher_attr_content_size_t *in_out_recipe_size); kern_return_t atm_command( ipc_voucher_attr_manager_t __assert_only manager, mach_voucher_attr_key_t __assert_only key, mach_voucher_attr_value_handle_array_t values, mach_msg_type_number_t value_count, mach_voucher_attr_command_t command, mach_voucher_attr_content_t in_content, mach_voucher_attr_content_size_t in_content_size, mach_voucher_attr_content_t out_content, mach_voucher_attr_content_size_t *in_out_content_size); void atm_release(ipc_voucher_attr_manager_t __assert_only manager); /* * communication channel from voucher system to ATM */ struct ipc_voucher_attr_manager atm_manager = { .ivam_release_value = atm_release_value, .ivam_get_value = atm_get_value, .ivam_extract_content = atm_extract_content, .ivam_command = atm_command, .ivam_release = atm_release, }; #if DEVELOPMENT || DEBUG decl_lck_mtx_data(, atm_descriptors_list_lock); decl_lck_mtx_data(, atm_values_list_lock); lck_grp_t atm_dev_lock_grp; lck_attr_t atm_dev_lock_attr; lck_grp_attr_t atm_dev_lock_grp_attr; #endif extern vm_map_t kernel_map; /* * Global aid. Incremented on each get_aid. */ aid_t global_aid; /* * Lock group attributes for atm sub system. */ lck_grp_t atm_lock_grp; lck_attr_t atm_lock_attr; lck_grp_attr_t atm_lock_grp_attr; /* * Routine: atm_init * Purpose: Initialize the atm subsystem. * Returns: None. */ void atm_init() { kern_return_t kr = KERN_SUCCESS; char temp_buf[20]; /* Disable atm if disable_atm present in device-tree properties or in boot-args */ if ((PE_get_default("kern.disable_atm", temp_buf, sizeof(temp_buf))) || (PE_parse_boot_argn("-disable_atm", temp_buf, sizeof(temp_buf)))) { disable_atm = TRUE; } /* setup zones for descriptors, values and link objects */ atm_value_zone = zinit(sizeof(struct atm_value), MAX_ATM_VALUES * sizeof(struct atm_value), sizeof(struct atm_value), "atm_values"); atm_descriptors_zone = zinit(sizeof(struct atm_task_descriptor), MAX_ATM_VALUES * sizeof(struct atm_task_descriptor), sizeof(struct atm_task_descriptor), "atm_task_descriptors"); atm_link_objects_zone = zinit(sizeof(struct atm_link_object), MAX_ATM_VALUES * sizeof(struct atm_link_object), sizeof(struct atm_link_object), "atm_link_objects"); /* Initialize atm lock group and lock attributes. */ lck_grp_attr_setdefault(&atm_lock_grp_attr); lck_grp_init(&atm_lock_grp, "atm_lock", &atm_lock_grp_attr); lck_attr_setdefault(&atm_lock_attr); global_aid = 1; atm_hash_table_init(); #if DEVELOPMENT || DEBUG /* Initialize global atm development lock group and lock attributes. */ lck_grp_attr_setdefault(&atm_dev_lock_grp_attr); lck_grp_init(&atm_dev_lock_grp, "atm_dev_lock", &atm_dev_lock_grp_attr); lck_attr_setdefault(&atm_dev_lock_attr); lck_mtx_init(&atm_descriptors_list_lock, &atm_dev_lock_grp, &atm_dev_lock_attr); lck_mtx_init(&atm_values_list_lock, &atm_dev_lock_grp, &atm_dev_lock_attr); queue_init(&atm_descriptors_list); queue_init(&atm_values_list); #endif /* Register the atm manager with the Vouchers sub system. */ kr = ipc_register_well_known_mach_voucher_attr_manager( &atm_manager, 0, MACH_VOUCHER_ATTR_KEY_ATM, &voucher_attr_control); if (kr != KERN_SUCCESS ) panic("ATM subsystem initialization failed"); kprintf("ATM subsystem is initialized\n"); return ; } /* * ATM Resource Manager Routines. */ /* * Routine: atm_release_value * Purpose: Release a value, if sync matches the sync count in value. * Returns: KERN_SUCCESS: on Successful deletion. * KERN_FAILURE: if sync value does not matches. */ kern_return_t atm_release_value( ipc_voucher_attr_manager_t __assert_only manager, mach_voucher_attr_key_t __assert_only key, mach_voucher_attr_value_handle_t value, mach_voucher_attr_value_reference_t sync) { atm_value_t atm_value = ATM_VALUE_NULL; assert(MACH_VOUCHER_ATTR_KEY_ATM == key); assert(manager == &atm_manager); atm_value = HANDLE_TO_ATM_VALUE(value); if (atm_value == VAM_DEFAULT_VALUE) { /* Return success for default value */ return KERN_SUCCESS; } if (atm_value->sync != sync) { return KERN_FAILURE; } /* Deallocate the atm value. */ atm_value_hash_table_delete(atm_value); atm_value_dealloc(atm_value); return KERN_SUCCESS; } /* * Routine: atm_get_value */ kern_return_t atm_get_value( ipc_voucher_attr_manager_t __assert_only manager, mach_voucher_attr_key_t __assert_only key, mach_voucher_attr_recipe_command_t command, mach_voucher_attr_value_handle_array_t prev_values, mach_msg_type_number_t __assert_only prev_value_count, mach_voucher_attr_content_t __unused recipe, mach_voucher_attr_content_size_t __unused recipe_size, mach_voucher_attr_value_handle_t *out_value, ipc_voucher_t *out_value_voucher) { atm_value_t atm_value = ATM_VALUE_NULL; mach_voucher_attr_value_handle_t atm_handle; atm_task_descriptor_t task_descriptor = ATM_TASK_DESCRIPTOR_NULL; task_t task; mailbox_offset_t mailbox_offset; natural_t i; kern_return_t kr = KERN_SUCCESS; assert(MACH_VOUCHER_ATTR_KEY_ATM == key); assert(manager == &atm_manager); /* never an out voucher */ *out_value_voucher = IPC_VOUCHER_NULL; if (disable_atm) return KERN_NOT_SUPPORTED; switch (command) { case MACH_VOUCHER_ATTR_ATM_REGISTER: for (i = 0; i < prev_value_count; i++) { atm_handle = prev_values[i]; atm_value = HANDLE_TO_ATM_VALUE(atm_handle); if (atm_value == VAM_DEFAULT_VALUE) continue; task = current_task(); task_descriptor = task->atm_context; if (task_descriptor != ATM_TASK_DESCRIPTOR_NULL) { if (recipe_size != sizeof(mailbox_offset_t)) { kr = KERN_INVALID_ARGUMENT; break; } memcpy(&mailbox_offset, recipe, sizeof(mailbox_offset_t)); if (mailbox_offset > task_descriptor->mailbox_array_size) { kr = KERN_INVALID_ARGUMENT; break; } kr = atm_listener_insert(atm_value, task_descriptor, mailbox_offset); if (kr != KERN_SUCCESS) { break; } } /* Increment sync value. */ lck_mtx_lock(&atm_value->listener_lock); atm_value->sync++; lck_mtx_unlock(&atm_value->listener_lock); *out_value = atm_handle; return kr; } *out_value = ATM_VALUE_TO_HANDLE(VAM_DEFAULT_VALUE); break; case MACH_VOUCHER_ATTR_ATM_CREATE: /* Allocate a new atm value. */ atm_value = atm_value_alloc_init(); atm_value_hash_table_insert(atm_value); if (atm_value == ATM_VALUE_NULL) { return KERN_RESOURCE_SHORTAGE; } *out_value = ATM_VALUE_TO_HANDLE(atm_value); break; case MACH_VOUCHER_ATTR_ATM_NULL: default: kr = KERN_INVALID_ARGUMENT; break; } return kr; } /* * Routine: atm_extract_content * Purpose: Extract a set of aid from an array of voucher values. * Returns: KERN_SUCCESS: on Success. * KERN_FAILURE: one of the value is not present in the hash. * KERN_NO_SPACE: insufficeint buffer provided to fill an array of aid. */ kern_return_t atm_extract_content( ipc_voucher_attr_manager_t __assert_only manager, mach_voucher_attr_key_t __assert_only key, mach_voucher_attr_value_handle_array_t values, mach_msg_type_number_t value_count, mach_voucher_attr_recipe_command_t *out_command, mach_voucher_attr_content_t out_recipe, mach_voucher_attr_content_size_t *in_out_recipe_size) { atm_value_t atm_value; mach_voucher_attr_value_handle_t atm_handle; natural_t i; assert(MACH_VOUCHER_ATTR_KEY_ATM == key); assert(manager == &atm_manager); for (i = 0; i < value_count; i++) { atm_handle = values[i]; atm_value = HANDLE_TO_ATM_VALUE(atm_handle); if (atm_value == VAM_DEFAULT_VALUE) continue; if (( sizeof(aid_t)) > *in_out_recipe_size) { *in_out_recipe_size = 0; return KERN_NO_SPACE; } memcpy(&out_recipe[0], &atm_value->aid, sizeof(aid_t)); *out_command = MACH_VOUCHER_ATTR_ATM_NULL; *in_out_recipe_size = sizeof(aid_t); return KERN_SUCCESS; } *in_out_recipe_size = 0; return KERN_SUCCESS; } /* * Routine: atm_command * Purpose: Execute a command against a set of ATM values. * Returns: KERN_SUCCESS: On successful execution of command. KERN_FAILURE: On failure. */ kern_return_t atm_command( ipc_voucher_attr_manager_t __assert_only manager, mach_voucher_attr_key_t __assert_only key, mach_voucher_attr_value_handle_array_t values, mach_msg_type_number_t value_count, mach_voucher_attr_command_t command, mach_voucher_attr_content_t in_content, mach_voucher_attr_content_size_t in_content_size, mach_voucher_attr_content_t out_content, mach_voucher_attr_content_size_t *out_content_size) { assert(MACH_VOUCHER_ATTR_KEY_ATM == key); assert(manager == &atm_manager); atm_value_t atm_value = ATM_VALUE_NULL; natural_t i = 0; aid_t *aid_array = NULL; mach_atm_subaid_t *subaid_array = NULL; uint32_t aid_array_count = 0; atm_task_descriptor_t task_descriptor = ATM_TASK_DESCRIPTOR_NULL; task_t task; uint32_t collection_flags = ATM_ACTION_LOGFAIL; kern_return_t kr = KERN_SUCCESS; switch (command) { case ATM_ACTION_COLLECT: collection_flags = ATM_ACTION_COLLECT; /* Fall through */ case ATM_ACTION_LOGFAIL: { mach_atm_subaid_t sub_aid = 0; /* find the first non-default atm_value */ for (i = 0; i < value_count; i++) { atm_value = HANDLE_TO_ATM_VALUE(values[i]); if (atm_value != VAM_DEFAULT_VALUE) break; } /* if we are not able to find any atm values * in stack then this call was made in error */ if (atm_value == NULL) { return KERN_FAILURE; } if (in_content == NULL || in_content_size < sizeof(mach_atm_subaid_t) ){ return KERN_INVALID_ARGUMENT; } sub_aid = *(mach_atm_subaid_t *)(void *)in_content; *out_content_size = 0; kr = atm_invoke_collection(atm_value, sub_aid, collection_flags); break; } case ATM_FIND_MIN_SUB_AID: if ((in_content_size/sizeof(aid_t)) > (*out_content_size/sizeof(mach_atm_subaid_t))) return KERN_FAILURE; aid_array_count = in_content_size / sizeof(aid_t); if (aid_array_count > AID_ARRAY_COUNT_MAX) return KERN_FAILURE; aid_array = (aid_t *) kalloc(aid_array_count * sizeof(aid_t)); if (aid_array == NULL) return KERN_NO_SPACE; subaid_array = (mach_atm_subaid_t *) kalloc(aid_array_count * sizeof(mach_atm_subaid_t)); if (subaid_array == NULL) { kfree(aid_array, aid_array_count * sizeof(aid_t)); return KERN_NO_SPACE; } memcpy(aid_array, in_content, aid_array_count * sizeof(aid_t)); atm_get_min_sub_aid_array(aid_array, subaid_array, aid_array_count); memcpy(out_content, subaid_array, aid_array_count * sizeof(mach_atm_subaid_t)); *out_content_size = aid_array_count * sizeof(mach_atm_subaid_t); kfree(aid_array, aid_array_count * sizeof(aid_t)); kfree(subaid_array, aid_array_count * sizeof(mach_atm_subaid_t)); kr = KERN_SUCCESS; break; case ATM_ACTION_UNREGISTER: /* find the first non-default atm_value */ for (i = 0; i < value_count; i++) { atm_value = HANDLE_TO_ATM_VALUE(values[i]); if (atm_value != VAM_DEFAULT_VALUE) break; } /* if we are not able to find any atm values * in stack then this call was made in error */ if (atm_value == NULL) { return KERN_FAILURE; } if (in_content == NULL || in_content_size != sizeof(mailbox_offset_t)){ return KERN_INVALID_ARGUMENT; } mailbox_offset_t mailbox_offset; memcpy(&mailbox_offset, in_content, sizeof(mailbox_offset_t)); task = current_task(); task_descriptor = task->atm_context; kr = atm_value_unregister(atm_value, task_descriptor, mailbox_offset); break; default: kr = KERN_INVALID_ARGUMENT; break; } return kr; } void atm_release( ipc_voucher_attr_manager_t __assert_only manager) { assert(manager == &atm_manager); } /* * Routine: atm_invoke_collection * Purpose: Sends a notification with array of memory buffer. * Note: may block till user daemon responds. */ kern_return_t atm_invoke_collection( atm_value_t atm_value, subaid_t sub_activity_id, uint32_t flags) { aid_t aid = atm_value->aid; kern_return_t kr = KERN_SUCCESS; uint32_t array_count = 0, i = 0, requestor_index = 0; uint64_t *sizes_array = NULL; atm_link_object_t link_object = NULL; mach_port_t *mem_array = NULL; boolean_t need_swap_first = FALSE; atm_task_descriptor_t requesting_descriptor = current_task()->atm_context; lck_mtx_lock(&atm_value->listener_lock); array_count = atm_value->listener_count; lck_mtx_unlock(&atm_value->listener_lock); if (array_count == 0){ return KERN_SUCCESS; } mem_array = kalloc(sizeof(mach_port_t) * array_count); if (mem_array == NULL){ return KERN_NO_SPACE; } sizes_array = kalloc(sizeof(uint64_t) * array_count); if (sizes_array == NULL){ kfree(mem_array, sizeof(mach_port_t) * array_count); return KERN_NO_SPACE; } lck_mtx_lock(&atm_value->listener_lock); queue_iterate(&atm_value->listeners, link_object, atm_link_object_t, listeners_element) { if (i >= array_count){ break; } if (!need_swap_first && requesting_descriptor == link_object->descriptor){ assert(requesting_descriptor != NULL); requestor_index = i; need_swap_first = TRUE; } sizes_array[i] = link_object->descriptor->trace_buffer_size; mem_array[i] = ipc_port_copy_send(link_object->descriptor->trace_buffer); if (!IPC_PORT_VALID(mem_array[i])){ mem_array[i] = NULL; } i++; } lck_mtx_unlock(&atm_value->listener_lock); /* * Swap the position of requesting task ahead, diagnostics can * process its buffers the first. */ if (need_swap_first && requestor_index != 0){ assert(requestor_index < array_count); mach_port_t tmp_port = mem_array[0]; uint64_t tmp_size = sizes_array[0]; mem_array[0] = mem_array[requestor_index]; sizes_array[0] = sizes_array[requestor_index]; mem_array[requestor_index] = tmp_port; sizes_array[requestor_index] = tmp_size; } if (i > 0) { kr = atm_send_user_notification(aid, sub_activity_id, mem_array, sizes_array, i, flags); } kfree(mem_array, sizeof(mach_port_t) * array_count); kfree(sizes_array, sizeof(uint64_t) * array_count); return kr; } /* * Routine: atm_send_user_notification * Purpose: Make an upcall to user space daemon if its listening for atm notifications. * Returns: KERN_SUCCESS for successful delivery. * KERN_FAILURE if port is dead or NULL. */ kern_return_t atm_send_user_notification( aid_t aid, subaid_t subaid, mach_port_t *buffers_array, uint64_t *sizes_array, mach_msg_type_number_t count, uint32_t flags) { mach_port_t user_port; int error; error = host_get_atm_notification_port(host_priv_self(), &user_port); if ((error != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) { return KERN_FAILURE; } return atm_collect_trace_info(user_port, aid, subaid, flags, buffers_array, count, sizes_array, count); } /* * Routine: atm_send_proc_inspect_notification * Purpose: Make an upcall to user space daemon if its listening for trace * notifications for per process inspection. * Returns: KERN_SUCCESS for successful delivery. * KERN_FAILURE if port is dead or NULL. */ kern_return_t atm_send_proc_inspect_notification( task_t task, int32_t traced_pid, uint64_t traced_uniqueid) { mach_port_t user_port = MACH_PORT_NULL; mach_port_t memory_port = MACH_PORT_NULL; atm_task_descriptor_t task_descriptor = ATM_TASK_DESCRIPTOR_NULL; uint64_t buffer_size = 0; int error; /* look for the requested memory in target task */ if (!task) return KERN_INVALID_TASK; task_lock(task); if (task->atm_context){ task_descriptor = task->atm_context; atm_descriptor_get_reference(task_descriptor); } task_unlock(task); if (task_descriptor == ATM_TASK_DESCRIPTOR_NULL){ return KERN_FAILURE; } memory_port = ipc_port_copy_send(task_descriptor->trace_buffer); buffer_size = task_descriptor->trace_buffer_size; atm_task_descriptor_dealloc(task_descriptor); /* get the communication port */ error = host_get_atm_notification_port(host_priv_self(), &user_port); if ((error != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) { ipc_port_release_send(memory_port); return KERN_FAILURE; } return atm_inspect_process_buffer(user_port, traced_pid, traced_uniqueid, buffer_size, memory_port); } /* * Routine: atm_value_alloc_init * Purpose: Allocates an atm value struct and initialize it. * Returns: atm_value_t: On Success with a sync count on atm_value. * ATM_VALUE_NULL: On failure. */ static atm_value_t atm_value_alloc_init() { atm_value_t new_atm_value = ATM_VALUE_NULL; new_atm_value = (atm_value_t) zalloc(atm_value_zone); if (new_atm_value == ATM_VALUE_NULL) panic("Ran out of ATM values structure.\n\n"); new_atm_value->aid = get_aid(); queue_init(&new_atm_value->listeners); new_atm_value->sync = 1; new_atm_value->listener_count = 0; new_atm_value->reference_count = 1; lck_mtx_init(&new_atm_value->listener_lock, &atm_lock_grp, &atm_lock_attr); #if DEVELOPMENT || DEBUG lck_mtx_lock(&atm_values_list_lock); queue_enter(&atm_values_list, new_atm_value, atm_value_t, value_elt); lck_mtx_unlock(&atm_values_list_lock); #endif return new_atm_value; } /* * Routine: get_aid * Purpose: Increment the global aid counter and return it. * Returns: aid */ static aid_t get_aid() { aid_t aid; aid = (aid_t)OSIncrementAtomic64((SInt64 *)&global_aid); return aid; } /* * Routine: atm_value_dealloc * Purpose: Drops the reference on atm value and deallocates. * Deletes all the listeners on deallocation. * Returns: None. */ static void atm_value_dealloc(atm_value_t atm_value) { lck_mtx_lock(&atm_value->listener_lock); atm_value->reference_count--; assert(atm_value->reference_count >= 0); if (atm_value->reference_count > 0) { lck_mtx_unlock(&atm_value->listener_lock); return; } lck_mtx_unlock(&atm_value->listener_lock); /* Free up the atm value and also remove all the listeners. */ atm_listener_delete_all(atm_value); lck_mtx_destroy(&atm_value->listener_lock, &atm_lock_grp); #if DEVELOPMENT || DEBUG lck_mtx_lock(&atm_values_list_lock); queue_remove(&atm_values_list, atm_value, atm_value_t, value_elt); lck_mtx_unlock(&atm_values_list_lock); #endif zfree(atm_value_zone, atm_value); return; } /* * Routine: atm_hash_table_init * Purpose: Initialize the atm aid hash table. * Returns: None. */ static void atm_hash_table_init() { int i; for (i = 0; i < ATM_MAX_HASH_TABLE_SIZE; i++) { queue_init(&atm_value_hash_table[i].hash_list); lck_mtx_init(&atm_value_hash_table[i].hash_list_lock, &atm_lock_grp, &atm_lock_attr); } } /* * Routine: atm_value_hash_table_insert * Purpose: Insert an atm value in the hash table. * Returns: None. */ static void atm_value_hash_table_insert(atm_value_t new_atm_value) { int hash_index; atm_value_hash_t hash_list_head; aid_t aid = new_atm_value->aid; hash_index = AID_TO_HASH(aid); hash_list_head = &atm_value_hash_table[hash_index]; lck_mtx_lock(&hash_list_head->hash_list_lock); queue_enter(&hash_list_head->hash_list, new_atm_value, atm_value_t, vid_hash_elt); lck_mtx_unlock(&hash_list_head->hash_list_lock); } /* * Routine: atm_value_hash_table_delete * Purpose: Delete the atm value from the hash table. * Returns: None. */ static void atm_value_hash_table_delete(atm_value_t atm_value) { int hash_index; atm_value_hash_t hash_list_head; aid_t aid = atm_value->aid; hash_index = AID_TO_HASH(aid); hash_list_head = &atm_value_hash_table[hash_index]; lck_mtx_lock(&hash_list_head->hash_list_lock); queue_remove(&hash_list_head->hash_list, atm_value, atm_value_t, vid_hash_elt); lck_mtx_unlock(&hash_list_head->hash_list_lock); } /* * Routine: get_atm_value_from_aid * Purpose: Search a given aid in atm value hash table and * return the atm value stucture. * Returns: atm value structure if aid found. * ATM_VALUE_NULL: If aid not found in atm value hash table. */ static atm_value_t get_atm_value_from_aid(aid_t aid) { int hash_index; atm_value_hash_t hash_list_head; atm_value_t next; hash_index = AID_TO_HASH(aid); hash_list_head = &atm_value_hash_table[hash_index]; /* Lock the atm list and search for the aid. */ lck_mtx_lock(&hash_list_head->hash_list_lock); queue_iterate(&hash_list_head->hash_list, next, atm_value_t, vid_hash_elt) { if (next->aid == aid) { /* * Aid found. Incerease ref count and return * the atm value structure. */ atm_value_get_ref(next); lck_mtx_unlock(&hash_list_head->hash_list_lock); return (next); } } lck_mtx_unlock(&hash_list_head->hash_list_lock); return ATM_VALUE_NULL; } /* * Routine: atm_value_get_ref * Purpose: Get a reference on atm value. * Returns: None. */ static void atm_value_get_ref(atm_value_t atm_value) { lck_mtx_lock(&atm_value->listener_lock); atm_value->reference_count++; lck_mtx_unlock(&atm_value->listener_lock); } /* * Routine: atm_listener_insert * Purpose: Insert a listener to an atm value. * Returns: KERN_SUCCESS on success. * KERN_FAILURE if the task is already present as a listener. */ static kern_return_t atm_listener_insert( atm_value_t atm_value, atm_task_descriptor_t task_descriptor, mailbox_offset_t mailbox_offset) { atm_link_object_t new_link_object; atm_link_object_t next; void *mailbox = (void *)((char *)task_descriptor->mailbox_kernel_addr + mailbox_offset); new_link_object = (atm_link_object_t) zalloc(atm_link_objects_zone); new_link_object->descriptor = task_descriptor; new_link_object->reference_count = 1; new_link_object->flags = 0; new_link_object->mailbox = mailbox; /* Get a reference on the task descriptor */ atm_descriptor_get_reference(task_descriptor); /* Check if the task mailbox is already on the listener list */ lck_mtx_lock(&atm_value->listener_lock); queue_iterate(&atm_value->listeners, next, atm_link_object_t, listeners_element) { if (next->descriptor == task_descriptor) { /* * Replace the mailbox with the new one, the old mailbox is anyways on unregister path. * There is a race when get_min_sub_aid would cache the mailbox, and this function will * replace it. It would just behave as if the get value call happened after get_min_sub_aid * was already completed. */ next->mailbox = mailbox; lck_mtx_unlock(&atm_value->listener_lock); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_GETVALUE_INFO, (ATM_VALUE_REPLACED))) | DBG_FUNC_NONE, atm_value, atm_value->aid, mailbox_offset, 0, 0); /* Drop the extra reference on task descriptor taken by this function. */ atm_task_descriptor_dealloc(task_descriptor); zfree(atm_link_objects_zone, new_link_object); return KERN_SUCCESS; } } KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_GETVALUE_INFO, (ATM_VALUE_ADDED))) | DBG_FUNC_NONE, atm_value, atm_value->aid, mailbox_offset, 0, 0); queue_enter(&atm_value->listeners, new_link_object, atm_link_object_t, listeners_element); atm_value->listener_count++; lck_mtx_unlock(&atm_value->listener_lock); return KERN_SUCCESS; } /* * Routine: atm_listener_delete_all * Purpose: Deletes all the listeners for an atm value. * Returns: None. */ static void atm_listener_delete_all(atm_value_t atm_value) { atm_link_object_t next; while(!queue_empty(&atm_value->listeners)) { queue_remove_first(&atm_value->listeners, next, atm_link_object_t, listeners_element); /* Drops the reference on the link object */ atm_link_dealloc(next); } } /* * Routine: atm_listener_delete * Purpose: Deletes a listerner for an atm value. * Returns: KERN_SUCCESS on successful unregister. * KERN_INVALID_VALUE on finding a different mailbox. * KERN_FAILURE on failure. */ static kern_return_t atm_listener_delete( atm_value_t atm_value, atm_task_descriptor_t task_descriptor, mailbox_offset_t mailbox_offset) { queue_head_t free_listeners; atm_link_object_t next, elem; void *mailbox = (void *)((char *)task_descriptor->mailbox_kernel_addr + mailbox_offset); kern_return_t kr = KERN_FAILURE; queue_init(&free_listeners); lck_mtx_lock(&atm_value->listener_lock); next = (atm_link_object_t)(void *) queue_first(&atm_value->listeners); while (!queue_end(&atm_value->listeners, (queue_entry_t)next)) { elem = next; next = (atm_link_object_t)(void *) queue_next(&next->listeners_element); if (elem->descriptor == task_descriptor) { if (elem->mailbox == mailbox) { KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_UNREGISTER_INFO, (ATM_VALUE_UNREGISTERED))) | DBG_FUNC_NONE, atm_value, atm_value->aid, mailbox_offset, 0, 0); queue_remove(&atm_value->listeners, elem, atm_link_object_t, listeners_element); queue_enter(&free_listeners, elem, atm_link_object_t, listeners_element); atm_value->listener_count--; kr = KERN_SUCCESS; break; } else { KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_UNREGISTER_INFO, (ATM_VALUE_DIFF_MAILBOX))) | DBG_FUNC_NONE, atm_value, atm_value->aid, 0, 0, 0); kr = KERN_INVALID_VALUE; break; } } } lck_mtx_unlock(&atm_value->listener_lock); while(!queue_empty(&free_listeners)) { queue_remove_first(&free_listeners, next, atm_link_object_t, listeners_element); /* Drops the reference on the link object */ atm_link_dealloc(next); } return kr; } /* * Routine: atm_descriptor_alloc_init * Purpose: Allocate an atm task descriptor and initialize it and takes a reference. * Returns: atm task descriptor: On success. * NULL: on error. */ static atm_task_descriptor_t atm_task_descriptor_alloc_init( mach_port_t trace_buffer, uint64_t buffer_size, void * mailbox_addr, uint64_t mailbox_array_size, task_t __assert_only task) { atm_task_descriptor_t new_task_descriptor; new_task_descriptor = (atm_task_descriptor_t) zalloc(atm_descriptors_zone); new_task_descriptor->trace_buffer = trace_buffer; new_task_descriptor->trace_buffer_size = buffer_size; new_task_descriptor->mailbox_array_size = mailbox_array_size; new_task_descriptor->mailbox_kernel_addr = mailbox_addr; new_task_descriptor->reference_count = 1; new_task_descriptor->flags = 0; lck_mtx_init(&new_task_descriptor->lock, &atm_lock_grp, &atm_lock_attr); #if DEVELOPMENT || DEBUG new_task_descriptor->task = task; lck_mtx_lock(&atm_descriptors_list_lock); queue_enter(&atm_descriptors_list, new_task_descriptor, atm_task_descriptor_t, descriptor_elt); lck_mtx_unlock(&atm_descriptors_list_lock); #endif return new_task_descriptor; } /* * Routine: atm_descriptor_get_reference * Purpose: Get a reference count on task descriptor. * Returns: None. */ static void atm_descriptor_get_reference(atm_task_descriptor_t task_descriptor) { lck_mtx_lock(&task_descriptor->lock); task_descriptor->reference_count++; lck_mtx_unlock(&task_descriptor->lock); } /* * Routine: atm_task_descriptor_dealloc * Prupose: Drops the reference on atm descriptor. * Returns: None. */ static void atm_task_descriptor_dealloc(atm_task_descriptor_t task_descriptor) { lck_mtx_lock(&task_descriptor->lock); task_descriptor->reference_count--; assert(task_descriptor->reference_count >= 0); if (task_descriptor->reference_count > 0) { lck_mtx_unlock(&task_descriptor->lock); return; } #if DEVELOPMENT || DEBUG lck_mtx_lock(&atm_descriptors_list_lock); queue_remove(&atm_descriptors_list, task_descriptor, atm_task_descriptor_t, descriptor_elt); lck_mtx_unlock(&atm_descriptors_list_lock); #endif mach_vm_deallocate(kernel_map, (mach_vm_address_t)task_descriptor->mailbox_kernel_addr, task_descriptor->mailbox_array_size); task_descriptor->mailbox_kernel_addr = NULL; task_descriptor->mailbox_array_size = 0; /* release the send right for the named memory entry */ ipc_port_release_send(task_descriptor->trace_buffer); lck_mtx_unlock(&task_descriptor->lock); lck_mtx_destroy(&task_descriptor->lock, &atm_lock_grp); zfree(atm_descriptors_zone, task_descriptor); return; } /* * Routine: atm_link_get_reference * Purpose: Get a reference count on atm link object. * Returns: None. */ static void atm_link_get_reference(atm_link_object_t link_object) { atm_link_object_reference_internal(link_object); } /* * Routine: atm_link_dealloc * Prupose: Drops the reference on link object. * Returns: None. */ static void atm_link_dealloc(atm_link_object_t link_object) { if (0 < atm_link_object_release_internal(link_object)) { return; } assert(link_object->reference_count == 0); /* Drop the reference on atm task descriptor. */ atm_task_descriptor_dealloc(link_object->descriptor); zfree(atm_link_objects_zone, link_object); } /* * Routine: atm_register_trace_memory * Purpose: Registers trace memory for a task. * Returns: KERN_SUCCESS: on Success. * KERN_FAILURE: on Error. */ kern_return_t atm_register_trace_memory( task_t task, uint64_t trace_buffer_address, uint64_t buffer_size, uint64_t mailbox_array_size) { atm_task_descriptor_t task_descriptor; mach_port_t trace_buffer = MACH_PORT_NULL; mach_vm_offset_t mailbox_kernel_ptr = 0; kern_return_t kr = KERN_SUCCESS; if (disable_atm) return KERN_NOT_SUPPORTED; if (task != current_task()) return KERN_INVALID_ARGUMENT; if (task->atm_context != NULL || (void *)trace_buffer_address == NULL || buffer_size == 0 || (buffer_size & PAGE_MASK) != 0 || buffer_size > MAX_TRACE_BUFFER_SIZE || mailbox_array_size == 0 || mailbox_array_size >= buffer_size || mailbox_array_size > MAX_MAILBOX_SIZE || mailbox_array_size & PAGE_MIN_MASK) { return KERN_INVALID_ARGUMENT; } vm_map_t map = current_map(); memory_object_size_t mo_size = (memory_object_size_t) buffer_size; kr = mach_make_memory_entry_64(map, &mo_size, (mach_vm_offset_t)trace_buffer_address, VM_PROT_READ, &trace_buffer, NULL); if (kr != KERN_SUCCESS) return kr; kr = mach_vm_map(kernel_map, &mailbox_kernel_ptr, mailbox_array_size, 0, VM_FLAGS_ANYWHERE, trace_buffer, 0, FALSE, VM_PROT_READ, VM_PROT_READ, VM_INHERIT_NONE ); if (kr != KERN_SUCCESS){ ipc_port_release_send(trace_buffer); return kr; } task_descriptor = atm_task_descriptor_alloc_init(trace_buffer, buffer_size, (void *)mailbox_kernel_ptr, mailbox_array_size, task); if (task_descriptor == ATM_TASK_DESCRIPTOR_NULL) { ipc_port_release_send(trace_buffer); mach_vm_deallocate(kernel_map, (mach_vm_address_t)mailbox_kernel_ptr, mailbox_array_size); return KERN_NO_SPACE; } task_lock(task); if (task->atm_context == NULL) { task->atm_context = task_descriptor; kr = KERN_SUCCESS; } else { kr = KERN_FAILURE; } task_unlock(task); if (kr != KERN_SUCCESS) { /* undo the mapping and allocations since we failed to hook descriptor to task */ atm_task_descriptor_dealloc(task_descriptor); } return KERN_SUCCESS; } /* * Routine: atm_get_min_sub_aid_array * Purpose: For an array of aid, lookup the atm value and fill the minimum subaid. * Returns: None. */ static void atm_get_min_sub_aid_array( aid_t *aid_array, mach_atm_subaid_t *subaid_array, uint32_t count) { atm_value_t atm_value; uint32_t i; KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_SUBAID_INFO, (ATM_MIN_CALLED))) | DBG_FUNC_START, 0, 0, 0, 0, 0); for (i = 0; i < count; i++) { atm_value = get_atm_value_from_aid(aid_array[i]); if (atm_value == ATM_VALUE_NULL) { subaid_array[i] = ATM_SUBAID32_MAX; continue; } subaid_array[i] = atm_get_min_sub_aid(atm_value); atm_value_dealloc(atm_value); } KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_SUBAID_INFO, (ATM_MIN_CALLED))) | DBG_FUNC_END, count, 0, 0, 0, 0); } /* * Routine: atm_get_min_sub_aid * Purpose: Walk the list of listeners and get the min sub-aid for an activity id. * Returns: Minimum sub-aid to keep. * Note: Unlock the listener lock before accessing the mailbox, since it may page fault and * might take long time. Also cleans the listeners list for the tasks which are dead * and atm_task_descriptors do not hold any useful data. */ static mach_atm_subaid_t atm_get_min_sub_aid(atm_value_t atm_value) { int32_t i = 0, j, freed_count = 0, dead_but_not_freed = 0; int32_t listener_count; atm_subaid32_t min_subaid = ATM_SUBAID32_MAX, subaid, max_subaid; atm_link_object_t *link_object_array = NULL; atm_link_object_t next, elem; queue_head_t free_listeners; KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_SUBAID_INFO, (ATM_MIN_LINK_LIST))) | DBG_FUNC_START, 0, 0, 0, 0, 0); lck_mtx_lock(&atm_value->listener_lock); listener_count = atm_value->listener_count; lck_mtx_unlock(&atm_value->listener_lock); /* separate memory access from locked iterate since memory read may fault */ link_object_array = (atm_link_object_t *) kalloc(sizeof(atm_link_object_t) * listener_count); if (link_object_array == NULL) { return 0; } /* Iterate the list and take a ref on link objects and store it in an array */ lck_mtx_lock(&atm_value->listener_lock); queue_iterate(&atm_value->listeners, next, atm_link_object_t, listeners_element) { /* Additional listener are added between the allocation of array and iterating the list */ if (i >= listener_count) break; /* Get a ref on the link object */ atm_link_get_reference(next); link_object_array[i] = (atm_link_object_t)next; i++; } lck_mtx_unlock(&atm_value->listener_lock); j = i; /* Iterate the array to find the min */ for (i = 0; i < j; i++) { /* Ignore the min value of the dead processes. */ if (link_object_array[i]->descriptor->flags == ATM_TASK_DEAD) continue; /* Dereference the mailbox to get the min subaid */ subaid = *((atm_subaid32_t *)link_object_array[i]->mailbox); if (subaid < min_subaid) min_subaid = subaid; } /* * Mark the link object that can be freed, and release the ref on the link object * Mark the link object of dead task free after the dead task descriptor count * increases than ATM_LIST_DEAD_MAX. */ for (i = j - 1; i >= 0; i--) { if (link_object_array[i]->descriptor->flags == ATM_TASK_DEAD) { if (dead_but_not_freed > ATM_LIST_DEAD_MAX) { link_object_array[i]->flags = ATM_LINK_REMOVE; freed_count++; } else { max_subaid = *(((atm_subaid32_t *)link_object_array[i]->mailbox) + 1); if (max_subaid < min_subaid) { link_object_array[i]->flags = ATM_LINK_REMOVE; freed_count++; } else { dead_but_not_freed++; } } } atm_link_dealloc(link_object_array[i]); link_object_array[i] = NULL; } /* Check if the number of live entries in list is less than maxproc */ assert((j - (freed_count + dead_but_not_freed)) <= maxproc); kfree(link_object_array, (sizeof(atm_link_object_t) * listener_count)); /* Remove the marked link objects from the list */ lck_mtx_lock(&atm_value->listener_lock); queue_init(&free_listeners); next = (atm_link_object_t)(void *) queue_first(&atm_value->listeners); while (!queue_end(&atm_value->listeners, (queue_entry_t)next)) { elem = next; next = (atm_link_object_t)(void *) queue_next(&next->listeners_element); if (elem->flags == ATM_LINK_REMOVE) { queue_remove(&atm_value->listeners, elem, atm_link_object_t, listeners_element); queue_enter(&free_listeners, elem, atm_link_object_t, listeners_element); atm_value->listener_count--; } } lck_mtx_unlock(&atm_value->listener_lock); /* Free the link objects */ while(!queue_empty(&free_listeners)) { queue_remove_first(&free_listeners, next, atm_link_object_t, listeners_element); /* Drops the reference on the link object */ atm_link_dealloc(next); } KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (ATM_CODE(ATM_SUBAID_INFO, (ATM_MIN_LINK_LIST))) | DBG_FUNC_END, j, freed_count, dead_but_not_freed, 0, 0); /* explicitly upgrade uint32_t to 64 bit mach size */ return CAST_DOWN(mach_atm_subaid_t, min_subaid); } /* * Routine: atm_value_unregister * Purpose: Unregisters a process from an activity id. * Returns: KERN_SUCCESS on successful unregister. * KERN_INVALID_VALUE on finding a diff mailbox. * KERN_FAILURE on failure. */ static kern_return_t atm_value_unregister( atm_value_t atm_value, atm_task_descriptor_t task_descriptor, mailbox_offset_t mailbox_offset) { kern_return_t kr; if (task_descriptor == ATM_TASK_DESCRIPTOR_NULL) return KERN_INVALID_ARGUMENT; if (mailbox_offset > task_descriptor->mailbox_array_size) return KERN_INVALID_ARGUMENT; kr = atm_listener_delete(atm_value, task_descriptor, mailbox_offset); return kr; } void atm_task_descriptor_destroy(atm_task_descriptor_t task_descriptor) { /* Mark the task dead in the task descriptor to make task descriptor eligible for cleanup. */ lck_mtx_lock(&task_descriptor->lock); task_descriptor->flags = ATM_TASK_DEAD; lck_mtx_unlock(&task_descriptor->lock); atm_task_descriptor_dealloc(task_descriptor); }