| /* |
| * CDDL HEADER START |
| * |
| * The contents of this file are subject to the terms of the |
| * Common Development and Distribution License (the "License"). |
| * You may not use this file except in compliance with the License. |
| * |
| * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
| * or http://www.opensolaris.org/os/licensing. |
| * See the License for the specific language governing permissions |
| * and limitations under the License. |
| * |
| * When distributing Covered Code, include this CDDL HEADER in each |
| * file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
| * If applicable, add the following below this CDDL HEADER, with the |
| * fields enclosed by brackets "[]" replaced with your own identifying |
| * information: Portions Copyright [yyyy] [name of copyright owner] |
| * |
| * CDDL HEADER END |
| */ |
| /* |
| * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. |
| */ |
| |
| #include <sys/zfs_context.h> |
| #include <sys/crypto/icp.h> |
| #include <sys/crypto/spi.h> |
| #include <modes/modes.h> |
| #include <aes/aes_impl.h> |
| #include <linux/simd.h> |
| |
| /* |
| * Initialize AES encryption and decryption key schedules. |
| * |
| * Parameters: |
| * cipherKey User key |
| * keyBits AES key size (128, 192, or 256 bits) |
| * keysched AES key schedule to be initialized, of type aes_key_t. |
| * Allocated by aes_alloc_keysched(). |
| */ |
| void |
| aes_init_keysched(const uint8_t *cipherKey, uint_t keyBits, void *keysched) |
| { |
| const aes_impl_ops_t *ops = aes_impl_get_ops(); |
| aes_key_t *newbie = keysched; |
| uint_t keysize, i, j; |
| union { |
| uint64_t ka64[4]; |
| uint32_t ka32[8]; |
| } keyarr; |
| |
| switch (keyBits) { |
| case 128: |
| newbie->nr = 10; |
| break; |
| |
| case 192: |
| newbie->nr = 12; |
| break; |
| |
| case 256: |
| newbie->nr = 14; |
| break; |
| |
| default: |
| /* should never get here */ |
| return; |
| } |
| keysize = CRYPTO_BITS2BYTES(keyBits); |
| |
| /* |
| * Generic C implementation requires byteswap for little endian |
| * machines, various accelerated implementations for various |
| * architectures may not. |
| */ |
| if (!ops->needs_byteswap) { |
| /* no byteswap needed */ |
| if (IS_P2ALIGNED(cipherKey, sizeof (uint64_t))) { |
| for (i = 0, j = 0; j < keysize; i++, j += 8) { |
| /* LINTED: pointer alignment */ |
| keyarr.ka64[i] = *((uint64_t *)&cipherKey[j]); |
| } |
| } else { |
| bcopy(cipherKey, keyarr.ka32, keysize); |
| } |
| } else { |
| /* byte swap */ |
| for (i = 0, j = 0; j < keysize; i++, j += 4) { |
| keyarr.ka32[i] = |
| htonl(*(uint32_t *)(void *)&cipherKey[j]); |
| } |
| } |
| |
| ops->generate(newbie, keyarr.ka32, keyBits); |
| newbie->ops = ops; |
| |
| /* |
| * Note: if there are systems that need the AES_64BIT_KS type in the |
| * future, move setting key schedule type to individual implementations |
| */ |
| newbie->type = AES_32BIT_KS; |
| } |
| |
| |
| /* |
| * Encrypt one block using AES. |
| * Align if needed and (for x86 32-bit only) byte-swap. |
| * |
| * Parameters: |
| * ks Key schedule, of type aes_key_t |
| * pt Input block (plain text) |
| * ct Output block (crypto text). Can overlap with pt |
| */ |
| int |
| aes_encrypt_block(const void *ks, const uint8_t *pt, uint8_t *ct) |
| { |
| aes_key_t *ksch = (aes_key_t *)ks; |
| const aes_impl_ops_t *ops = ksch->ops; |
| |
| if (IS_P2ALIGNED2(pt, ct, sizeof (uint32_t)) && !ops->needs_byteswap) { |
| /* LINTED: pointer alignment */ |
| ops->encrypt(&ksch->encr_ks.ks32[0], ksch->nr, |
| /* LINTED: pointer alignment */ |
| (uint32_t *)pt, (uint32_t *)ct); |
| } else { |
| uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)]; |
| |
| /* Copy input block into buffer */ |
| if (ops->needs_byteswap) { |
| buffer[0] = htonl(*(uint32_t *)(void *)&pt[0]); |
| buffer[1] = htonl(*(uint32_t *)(void *)&pt[4]); |
| buffer[2] = htonl(*(uint32_t *)(void *)&pt[8]); |
| buffer[3] = htonl(*(uint32_t *)(void *)&pt[12]); |
| } else |
| bcopy(pt, &buffer, AES_BLOCK_LEN); |
| |
| ops->encrypt(&ksch->encr_ks.ks32[0], ksch->nr, buffer, buffer); |
| |
| /* Copy result from buffer to output block */ |
| if (ops->needs_byteswap) { |
| *(uint32_t *)(void *)&ct[0] = htonl(buffer[0]); |
| *(uint32_t *)(void *)&ct[4] = htonl(buffer[1]); |
| *(uint32_t *)(void *)&ct[8] = htonl(buffer[2]); |
| *(uint32_t *)(void *)&ct[12] = htonl(buffer[3]); |
| } else |
| bcopy(&buffer, ct, AES_BLOCK_LEN); |
| } |
| return (CRYPTO_SUCCESS); |
| } |
| |
| |
| /* |
| * Decrypt one block using AES. |
| * Align and byte-swap if needed. |
| * |
| * Parameters: |
| * ks Key schedule, of type aes_key_t |
| * ct Input block (crypto text) |
| * pt Output block (plain text). Can overlap with pt |
| */ |
| int |
| aes_decrypt_block(const void *ks, const uint8_t *ct, uint8_t *pt) |
| { |
| aes_key_t *ksch = (aes_key_t *)ks; |
| const aes_impl_ops_t *ops = ksch->ops; |
| |
| if (IS_P2ALIGNED2(ct, pt, sizeof (uint32_t)) && !ops->needs_byteswap) { |
| /* LINTED: pointer alignment */ |
| ops->decrypt(&ksch->decr_ks.ks32[0], ksch->nr, |
| /* LINTED: pointer alignment */ |
| (uint32_t *)ct, (uint32_t *)pt); |
| } else { |
| uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)]; |
| |
| /* Copy input block into buffer */ |
| if (ops->needs_byteswap) { |
| buffer[0] = htonl(*(uint32_t *)(void *)&ct[0]); |
| buffer[1] = htonl(*(uint32_t *)(void *)&ct[4]); |
| buffer[2] = htonl(*(uint32_t *)(void *)&ct[8]); |
| buffer[3] = htonl(*(uint32_t *)(void *)&ct[12]); |
| } else |
| bcopy(ct, &buffer, AES_BLOCK_LEN); |
| |
| ops->decrypt(&ksch->decr_ks.ks32[0], ksch->nr, buffer, buffer); |
| |
| /* Copy result from buffer to output block */ |
| if (ops->needs_byteswap) { |
| *(uint32_t *)(void *)&pt[0] = htonl(buffer[0]); |
| *(uint32_t *)(void *)&pt[4] = htonl(buffer[1]); |
| *(uint32_t *)(void *)&pt[8] = htonl(buffer[2]); |
| *(uint32_t *)(void *)&pt[12] = htonl(buffer[3]); |
| } else |
| bcopy(&buffer, pt, AES_BLOCK_LEN); |
| } |
| return (CRYPTO_SUCCESS); |
| } |
| |
| |
| /* |
| * Allocate key schedule for AES. |
| * |
| * Return the pointer and set size to the number of bytes allocated. |
| * Memory allocated must be freed by the caller when done. |
| * |
| * Parameters: |
| * size Size of key schedule allocated, in bytes |
| * kmflag Flag passed to kmem_alloc(9F); ignored in userland. |
| */ |
| /* ARGSUSED */ |
| void * |
| aes_alloc_keysched(size_t *size, int kmflag) |
| { |
| aes_key_t *keysched; |
| |
| keysched = (aes_key_t *)kmem_alloc(sizeof (aes_key_t), kmflag); |
| if (keysched != NULL) { |
| *size = sizeof (aes_key_t); |
| return (keysched); |
| } |
| return (NULL); |
| } |
| |
| /* AES implementation that contains the fastest methods */ |
| static aes_impl_ops_t aes_fastest_impl = { |
| .name = "fastest" |
| }; |
| |
| /* All compiled in implementations */ |
| const aes_impl_ops_t *aes_all_impl[] = { |
| &aes_generic_impl, |
| #if defined(__x86_64) |
| &aes_x86_64_impl, |
| #endif |
| #if defined(__x86_64) && defined(HAVE_AES) |
| &aes_aesni_impl, |
| #endif |
| }; |
| |
| /* Indicate that benchmark has been completed */ |
| static boolean_t aes_impl_initialized = B_FALSE; |
| |
| /* Select aes implementation */ |
| #define IMPL_FASTEST (UINT32_MAX) |
| #define IMPL_CYCLE (UINT32_MAX-1) |
| |
| #define AES_IMPL_READ(i) (*(volatile uint32_t *) &(i)) |
| |
| static uint32_t icp_aes_impl = IMPL_FASTEST; |
| static uint32_t user_sel_impl = IMPL_FASTEST; |
| |
| /* Hold all supported implementations */ |
| static size_t aes_supp_impl_cnt = 0; |
| static aes_impl_ops_t *aes_supp_impl[ARRAY_SIZE(aes_all_impl)]; |
| |
| /* |
| * Returns the AES operations for encrypt/decrypt/key setup. When a |
| * SIMD implementation is not allowed in the current context, then |
| * fallback to the fastest generic implementation. |
| */ |
| const aes_impl_ops_t * |
| aes_impl_get_ops(void) |
| { |
| if (!kfpu_allowed()) |
| return (&aes_generic_impl); |
| |
| const aes_impl_ops_t *ops = NULL; |
| const uint32_t impl = AES_IMPL_READ(icp_aes_impl); |
| |
| switch (impl) { |
| case IMPL_FASTEST: |
| ASSERT(aes_impl_initialized); |
| ops = &aes_fastest_impl; |
| break; |
| case IMPL_CYCLE: |
| /* Cycle through supported implementations */ |
| ASSERT(aes_impl_initialized); |
| ASSERT3U(aes_supp_impl_cnt, >, 0); |
| static size_t cycle_impl_idx = 0; |
| size_t idx = (++cycle_impl_idx) % aes_supp_impl_cnt; |
| ops = aes_supp_impl[idx]; |
| break; |
| default: |
| ASSERT3U(impl, <, aes_supp_impl_cnt); |
| ASSERT3U(aes_supp_impl_cnt, >, 0); |
| if (impl < ARRAY_SIZE(aes_all_impl)) |
| ops = aes_supp_impl[impl]; |
| break; |
| } |
| |
| ASSERT3P(ops, !=, NULL); |
| |
| return (ops); |
| } |
| |
| /* |
| * Initialize all supported implementations. |
| */ |
| void |
| aes_impl_init(void) |
| { |
| aes_impl_ops_t *curr_impl; |
| int i, c; |
| |
| /* Move supported implementations into aes_supp_impls */ |
| for (i = 0, c = 0; i < ARRAY_SIZE(aes_all_impl); i++) { |
| curr_impl = (aes_impl_ops_t *)aes_all_impl[i]; |
| |
| if (curr_impl->is_supported()) |
| aes_supp_impl[c++] = (aes_impl_ops_t *)curr_impl; |
| } |
| aes_supp_impl_cnt = c; |
| |
| /* |
| * Set the fastest implementation given the assumption that the |
| * hardware accelerated version is the fastest. |
| */ |
| #if defined(__x86_64) |
| #if defined(HAVE_AES) |
| if (aes_aesni_impl.is_supported()) { |
| memcpy(&aes_fastest_impl, &aes_aesni_impl, |
| sizeof (aes_fastest_impl)); |
| } else |
| #endif |
| { |
| memcpy(&aes_fastest_impl, &aes_x86_64_impl, |
| sizeof (aes_fastest_impl)); |
| } |
| #else |
| memcpy(&aes_fastest_impl, &aes_generic_impl, |
| sizeof (aes_fastest_impl)); |
| #endif |
| |
| strlcpy(aes_fastest_impl.name, "fastest", AES_IMPL_NAME_MAX); |
| |
| /* Finish initialization */ |
| atomic_swap_32(&icp_aes_impl, user_sel_impl); |
| aes_impl_initialized = B_TRUE; |
| } |
| |
| static const struct { |
| char *name; |
| uint32_t sel; |
| } aes_impl_opts[] = { |
| { "cycle", IMPL_CYCLE }, |
| { "fastest", IMPL_FASTEST }, |
| }; |
| |
| /* |
| * Function sets desired aes implementation. |
| * |
| * If we are called before init(), user preference will be saved in |
| * user_sel_impl, and applied in later init() call. This occurs when module |
| * parameter is specified on module load. Otherwise, directly update |
| * icp_aes_impl. |
| * |
| * @val Name of aes implementation to use |
| * @param Unused. |
| */ |
| int |
| aes_impl_set(const char *val) |
| { |
| int err = -EINVAL; |
| char req_name[AES_IMPL_NAME_MAX]; |
| uint32_t impl = AES_IMPL_READ(user_sel_impl); |
| size_t i; |
| |
| /* sanitize input */ |
| i = strnlen(val, AES_IMPL_NAME_MAX); |
| if (i == 0 || i >= AES_IMPL_NAME_MAX) |
| return (err); |
| |
| strlcpy(req_name, val, AES_IMPL_NAME_MAX); |
| while (i > 0 && isspace(req_name[i-1])) |
| i--; |
| req_name[i] = '\0'; |
| |
| /* Check mandatory options */ |
| for (i = 0; i < ARRAY_SIZE(aes_impl_opts); i++) { |
| if (strcmp(req_name, aes_impl_opts[i].name) == 0) { |
| impl = aes_impl_opts[i].sel; |
| err = 0; |
| break; |
| } |
| } |
| |
| /* check all supported impl if init() was already called */ |
| if (err != 0 && aes_impl_initialized) { |
| /* check all supported implementations */ |
| for (i = 0; i < aes_supp_impl_cnt; i++) { |
| if (strcmp(req_name, aes_supp_impl[i]->name) == 0) { |
| impl = i; |
| err = 0; |
| break; |
| } |
| } |
| } |
| |
| if (err == 0) { |
| if (aes_impl_initialized) |
| atomic_swap_32(&icp_aes_impl, impl); |
| else |
| atomic_swap_32(&user_sel_impl, impl); |
| } |
| |
| return (err); |
| } |
| |
| #if defined(_KERNEL) && defined(__linux__) |
| #include <linux/mod_compat.h> |
| |
| static int |
| icp_aes_impl_set(const char *val, zfs_kernel_param_t *kp) |
| { |
| return (aes_impl_set(val)); |
| } |
| |
| static int |
| icp_aes_impl_get(char *buffer, zfs_kernel_param_t *kp) |
| { |
| int i, cnt = 0; |
| char *fmt; |
| const uint32_t impl = AES_IMPL_READ(icp_aes_impl); |
| |
| ASSERT(aes_impl_initialized); |
| |
| /* list mandatory options */ |
| for (i = 0; i < ARRAY_SIZE(aes_impl_opts); i++) { |
| fmt = (impl == aes_impl_opts[i].sel) ? "[%s] " : "%s "; |
| cnt += sprintf(buffer + cnt, fmt, aes_impl_opts[i].name); |
| } |
| |
| /* list all supported implementations */ |
| for (i = 0; i < aes_supp_impl_cnt; i++) { |
| fmt = (i == impl) ? "[%s] " : "%s "; |
| cnt += sprintf(buffer + cnt, fmt, aes_supp_impl[i]->name); |
| } |
| |
| return (cnt); |
| } |
| |
| module_param_call(icp_aes_impl, icp_aes_impl_set, icp_aes_impl_get, |
| NULL, 0644); |
| MODULE_PARM_DESC(icp_aes_impl, "Select aes implementation."); |
| #endif |