From 80d663a42127b839e820ed934cead51cff6196d3 Mon Sep 17 00:00:00 2001 From: Jan Glauber Date: Fri, 21 May 2010 22:04:08 +1000 Subject: crypto: des_s390: remove des3_ede128 mode des_s390 implements support for 3DES with a 128 bit key. This mode is probably not used anywhere, less secure than 3DES with a 192 bit key and not implemented in the generic des version. Removing this mode seems to be low risk and will ease maintenance of the code. Signed-off-by: Jan Glauber Signed-off-by: Herbert Xu --- arch/s390/crypto/des_s390.c | 191 +------------------------------------------- 1 file changed, 1 insertion(+), 190 deletions(-) diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c index 2bc479a..a1a18b3 100644 --- a/arch/s390/crypto/des_s390.c +++ b/arch/s390/crypto/des_s390.c @@ -24,9 +24,6 @@ #define DES_BLOCK_SIZE 8 #define DES_KEY_SIZE 8 -#define DES3_128_KEY_SIZE (2 * DES_KEY_SIZE) -#define DES3_128_BLOCK_SIZE DES_BLOCK_SIZE - #define DES3_192_KEY_SIZE (3 * DES_KEY_SIZE) #define DES3_192_BLOCK_SIZE DES_BLOCK_SIZE @@ -35,11 +32,6 @@ struct crypt_s390_des_ctx { u8 key[DES_KEY_SIZE]; }; -struct crypt_s390_des3_128_ctx { - u8 iv[DES_BLOCK_SIZE]; - u8 key[DES3_128_KEY_SIZE]; -}; - struct crypt_s390_des3_192_ctx { u8 iv[DES_BLOCK_SIZE]; u8 key[DES3_192_KEY_SIZE]; @@ -237,165 +229,6 @@ static struct crypto_alg cbc_des_alg = { * complementation keys. Any weakness is obviated by the use of * multiple keys. * - * However, if the two independent 64-bit keys are equal, - * then the DES3 operation is simply the same as DES. - * Implementers MUST reject keys that exhibit this property. - * - */ -static int des3_128_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) -{ - int i, ret; - struct crypt_s390_des3_128_ctx *dctx = crypto_tfm_ctx(tfm); - const u8 *temp_key = key; - u32 *flags = &tfm->crt_flags; - - if (!(memcmp(key, &key[DES_KEY_SIZE], DES_KEY_SIZE)) && - (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { - *flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; - } - for (i = 0; i < 2; i++, temp_key += DES_KEY_SIZE) { - ret = crypto_des_check_key(temp_key, DES_KEY_SIZE, flags); - if (ret < 0) - return ret; - } - memcpy(dctx->key, key, keylen); - return 0; -} - -static void des3_128_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - struct crypt_s390_des3_128_ctx *dctx = crypto_tfm_ctx(tfm); - - crypt_s390_km(KM_TDEA_128_ENCRYPT, dctx->key, dst, (void*)src, - DES3_128_BLOCK_SIZE); -} - -static void des3_128_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - struct crypt_s390_des3_128_ctx *dctx = crypto_tfm_ctx(tfm); - - crypt_s390_km(KM_TDEA_128_DECRYPT, dctx->key, dst, (void*)src, - DES3_128_BLOCK_SIZE); -} - -static struct crypto_alg des3_128_alg = { - .cra_name = "des3_ede128", - .cra_driver_name = "des3_ede128-s390", - .cra_priority = CRYPT_S390_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = DES3_128_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypt_s390_des3_128_ctx), - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(des3_128_alg.cra_list), - .cra_u = { - .cipher = { - .cia_min_keysize = DES3_128_KEY_SIZE, - .cia_max_keysize = DES3_128_KEY_SIZE, - .cia_setkey = des3_128_setkey, - .cia_encrypt = des3_128_encrypt, - .cia_decrypt = des3_128_decrypt, - } - } -}; - -static int ecb_des3_128_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) -{ - struct crypt_s390_des3_128_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; - - blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, KM_TDEA_128_ENCRYPT, sctx->key, &walk); -} - -static int ecb_des3_128_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) -{ - struct crypt_s390_des3_128_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; - - blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_desall_crypt(desc, KM_TDEA_128_DECRYPT, sctx->key, &walk); -} - -static struct crypto_alg ecb_des3_128_alg = { - .cra_name = "ecb(des3_ede128)", - .cra_driver_name = "ecb-des3_ede128-s390", - .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = DES3_128_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypt_s390_des3_128_ctx), - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT( - ecb_des3_128_alg.cra_list), - .cra_u = { - .blkcipher = { - .min_keysize = DES3_128_KEY_SIZE, - .max_keysize = DES3_128_KEY_SIZE, - .setkey = des3_128_setkey, - .encrypt = ecb_des3_128_encrypt, - .decrypt = ecb_des3_128_decrypt, - } - } -}; - -static int cbc_des3_128_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) -{ - struct crypt_s390_des3_128_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; - - blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, KMC_TDEA_128_ENCRYPT, sctx->iv, &walk); -} - -static int cbc_des3_128_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) -{ - struct crypt_s390_des3_128_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; - - blkcipher_walk_init(&walk, dst, src, nbytes); - return cbc_desall_crypt(desc, KMC_TDEA_128_DECRYPT, sctx->iv, &walk); -} - -static struct crypto_alg cbc_des3_128_alg = { - .cra_name = "cbc(des3_ede128)", - .cra_driver_name = "cbc-des3_ede128-s390", - .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = DES3_128_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypt_s390_des3_128_ctx), - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT( - cbc_des3_128_alg.cra_list), - .cra_u = { - .blkcipher = { - .min_keysize = DES3_128_KEY_SIZE, - .max_keysize = DES3_128_KEY_SIZE, - .ivsize = DES3_128_BLOCK_SIZE, - .setkey = des3_128_setkey, - .encrypt = cbc_des3_128_encrypt, - .decrypt = cbc_des3_128_decrypt, - } - } -}; - -/* - * RFC2451: - * - * For DES-EDE3, there is no known need to reject weak or - * complementation keys. Any weakness is obviated by the use of - * multiple keys. - * * However, if the first two or last two independent 64-bit keys are * equal (k1 == k2 or k2 == k3), then the DES3 operation is simply the * same as DES. Implementers MUST reject keys that exhibit this @@ -553,10 +386,9 @@ static struct crypto_alg cbc_des3_192_alg = { static int des_s390_init(void) { - int ret = 0; + int ret; if (!crypt_s390_func_available(KM_DEA_ENCRYPT) || - !crypt_s390_func_available(KM_TDEA_128_ENCRYPT) || !crypt_s390_func_available(KM_TDEA_192_ENCRYPT)) return -EOPNOTSUPP; @@ -569,17 +401,6 @@ static int des_s390_init(void) ret = crypto_register_alg(&cbc_des_alg); if (ret) goto cbc_des_err; - - ret = crypto_register_alg(&des3_128_alg); - if (ret) - goto des3_128_err; - ret = crypto_register_alg(&ecb_des3_128_alg); - if (ret) - goto ecb_des3_128_err; - ret = crypto_register_alg(&cbc_des3_128_alg); - if (ret) - goto cbc_des3_128_err; - ret = crypto_register_alg(&des3_192_alg); if (ret) goto des3_192_err; @@ -589,7 +410,6 @@ static int des_s390_init(void) ret = crypto_register_alg(&cbc_des3_192_alg); if (ret) goto cbc_des3_192_err; - out: return ret; @@ -598,12 +418,6 @@ cbc_des3_192_err: ecb_des3_192_err: crypto_unregister_alg(&des3_192_alg); des3_192_err: - crypto_unregister_alg(&cbc_des3_128_alg); -cbc_des3_128_err: - crypto_unregister_alg(&ecb_des3_128_alg); -ecb_des3_128_err: - crypto_unregister_alg(&des3_128_alg); -des3_128_err: crypto_unregister_alg(&cbc_des_alg); cbc_des_err: crypto_unregister_alg(&ecb_des_alg); @@ -618,9 +432,6 @@ static void __exit des_s390_fini(void) crypto_unregister_alg(&cbc_des3_192_alg); crypto_unregister_alg(&ecb_des3_192_alg); crypto_unregister_alg(&des3_192_alg); - crypto_unregister_alg(&cbc_des3_128_alg); - crypto_unregister_alg(&ecb_des3_128_alg); - crypto_unregister_alg(&des3_128_alg); crypto_unregister_alg(&cbc_des_alg); crypto_unregister_alg(&ecb_des_alg); crypto_unregister_alg(&des_alg); -- cgit v1.1 From 1efbd15c3bc2b79d33e033e898211109c32159fa Mon Sep 17 00:00:00 2001 From: Jan Glauber Date: Fri, 21 May 2010 22:04:46 +1000 Subject: crypto: des_s390: use generic weak key check Get rid of the des_s390 specific key check module and use the generic DES weak key check instead. Also use the generic DES header and remove the weak key check in 3DES mode, as RFC2451 mentions that the DES weak keys are not relevant for 3DES. Signed-off-by: Jan Glauber Signed-off-by: Herbert Xu --- arch/s390/crypto/Makefile | 2 +- arch/s390/crypto/des_s390.c | 47 +++++++++++++++++++-------------------------- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 6a1157f..1cf81d7 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -5,6 +5,6 @@ obj-$(CONFIG_CRYPTO_SHA1_S390) += sha1_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA256_S390) += sha256_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA512_S390) += sha512_s390.o sha_common.o -obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o des_check_key.o +obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o obj-$(CONFIG_S390_PRNG) += prng.o diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c index a1a18b3..cc54201 100644 --- a/arch/s390/crypto/des_s390.c +++ b/arch/s390/crypto/des_s390.c @@ -14,18 +14,15 @@ * */ -#include #include #include +#include +#include +#include #include "crypt_s390.h" -#include "crypto_des.h" - -#define DES_BLOCK_SIZE 8 -#define DES_KEY_SIZE 8 #define DES3_192_KEY_SIZE (3 * DES_KEY_SIZE) -#define DES3_192_BLOCK_SIZE DES_BLOCK_SIZE struct crypt_s390_des_ctx { u8 iv[DES_BLOCK_SIZE]; @@ -42,13 +39,16 @@ static int des_setkey(struct crypto_tfm *tfm, const u8 *key, { struct crypt_s390_des_ctx *dctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; - int ret; + u32 tmp[DES_EXPKEY_WORDS]; - /* test if key is valid (not a weak key) */ - ret = crypto_des_check_key(key, keylen, flags); - if (ret == 0) - memcpy(dctx->key, key, keylen); - return ret; + /* check for weak keys */ + if (!des_ekey(tmp, key) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + *flags |= CRYPTO_TFM_RES_WEAK_KEY; + return -EINVAL; + } + + memcpy(dctx->key, key, keylen); + return 0; } static void des_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) @@ -238,9 +238,7 @@ static struct crypto_alg cbc_des_alg = { static int des3_192_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) { - int i, ret; struct crypt_s390_des3_192_ctx *dctx = crypto_tfm_ctx(tfm); - const u8 *temp_key = key; u32 *flags = &tfm->crt_flags; if (!(memcmp(key, &key[DES_KEY_SIZE], DES_KEY_SIZE) && @@ -250,11 +248,6 @@ static int des3_192_setkey(struct crypto_tfm *tfm, const u8 *key, *flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } - for (i = 0; i < 3; i++, temp_key += DES_KEY_SIZE) { - ret = crypto_des_check_key(temp_key, DES_KEY_SIZE, flags); - if (ret < 0) - return ret; - } memcpy(dctx->key, key, keylen); return 0; } @@ -264,7 +257,7 @@ static void des3_192_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) struct crypt_s390_des3_192_ctx *dctx = crypto_tfm_ctx(tfm); crypt_s390_km(KM_TDEA_192_ENCRYPT, dctx->key, dst, (void*)src, - DES3_192_BLOCK_SIZE); + DES_BLOCK_SIZE); } static void des3_192_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) @@ -272,7 +265,7 @@ static void des3_192_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) struct crypt_s390_des3_192_ctx *dctx = crypto_tfm_ctx(tfm); crypt_s390_km(KM_TDEA_192_DECRYPT, dctx->key, dst, (void*)src, - DES3_192_BLOCK_SIZE); + DES_BLOCK_SIZE); } static struct crypto_alg des3_192_alg = { @@ -280,7 +273,7 @@ static struct crypto_alg des3_192_alg = { .cra_driver_name = "des3_ede-s390", .cra_priority = CRYPT_S390_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = DES3_192_BLOCK_SIZE, + .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypt_s390_des3_192_ctx), .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(des3_192_alg.cra_list), @@ -322,7 +315,7 @@ static struct crypto_alg ecb_des3_192_alg = { .cra_driver_name = "ecb-des3_ede-s390", .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = DES3_192_BLOCK_SIZE, + .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypt_s390_des3_192_ctx), .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, @@ -366,7 +359,7 @@ static struct crypto_alg cbc_des3_192_alg = { .cra_driver_name = "cbc-des3_ede-s390", .cra_priority = CRYPT_S390_COMPOSITE_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = DES3_192_BLOCK_SIZE, + .cra_blocksize = DES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypt_s390_des3_192_ctx), .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, @@ -376,7 +369,7 @@ static struct crypto_alg cbc_des3_192_alg = { .blkcipher = { .min_keysize = DES3_192_KEY_SIZE, .max_keysize = DES3_192_KEY_SIZE, - .ivsize = DES3_192_BLOCK_SIZE, + .ivsize = DES_BLOCK_SIZE, .setkey = des3_192_setkey, .encrypt = cbc_des3_192_encrypt, .decrypt = cbc_des3_192_decrypt, @@ -427,7 +420,7 @@ des_err: goto out; } -static void __exit des_s390_fini(void) +static void __exit des_s390_exit(void) { crypto_unregister_alg(&cbc_des3_192_alg); crypto_unregister_alg(&ecb_des3_192_alg); @@ -438,7 +431,7 @@ static void __exit des_s390_fini(void) } module_init(des_s390_init); -module_exit(des_s390_fini); +module_exit(des_s390_exit); MODULE_ALIAS("des"); MODULE_ALIAS("des3_ede"); -- cgit v1.1 From 8db050786aaf7a43f4c7a900a103f99ea0fb493f Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Wed, 26 May 2010 10:36:51 +1000 Subject: crypto: Use ERR_CAST Use ERR_CAST(x) rather than ERR_PTR(PTR_ERR(x)). The former makes more clear what is the purpose of the operation, which otherwise looks like a no-op. The semantic patch that makes this change is as follows: (http://coccinelle.lip6.fr/) // @@ type T; T x; identifier f; @@ T f (...) { <+... - ERR_PTR(PTR_ERR(x)) + x ...+> } @@ expression x; @@ - ERR_PTR(PTR_ERR(x)) + ERR_CAST(x) // Signed-off-by: Julia Lawall Signed-off-by: Herbert Xu --- crypto/authenc.c | 2 +- crypto/ctr.c | 2 +- crypto/xts.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/authenc.c b/crypto/authenc.c index b9884ee..a5a22cf 100644 --- a/crypto/authenc.c +++ b/crypto/authenc.c @@ -616,7 +616,7 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH, CRYPTO_ALG_TYPE_AHASH_MASK); if (IS_ERR(auth)) - return ERR_PTR(PTR_ERR(auth)); + return ERR_CAST(auth); auth_base = &auth->base; diff --git a/crypto/ctr.c b/crypto/ctr.c index 6c3bfab..4ca7222 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -185,7 +185,7 @@ static struct crypto_instance *crypto_ctr_alloc(struct rtattr **tb) alg = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) - return ERR_PTR(PTR_ERR(alg)); + return ERR_CAST(alg); /* Block size must be >= 4 bytes. */ err = -EINVAL; diff --git a/crypto/xts.c b/crypto/xts.c index d87b0f3..555ecaa 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -224,7 +224,7 @@ static struct crypto_instance *alloc(struct rtattr **tb) alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) - return ERR_PTR(PTR_ERR(alg)); + return ERR_CAST(alg); inst = crypto_alloc_instance("xts", alg); if (IS_ERR(inst)) -- cgit v1.1 From 65a23d6706ce2d58e302970971e41688783bf63f Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sat, 22 May 2010 01:11:03 -0700 Subject: n2_crypto: Kill n2_base_ctx and helpers. Unused, and we'll do this via the request context. Signed-off-by: David S. Miller --- drivers/crypto/n2_core.c | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 23163fd..3a4eed4 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -239,18 +239,7 @@ static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) } #endif -struct n2_base_ctx { - struct list_head list; -}; - -static void n2_base_ctx_init(struct n2_base_ctx *ctx) -{ - INIT_LIST_HEAD(&ctx->list); -} - struct n2_hash_ctx { - struct n2_base_ctx base; - struct crypto_ahash *fallback_tfm; }; @@ -390,7 +379,6 @@ static int n2_hash_async_digest(struct ahash_request *req, unsigned int result_size, void *hash_loc) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); struct cwq_initial_entry *ent; struct crypto_hash_walk walk; struct spu_queue *qp; @@ -403,6 +391,7 @@ static int n2_hash_async_digest(struct ahash_request *req, */ if (unlikely(req->nbytes > (1 << 16))) { struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); + struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); rctx->fallback_req.base.flags = @@ -414,8 +403,6 @@ static int n2_hash_async_digest(struct ahash_request *req, return crypto_ahash_digest(&rctx->fallback_req); } - n2_base_ctx_init(&ctx->base); - nbytes = crypto_hash_walk_first(req, &walk); cpu = get_cpu(); -- cgit v1.1 From 38511108a37e5551b2bfe143a957132d46f9e6e7 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Wed, 19 May 2010 23:16:05 -0700 Subject: n2_crypto: Log algorithm success/failure in kernel log. Signed-off-by: David S. Miller --- drivers/crypto/n2_core.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 3a4eed4..77939e4 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -1277,8 +1277,11 @@ static int __devinit __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl) list_add(&p->entry, &cipher_algs); err = crypto_register_alg(alg); if (err) { + pr_err("%s alg registration failed\n", alg->cra_name); list_del(&p->entry); kfree(p); + } else { + pr_info("%s alg registered\n", alg->cra_name); } return err; } @@ -1318,8 +1321,11 @@ static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl) list_add(&p->entry, &ahash_algs); err = crypto_register_ahash(ahash); if (err) { + pr_err("%s alg registration failed\n", base->cra_name); list_del(&p->entry); kfree(p); + } else { + pr_info("%s alg registered\n", base->cra_name); } return err; } -- cgit v1.1 From 3a2c034697558602a72e51897c6d3665bc515927 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sat, 22 May 2010 02:45:56 -0700 Subject: n2_crypto: Make ahash parameterization explicit. All of the ahash ->digest() ops do essentially the same thing, just using different parameters. So instead, have a single n2_hash_async_digest() and use an n2_ahash_alg container that provides the parameters. Signed-off-by: David S. Miller --- drivers/crypto/n2_core.c | 216 ++++++++++++++++++++++------------------------- 1 file changed, 99 insertions(+), 117 deletions(-) diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 77939e4..d01a2af 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -239,6 +239,26 @@ static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) } #endif +struct n2_ahash_alg { + struct list_head entry; + const char *hash_zero; + const u32 *hash_init; + u8 hw_op_hashsz; + u8 digest_size; + u8 auth_type; + struct ahash_alg alg; +}; + +static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct ahash_alg *ahash_alg; + + ahash_alg = container_of(alg, struct ahash_alg, halg.base); + + return container_of(ahash_alg, struct n2_ahash_alg, alg); +} + struct n2_hash_ctx { struct crypto_ahash *fallback_tfm; }; @@ -250,9 +270,6 @@ struct n2_hash_req_ctx { struct sha256_state sha256; } u; - unsigned char hash_key[64]; - unsigned char keyed_zero_hash[32]; - struct ahash_request fallback_req; }; @@ -374,9 +391,9 @@ static unsigned long submit_and_wait_for_tail(struct spu_queue *qp, return hv_ret; } -static int n2_hash_async_digest(struct ahash_request *req, - unsigned int auth_type, unsigned int digest_size, - unsigned int result_size, void *hash_loc) +static int n2_do_async_digest(struct ahash_request *req, + unsigned int auth_type, unsigned int digest_size, + unsigned int result_size, void *hash_loc) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct cwq_initial_entry *ent; @@ -462,114 +479,22 @@ out: return err; } -static int n2_md5_async_digest(struct ahash_request *req) -{ - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct md5_state *m = &rctx->u.md5; - - if (unlikely(req->nbytes == 0)) { - static const char md5_zero[MD5_DIGEST_SIZE] = { - 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, - 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e, - }; - - memcpy(req->result, md5_zero, MD5_DIGEST_SIZE); - return 0; - } - m->hash[0] = cpu_to_le32(0x67452301); - m->hash[1] = cpu_to_le32(0xefcdab89); - m->hash[2] = cpu_to_le32(0x98badcfe); - m->hash[3] = cpu_to_le32(0x10325476); - - return n2_hash_async_digest(req, AUTH_TYPE_MD5, - MD5_DIGEST_SIZE, MD5_DIGEST_SIZE, - m->hash); -} - -static int n2_sha1_async_digest(struct ahash_request *req) +static int n2_hash_async_digest(struct ahash_request *req) { + struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm); struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct sha1_state *s = &rctx->u.sha1; + int ds; + ds = n2alg->digest_size; if (unlikely(req->nbytes == 0)) { - static const char sha1_zero[SHA1_DIGEST_SIZE] = { - 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, - 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, - 0x07, 0x09 - }; - - memcpy(req->result, sha1_zero, SHA1_DIGEST_SIZE); - return 0; - } - s->state[0] = SHA1_H0; - s->state[1] = SHA1_H1; - s->state[2] = SHA1_H2; - s->state[3] = SHA1_H3; - s->state[4] = SHA1_H4; - - return n2_hash_async_digest(req, AUTH_TYPE_SHA1, - SHA1_DIGEST_SIZE, SHA1_DIGEST_SIZE, - s->state); -} - -static int n2_sha256_async_digest(struct ahash_request *req) -{ - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct sha256_state *s = &rctx->u.sha256; - - if (req->nbytes == 0) { - static const char sha256_zero[SHA256_DIGEST_SIZE] = { - 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, - 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, - 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, - 0x1b, 0x78, 0x52, 0xb8, 0x55 - }; - - memcpy(req->result, sha256_zero, SHA256_DIGEST_SIZE); - return 0; - } - s->state[0] = SHA256_H0; - s->state[1] = SHA256_H1; - s->state[2] = SHA256_H2; - s->state[3] = SHA256_H3; - s->state[4] = SHA256_H4; - s->state[5] = SHA256_H5; - s->state[6] = SHA256_H6; - s->state[7] = SHA256_H7; - - return n2_hash_async_digest(req, AUTH_TYPE_SHA256, - SHA256_DIGEST_SIZE, SHA256_DIGEST_SIZE, - s->state); -} - -static int n2_sha224_async_digest(struct ahash_request *req) -{ - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); - struct sha256_state *s = &rctx->u.sha256; - - if (req->nbytes == 0) { - static const char sha224_zero[SHA224_DIGEST_SIZE] = { - 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47, - 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2, - 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4, - 0x2f - }; - - memcpy(req->result, sha224_zero, SHA224_DIGEST_SIZE); + memcpy(req->result, n2alg->hash_zero, ds); return 0; } - s->state[0] = SHA224_H0; - s->state[1] = SHA224_H1; - s->state[2] = SHA224_H2; - s->state[3] = SHA224_H3; - s->state[4] = SHA224_H4; - s->state[5] = SHA224_H5; - s->state[6] = SHA224_H6; - s->state[7] = SHA224_H7; + memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz); - return n2_hash_async_digest(req, AUTH_TYPE_SHA256, - SHA256_DIGEST_SIZE, SHA224_DIGEST_SIZE, - s->state); + return n2_do_async_digest(req, n2alg->auth_type, + n2alg->hw_op_hashsz, ds, + &rctx->u); } struct n2_cipher_context { @@ -1196,34 +1121,85 @@ static LIST_HEAD(cipher_algs); struct n2_hash_tmpl { const char *name; - int (*digest)(struct ahash_request *req); + const char *hash_zero; + const u32 *hash_init; + u8 hw_op_hashsz; u8 digest_size; u8 block_size; + u8 auth_type; +}; + +static const char md5_zero[MD5_DIGEST_SIZE] = { + 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, + 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e, +}; +static const u32 md5_init[MD5_HASH_WORDS] = { + cpu_to_le32(0x67452301), + cpu_to_le32(0xefcdab89), + cpu_to_le32(0x98badcfe), + cpu_to_le32(0x10325476), +}; +static const char sha1_zero[SHA1_DIGEST_SIZE] = { + 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, + 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, + 0x07, 0x09 +}; +static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = { + SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, +}; +static const char sha256_zero[SHA256_DIGEST_SIZE] = { + 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, + 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, + 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, + 0x1b, 0x78, 0x52, 0xb8, 0x55 +}; +static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = { + SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, + SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, +}; +static const char sha224_zero[SHA224_DIGEST_SIZE] = { + 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47, + 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2, + 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4, + 0x2f }; +static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = { + SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, + SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, +}; + static const struct n2_hash_tmpl hash_tmpls[] = { { .name = "md5", - .digest = n2_md5_async_digest, + .hash_zero = md5_zero, + .hash_init = md5_init, + .auth_type = AUTH_TYPE_MD5, + .hw_op_hashsz = MD5_DIGEST_SIZE, .digest_size = MD5_DIGEST_SIZE, .block_size = MD5_HMAC_BLOCK_SIZE }, { .name = "sha1", - .digest = n2_sha1_async_digest, + .hash_zero = sha1_zero, + .hash_init = sha1_init, + .auth_type = AUTH_TYPE_SHA1, + .hw_op_hashsz = SHA1_DIGEST_SIZE, .digest_size = SHA1_DIGEST_SIZE, .block_size = SHA1_BLOCK_SIZE }, { .name = "sha256", - .digest = n2_sha256_async_digest, + .hash_zero = sha256_zero, + .hash_init = sha256_init, + .auth_type = AUTH_TYPE_SHA256, + .hw_op_hashsz = SHA256_DIGEST_SIZE, .digest_size = SHA256_DIGEST_SIZE, .block_size = SHA256_BLOCK_SIZE }, { .name = "sha224", - .digest = n2_sha224_async_digest, + .hash_zero = sha224_zero, + .hash_init = sha224_init, + .auth_type = AUTH_TYPE_SHA256, + .hw_op_hashsz = SHA256_DIGEST_SIZE, .digest_size = SHA224_DIGEST_SIZE, .block_size = SHA224_BLOCK_SIZE }, }; #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls) -struct n2_ahash_alg { - struct list_head entry; - struct ahash_alg alg; -}; static LIST_HEAD(ahash_algs); static int algs_registered; @@ -1297,12 +1273,18 @@ static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl) if (!p) return -ENOMEM; + p->hash_zero = tmpl->hash_zero; + p->hash_init = tmpl->hash_init; + p->auth_type = tmpl->auth_type; + p->hw_op_hashsz = tmpl->hw_op_hashsz; + p->digest_size = tmpl->digest_size; + ahash = &p->alg; ahash->init = n2_hash_async_init; ahash->update = n2_hash_async_update; ahash->final = n2_hash_async_final; ahash->finup = n2_hash_async_finup; - ahash->digest = tmpl->digest; + ahash->digest = n2_hash_async_digest; halg = &ahash->halg; halg->digestsize = tmpl->digest_size; -- cgit v1.1 From dc4ccfd15d4fc7a91ddf222bc5eed5cc4bcf10e6 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sat, 22 May 2010 22:53:09 -0700 Subject: n2_crypto: Add HMAC support. One note is that, unlike with non-HMAC hashes, we can't support hmac(sha224) using the HMAC_SHA256 opcode. Signed-off-by: David S. Miller --- drivers/crypto/n2_core.c | 210 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 206 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index d01a2af..b99c38f 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -246,6 +246,7 @@ struct n2_ahash_alg { u8 hw_op_hashsz; u8 digest_size; u8 auth_type; + u8 hmac_type; struct ahash_alg alg; }; @@ -259,10 +260,36 @@ static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm) return container_of(ahash_alg, struct n2_ahash_alg, alg); } +struct n2_hmac_alg { + const char *child_alg; + struct n2_ahash_alg derived; +}; + +static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct ahash_alg *ahash_alg; + + ahash_alg = container_of(alg, struct ahash_alg, halg.base); + + return container_of(ahash_alg, struct n2_hmac_alg, derived.alg); +} + struct n2_hash_ctx { struct crypto_ahash *fallback_tfm; }; +#define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */ + +struct n2_hmac_ctx { + struct n2_hash_ctx base; + + struct crypto_shash *child_shash; + + int hash_key_len; + unsigned char hash_key[N2_HASH_KEY_MAX]; +}; + struct n2_hash_req_ctx { union { struct md5_state md5; @@ -362,6 +389,94 @@ static void n2_hash_cra_exit(struct crypto_tfm *tfm) crypto_free_ahash(ctx->fallback_tfm); } +static int n2_hmac_cra_init(struct crypto_tfm *tfm) +{ + const char *fallback_driver_name = tfm->__crt_alg->cra_name; + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); + struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm); + struct crypto_ahash *fallback_tfm; + struct crypto_shash *child_shash; + int err; + + fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback_tfm)) { + pr_warning("Fallback driver '%s' could not be loaded!\n", + fallback_driver_name); + err = PTR_ERR(fallback_tfm); + goto out; + } + + child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0); + if (IS_ERR(child_shash)) { + pr_warning("Child shash '%s' could not be loaded!\n", + n2alg->child_alg); + err = PTR_ERR(child_shash); + goto out_free_fallback; + } + + crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) + + crypto_ahash_reqsize(fallback_tfm))); + + ctx->child_shash = child_shash; + ctx->base.fallback_tfm = fallback_tfm; + return 0; + +out_free_fallback: + crypto_free_ahash(fallback_tfm); + +out: + return err; +} + +static void n2_hmac_cra_exit(struct crypto_tfm *tfm) +{ + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); + + crypto_free_ahash(ctx->base.fallback_tfm); + crypto_free_shash(ctx->child_shash); +} + +static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); + struct crypto_shash *child_shash = ctx->child_shash; + struct crypto_ahash *fallback_tfm; + struct { + struct shash_desc shash; + char ctx[crypto_shash_descsize(child_shash)]; + } desc; + int err, bs, ds; + + fallback_tfm = ctx->base.fallback_tfm; + err = crypto_ahash_setkey(fallback_tfm, key, keylen); + if (err) + return err; + + desc.shash.tfm = child_shash; + desc.shash.flags = crypto_ahash_get_flags(tfm) & + CRYPTO_TFM_REQ_MAY_SLEEP; + + bs = crypto_shash_blocksize(child_shash); + ds = crypto_shash_digestsize(child_shash); + BUG_ON(ds > N2_HASH_KEY_MAX); + if (keylen > bs) { + err = crypto_shash_digest(&desc.shash, key, keylen, + ctx->hash_key); + if (err) + return err; + keylen = ds; + } else if (keylen <= N2_HASH_KEY_MAX) + memcpy(ctx->hash_key, key, keylen); + + ctx->hash_key_len = keylen; + + return err; +} + static unsigned long wait_for_tail(struct spu_queue *qp) { unsigned long head, hv_ret; @@ -393,7 +508,8 @@ static unsigned long submit_and_wait_for_tail(struct spu_queue *qp, static int n2_do_async_digest(struct ahash_request *req, unsigned int auth_type, unsigned int digest_size, - unsigned int result_size, void *hash_loc) + unsigned int result_size, void *hash_loc, + unsigned long auth_key, unsigned int auth_key_len) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct cwq_initial_entry *ent; @@ -434,13 +550,13 @@ static int n2_do_async_digest(struct ahash_request *req, */ ent = qp->q + qp->tail; - ent->control = control_word_base(nbytes, 0, 0, + ent->control = control_word_base(nbytes, auth_key_len, 0, auth_type, digest_size, false, true, false, false, OPCODE_INPLACE_BIT | OPCODE_AUTH_MAC); ent->src_addr = __pa(walk.data); - ent->auth_key_addr = 0UL; + ent->auth_key_addr = auth_key; ent->auth_iv_addr = __pa(hash_loc); ent->final_auth_state_addr = 0UL; ent->enc_key_addr = 0UL; @@ -494,7 +610,40 @@ static int n2_hash_async_digest(struct ahash_request *req) return n2_do_async_digest(req, n2alg->auth_type, n2alg->hw_op_hashsz, ds, - &rctx->u); + &rctx->u, 0UL, 0); +} + +static int n2_hmac_async_digest(struct ahash_request *req) +{ + struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm); + struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); + int ds; + + ds = n2alg->derived.digest_size; + if (unlikely(req->nbytes == 0) || + unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) { + struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); + struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.nbytes = req->nbytes; + rctx->fallback_req.src = req->src; + rctx->fallback_req.result = req->result; + + return crypto_ahash_digest(&rctx->fallback_req); + } + memcpy(&rctx->u, n2alg->derived.hash_init, + n2alg->derived.hw_op_hashsz); + + return n2_do_async_digest(req, n2alg->derived.hmac_type, + n2alg->derived.hw_op_hashsz, ds, + &rctx->u, + __pa(&ctx->hash_key), + ctx->hash_key_len); } struct n2_cipher_context { @@ -1127,6 +1276,7 @@ struct n2_hash_tmpl { u8 digest_size; u8 block_size; u8 auth_type; + u8 hmac_type; }; static const char md5_zero[MD5_DIGEST_SIZE] = { @@ -1173,6 +1323,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .hash_zero = md5_zero, .hash_init = md5_init, .auth_type = AUTH_TYPE_MD5, + .hmac_type = AUTH_TYPE_HMAC_MD5, .hw_op_hashsz = MD5_DIGEST_SIZE, .digest_size = MD5_DIGEST_SIZE, .block_size = MD5_HMAC_BLOCK_SIZE }, @@ -1180,6 +1331,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .hash_zero = sha1_zero, .hash_init = sha1_init, .auth_type = AUTH_TYPE_SHA1, + .hmac_type = AUTH_TYPE_HMAC_SHA1, .hw_op_hashsz = SHA1_DIGEST_SIZE, .digest_size = SHA1_DIGEST_SIZE, .block_size = SHA1_BLOCK_SIZE }, @@ -1187,6 +1339,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .hash_zero = sha256_zero, .hash_init = sha256_init, .auth_type = AUTH_TYPE_SHA256, + .hmac_type = AUTH_TYPE_HMAC_SHA256, .hw_op_hashsz = SHA256_DIGEST_SIZE, .digest_size = SHA256_DIGEST_SIZE, .block_size = SHA256_BLOCK_SIZE }, @@ -1194,6 +1347,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .hash_zero = sha224_zero, .hash_init = sha224_init, .auth_type = AUTH_TYPE_SHA256, + .hmac_type = AUTH_TYPE_RESERVED, .hw_op_hashsz = SHA256_DIGEST_SIZE, .digest_size = SHA224_DIGEST_SIZE, .block_size = SHA224_BLOCK_SIZE }, @@ -1201,6 +1355,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls) static LIST_HEAD(ahash_algs); +static LIST_HEAD(hmac_algs); static int algs_registered; @@ -1208,12 +1363,18 @@ static void __n2_unregister_algs(void) { struct n2_cipher_alg *cipher, *cipher_tmp; struct n2_ahash_alg *alg, *alg_tmp; + struct n2_hmac_alg *hmac, *hmac_tmp; list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) { crypto_unregister_alg(&cipher->alg); list_del(&cipher->entry); kfree(cipher); } + list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) { + crypto_unregister_ahash(&hmac->derived.alg); + list_del(&hmac->derived.entry); + kfree(hmac); + } list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) { crypto_unregister_ahash(&alg->alg); list_del(&alg->entry); @@ -1262,6 +1423,44 @@ static int __devinit __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl) return err; } +static int __devinit __n2_register_one_hmac(struct n2_ahash_alg *n2ahash) +{ + struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); + struct ahash_alg *ahash; + struct crypto_alg *base; + int err; + + if (!p) + return -ENOMEM; + + p->child_alg = n2ahash->alg.halg.base.cra_name; + memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg)); + INIT_LIST_HEAD(&p->derived.entry); + + ahash = &p->derived.alg; + ahash->digest = n2_hmac_async_digest; + ahash->setkey = n2_hmac_async_setkey; + + base = &ahash->halg.base; + snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg); + snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg); + + base->cra_ctxsize = sizeof(struct n2_hmac_ctx); + base->cra_init = n2_hmac_cra_init; + base->cra_exit = n2_hmac_cra_exit; + + list_add(&p->derived.entry, &hmac_algs); + err = crypto_register_ahash(ahash); + if (err) { + pr_err("%s alg registration failed\n", base->cra_name); + list_del(&p->derived.entry); + kfree(p); + } else { + pr_info("%s alg registered\n", base->cra_name); + } + return err; +} + static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl) { struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); @@ -1276,6 +1475,7 @@ static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl) p->hash_zero = tmpl->hash_zero; p->hash_init = tmpl->hash_init; p->auth_type = tmpl->auth_type; + p->hmac_type = tmpl->hmac_type; p->hw_op_hashsz = tmpl->hw_op_hashsz; p->digest_size = tmpl->digest_size; @@ -1309,6 +1509,8 @@ static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl) } else { pr_info("%s alg registered\n", base->cra_name); } + if (!err && p->hmac_type != AUTH_TYPE_RESERVED) + err = __n2_register_one_hmac(p); return err; } -- cgit v1.1 From 0efbaabd1ec91476c020e96240d6ab858e9a4871 Mon Sep 17 00:00:00 2001 From: Huang Weiyi Date: Wed, 26 May 2010 10:37:52 +1000 Subject: crypto: omap - remove unused #include Remove unused #include ('s) in drivers/crypto/omap-sham.c Signed-off-by: Huang Weiyi Signed-off-by: Herbert Xu --- drivers/crypto/omap-sham.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 8b03433..7d14856 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -15,7 +15,6 @@ #define pr_fmt(fmt) "%s: " fmt, __func__ -#include #include #include #include -- cgit v1.1 From 026c8ec69d6077846ba8000b3301e8a62c702dc8 Mon Sep 17 00:00:00 2001 From: Andrea Gelmini Date: Wed, 26 May 2010 10:41:03 +1000 Subject: crypto: des_s390 - Checkpatch cleanup arch/s390/crypto/crypto_des.h:18: ERROR: do not use C99 // comments Signed-off-by: Andrea Gelmini Signed-off-by: Herbert Xu --- arch/s390/crypto/crypto_des.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/crypto/crypto_des.h b/arch/s390/crypto/crypto_des.h index c964b64..6210457 100644 --- a/arch/s390/crypto/crypto_des.h +++ b/arch/s390/crypto/crypto_des.h @@ -15,4 +15,4 @@ extern int crypto_des_check_key(const u8*, unsigned int, u32*); -#endif //__CRYPTO_DES_H__ +#endif /*__CRYPTO_DES_H__*/ -- cgit v1.1 From 7cc2835083aedfde42de02301005a5555e00c4b1 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 26 May 2010 10:45:22 +1000 Subject: crypto: mv_cesa - fixup error handling in mv_probe() The error handling in mv_probe() was a bit messed up. There were some gotos to the wrong labels so it ended up releasing stuff that that hadn't been aquired and not releasing stuff that was meant to be released. I shuffled it around a bit to fix it and make it clearer. Signed-off-by: Dan Carpenter Signed-off-by: Herbert Xu --- drivers/crypto/mv_cesa.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c index e095422..7d279e5 100644 --- a/drivers/crypto/mv_cesa.c +++ b/drivers/crypto/mv_cesa.c @@ -1055,20 +1055,20 @@ static int mv_probe(struct platform_device *pdev) cp->queue_th = kthread_run(queue_manag, cp, "mv_crypto"); if (IS_ERR(cp->queue_th)) { ret = PTR_ERR(cp->queue_th); - goto err_thread; + goto err_unmap_sram; } ret = request_irq(irq, crypto_int, IRQF_DISABLED, dev_name(&pdev->dev), cp); if (ret) - goto err_unmap_sram; + goto err_thread; writel(SEC_INT_ACCEL0_DONE, cpg->reg + SEC_ACCEL_INT_MASK); writel(SEC_CFG_STOP_DIG_ERR, cpg->reg + SEC_ACCEL_CFG); ret = crypto_register_alg(&mv_aes_alg_ecb); if (ret) - goto err_reg; + goto err_irq; ret = crypto_register_alg(&mv_aes_alg_cbc); if (ret) @@ -1091,9 +1091,9 @@ static int mv_probe(struct platform_device *pdev) return 0; err_unreg_ecb: crypto_unregister_alg(&mv_aes_alg_ecb); -err_thread: +err_irq: free_irq(irq, cp); -err_reg: +err_thread: kthread_stop(cp->queue_th); err_unmap_sram: iounmap(cp->sram); -- cgit v1.1 From 749d811f10a410b64cf4c674c498ec04316ec373 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Thu, 3 Jun 2010 20:19:28 +1000 Subject: padata: add parenthesis in MAX_SEQ_NR macro MAX_SEQ_NR is used in padata_alloc_pd() like this: pd->max_seq_nr = (MAX_SEQ_NR / num_cpus) * num_cpus - 1; It needs parenthesis or the divide by num_cpus takes precedence over the subtraction. Signed-off-by: Dan Carpenter Acked-by: Steffen Klassert Signed-off-by: Herbert Xu --- kernel/padata.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/padata.c b/kernel/padata.c index b1c9857..ff8de1b 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -28,7 +28,7 @@ #include #include -#define MAX_SEQ_NR INT_MAX - NR_CPUS +#define MAX_SEQ_NR (INT_MAX - NR_CPUS) #define MAX_OBJ_NUM 1000 static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index) -- cgit v1.1 From bc94e59662c13516d13e117b6edab4bec487d5a0 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 3 Jun 2010 20:33:06 +1000 Subject: crypto: pcomp - Fix illegal Kconfig configuration The PCOMP Kconfig entry current allows the following combination which is illegal: ZLIB=y PCOMP=y ALGAPI=m ALGAPI2=y MANAGER=m MANAGER2=m This patch fixes this by adding PCOMP2 so that PCOMP can select ALGAPI to propagate the setting to MANAGER2. Signed-off-by: Herbert Xu --- crypto/Kconfig | 7 ++++++- crypto/Makefile | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 9d9434f..11d0f21 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -80,6 +80,11 @@ config CRYPTO_RNG2 config CRYPTO_PCOMP tristate + select CRYPTO_PCOMP2 + select CRYPTO_ALGAPI + +config CRYPTO_PCOMP2 + tristate select CRYPTO_ALGAPI2 config CRYPTO_MANAGER @@ -94,7 +99,7 @@ config CRYPTO_MANAGER2 select CRYPTO_AEAD2 select CRYPTO_HASH2 select CRYPTO_BLKCIPHER2 - select CRYPTO_PCOMP + select CRYPTO_PCOMP2 config CRYPTO_GF128MUL tristate "GF(2^128) multiplication functions (EXPERIMENTAL)" diff --git a/crypto/Makefile b/crypto/Makefile index d7e6441..d1cfa05 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -26,7 +26,7 @@ crypto_hash-objs += ahash.o crypto_hash-objs += shash.o obj-$(CONFIG_CRYPTO_HASH2) += crypto_hash.o -obj-$(CONFIG_CRYPTO_PCOMP) += pcompress.o +obj-$(CONFIG_CRYPTO_PCOMP2) += pcompress.o cryptomgr-objs := algboss.o testmgr.o -- cgit v1.1 From 0b767f96164b2b27488e3daa722ff16e89d49314 Mon Sep 17 00:00:00 2001 From: Alexander Shishkin Date: Thu, 3 Jun 2010 20:53:43 +1000 Subject: crypto: testmgr - add an option to disable cryptoalgos' self-tests By default, CONFIG_CRYPTO_MANAGER_TESTS will be enabled and thus self-tests will still run, but it is now possible to disable them to gain some time during bootup. Signed-off-by: Alexander Shishkin Signed-off-by: Herbert Xu --- crypto/Kconfig | 8 ++++++++ crypto/algboss.c | 4 ++++ crypto/testmgr.c | 14 ++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/crypto/Kconfig b/crypto/Kconfig index 11d0f21..1cd497d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -101,6 +101,14 @@ config CRYPTO_MANAGER2 select CRYPTO_BLKCIPHER2 select CRYPTO_PCOMP2 +config CRYPTO_MANAGER_TESTS + bool "Run algolithms' self-tests" + default y + depends on CRYPTO_MANAGER2 + help + Run cryptomanager's tests for the new crypto algorithms being + registered. + config CRYPTO_GF128MUL tristate "GF(2^128) multiplication functions (EXPERIMENTAL)" depends on EXPERIMENTAL diff --git a/crypto/algboss.c b/crypto/algboss.c index c3c196b..40bd391 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -206,6 +206,7 @@ err: return NOTIFY_OK; } +#ifdef CONFIG_CRYPTO_MANAGER_TESTS static int cryptomgr_test(void *data) { struct crypto_test_param *param = data; @@ -266,6 +267,7 @@ err_put_module: err: return NOTIFY_OK; } +#endif /* CONFIG_CRYPTO_MANAGER_TESTS */ static int cryptomgr_notify(struct notifier_block *this, unsigned long msg, void *data) @@ -273,8 +275,10 @@ static int cryptomgr_notify(struct notifier_block *this, unsigned long msg, switch (msg) { case CRYPTO_MSG_ALG_REQUEST: return cryptomgr_schedule_probe(data); +#ifdef CONFIG_CRYPTO_MANAGER_TESTS case CRYPTO_MSG_ALG_REGISTER: return cryptomgr_schedule_test(data); +#endif } return NOTIFY_DONE; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 5c8aaa0..abd980c 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -22,6 +22,17 @@ #include #include "internal.h" + +#ifndef CONFIG_CRYPTO_MANAGER_TESTS + +/* a perfect nop */ +int alg_test(const char *driver, const char *alg, u32 type, u32 mask) +{ + return 0; +} + +#else + #include "testmgr.h" /* @@ -2530,4 +2541,7 @@ notest: non_fips_alg: return -EINVAL; } + +#endif /* CONFIG_CRYPTO_MANAGER_TESTS */ + EXPORT_SYMBOL_GPL(alg_test); -- cgit v1.1 From 5b04fc170382195d7d33fd08e3ccc2ad8e50e782 Mon Sep 17 00:00:00 2001 From: Joachim Fritschi Date: Thu, 3 Jun 2010 21:02:51 +1000 Subject: crypto: twofish: Rename twofish to twofish_generic and add an alias This fixes the broken autoloading of the corresponding twofish assembler ciphers on x86 and x86_64 if they are available. The module name of the generic implementation was in conflict with the alias in the assembler modules. The generic twofish c implementation is renamed to twofish_generic according to the other algorithms with assembler implementations and an module alias is added for 'twofish'. You can now load 'twofish' giving you the best implementation by priority, 'twofish-generic' to get the c implementation or 'twofish-asm' to get the assembler version of cipher. Signed-off-by: Joachim Fritschi Signed-off-by: Herbert Xu --- crypto/Makefile | 2 +- crypto/twofish.c | 214 ---------------------------------------------- crypto/twofish_generic.c | 215 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 215 deletions(-) delete mode 100644 crypto/twofish.c create mode 100644 crypto/twofish_generic.c diff --git a/crypto/Makefile b/crypto/Makefile index d1cfa05..423b7de 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -61,7 +61,7 @@ obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o obj-$(CONFIG_CRYPTO_DES) += des_generic.o obj-$(CONFIG_CRYPTO_FCRYPT) += fcrypt.o obj-$(CONFIG_CRYPTO_BLOWFISH) += blowfish.o -obj-$(CONFIG_CRYPTO_TWOFISH) += twofish.o +obj-$(CONFIG_CRYPTO_TWOFISH) += twofish_generic.o obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o obj-$(CONFIG_CRYPTO_SERPENT) += serpent.o obj-$(CONFIG_CRYPTO_AES) += aes_generic.o diff --git a/crypto/twofish.c b/crypto/twofish.c deleted file mode 100644 index dfcda23..0000000 --- a/crypto/twofish.c +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Twofish for CryptoAPI - * - * Originally Twofish for GPG - * By Matthew Skala , July 26, 1998 - * 256-bit key length added March 20, 1999 - * Some modifications to reduce the text size by Werner Koch, April, 1998 - * Ported to the kerneli patch by Marc Mutz - * Ported to CryptoAPI by Colin Slater - * - * The original author has disclaimed all copyright interest in this - * code and thus put it in the public domain. The subsequent authors - * have put this under the GNU General Public License. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA - * - * This code is a "clean room" implementation, written from the paper - * _Twofish: A 128-Bit Block Cipher_ by Bruce Schneier, John Kelsey, - * Doug Whiting, David Wagner, Chris Hall, and Niels Ferguson, available - * through http://www.counterpane.com/twofish.html - * - * For background information on multiplication in finite fields, used for - * the matrix operations in the key schedule, see the book _Contemporary - * Abstract Algebra_ by Joseph A. Gallian, especially chapter 22 in the - * Third Edition. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -/* Macros to compute the g() function in the encryption and decryption - * rounds. G1 is the straight g() function; G2 includes the 8-bit - * rotation for the high 32-bit word. */ - -#define G1(a) \ - (ctx->s[0][(a) & 0xFF]) ^ (ctx->s[1][((a) >> 8) & 0xFF]) \ - ^ (ctx->s[2][((a) >> 16) & 0xFF]) ^ (ctx->s[3][(a) >> 24]) - -#define G2(b) \ - (ctx->s[1][(b) & 0xFF]) ^ (ctx->s[2][((b) >> 8) & 0xFF]) \ - ^ (ctx->s[3][((b) >> 16) & 0xFF]) ^ (ctx->s[0][(b) >> 24]) - -/* Encryption and decryption Feistel rounds. Each one calls the two g() - * macros, does the PHT, and performs the XOR and the appropriate bit - * rotations. The parameters are the round number (used to select subkeys), - * and the four 32-bit chunks of the text. */ - -#define ENCROUND(n, a, b, c, d) \ - x = G1 (a); y = G2 (b); \ - x += y; y += x + ctx->k[2 * (n) + 1]; \ - (c) ^= x + ctx->k[2 * (n)]; \ - (c) = ror32((c), 1); \ - (d) = rol32((d), 1) ^ y - -#define DECROUND(n, a, b, c, d) \ - x = G1 (a); y = G2 (b); \ - x += y; y += x; \ - (d) ^= y + ctx->k[2 * (n) + 1]; \ - (d) = ror32((d), 1); \ - (c) = rol32((c), 1); \ - (c) ^= (x + ctx->k[2 * (n)]) - -/* Encryption and decryption cycles; each one is simply two Feistel rounds - * with the 32-bit chunks re-ordered to simulate the "swap" */ - -#define ENCCYCLE(n) \ - ENCROUND (2 * (n), a, b, c, d); \ - ENCROUND (2 * (n) + 1, c, d, a, b) - -#define DECCYCLE(n) \ - DECROUND (2 * (n) + 1, c, d, a, b); \ - DECROUND (2 * (n), a, b, c, d) - -/* Macros to convert the input and output bytes into 32-bit words, - * and simultaneously perform the whitening step. INPACK packs word - * number n into the variable named by x, using whitening subkey number m. - * OUTUNPACK unpacks word number n from the variable named by x, using - * whitening subkey number m. */ - -#define INPACK(n, x, m) \ - x = le32_to_cpu(src[n]) ^ ctx->w[m] - -#define OUTUNPACK(n, x, m) \ - x ^= ctx->w[m]; \ - dst[n] = cpu_to_le32(x) - - - -/* Encrypt one block. in and out may be the same. */ -static void twofish_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - struct twofish_ctx *ctx = crypto_tfm_ctx(tfm); - const __le32 *src = (const __le32 *)in; - __le32 *dst = (__le32 *)out; - - /* The four 32-bit chunks of the text. */ - u32 a, b, c, d; - - /* Temporaries used by the round function. */ - u32 x, y; - - /* Input whitening and packing. */ - INPACK (0, a, 0); - INPACK (1, b, 1); - INPACK (2, c, 2); - INPACK (3, d, 3); - - /* Encryption Feistel cycles. */ - ENCCYCLE (0); - ENCCYCLE (1); - ENCCYCLE (2); - ENCCYCLE (3); - ENCCYCLE (4); - ENCCYCLE (5); - ENCCYCLE (6); - ENCCYCLE (7); - - /* Output whitening and unpacking. */ - OUTUNPACK (0, c, 4); - OUTUNPACK (1, d, 5); - OUTUNPACK (2, a, 6); - OUTUNPACK (3, b, 7); - -} - -/* Decrypt one block. in and out may be the same. */ -static void twofish_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - struct twofish_ctx *ctx = crypto_tfm_ctx(tfm); - const __le32 *src = (const __le32 *)in; - __le32 *dst = (__le32 *)out; - - /* The four 32-bit chunks of the text. */ - u32 a, b, c, d; - - /* Temporaries used by the round function. */ - u32 x, y; - - /* Input whitening and packing. */ - INPACK (0, c, 4); - INPACK (1, d, 5); - INPACK (2, a, 6); - INPACK (3, b, 7); - - /* Encryption Feistel cycles. */ - DECCYCLE (7); - DECCYCLE (6); - DECCYCLE (5); - DECCYCLE (4); - DECCYCLE (3); - DECCYCLE (2); - DECCYCLE (1); - DECCYCLE (0); - - /* Output whitening and unpacking. */ - OUTUNPACK (0, a, 0); - OUTUNPACK (1, b, 1); - OUTUNPACK (2, c, 2); - OUTUNPACK (3, d, 3); - -} - -static struct crypto_alg alg = { - .cra_name = "twofish", - .cra_driver_name = "twofish-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = TF_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct twofish_ctx), - .cra_alignmask = 3, - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(alg.cra_list), - .cra_u = { .cipher = { - .cia_min_keysize = TF_MIN_KEY_SIZE, - .cia_max_keysize = TF_MAX_KEY_SIZE, - .cia_setkey = twofish_setkey, - .cia_encrypt = twofish_encrypt, - .cia_decrypt = twofish_decrypt } } -}; - -static int __init twofish_mod_init(void) -{ - return crypto_register_alg(&alg); -} - -static void __exit twofish_mod_fini(void) -{ - crypto_unregister_alg(&alg); -} - -module_init(twofish_mod_init); -module_exit(twofish_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION ("Twofish Cipher Algorithm"); diff --git a/crypto/twofish_generic.c b/crypto/twofish_generic.c new file mode 100644 index 0000000..1f07b84 --- /dev/null +++ b/crypto/twofish_generic.c @@ -0,0 +1,215 @@ +/* + * Twofish for CryptoAPI + * + * Originally Twofish for GPG + * By Matthew Skala , July 26, 1998 + * 256-bit key length added March 20, 1999 + * Some modifications to reduce the text size by Werner Koch, April, 1998 + * Ported to the kerneli patch by Marc Mutz + * Ported to CryptoAPI by Colin Slater + * + * The original author has disclaimed all copyright interest in this + * code and thus put it in the public domain. The subsequent authors + * have put this under the GNU General Public License. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * + * This code is a "clean room" implementation, written from the paper + * _Twofish: A 128-Bit Block Cipher_ by Bruce Schneier, John Kelsey, + * Doug Whiting, David Wagner, Chris Hall, and Niels Ferguson, available + * through http://www.counterpane.com/twofish.html + * + * For background information on multiplication in finite fields, used for + * the matrix operations in the key schedule, see the book _Contemporary + * Abstract Algebra_ by Joseph A. Gallian, especially chapter 22 in the + * Third Edition. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Macros to compute the g() function in the encryption and decryption + * rounds. G1 is the straight g() function; G2 includes the 8-bit + * rotation for the high 32-bit word. */ + +#define G1(a) \ + (ctx->s[0][(a) & 0xFF]) ^ (ctx->s[1][((a) >> 8) & 0xFF]) \ + ^ (ctx->s[2][((a) >> 16) & 0xFF]) ^ (ctx->s[3][(a) >> 24]) + +#define G2(b) \ + (ctx->s[1][(b) & 0xFF]) ^ (ctx->s[2][((b) >> 8) & 0xFF]) \ + ^ (ctx->s[3][((b) >> 16) & 0xFF]) ^ (ctx->s[0][(b) >> 24]) + +/* Encryption and decryption Feistel rounds. Each one calls the two g() + * macros, does the PHT, and performs the XOR and the appropriate bit + * rotations. The parameters are the round number (used to select subkeys), + * and the four 32-bit chunks of the text. */ + +#define ENCROUND(n, a, b, c, d) \ + x = G1 (a); y = G2 (b); \ + x += y; y += x + ctx->k[2 * (n) + 1]; \ + (c) ^= x + ctx->k[2 * (n)]; \ + (c) = ror32((c), 1); \ + (d) = rol32((d), 1) ^ y + +#define DECROUND(n, a, b, c, d) \ + x = G1 (a); y = G2 (b); \ + x += y; y += x; \ + (d) ^= y + ctx->k[2 * (n) + 1]; \ + (d) = ror32((d), 1); \ + (c) = rol32((c), 1); \ + (c) ^= (x + ctx->k[2 * (n)]) + +/* Encryption and decryption cycles; each one is simply two Feistel rounds + * with the 32-bit chunks re-ordered to simulate the "swap" */ + +#define ENCCYCLE(n) \ + ENCROUND (2 * (n), a, b, c, d); \ + ENCROUND (2 * (n) + 1, c, d, a, b) + +#define DECCYCLE(n) \ + DECROUND (2 * (n) + 1, c, d, a, b); \ + DECROUND (2 * (n), a, b, c, d) + +/* Macros to convert the input and output bytes into 32-bit words, + * and simultaneously perform the whitening step. INPACK packs word + * number n into the variable named by x, using whitening subkey number m. + * OUTUNPACK unpacks word number n from the variable named by x, using + * whitening subkey number m. */ + +#define INPACK(n, x, m) \ + x = le32_to_cpu(src[n]) ^ ctx->w[m] + +#define OUTUNPACK(n, x, m) \ + x ^= ctx->w[m]; \ + dst[n] = cpu_to_le32(x) + + + +/* Encrypt one block. in and out may be the same. */ +static void twofish_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +{ + struct twofish_ctx *ctx = crypto_tfm_ctx(tfm); + const __le32 *src = (const __le32 *)in; + __le32 *dst = (__le32 *)out; + + /* The four 32-bit chunks of the text. */ + u32 a, b, c, d; + + /* Temporaries used by the round function. */ + u32 x, y; + + /* Input whitening and packing. */ + INPACK (0, a, 0); + INPACK (1, b, 1); + INPACK (2, c, 2); + INPACK (3, d, 3); + + /* Encryption Feistel cycles. */ + ENCCYCLE (0); + ENCCYCLE (1); + ENCCYCLE (2); + ENCCYCLE (3); + ENCCYCLE (4); + ENCCYCLE (5); + ENCCYCLE (6); + ENCCYCLE (7); + + /* Output whitening and unpacking. */ + OUTUNPACK (0, c, 4); + OUTUNPACK (1, d, 5); + OUTUNPACK (2, a, 6); + OUTUNPACK (3, b, 7); + +} + +/* Decrypt one block. in and out may be the same. */ +static void twofish_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +{ + struct twofish_ctx *ctx = crypto_tfm_ctx(tfm); + const __le32 *src = (const __le32 *)in; + __le32 *dst = (__le32 *)out; + + /* The four 32-bit chunks of the text. */ + u32 a, b, c, d; + + /* Temporaries used by the round function. */ + u32 x, y; + + /* Input whitening and packing. */ + INPACK (0, c, 4); + INPACK (1, d, 5); + INPACK (2, a, 6); + INPACK (3, b, 7); + + /* Encryption Feistel cycles. */ + DECCYCLE (7); + DECCYCLE (6); + DECCYCLE (5); + DECCYCLE (4); + DECCYCLE (3); + DECCYCLE (2); + DECCYCLE (1); + DECCYCLE (0); + + /* Output whitening and unpacking. */ + OUTUNPACK (0, a, 0); + OUTUNPACK (1, b, 1); + OUTUNPACK (2, c, 2); + OUTUNPACK (3, d, 3); + +} + +static struct crypto_alg alg = { + .cra_name = "twofish", + .cra_driver_name = "twofish-generic", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(alg.cra_list), + .cra_u = { .cipher = { + .cia_min_keysize = TF_MIN_KEY_SIZE, + .cia_max_keysize = TF_MAX_KEY_SIZE, + .cia_setkey = twofish_setkey, + .cia_encrypt = twofish_encrypt, + .cia_decrypt = twofish_decrypt } } +}; + +static int __init twofish_mod_init(void) +{ + return crypto_register_alg(&alg); +} + +static void __exit twofish_mod_fini(void) +{ + crypto_unregister_alg(&alg); +} + +module_init(twofish_mod_init); +module_exit(twofish_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION ("Twofish Cipher Algorithm"); +MODULE_ALIAS("twofish"); -- cgit v1.1 From 5e833bc4166ea524c00a95e777e4344844ed189f Mon Sep 17 00:00:00 2001 From: Lee Nipper Date: Wed, 16 Jun 2010 15:29:15 +1000 Subject: crypto: talitos - fix ahash for multiple of blocksize Correct ahash_process_req() to properly handle cases where the total hash amount is a multiple of the blocksize. The SEC must have some data to hash during the very last descriptor operation; so up to one whole blocksize of data is buffered until the final hash. Signed-off-by: Lee Nipper Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 77 +++++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 637c105..0f2483e 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -720,7 +720,6 @@ struct talitos_ctx { #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 struct talitos_ahash_req_ctx { - u64 count; u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; unsigned int hw_context_size; u8 buf[HASH_MAX_BLOCK_SIZE]; @@ -729,6 +728,7 @@ struct talitos_ahash_req_ctx { unsigned int first; unsigned int last; unsigned int to_hash_later; + u64 nbuf; struct scatterlist bufsl[2]; struct scatterlist *psrc; }; @@ -1609,6 +1609,7 @@ static void ahash_done(struct device *dev, if (!req_ctx->last && req_ctx->to_hash_later) { /* Position any partial block for next update/final/finup */ memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later); + req_ctx->nbuf = req_ctx->to_hash_later; } common_nonsnoop_hash_unmap(dev, edesc, areq); @@ -1724,7 +1725,7 @@ static int ahash_init(struct ahash_request *areq) struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); /* Initialize the context */ - req_ctx->count = 0; + req_ctx->nbuf = 0; req_ctx->first = 1; /* first indicates h/w must init its context */ req_ctx->swinit = 0; /* assume h/w init of context */ req_ctx->hw_context_size = @@ -1772,52 +1773,54 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); unsigned int nbytes_to_hash; unsigned int to_hash_later; - unsigned int index; + unsigned int nsg; int chained; - index = req_ctx->count & (blocksize - 1); - req_ctx->count += nbytes; - - if (!req_ctx->last && (index + nbytes) < blocksize) { - /* Buffer the partial block */ + if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { + /* Buffer up to one whole block */ sg_copy_to_buffer(areq->src, sg_count(areq->src, nbytes, &chained), - req_ctx->buf + index, nbytes); + req_ctx->buf + req_ctx->nbuf, nbytes); + req_ctx->nbuf += nbytes; return 0; } - if (index) { - /* partial block from previous update; chain it in. */ - sg_init_table(req_ctx->bufsl, (nbytes) ? 2 : 1); - sg_set_buf(req_ctx->bufsl, req_ctx->buf, index); - if (nbytes) - scatterwalk_sg_chain(req_ctx->bufsl, 2, - areq->src); + /* At least (blocksize + 1) bytes are available to hash */ + nbytes_to_hash = nbytes + req_ctx->nbuf; + to_hash_later = nbytes_to_hash & (blocksize - 1); + + if (req_ctx->last) + to_hash_later = 0; + else if (to_hash_later) + /* There is a partial block. Hash the full block(s) now */ + nbytes_to_hash -= to_hash_later; + else { + /* Keep one block buffered */ + nbytes_to_hash -= blocksize; + to_hash_later = blocksize; + } + + /* Chain in any previously buffered data */ + if (req_ctx->nbuf) { + nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; + sg_init_table(req_ctx->bufsl, nsg); + sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf); + if (nsg > 1) + scatterwalk_sg_chain(req_ctx->bufsl, 2, areq->src); req_ctx->psrc = req_ctx->bufsl; - } else { + } else req_ctx->psrc = areq->src; + + if (to_hash_later) { + int nents = sg_count(areq->src, nbytes, &chained); + sg_copy_end_to_buffer(areq->src, nents, + req_ctx->bufnext, + to_hash_later, + nbytes - to_hash_later); } - nbytes_to_hash = index + nbytes; - if (!req_ctx->last) { - to_hash_later = (nbytes_to_hash & (blocksize - 1)); - if (to_hash_later) { - int nents; - /* Must copy to_hash_later bytes from the end - * to bufnext (a partial block) for later. - */ - nents = sg_count(areq->src, nbytes, &chained); - sg_copy_end_to_buffer(areq->src, nents, - req_ctx->bufnext, - to_hash_later, - nbytes - to_hash_later); - - /* Adjust count for what will be hashed now */ - nbytes_to_hash -= to_hash_later; - } - req_ctx->to_hash_later = to_hash_later; - } + req_ctx->to_hash_later = to_hash_later; - /* allocate extended descriptor */ + /* Allocate extended descriptor */ edesc = ahash_edesc_alloc(areq, nbytes_to_hash); if (IS_ERR(edesc)) return PTR_ERR(edesc); -- cgit v1.1 From 8b9cfdca9c52f7d39c3ccfac1668e31c20c9f42e Mon Sep 17 00:00:00 2001 From: Kulikov Vasiliy Date: Mon, 5 Jul 2010 21:25:06 +0800 Subject: hwrng: n2-drv - remove casts from void* Remove unnesessary casts from void*. Signed-off-by: Kulikov Vasiliy Signed-off-by: Herbert Xu --- drivers/char/hw_random/n2-drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/char/hw_random/n2-drv.c b/drivers/char/hw_random/n2-drv.c index 0f9cbf1..101d5f2 100644 --- a/drivers/char/hw_random/n2-drv.c +++ b/drivers/char/hw_random/n2-drv.c @@ -387,7 +387,7 @@ static int n2rng_init_control(struct n2rng *np) static int n2rng_data_read(struct hwrng *rng, u32 *data) { - struct n2rng *np = (struct n2rng *) rng->priv; + struct n2rng *np = rng->priv; unsigned long ra = __pa(&np->test_data); int len; -- cgit v1.1 From 7e3de7b1be6ce0643f60aed697070e2286db32cd Mon Sep 17 00:00:00 2001 From: "Justin P. Mattock" Date: Wed, 14 Jul 2010 20:11:39 +0800 Subject: crypto: hifn_795x - Remove unused ctx variable The below patch gets rid of an unused variable ctx reported by GCC when building the kernel. CC [M] drivers/crypto/hifn_795x.o drivers/crypto/hifn_795x.c: In function 'hifn_flush': drivers/crypto/hifn_795x.c:2021:23: warning: variable 'ctx' set but not used drivers/crypto/hifn_795x.c: In function 'hifn_process_queue': drivers/crypto/hifn_795x.c:2142:23: warning: variable 'ctx' set but not used Signed-off-by: Justin P. Mattock Signed-off-by: Herbert Xu --- drivers/crypto/hifn_795x.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 16fce3a..e449ac5 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -2018,7 +2018,6 @@ static void hifn_flush(struct hifn_device *dev) { unsigned long flags; struct crypto_async_request *async_req; - struct hifn_context *ctx; struct ablkcipher_request *req; struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int i; @@ -2035,7 +2034,6 @@ static void hifn_flush(struct hifn_device *dev) spin_lock_irqsave(&dev->lock, flags); while ((async_req = crypto_dequeue_request(&dev->queue))) { - ctx = crypto_tfm_ctx(async_req->tfm); req = container_of(async_req, struct ablkcipher_request, base); spin_unlock_irqrestore(&dev->lock, flags); @@ -2139,7 +2137,6 @@ static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op, static int hifn_process_queue(struct hifn_device *dev) { struct crypto_async_request *async_req, *backlog; - struct hifn_context *ctx; struct ablkcipher_request *req; unsigned long flags; int err = 0; @@ -2156,7 +2153,6 @@ static int hifn_process_queue(struct hifn_device *dev) if (backlog) backlog->complete(backlog, -EINPROGRESS); - ctx = crypto_tfm_ctx(async_req->tfm); req = container_of(async_req, struct ablkcipher_request, base); err = hifn_handle_req(req); -- cgit v1.1 From 4c879170296174bde05cd1c643dac16594edee77 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Wed, 7 Jul 2010 15:30:10 +0200 Subject: padata: Check for valid padata instance on start This patch introduces the PADATA_INVALID flag which is checked on padata start. This will be used to mark a padata instance as invalid, if the padata cpumask does not intersect with the active cpumask. we change padata_start to return an error if the PADATA_INVALID is set. Also we adapt the only padata user, pcrypt to this change. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 19 ++++++++++++++----- include/linux/padata.h | 3 ++- kernel/padata.c | 18 ++++++++++++++++-- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 247178c..71ae2b2 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -385,6 +385,7 @@ static struct crypto_template pcrypt_tmpl = { static int __init pcrypt_init(void) { + int err = -ENOMEM; encwq = create_workqueue("pencrypt"); if (!encwq) goto err; @@ -400,14 +401,22 @@ static int __init pcrypt_init(void) pcrypt_dec_padata = padata_alloc(cpu_possible_mask, decwq); if (!pcrypt_dec_padata) - goto err_free_padata; + goto err_free_enc_padata; - padata_start(pcrypt_enc_padata); - padata_start(pcrypt_dec_padata); + err = padata_start(pcrypt_enc_padata); + if (err) + goto err_free_dec_padata; + + err = padata_start(pcrypt_dec_padata); + if (err) + goto err_free_dec_padata; return crypto_register_template(&pcrypt_tmpl); -err_free_padata: +err_free_dec_padata: + padata_free(pcrypt_dec_padata); + +err_free_enc_padata: padata_free(pcrypt_enc_padata); err_destroy_decwq: @@ -417,7 +426,7 @@ err_destroy_encwq: destroy_workqueue(encwq); err: - return -ENOMEM; + return err; } static void __exit pcrypt_exit(void) diff --git a/include/linux/padata.h b/include/linux/padata.h index 8d84062..e4c17f9 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -126,6 +126,7 @@ struct padata_instance { u8 flags; #define PADATA_INIT 1 #define PADATA_RESET 2 +#define PADATA_INVALID 4 }; extern struct padata_instance *padata_alloc(const struct cpumask *cpumask, @@ -138,6 +139,6 @@ extern int padata_set_cpumask(struct padata_instance *pinst, cpumask_var_t cpumask); extern int padata_add_cpu(struct padata_instance *pinst, int cpu); extern int padata_remove_cpu(struct padata_instance *pinst, int cpu); -extern void padata_start(struct padata_instance *pinst); +extern int padata_start(struct padata_instance *pinst); extern void padata_stop(struct padata_instance *pinst); #endif diff --git a/kernel/padata.c b/kernel/padata.c index ff8de1b..e7d723a 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -485,6 +485,11 @@ static void padata_flush_queues(struct parallel_data *pd) BUG_ON(atomic_read(&pd->refcnt) != 0); } +static void __padata_start(struct padata_instance *pinst) +{ + pinst->flags |= PADATA_INIT; +} + /* Replace the internal control stucture with a new one. */ static void padata_replace(struct padata_instance *pinst, struct parallel_data *pd_new) @@ -619,11 +624,20 @@ EXPORT_SYMBOL(padata_remove_cpu); * * @pinst: padata instance to start */ -void padata_start(struct padata_instance *pinst) +int padata_start(struct padata_instance *pinst) { + int err = 0; + mutex_lock(&pinst->lock); - pinst->flags |= PADATA_INIT; + + if (pinst->flags & PADATA_INVALID) + err =-EINVAL; + + __padata_start(pinst); + mutex_unlock(&pinst->lock); + + return err; } EXPORT_SYMBOL(padata_start); -- cgit v1.1 From ee836555120140f770005b8ce6673c913d1b9a98 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Wed, 7 Jul 2010 15:30:47 +0200 Subject: padata: Block until the instance is unused on stop This patch makes padata_stop to block until the padata instance is unused. Also we split padata_stop to a locked and a unlocked version. This is in preparation to be able to change the cpumask after a call to patata stop. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- kernel/padata.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/kernel/padata.c b/kernel/padata.c index e7d723a..9e18dfa 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -490,6 +490,20 @@ static void __padata_start(struct padata_instance *pinst) pinst->flags |= PADATA_INIT; } +static void __padata_stop(struct padata_instance *pinst) +{ + if (!(pinst->flags & PADATA_INIT)) + return; + + pinst->flags &= ~PADATA_INIT; + + synchronize_rcu(); + + get_online_cpus(); + padata_flush_queues(pinst->pd); + put_online_cpus(); +} + /* Replace the internal control stucture with a new one. */ static void padata_replace(struct padata_instance *pinst, struct parallel_data *pd_new) @@ -649,7 +663,7 @@ EXPORT_SYMBOL(padata_start); void padata_stop(struct padata_instance *pinst) { mutex_lock(&pinst->lock); - pinst->flags &= ~PADATA_INIT; + __padata_stop(pinst); mutex_unlock(&pinst->lock); } EXPORT_SYMBOL(padata_stop); @@ -770,17 +784,11 @@ EXPORT_SYMBOL(padata_alloc); */ void padata_free(struct padata_instance *pinst) { - padata_stop(pinst); - - synchronize_rcu(); - #ifdef CONFIG_HOTPLUG_CPU unregister_hotcpu_notifier(&pinst->cpu_notifier); #endif - get_online_cpus(); - padata_flush_queues(pinst->pd); - put_online_cpus(); + padata_stop(pinst); padata_free_pd(pinst->pd); free_cpumask_var(pinst->cpumask); kfree(pinst); -- cgit v1.1 From 33e54450683c5e970ac007489d7921ba792d093c Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Wed, 7 Jul 2010 15:31:26 +0200 Subject: padata: Handle empty padata cpumasks This patch fixes a bug when the padata cpumask does not intersect with the active cpumask. In this case we get a division by zero in padata_alloc_pd and we end up with a useless padata instance. Padata can end up with an empty cpumask for two reasons: 1. A user removed the last cpu that belongs to the padata cpumask and the active cpumask. 2. The last cpu that belongs to the padata cpumask and the active cpumask goes offline. We introduce a function padata_validate_cpumask to check if the padata cpumask does intersect with the active cpumask. If the cpumasks do not intersect we mark the instance as invalid, so it can't be used. We do not allocate the cpumask dependend recources in this case. This fixes the division by zero and keeps the padate instance in a consistent state. It's not possible to trigger this bug by now because the only padata user, pcrypt uses always the possible cpumask. Reported-by: Dan Kruchinin Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- kernel/padata.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/kernel/padata.c b/kernel/padata.c index 9e18dfa..57ec4eb 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -516,12 +516,27 @@ static void padata_replace(struct padata_instance *pinst, synchronize_rcu(); - padata_flush_queues(pd_old); - padata_free_pd(pd_old); + if (pd_old) { + padata_flush_queues(pd_old); + padata_free_pd(pd_old); + } pinst->flags &= ~PADATA_RESET; } +/* If cpumask contains no active cpu, we mark the instance as invalid. */ +static bool padata_validate_cpumask(struct padata_instance *pinst, + const struct cpumask *cpumask) +{ + if (!cpumask_intersects(cpumask, cpu_active_mask)) { + pinst->flags |= PADATA_INVALID; + return false; + } + + pinst->flags &= ~PADATA_INVALID; + return true; +} + /** * padata_set_cpumask - set the cpumask that padata should use * @@ -531,11 +546,18 @@ static void padata_replace(struct padata_instance *pinst, int padata_set_cpumask(struct padata_instance *pinst, cpumask_var_t cpumask) { - struct parallel_data *pd; + int valid; int err = 0; + struct parallel_data *pd = NULL; mutex_lock(&pinst->lock); + valid = padata_validate_cpumask(pinst, cpumask); + if (!valid) { + __padata_stop(pinst); + goto out_replace; + } + get_online_cpus(); pd = padata_alloc_pd(pinst, cpumask); @@ -544,10 +566,14 @@ int padata_set_cpumask(struct padata_instance *pinst, goto out; } +out_replace: cpumask_copy(pinst->cpumask, cpumask); padata_replace(pinst, pd); + if (valid) + __padata_start(pinst); + out: put_online_cpus(); @@ -567,6 +593,9 @@ static int __padata_add_cpu(struct padata_instance *pinst, int cpu) return -ENOMEM; padata_replace(pinst, pd); + + if (padata_validate_cpumask(pinst, pinst->cpumask)) + __padata_start(pinst); } return 0; @@ -597,9 +626,16 @@ EXPORT_SYMBOL(padata_add_cpu); static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) { - struct parallel_data *pd; + struct parallel_data *pd = NULL; if (cpumask_test_cpu(cpu, cpu_online_mask)) { + + if (!padata_validate_cpumask(pinst, pinst->cpumask)) { + __padata_stop(pinst); + padata_replace(pinst, pd); + goto out; + } + pd = padata_alloc_pd(pinst, pinst->cpumask); if (!pd) return -ENOMEM; @@ -607,6 +643,7 @@ static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) padata_replace(pinst, pd); } +out: return 0; } @@ -732,7 +769,7 @@ struct padata_instance *padata_alloc(const struct cpumask *cpumask, struct workqueue_struct *wq) { struct padata_instance *pinst; - struct parallel_data *pd; + struct parallel_data *pd = NULL; pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL); if (!pinst) @@ -740,12 +777,14 @@ struct padata_instance *padata_alloc(const struct cpumask *cpumask, get_online_cpus(); - pd = padata_alloc_pd(pinst, cpumask); - if (!pd) + if (!alloc_cpumask_var(&pinst->cpumask, GFP_KERNEL)) goto err_free_inst; - if (!alloc_cpumask_var(&pinst->cpumask, GFP_KERNEL)) - goto err_free_pd; + if (padata_validate_cpumask(pinst, cpumask)) { + pd = padata_alloc_pd(pinst, cpumask); + if (!pd) + goto err_free_mask; + } rcu_assign_pointer(pinst->pd, pd); @@ -767,8 +806,8 @@ struct padata_instance *padata_alloc(const struct cpumask *cpumask, return pinst; -err_free_pd: - padata_free_pd(pd); +err_free_mask: + free_cpumask_var(pinst->cpumask); err_free_inst: kfree(pinst); put_online_cpus(); -- cgit v1.1 From 83f619f3c8abb82cac9158cf23c656ec5c184607 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Wed, 7 Jul 2010 15:32:02 +0200 Subject: padata: make padata_do_parallel to return zero on success To return -EINPROGRESS on success in padata_do_parallel was considered to be odd. This patch changes this to return zero on success. Also the only user of padata, pcrypt is adapted to convert a return of zero to -EINPROGRESS within the crypto layer. This also removes the pcrypt fallback if padata_do_parallel was called on a not running padata instance as we can't handle it anymore. This fallback was unused, so it's save to remove it. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 18 ++++++------------ kernel/padata.c | 11 +++++------ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 71ae2b2..6036b6d 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -143,10 +143,8 @@ static int pcrypt_aead_encrypt(struct aead_request *req) aead_request_set_assoc(creq, req->assoc, req->assoclen); err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_enc_padata); - if (err) - return err; - else - err = crypto_aead_encrypt(creq); + if (!err) + return -EINPROGRESS; return err; } @@ -187,10 +185,8 @@ static int pcrypt_aead_decrypt(struct aead_request *req) aead_request_set_assoc(creq, req->assoc, req->assoclen); err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_dec_padata); - if (err) - return err; - else - err = crypto_aead_decrypt(creq); + if (!err) + return -EINPROGRESS; return err; } @@ -233,10 +229,8 @@ static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req) aead_givcrypt_set_giv(creq, req->giv, req->seq); err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_enc_padata); - if (err) - return err; - else - err = crypto_aead_givencrypt(creq); + if (!err) + return -EINPROGRESS; return err; } diff --git a/kernel/padata.c b/kernel/padata.c index 57ec4eb..ae8defc 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -111,10 +111,13 @@ int padata_do_parallel(struct padata_instance *pinst, pd = rcu_dereference(pinst->pd); - err = 0; + err = -EINVAL; if (!(pinst->flags & PADATA_INIT)) goto out; + if (!cpumask_test_cpu(cb_cpu, pd->cpumask)) + goto out; + err = -EBUSY; if ((pinst->flags & PADATA_RESET)) goto out; @@ -122,11 +125,7 @@ int padata_do_parallel(struct padata_instance *pinst, if (atomic_read(&pd->refcnt) >= MAX_OBJ_NUM) goto out; - err = -EINVAL; - if (!cpumask_test_cpu(cb_cpu, pd->cpumask)) - goto out; - - err = -EINPROGRESS; + err = 0; atomic_inc(&pd->refcnt); padata->pd = pd; padata->cb_cpu = cb_cpu; -- cgit v1.1 From 5f1a8c1bc724498ff32acbd59ed5263275676b9d Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Wed, 7 Jul 2010 15:32:39 +0200 Subject: padata: simplify serialization mechanism We count the number of processed objects on a percpu basis, so we need to go through all the percpu reorder queues to calculate the sequence number of the next object that needs serialization. This patch changes this to count the number of processed objects global. So we can calculate the sequence number and the percpu reorder queue of the next object that needs serialization without searching through the percpu reorder queues. This avoids some accesses to memory of foreign cpus. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- include/linux/padata.h | 6 ++--- kernel/padata.c | 71 ++++++++++++++------------------------------------ 2 files changed, 22 insertions(+), 55 deletions(-) diff --git a/include/linux/padata.h b/include/linux/padata.h index e4c17f9..8844b85 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -67,7 +67,6 @@ struct padata_list { * @pwork: work struct for parallelization. * @swork: work struct for serialization. * @pd: Backpointer to the internal control structure. - * @num_obj: Number of objects that are processed by this cpu. * @cpu_index: Index of the cpu. */ struct padata_queue { @@ -77,7 +76,6 @@ struct padata_queue { struct work_struct pwork; struct work_struct swork; struct parallel_data *pd; - atomic_t num_obj; int cpu_index; }; @@ -93,6 +91,7 @@ struct padata_queue { * @max_seq_nr: Maximal used sequence number. * @cpumask: cpumask in use. * @lock: Reorder lock. + * @processed: Number of already processed objects. * @timer: Reorder timer. */ struct parallel_data { @@ -103,7 +102,8 @@ struct parallel_data { atomic_t refcnt; unsigned int max_seq_nr; cpumask_var_t cpumask; - spinlock_t lock; + spinlock_t lock ____cacheline_aligned; + unsigned int processed; struct timer_list timer; }; diff --git a/kernel/padata.c b/kernel/padata.c index ae8defc..450d67d 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -170,79 +170,47 @@ EXPORT_SYMBOL(padata_do_parallel); */ static struct padata_priv *padata_get_next(struct parallel_data *pd) { - int cpu, num_cpus, empty, calc_seq_nr; - int seq_nr, next_nr, overrun, next_overrun; + int cpu, num_cpus; + int next_nr, next_index; struct padata_queue *queue, *next_queue; struct padata_priv *padata; struct padata_list *reorder; - empty = 0; - next_nr = -1; - next_overrun = 0; - next_queue = NULL; - num_cpus = cpumask_weight(pd->cpumask); - for_each_cpu(cpu, pd->cpumask) { - queue = per_cpu_ptr(pd->queue, cpu); - reorder = &queue->reorder; - - /* - * Calculate the seq_nr of the object that should be - * next in this reorder queue. - */ - overrun = 0; - calc_seq_nr = (atomic_read(&queue->num_obj) * num_cpus) - + queue->cpu_index; - - if (unlikely(calc_seq_nr > pd->max_seq_nr)) { - calc_seq_nr = calc_seq_nr - pd->max_seq_nr - 1; - overrun = 1; - } - - if (!list_empty(&reorder->list)) { - padata = list_entry(reorder->list.next, - struct padata_priv, list); - - seq_nr = padata->seq_nr; - BUG_ON(calc_seq_nr != seq_nr); - } else { - seq_nr = calc_seq_nr; - empty++; - } - - if (next_nr < 0 || seq_nr < next_nr - || (next_overrun && !overrun)) { - next_nr = seq_nr; - next_overrun = overrun; - next_queue = queue; - } + /* + * Calculate the percpu reorder queue and the sequence + * number of the next object. + */ + next_nr = pd->processed; + next_index = next_nr % num_cpus; + cpu = padata_index_to_cpu(pd, next_index); + next_queue = per_cpu_ptr(pd->queue, cpu); + + if (unlikely(next_nr > pd->max_seq_nr)) { + next_nr = next_nr - pd->max_seq_nr - 1; + next_index = next_nr % num_cpus; + cpu = padata_index_to_cpu(pd, next_index); + next_queue = per_cpu_ptr(pd->queue, cpu); + pd->processed = 0; } padata = NULL; - if (empty == num_cpus) - goto out; - reorder = &next_queue->reorder; if (!list_empty(&reorder->list)) { padata = list_entry(reorder->list.next, struct padata_priv, list); - if (unlikely(next_overrun)) { - for_each_cpu(cpu, pd->cpumask) { - queue = per_cpu_ptr(pd->queue, cpu); - atomic_set(&queue->num_obj, 0); - } - } + BUG_ON(next_nr != padata->seq_nr); spin_lock(&reorder->lock); list_del_init(&padata->list); atomic_dec(&pd->reorder_objects); spin_unlock(&reorder->lock); - atomic_inc(&next_queue->num_obj); + pd->processed++; goto out; } @@ -430,7 +398,6 @@ static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst, INIT_WORK(&queue->pwork, padata_parallel_worker); INIT_WORK(&queue->swork, padata_serial_worker); - atomic_set(&queue->num_obj, 0); } num_cpus = cpumask_weight(pd->cpumask); -- cgit v1.1 From 2197f9a16df9de94655992941d80953ba47042c2 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Wed, 7 Jul 2010 15:34:03 +0200 Subject: padata: update documentation This patch updates the padata documentation to the changed API of padata_start/padata_stop and padata_do parallel. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- Documentation/padata.txt | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Documentation/padata.txt b/Documentation/padata.txt index 269d7d0..93dd4e6 100644 --- a/Documentation/padata.txt +++ b/Documentation/padata.txt @@ -22,12 +22,15 @@ actually be done; it should be a multithreaded queue, naturally. There are functions for enabling and disabling the instance: - void padata_start(struct padata_instance *pinst); + int padata_start(struct padata_instance *pinst); void padata_stop(struct padata_instance *pinst); -These functions literally do nothing beyond setting or clearing the -"padata_start() was called" flag; if that flag is not set, other functions -will refuse to work. +These functions are setting or clearing the "PADATA_INIT" flag; +if that flag is not set, other functions will refuse to work. +padata_start returns zero on success (flag set) or -EINVAL if the +padata cpumask contains no active cpu (flag not set). +padata_stop clears the flag and blocks until the padata instance +is unused. The list of CPUs to be used can be adjusted with these functions: @@ -63,12 +66,10 @@ The submission of work is done with: The pinst and padata structures must be set up as described above; cb_cpu specifies which CPU will be used for the final callback when the work is done; it must be in the current instance's CPU mask. The return value from -padata_do_parallel() is a little strange; zero is an error return -indicating that the caller forgot the padata_start() formalities. -EBUSY -means that somebody, somewhere else is messing with the instance's CPU -mask, while -EINVAL is a complaint about cb_cpu not being in that CPU mask. -If all goes well, this function will return -EINPROGRESS, indicating that -the work is in progress. +padata_do_parallel() is zero on success, indicating that the work is in +progress. -EBUSY means that somebody, somewhere else is messing with the +instance's CPU mask, while -EINVAL is a complaint about cb_cpu not being +in that CPU mask or about a not running instance. Each task submitted to padata_do_parallel() will, in turn, be passed to exactly one call to the above-mentioned parallel() function, on one CPU, so -- cgit v1.1 From e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8 Mon Sep 17 00:00:00 2001 From: Dan Kruchinin Date: Wed, 14 Jul 2010 14:31:57 +0400 Subject: padata: Make two separate cpumasks The aim of this patch is to make two separate cpumasks for padata parallel and serial workers respectively. It allows user to make more thin and sophisticated configurations of padata framework. For example user may bind parallel and serial workers to non-intersecting CPU groups to gain better performance. Also each padata instance has notifiers chain for its cpumasks now. If either parallel or serial or both masks were changed all interested subsystems will get notification about that. It's especially useful if padata user uses algorithm for callback CPU selection according to serial cpumask. Signed-off-by: Dan Kruchinin Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 191 ++++++++++++++------ include/linux/padata.h | 116 ++++++++---- kernel/padata.c | 471 ++++++++++++++++++++++++++++++++++++------------- 3 files changed, 564 insertions(+), 214 deletions(-) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 6036b6d..c9662e2 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -24,12 +24,38 @@ #include #include #include +#include #include -static struct padata_instance *pcrypt_enc_padata; -static struct padata_instance *pcrypt_dec_padata; -static struct workqueue_struct *encwq; -static struct workqueue_struct *decwq; +struct pcrypt_instance { + struct padata_instance *pinst; + struct workqueue_struct *wq; + + /* + * Cpumask for callback CPUs. It should be + * equal to serial cpumask of corresponding padata instance, + * so it is updated when padata notifies us about serial + * cpumask change. + * + * cb_cpumask is protected by RCU. This fact prevents us from + * using cpumask_var_t directly because the actual type of + * cpumsak_var_t depends on kernel configuration(particularly on + * CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration + * cpumask_var_t may be either a pointer to the struct cpumask + * or a variable allocated on the stack. Thus we can not safely use + * cpumask_var_t with RCU operations such as rcu_assign_pointer or + * rcu_dereference. So cpumask_var_t is wrapped with struct + * pcrypt_cpumask which makes possible to use it with RCU. + */ + struct pcrypt_cpumask { + cpumask_var_t mask; + } *cb_cpumask; + struct notifier_block nblock; +}; + +static struct pcrypt_instance pencrypt; +static struct pcrypt_instance pdecrypt; + struct pcrypt_instance_ctx { struct crypto_spawn spawn; @@ -42,25 +68,29 @@ struct pcrypt_aead_ctx { }; static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, - struct padata_instance *pinst) + struct pcrypt_instance *pcrypt) { unsigned int cpu_index, cpu, i; + struct pcrypt_cpumask *cpumask; cpu = *cb_cpu; - if (cpumask_test_cpu(cpu, cpu_active_mask)) + rcu_read_lock_bh(); + cpumask = rcu_dereference(pcrypt->cb_cpumask); + if (cpumask_test_cpu(cpu, cpumask->mask)) goto out; - cpu_index = cpu % cpumask_weight(cpu_active_mask); + cpu_index = cpu % cpumask_weight(cpumask->mask); - cpu = cpumask_first(cpu_active_mask); + cpu = cpumask_first(cpumask->mask); for (i = 0; i < cpu_index; i++) - cpu = cpumask_next(cpu, cpu_active_mask); + cpu = cpumask_next(cpu, cpumask->mask); *cb_cpu = cpu; out: - return padata_do_parallel(pinst, padata, cpu); + rcu_read_unlock_bh(); + return padata_do_parallel(pcrypt->pinst, padata, cpu); } static int pcrypt_aead_setkey(struct crypto_aead *parent, @@ -142,7 +172,7 @@ static int pcrypt_aead_encrypt(struct aead_request *req) req->cryptlen, req->iv); aead_request_set_assoc(creq, req->assoc, req->assoclen); - err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_enc_padata); + err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); if (!err) return -EINPROGRESS; @@ -184,7 +214,7 @@ static int pcrypt_aead_decrypt(struct aead_request *req) req->cryptlen, req->iv); aead_request_set_assoc(creq, req->assoc, req->assoclen); - err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_dec_padata); + err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt); if (!err) return -EINPROGRESS; @@ -228,7 +258,7 @@ static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req) aead_givcrypt_set_assoc(creq, areq->assoc, areq->assoclen); aead_givcrypt_set_giv(creq, req->giv, req->seq); - err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_enc_padata); + err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); if (!err) return -EINPROGRESS; @@ -370,6 +400,88 @@ static void pcrypt_free(struct crypto_instance *inst) kfree(inst); } +static int pcrypt_cpumask_change_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + struct pcrypt_instance *pcrypt; + struct pcrypt_cpumask *new_mask, *old_mask; + + if (!(val & PADATA_CPU_SERIAL)) + return 0; + + pcrypt = container_of(self, struct pcrypt_instance, nblock); + new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL); + if (!new_mask) + return -ENOMEM; + if (!alloc_cpumask_var(&new_mask->mask, GFP_KERNEL)) { + kfree(new_mask); + return -ENOMEM; + } + + old_mask = pcrypt->cb_cpumask; + + padata_get_cpumask(pcrypt->pinst, PADATA_CPU_SERIAL, new_mask->mask); + rcu_assign_pointer(pcrypt->cb_cpumask, new_mask); + synchronize_rcu_bh(); + + free_cpumask_var(old_mask->mask); + kfree(old_mask); + return 0; +} + +static int __pcrypt_init_instance(struct pcrypt_instance *pcrypt, + const char *name) +{ + int ret = -ENOMEM; + struct pcrypt_cpumask *mask; + + pcrypt->wq = create_workqueue(name); + if (!pcrypt->wq) + goto err; + + pcrypt->pinst = padata_alloc(pcrypt->wq); + if (!pcrypt->pinst) + goto err_destroy_workqueue; + + mask = kmalloc(sizeof(*mask), GFP_KERNEL); + if (!mask) + goto err_free_padata; + if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) { + kfree(mask); + goto err_free_padata; + } + + padata_get_cpumask(pcrypt->pinst, PADATA_CPU_SERIAL, mask->mask); + rcu_assign_pointer(pcrypt->cb_cpumask, mask); + + pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify; + ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); + if (ret) + goto err_free_cpumask; + + return ret; +err_free_cpumask: + free_cpumask_var(mask->mask); + kfree(mask); +err_free_padata: + padata_free(pcrypt->pinst); +err_destroy_workqueue: + destroy_workqueue(pcrypt->wq); +err: + return ret; +} + +static void __pcrypt_deinit_instance(struct pcrypt_instance *pcrypt) +{ + free_cpumask_var(pcrypt->cb_cpumask->mask); + kfree(pcrypt->cb_cpumask); + + padata_stop(pcrypt->pinst); + padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); + destroy_workqueue(pcrypt->wq); + padata_free(pcrypt->pinst); +} + static struct crypto_template pcrypt_tmpl = { .name = "pcrypt", .alloc = pcrypt_alloc, @@ -379,60 +491,31 @@ static struct crypto_template pcrypt_tmpl = { static int __init pcrypt_init(void) { - int err = -ENOMEM; - encwq = create_workqueue("pencrypt"); - if (!encwq) - goto err; - - decwq = create_workqueue("pdecrypt"); - if (!decwq) - goto err_destroy_encwq; - - - pcrypt_enc_padata = padata_alloc(cpu_possible_mask, encwq); - if (!pcrypt_enc_padata) - goto err_destroy_decwq; - - pcrypt_dec_padata = padata_alloc(cpu_possible_mask, decwq); - if (!pcrypt_dec_padata) - goto err_free_enc_padata; + int err; - err = padata_start(pcrypt_enc_padata); + err = __pcrypt_init_instance(&pencrypt, "pencrypt"); if (err) - goto err_free_dec_padata; + goto err; - err = padata_start(pcrypt_dec_padata); + err = __pcrypt_init_instance(&pdecrypt, "pdecrypt"); if (err) - goto err_free_dec_padata; - - return crypto_register_template(&pcrypt_tmpl); - -err_free_dec_padata: - padata_free(pcrypt_dec_padata); + goto err_deinit_pencrypt; -err_free_enc_padata: - padata_free(pcrypt_enc_padata); + padata_start(pencrypt.pinst); + padata_start(pdecrypt.pinst); -err_destroy_decwq: - destroy_workqueue(decwq); - -err_destroy_encwq: - destroy_workqueue(encwq); + return crypto_register_template(&pcrypt_tmpl); +err_deinit_pencrypt: + __pcrypt_deinit_instance(&pencrypt); err: return err; } static void __exit pcrypt_exit(void) { - padata_stop(pcrypt_enc_padata); - padata_stop(pcrypt_dec_padata); - - destroy_workqueue(encwq); - destroy_workqueue(decwq); - - padata_free(pcrypt_enc_padata); - padata_free(pcrypt_dec_padata); + __pcrypt_deinit_instance(&pencrypt); + __pcrypt_deinit_instance(&pdecrypt); crypto_unregister_template(&pcrypt_tmpl); } diff --git a/include/linux/padata.h b/include/linux/padata.h index 8844b85..621e773 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -25,6 +25,10 @@ #include #include #include +#include + +#define PADATA_CPU_SERIAL 0x01 +#define PADATA_CPU_PARALLEL 0x02 /** * struct padata_priv - Embedded to the users data structure. @@ -59,7 +63,20 @@ struct padata_list { }; /** - * struct padata_queue - The percpu padata queues. +* struct padata_serial_queue - The percpu padata serial queue +* +* @serial: List to wait for serialization after reordering. +* @work: work struct for serialization. +* @pd: Backpointer to the internal control structure. +*/ +struct padata_serial_queue { + struct padata_list serial; + struct work_struct work; + struct parallel_data *pd; +}; + +/** + * struct padata_parallel_queue - The percpu padata parallel queue * * @parallel: List to wait for parallelization. * @reorder: List to wait for reordering after parallel processing. @@ -67,44 +84,52 @@ struct padata_list { * @pwork: work struct for parallelization. * @swork: work struct for serialization. * @pd: Backpointer to the internal control structure. + * @work: work struct for parallelization. + * @num_obj: Number of objects that are processed by this cpu. * @cpu_index: Index of the cpu. */ -struct padata_queue { - struct padata_list parallel; - struct padata_list reorder; - struct padata_list serial; - struct work_struct pwork; - struct work_struct swork; - struct parallel_data *pd; - int cpu_index; +struct padata_parallel_queue { + struct padata_list parallel; + struct padata_list reorder; + struct parallel_data *pd; + struct work_struct work; + atomic_t num_obj; + int cpu_index; }; + /** * struct parallel_data - Internal control structure, covers everything * that depends on the cpumask in use. * * @pinst: padata instance. - * @queue: percpu padata queues. + * @pqueue: percpu padata queues used for parallelization. + * @squeue: percpu padata queues used for serialuzation. * @seq_nr: The sequence number that will be attached to the next object. * @reorder_objects: Number of objects waiting in the reorder queues. * @refcnt: Number of objects holding a reference on this parallel_data. * @max_seq_nr: Maximal used sequence number. - * @cpumask: cpumask in use. + * @cpumask: Contains two cpumasks: pcpu and cbcpu for + * parallel and serial workers respectively. * @lock: Reorder lock. * @processed: Number of already processed objects. * @timer: Reorder timer. */ struct parallel_data { - struct padata_instance *pinst; - struct padata_queue *queue; - atomic_t seq_nr; - atomic_t reorder_objects; - atomic_t refcnt; - unsigned int max_seq_nr; - cpumask_var_t cpumask; - spinlock_t lock ____cacheline_aligned; - unsigned int processed; - struct timer_list timer; + struct padata_instance *pinst; + struct padata_parallel_queue *pqueue; + struct padata_serial_queue *squeue; + atomic_t seq_nr; + atomic_t reorder_objects; + atomic_t refcnt; + unsigned int max_seq_nr; + struct { + cpumask_var_t pcpu; + cpumask_var_t cbcpu; + } cpumask; + spinlock_t lock ____cacheline_aligned; + unsigned int processed; + struct timer_list timer; }; /** @@ -113,32 +138,51 @@ struct parallel_data { * @cpu_notifier: cpu hotplug notifier. * @wq: The workqueue in use. * @pd: The internal control structure. - * @cpumask: User supplied cpumask. + * @cpumask: User supplied cpumask. Contains two cpumasks: pcpu and + * cbcpu for parallel and serial works respectivly. + * @cpumask_change_notifier: Notifiers chain for user-defined notify + * callbacks that will be called when either @pcpu or @cbcpu + * or both cpumasks change. * @lock: padata instance lock. * @flags: padata flags. */ struct padata_instance { - struct notifier_block cpu_notifier; - struct workqueue_struct *wq; - struct parallel_data *pd; - cpumask_var_t cpumask; - struct mutex lock; - u8 flags; -#define PADATA_INIT 1 -#define PADATA_RESET 2 -#define PADATA_INVALID 4 + struct notifier_block cpu_notifier; + struct workqueue_struct *wq; + struct parallel_data *pd; + struct { + cpumask_var_t pcpu; + cpumask_var_t cbcpu; + } cpumask; + struct blocking_notifier_head cpumask_change_notifier; + struct mutex lock; + u8 flags; +#define PADATA_INIT 1 +#define PADATA_RESET 2 +#define PADATA_INVALID 4 }; -extern struct padata_instance *padata_alloc(const struct cpumask *cpumask, - struct workqueue_struct *wq); +extern struct padata_instance *padata_alloc(struct workqueue_struct *wq); +extern struct padata_instance *__padata_alloc(struct workqueue_struct *wq, + const struct cpumask *pcpumask, + const struct cpumask *cbcpumask); extern void padata_free(struct padata_instance *pinst); extern int padata_do_parallel(struct padata_instance *pinst, struct padata_priv *padata, int cb_cpu); extern void padata_do_serial(struct padata_priv *padata); -extern int padata_set_cpumask(struct padata_instance *pinst, +extern int padata_get_cpumask(struct padata_instance *pinst, + int cpumask_type, struct cpumask *out_mask); +extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask); -extern int padata_add_cpu(struct padata_instance *pinst, int cpu); -extern int padata_remove_cpu(struct padata_instance *pinst, int cpu); +extern int __padata_set_cpumasks(struct padata_instance *pinst, + cpumask_var_t pcpumask, + cpumask_var_t cbcpumask); +extern int padata_add_cpu(struct padata_instance *pinst, int cpu, int mask); +extern int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask); extern int padata_start(struct padata_instance *pinst); extern void padata_stop(struct padata_instance *pinst); +extern int padata_register_cpumask_notifier(struct padata_instance *pinst, + struct notifier_block *nblock); +extern int padata_unregister_cpumask_notifier(struct padata_instance *pinst, + struct notifier_block *nblock); #endif diff --git a/kernel/padata.c b/kernel/padata.c index 450d67d..84d0ca9 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -35,9 +35,9 @@ static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index) { int cpu, target_cpu; - target_cpu = cpumask_first(pd->cpumask); + target_cpu = cpumask_first(pd->cpumask.pcpu); for (cpu = 0; cpu < cpu_index; cpu++) - target_cpu = cpumask_next(target_cpu, pd->cpumask); + target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu); return target_cpu; } @@ -53,26 +53,27 @@ static int padata_cpu_hash(struct padata_priv *padata) * Hash the sequence numbers to the cpus by taking * seq_nr mod. number of cpus in use. */ - cpu_index = padata->seq_nr % cpumask_weight(pd->cpumask); + cpu_index = padata->seq_nr % cpumask_weight(pd->cpumask.pcpu); return padata_index_to_cpu(pd, cpu_index); } -static void padata_parallel_worker(struct work_struct *work) +static void padata_parallel_worker(struct work_struct *parallel_work) { - struct padata_queue *queue; + struct padata_parallel_queue *pqueue; struct parallel_data *pd; struct padata_instance *pinst; LIST_HEAD(local_list); local_bh_disable(); - queue = container_of(work, struct padata_queue, pwork); - pd = queue->pd; + pqueue = container_of(parallel_work, + struct padata_parallel_queue, work); + pd = pqueue->pd; pinst = pd->pinst; - spin_lock(&queue->parallel.lock); - list_replace_init(&queue->parallel.list, &local_list); - spin_unlock(&queue->parallel.lock); + spin_lock(&pqueue->parallel.lock); + list_replace_init(&pqueue->parallel.list, &local_list); + spin_unlock(&pqueue->parallel.lock); while (!list_empty(&local_list)) { struct padata_priv *padata; @@ -94,7 +95,7 @@ static void padata_parallel_worker(struct work_struct *work) * @pinst: padata instance * @padata: object to be parallelized * @cb_cpu: cpu the serialization callback function will run on, - * must be in the cpumask of padata. + * must be in the serial cpumask of padata(i.e. cpumask.cbcpu). * * The parallelization callback function will run with BHs off. * Note: Every object which is parallelized by padata_do_parallel @@ -104,7 +105,7 @@ int padata_do_parallel(struct padata_instance *pinst, struct padata_priv *padata, int cb_cpu) { int target_cpu, err; - struct padata_queue *queue; + struct padata_parallel_queue *queue; struct parallel_data *pd; rcu_read_lock_bh(); @@ -115,7 +116,7 @@ int padata_do_parallel(struct padata_instance *pinst, if (!(pinst->flags & PADATA_INIT)) goto out; - if (!cpumask_test_cpu(cb_cpu, pd->cpumask)) + if (!cpumask_test_cpu(cb_cpu, pd->cpumask.cbcpu)) goto out; err = -EBUSY; @@ -136,13 +137,13 @@ int padata_do_parallel(struct padata_instance *pinst, padata->seq_nr = atomic_inc_return(&pd->seq_nr); target_cpu = padata_cpu_hash(padata); - queue = per_cpu_ptr(pd->queue, target_cpu); + queue = per_cpu_ptr(pd->pqueue, target_cpu); spin_lock(&queue->parallel.lock); list_add_tail(&padata->list, &queue->parallel.list); spin_unlock(&queue->parallel.lock); - queue_work_on(target_cpu, pinst->wq, &queue->pwork); + queue_work_on(target_cpu, pinst->wq, &queue->work); out: rcu_read_unlock_bh(); @@ -172,11 +173,11 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd) { int cpu, num_cpus; int next_nr, next_index; - struct padata_queue *queue, *next_queue; + struct padata_parallel_queue *queue, *next_queue; struct padata_priv *padata; struct padata_list *reorder; - num_cpus = cpumask_weight(pd->cpumask); + num_cpus = cpumask_weight(pd->cpumask.pcpu); /* * Calculate the percpu reorder queue and the sequence @@ -185,13 +186,13 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd) next_nr = pd->processed; next_index = next_nr % num_cpus; cpu = padata_index_to_cpu(pd, next_index); - next_queue = per_cpu_ptr(pd->queue, cpu); + next_queue = per_cpu_ptr(pd->pqueue, cpu); if (unlikely(next_nr > pd->max_seq_nr)) { next_nr = next_nr - pd->max_seq_nr - 1; next_index = next_nr % num_cpus; cpu = padata_index_to_cpu(pd, next_index); - next_queue = per_cpu_ptr(pd->queue, cpu); + next_queue = per_cpu_ptr(pd->pqueue, cpu); pd->processed = 0; } @@ -215,7 +216,7 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd) goto out; } - queue = per_cpu_ptr(pd->queue, smp_processor_id()); + queue = per_cpu_ptr(pd->pqueue, smp_processor_id()); if (queue->cpu_index == next_queue->cpu_index) { padata = ERR_PTR(-ENODATA); goto out; @@ -229,7 +230,7 @@ out: static void padata_reorder(struct parallel_data *pd) { struct padata_priv *padata; - struct padata_queue *queue; + struct padata_serial_queue *squeue; struct padata_instance *pinst = pd->pinst; /* @@ -268,13 +269,13 @@ static void padata_reorder(struct parallel_data *pd) return; } - queue = per_cpu_ptr(pd->queue, padata->cb_cpu); + squeue = per_cpu_ptr(pd->squeue, padata->cb_cpu); - spin_lock(&queue->serial.lock); - list_add_tail(&padata->list, &queue->serial.list); - spin_unlock(&queue->serial.lock); + spin_lock(&squeue->serial.lock); + list_add_tail(&padata->list, &squeue->serial.list); + spin_unlock(&squeue->serial.lock); - queue_work_on(padata->cb_cpu, pinst->wq, &queue->swork); + queue_work_on(padata->cb_cpu, pinst->wq, &squeue->work); } spin_unlock_bh(&pd->lock); @@ -300,19 +301,19 @@ static void padata_reorder_timer(unsigned long arg) padata_reorder(pd); } -static void padata_serial_worker(struct work_struct *work) +static void padata_serial_worker(struct work_struct *serial_work) { - struct padata_queue *queue; + struct padata_serial_queue *squeue; struct parallel_data *pd; LIST_HEAD(local_list); local_bh_disable(); - queue = container_of(work, struct padata_queue, swork); - pd = queue->pd; + squeue = container_of(serial_work, struct padata_serial_queue, work); + pd = squeue->pd; - spin_lock(&queue->serial.lock); - list_replace_init(&queue->serial.list, &local_list); - spin_unlock(&queue->serial.lock); + spin_lock(&squeue->serial.lock); + list_replace_init(&squeue->serial.list, &local_list); + spin_unlock(&squeue->serial.lock); while (!list_empty(&local_list)) { struct padata_priv *padata; @@ -339,18 +340,18 @@ static void padata_serial_worker(struct work_struct *work) void padata_do_serial(struct padata_priv *padata) { int cpu; - struct padata_queue *queue; + struct padata_parallel_queue *pqueue; struct parallel_data *pd; pd = padata->pd; cpu = get_cpu(); - queue = per_cpu_ptr(pd->queue, cpu); + pqueue = per_cpu_ptr(pd->pqueue, cpu); - spin_lock(&queue->reorder.lock); + spin_lock(&pqueue->reorder.lock); atomic_inc(&pd->reorder_objects); - list_add_tail(&padata->list, &queue->reorder.list); - spin_unlock(&queue->reorder.lock); + list_add_tail(&padata->list, &pqueue->reorder.list); + spin_unlock(&pqueue->reorder.lock); put_cpu(); @@ -358,51 +359,88 @@ void padata_do_serial(struct padata_priv *padata) } EXPORT_SYMBOL(padata_do_serial); -/* Allocate and initialize the internal cpumask dependend resources. */ -static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst, - const struct cpumask *cpumask) +static int padata_setup_cpumasks(struct parallel_data *pd, + const struct cpumask *pcpumask, + const struct cpumask *cbcpumask) { - int cpu, cpu_index, num_cpus; - struct padata_queue *queue; - struct parallel_data *pd; + if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL)) + return -ENOMEM; - cpu_index = 0; + cpumask_and(pd->cpumask.pcpu, pcpumask, cpu_active_mask); + if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL)) { + free_cpumask_var(pd->cpumask.cbcpu); + return -ENOMEM; + } - pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL); - if (!pd) - goto err; + cpumask_and(pd->cpumask.cbcpu, cbcpumask, cpu_active_mask); + return 0; +} - pd->queue = alloc_percpu(struct padata_queue); - if (!pd->queue) - goto err_free_pd; +static void __padata_list_init(struct padata_list *pd_list) +{ + INIT_LIST_HEAD(&pd_list->list); + spin_lock_init(&pd_list->lock); +} - if (!alloc_cpumask_var(&pd->cpumask, GFP_KERNEL)) - goto err_free_queue; +/* Initialize all percpu queues used by serial workers */ +static void padata_init_squeues(struct parallel_data *pd) +{ + int cpu; + struct padata_serial_queue *squeue; - cpumask_and(pd->cpumask, cpumask, cpu_active_mask); + for_each_cpu(cpu, pd->cpumask.cbcpu) { + squeue = per_cpu_ptr(pd->squeue, cpu); + squeue->pd = pd; + __padata_list_init(&squeue->serial); + INIT_WORK(&squeue->work, padata_serial_worker); + } +} - for_each_cpu(cpu, pd->cpumask) { - queue = per_cpu_ptr(pd->queue, cpu); +/* Initialize all percpu queues used by parallel workers */ +static void padata_init_pqueues(struct parallel_data *pd) +{ + int cpu_index, num_cpus, cpu; + struct padata_parallel_queue *pqueue; - queue->pd = pd; + cpu_index = 0; + for_each_cpu(cpu, pd->cpumask.pcpu) { + pqueue = per_cpu_ptr(pd->pqueue, cpu); + pqueue->pd = pd; + pqueue->cpu_index = cpu_index; + + __padata_list_init(&pqueue->reorder); + __padata_list_init(&pqueue->parallel); + INIT_WORK(&pqueue->work, padata_parallel_worker); + atomic_set(&pqueue->num_obj, 0); + } - queue->cpu_index = cpu_index; - cpu_index++; + num_cpus = cpumask_weight(pd->cpumask.pcpu); + pd->max_seq_nr = (MAX_SEQ_NR / num_cpus) * num_cpus - 1; +} - INIT_LIST_HEAD(&queue->reorder.list); - INIT_LIST_HEAD(&queue->parallel.list); - INIT_LIST_HEAD(&queue->serial.list); - spin_lock_init(&queue->reorder.lock); - spin_lock_init(&queue->parallel.lock); - spin_lock_init(&queue->serial.lock); +/* Allocate and initialize the internal cpumask dependend resources. */ +static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst, + const struct cpumask *pcpumask, + const struct cpumask *cbcpumask) +{ + struct parallel_data *pd; - INIT_WORK(&queue->pwork, padata_parallel_worker); - INIT_WORK(&queue->swork, padata_serial_worker); - } + pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL); + if (!pd) + goto err; - num_cpus = cpumask_weight(pd->cpumask); - pd->max_seq_nr = (MAX_SEQ_NR / num_cpus) * num_cpus - 1; + pd->pqueue = alloc_percpu(struct padata_parallel_queue); + if (!pd->pqueue) + goto err_free_pd; + + pd->squeue = alloc_percpu(struct padata_serial_queue); + if (!pd->squeue) + goto err_free_pqueue; + if (padata_setup_cpumasks(pd, pcpumask, cbcpumask) < 0) + goto err_free_squeue; + padata_init_pqueues(pd); + padata_init_squeues(pd); setup_timer(&pd->timer, padata_reorder_timer, (unsigned long)pd); atomic_set(&pd->seq_nr, -1); atomic_set(&pd->reorder_objects, 0); @@ -412,8 +450,10 @@ static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst, return pd; -err_free_queue: - free_percpu(pd->queue); +err_free_squeue: + free_percpu(pd->squeue); +err_free_pqueue: + free_percpu(pd->pqueue); err_free_pd: kfree(pd); err: @@ -422,8 +462,10 @@ err: static void padata_free_pd(struct parallel_data *pd) { - free_cpumask_var(pd->cpumask); - free_percpu(pd->queue); + free_cpumask_var(pd->cpumask.pcpu); + free_cpumask_var(pd->cpumask.cbcpu); + free_percpu(pd->pqueue); + free_percpu(pd->squeue); kfree(pd); } @@ -431,11 +473,12 @@ static void padata_free_pd(struct parallel_data *pd) static void padata_flush_queues(struct parallel_data *pd) { int cpu; - struct padata_queue *queue; + struct padata_parallel_queue *pqueue; + struct padata_serial_queue *squeue; - for_each_cpu(cpu, pd->cpumask) { - queue = per_cpu_ptr(pd->queue, cpu); - flush_work(&queue->pwork); + for_each_cpu(cpu, pd->cpumask.pcpu) { + pqueue = per_cpu_ptr(pd->pqueue, cpu); + flush_work(&pqueue->work); } del_timer_sync(&pd->timer); @@ -443,9 +486,9 @@ static void padata_flush_queues(struct parallel_data *pd) if (atomic_read(&pd->reorder_objects)) padata_reorder(pd); - for_each_cpu(cpu, pd->cpumask) { - queue = per_cpu_ptr(pd->queue, cpu); - flush_work(&queue->swork); + for_each_cpu(cpu, pd->cpumask.cbcpu) { + squeue = per_cpu_ptr(pd->squeue, cpu); + flush_work(&squeue->work); } BUG_ON(atomic_read(&pd->refcnt) != 0); @@ -475,21 +518,63 @@ static void padata_replace(struct padata_instance *pinst, struct parallel_data *pd_new) { struct parallel_data *pd_old = pinst->pd; + int notification_mask = 0; pinst->flags |= PADATA_RESET; rcu_assign_pointer(pinst->pd, pd_new); synchronize_rcu(); + if (!pd_old) + goto out; - if (pd_old) { - padata_flush_queues(pd_old); - padata_free_pd(pd_old); - } + padata_flush_queues(pd_old); + if (!cpumask_equal(pd_old->cpumask.pcpu, pd_new->cpumask.pcpu)) + notification_mask |= PADATA_CPU_PARALLEL; + if (!cpumask_equal(pd_old->cpumask.cbcpu, pd_new->cpumask.cbcpu)) + notification_mask |= PADATA_CPU_SERIAL; + + padata_free_pd(pd_old); + if (notification_mask) + blocking_notifier_call_chain(&pinst->cpumask_change_notifier, + notification_mask, pinst); +out: pinst->flags &= ~PADATA_RESET; } +/** + * padata_register_cpumask_notifier - Registers a notifier that will be called + * if either pcpu or cbcpu or both cpumasks change. + * + * @pinst: A poineter to padata instance + * @nblock: A pointer to notifier block. + */ +int padata_register_cpumask_notifier(struct padata_instance *pinst, + struct notifier_block *nblock) +{ + return blocking_notifier_chain_register(&pinst->cpumask_change_notifier, + nblock); +} +EXPORT_SYMBOL(padata_register_cpumask_notifier); + +/** + * padata_unregister_cpumask_notifier - Unregisters cpumask notifier + * registered earlier using padata_register_cpumask_notifier + * + * @pinst: A pointer to data instance. + * @nlock: A pointer to notifier block. + */ +int padata_unregister_cpumask_notifier(struct padata_instance *pinst, + struct notifier_block *nblock) +{ + return blocking_notifier_chain_unregister( + &pinst->cpumask_change_notifier, + nblock); +} +EXPORT_SYMBOL(padata_unregister_cpumask_notifier); + + /* If cpumask contains no active cpu, we mark the instance as invalid. */ static bool padata_validate_cpumask(struct padata_instance *pinst, const struct cpumask *cpumask) @@ -504,13 +589,82 @@ static bool padata_validate_cpumask(struct padata_instance *pinst, } /** - * padata_set_cpumask - set the cpumask that padata should use + * padata_get_cpumask: Fetch serial or parallel cpumask from the + * given padata instance and copy it to @out_mask + * + * @pinst: A pointer to padata instance + * @cpumask_type: Specifies which cpumask will be copied. + * Possible values are PADATA_CPU_SERIAL *or* PADATA_CPU_PARALLEL + * corresponding to serial and parallel cpumask respectively. + * @out_mask: A pointer to cpumask structure where selected + * cpumask will be copied. + */ +int padata_get_cpumask(struct padata_instance *pinst, + int cpumask_type, struct cpumask *out_mask) +{ + struct parallel_data *pd; + int ret = 0; + + rcu_read_lock_bh(); + pd = rcu_dereference(pinst->pd); + switch (cpumask_type) { + case PADATA_CPU_SERIAL: + cpumask_copy(out_mask, pd->cpumask.cbcpu); + break; + case PADATA_CPU_PARALLEL: + cpumask_copy(out_mask, pd->cpumask.pcpu); + break; + default: + ret = -EINVAL; + } + + rcu_read_unlock_bh(); + return ret; +} +EXPORT_SYMBOL(padata_get_cpumask); + +/** + * padata_set_cpumask: Sets specified by @cpumask_type cpumask to the value + * equivalent to @cpumask. * * @pinst: padata instance + * @cpumask_type: PADATA_CPU_SERIAL or PADATA_CPU_PARALLEL corresponding + * to parallel and serial cpumasks respectively. * @cpumask: the cpumask to use */ -int padata_set_cpumask(struct padata_instance *pinst, - cpumask_var_t cpumask) +int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, + cpumask_var_t cpumask) +{ + struct cpumask *serial_mask, *parallel_mask; + + switch (cpumask_type) { + case PADATA_CPU_PARALLEL: + serial_mask = pinst->cpumask.cbcpu; + parallel_mask = cpumask; + break; + case PADATA_CPU_SERIAL: + parallel_mask = pinst->cpumask.pcpu; + serial_mask = cpumask; + break; + default: + return -EINVAL; + } + + return __padata_set_cpumasks(pinst, parallel_mask, serial_mask); +} +EXPORT_SYMBOL(padata_set_cpumask); + +/** + * __padata_set_cpumasks - Set both parallel and serial cpumasks. The first + * one is used by parallel workers and the second one + * by the wokers doing serialization. + * + * @pinst: padata instance + * @pcpumask: the cpumask to use for parallel workers + * @cbcpumask: the cpumsak to use for serial workers + */ +int __padata_set_cpumasks(struct padata_instance *pinst, + cpumask_var_t pcpumask, cpumask_var_t cbcpumask) { int valid; int err = 0; @@ -518,7 +672,13 @@ int padata_set_cpumask(struct padata_instance *pinst, mutex_lock(&pinst->lock); - valid = padata_validate_cpumask(pinst, cpumask); + valid = padata_validate_cpumask(pinst, pcpumask); + if (!valid) { + __padata_stop(pinst); + goto out_replace; + } + + valid = padata_validate_cpumask(pinst, cbcpumask); if (!valid) { __padata_stop(pinst); goto out_replace; @@ -526,14 +686,15 @@ int padata_set_cpumask(struct padata_instance *pinst, get_online_cpus(); - pd = padata_alloc_pd(pinst, cpumask); + pd = padata_alloc_pd(pinst, pcpumask, cbcpumask); if (!pd) { err = -ENOMEM; goto out; } out_replace: - cpumask_copy(pinst->cpumask, cpumask); + cpumask_copy(pinst->cpumask.pcpu, pcpumask); + cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); padata_replace(pinst, pd); @@ -546,41 +707,57 @@ out: mutex_unlock(&pinst->lock); return err; + } -EXPORT_SYMBOL(padata_set_cpumask); +EXPORT_SYMBOL(__padata_set_cpumasks); static int __padata_add_cpu(struct padata_instance *pinst, int cpu) { struct parallel_data *pd; if (cpumask_test_cpu(cpu, cpu_active_mask)) { - pd = padata_alloc_pd(pinst, pinst->cpumask); + pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu, + pinst->cpumask.cbcpu); if (!pd) return -ENOMEM; padata_replace(pinst, pd); - if (padata_validate_cpumask(pinst, pinst->cpumask)) + if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) && + padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) __padata_start(pinst); } return 0; } -/** - * padata_add_cpu - add a cpu to the padata cpumask + /** + * padata_add_cpu - add a cpu to one or both(parallel and serial) + * padata cpumasks. * * @pinst: padata instance * @cpu: cpu to add + * @mask: bitmask of flags specifying to which cpumask @cpu shuld be added. + * The @mask may be any combination of the following flags: + * PADATA_CPU_SERIAL - serial cpumask + * PADATA_CPU_PARALLEL - parallel cpumask */ -int padata_add_cpu(struct padata_instance *pinst, int cpu) + +int padata_add_cpu(struct padata_instance *pinst, int cpu, int mask) { int err; + if (!(mask & (PADATA_CPU_SERIAL | PADATA_CPU_PARALLEL))) + return -EINVAL; + mutex_lock(&pinst->lock); get_online_cpus(); - cpumask_set_cpu(cpu, pinst->cpumask); + if (mask & PADATA_CPU_SERIAL) + cpumask_set_cpu(cpu, pinst->cpumask.cbcpu); + if (mask & PADATA_CPU_PARALLEL) + cpumask_set_cpu(cpu, pinst->cpumask.pcpu); + err = __padata_add_cpu(pinst, cpu); put_online_cpus(); @@ -596,13 +773,15 @@ static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) if (cpumask_test_cpu(cpu, cpu_online_mask)) { - if (!padata_validate_cpumask(pinst, pinst->cpumask)) { + if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) || + !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) { __padata_stop(pinst); padata_replace(pinst, pd); goto out; } - pd = padata_alloc_pd(pinst, pinst->cpumask); + pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu, + pinst->cpumask.cbcpu); if (!pd) return -ENOMEM; @@ -613,20 +792,32 @@ out: return 0; } -/** - * padata_remove_cpu - remove a cpu from the padata cpumask + /** + * padata_remove_cpu - remove a cpu from the one or both(serial and paralell) + * padata cpumasks. * * @pinst: padata instance * @cpu: cpu to remove + * @mask: bitmask specifying from which cpumask @cpu should be removed + * The @mask may be any combination of the following flags: + * PADATA_CPU_SERIAL - serial cpumask + * PADATA_CPU_PARALLEL - parallel cpumask */ -int padata_remove_cpu(struct padata_instance *pinst, int cpu) +int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask) { int err; + if (!(mask & (PADATA_CPU_SERIAL | PADATA_CPU_PARALLEL))) + return -EINVAL; + mutex_lock(&pinst->lock); get_online_cpus(); - cpumask_clear_cpu(cpu, pinst->cpumask); + if (mask & PADATA_CPU_SERIAL) + cpumask_clear_cpu(cpu, pinst->cpumask.cbcpu); + if (mask & PADATA_CPU_PARALLEL) + cpumask_clear_cpu(cpu, pinst->cpumask.pcpu); + err = __padata_remove_cpu(pinst, cpu); put_online_cpus(); @@ -672,6 +863,14 @@ void padata_stop(struct padata_instance *pinst) EXPORT_SYMBOL(padata_stop); #ifdef CONFIG_HOTPLUG_CPU + +static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu) +{ + return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) || + cpumask_test_cpu(cpu, pinst->cpumask.cbcpu); +} + + static int padata_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { @@ -684,7 +883,7 @@ static int padata_cpu_callback(struct notifier_block *nfb, switch (action) { case CPU_ONLINE: case CPU_ONLINE_FROZEN: - if (!cpumask_test_cpu(cpu, pinst->cpumask)) + if (!pinst_has_cpu(pinst, cpu)) break; mutex_lock(&pinst->lock); err = __padata_add_cpu(pinst, cpu); @@ -695,7 +894,7 @@ static int padata_cpu_callback(struct notifier_block *nfb, case CPU_DOWN_PREPARE: case CPU_DOWN_PREPARE_FROZEN: - if (!cpumask_test_cpu(cpu, pinst->cpumask)) + if (!pinst_has_cpu(pinst, cpu)) break; mutex_lock(&pinst->lock); err = __padata_remove_cpu(pinst, cpu); @@ -706,7 +905,7 @@ static int padata_cpu_callback(struct notifier_block *nfb, case CPU_UP_CANCELED: case CPU_UP_CANCELED_FROZEN: - if (!cpumask_test_cpu(cpu, pinst->cpumask)) + if (!pinst_has_cpu(pinst, cpu)) break; mutex_lock(&pinst->lock); __padata_remove_cpu(pinst, cpu); @@ -714,7 +913,7 @@ static int padata_cpu_callback(struct notifier_block *nfb, case CPU_DOWN_FAILED: case CPU_DOWN_FAILED_FROZEN: - if (!cpumask_test_cpu(cpu, pinst->cpumask)) + if (!pinst_has_cpu(pinst, cpu)) break; mutex_lock(&pinst->lock); __padata_add_cpu(pinst, cpu); @@ -726,13 +925,29 @@ static int padata_cpu_callback(struct notifier_block *nfb, #endif /** - * padata_alloc - allocate and initialize a padata instance + * padata_alloc - Allocate and initialize padata instance. + * Use default cpumask(cpu_possible_mask) + * for serial and parallel workes. + * + * @wq: workqueue to use for the allocated padata instance + */ +struct padata_instance *padata_alloc(struct workqueue_struct *wq) +{ + return __padata_alloc(wq, cpu_possible_mask, cpu_possible_mask); +} +EXPORT_SYMBOL(padata_alloc); + +/** + * __padata_alloc - allocate and initialize a padata instance + * and specify cpumasks for serial and parallel workers. * - * @cpumask: cpumask that padata uses for parallelization * @wq: workqueue to use for the allocated padata instance + * @pcpumask: cpumask that will be used for padata parallelization + * @cbcpumask: cpumask that will be used for padata serialization */ -struct padata_instance *padata_alloc(const struct cpumask *cpumask, - struct workqueue_struct *wq) +struct padata_instance *__padata_alloc(struct workqueue_struct *wq, + const struct cpumask *pcpumask, + const struct cpumask *cbcpumask) { struct padata_instance *pinst; struct parallel_data *pd = NULL; @@ -742,21 +957,26 @@ struct padata_instance *padata_alloc(const struct cpumask *cpumask, goto err; get_online_cpus(); - - if (!alloc_cpumask_var(&pinst->cpumask, GFP_KERNEL)) + if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL)) + goto err_free_inst; + if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) { + free_cpumask_var(pinst->cpumask.pcpu); goto err_free_inst; - - if (padata_validate_cpumask(pinst, cpumask)) { - pd = padata_alloc_pd(pinst, cpumask); - if (!pd) - goto err_free_mask; } + if (!padata_validate_cpumask(pinst, pcpumask) || + !padata_validate_cpumask(pinst, cbcpumask)) + goto err_free_masks; + + pd = padata_alloc_pd(pinst, pcpumask, cbcpumask); + if (!pd) + goto err_free_masks; rcu_assign_pointer(pinst->pd, pd); pinst->wq = wq; - cpumask_copy(pinst->cpumask, cpumask); + cpumask_copy(pinst->cpumask.pcpu, pcpumask); + cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); pinst->flags = 0; @@ -768,19 +988,21 @@ struct padata_instance *padata_alloc(const struct cpumask *cpumask, put_online_cpus(); + BLOCKING_INIT_NOTIFIER_HEAD(&pinst->cpumask_change_notifier); mutex_init(&pinst->lock); return pinst; -err_free_mask: - free_cpumask_var(pinst->cpumask); +err_free_masks: + free_cpumask_var(pinst->cpumask.pcpu); + free_cpumask_var(pinst->cpumask.cbcpu); err_free_inst: kfree(pinst); put_online_cpus(); err: return NULL; } -EXPORT_SYMBOL(padata_alloc); +EXPORT_SYMBOL(__padata_alloc); /** * padata_free - free a padata instance @@ -795,7 +1017,8 @@ void padata_free(struct padata_instance *pinst) padata_stop(pinst); padata_free_pd(pinst->pd); - free_cpumask_var(pinst->cpumask); + free_cpumask_var(pinst->cpumask.pcpu); + free_cpumask_var(pinst->cpumask.cbcpu); kfree(pinst); } EXPORT_SYMBOL(padata_free); -- cgit v1.1 From 5e017dc3f8bc9e4a28983666e6bc00114a2018bb Mon Sep 17 00:00:00 2001 From: Dan Kruchinin Date: Wed, 14 Jul 2010 14:33:08 +0400 Subject: padata: Added sysfs primitives to padata subsystem Added sysfs primitives to padata subsystem. Now API user may embedded kobject each padata instance contains into any sysfs hierarchy. For now padata sysfs interface provides only two objects: serial_cpumask [RW] - cpumask for serial workers parallel_cpumask [RW] - cpumask for parallel workers Signed-off-by: Dan Kruchinin Signed-off-by: Herbert Xu --- include/linux/padata.h | 5 +- kernel/padata.c | 155 ++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 150 insertions(+), 10 deletions(-) diff --git a/include/linux/padata.h b/include/linux/padata.h index 621e773..293ad46 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -26,6 +26,7 @@ #include #include #include +#include #define PADATA_CPU_SERIAL 0x01 #define PADATA_CPU_PARALLEL 0x02 @@ -142,7 +143,8 @@ struct parallel_data { * cbcpu for parallel and serial works respectivly. * @cpumask_change_notifier: Notifiers chain for user-defined notify * callbacks that will be called when either @pcpu or @cbcpu - * or both cpumasks change. + * or both cpumasks change. + * @kobj: padata instance kernel object. * @lock: padata instance lock. * @flags: padata flags. */ @@ -155,6 +157,7 @@ struct padata_instance { cpumask_var_t cbcpu; } cpumask; struct blocking_notifier_head cpumask_change_notifier; + struct kobject kobj; struct mutex lock; u8 flags; #define PADATA_INIT 1 diff --git a/kernel/padata.c b/kernel/padata.c index 84d0ca9..526f9ea 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #define MAX_SEQ_NR (INT_MAX - NR_CPUS) @@ -924,6 +925,149 @@ static int padata_cpu_callback(struct notifier_block *nfb, } #endif +static void __padata_free(struct padata_instance *pinst) +{ +#ifdef CONFIG_HOTPLUG_CPU + unregister_hotcpu_notifier(&pinst->cpu_notifier); +#endif + + padata_stop(pinst); + padata_free_pd(pinst->pd); + free_cpumask_var(pinst->cpumask.pcpu); + free_cpumask_var(pinst->cpumask.cbcpu); + kfree(pinst); +} + +#define kobj2pinst(_kobj) \ + container_of(_kobj, struct padata_instance, kobj) +#define attr2pentry(_attr) \ + container_of(_attr, struct padata_sysfs_entry, attr) + +static void padata_sysfs_release(struct kobject *kobj) +{ + struct padata_instance *pinst = kobj2pinst(kobj); + __padata_free(pinst); +} + +struct padata_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct padata_instance *, struct attribute *, char *); + ssize_t (*store)(struct padata_instance *, struct attribute *, + const char *, size_t); +}; + +static ssize_t show_cpumask(struct padata_instance *pinst, + struct attribute *attr, char *buf) +{ + struct cpumask *cpumask; + ssize_t len; + + mutex_lock(&pinst->lock); + if (!strcmp(attr->name, "serial_cpumask")) + cpumask = pinst->cpumask.cbcpu; + else + cpumask = pinst->cpumask.pcpu; + + len = bitmap_scnprintf(buf, PAGE_SIZE, cpumask_bits(cpumask), + nr_cpu_ids); + if (PAGE_SIZE - len < 2) + len = -EINVAL; + else + len += sprintf(buf + len, "\n"); + + mutex_unlock(&pinst->lock); + return len; +} + +static ssize_t store_cpumask(struct padata_instance *pinst, + struct attribute *attr, + const char *buf, size_t count) +{ + cpumask_var_t new_cpumask; + ssize_t ret; + int mask_type; + + if (!alloc_cpumask_var(&new_cpumask, GFP_KERNEL)) + return -ENOMEM; + + ret = bitmap_parse(buf, count, cpumask_bits(new_cpumask), + nr_cpumask_bits); + if (ret < 0) + goto out; + + mask_type = !strcmp(attr->name, "serial_cpumask") ? + PADATA_CPU_SERIAL : PADATA_CPU_PARALLEL; + ret = padata_set_cpumask(pinst, mask_type, new_cpumask); + if (!ret) + ret = count; + +out: + free_cpumask_var(new_cpumask); + return ret; +} + +#define PADATA_ATTR_RW(_name, _show_name, _store_name) \ + static struct padata_sysfs_entry _name##_attr = \ + __ATTR(_name, 0644, _show_name, _store_name) +#define PADATA_ATTR_RO(_name, _show_name) \ + static struct padata_sysfs_entry _name##_attr = \ + __ATTR(_name, 0400, _show_name, NULL) + +PADATA_ATTR_RW(serial_cpumask, show_cpumask, store_cpumask); +PADATA_ATTR_RW(parallel_cpumask, show_cpumask, store_cpumask); + +/* + * Padata sysfs provides the following objects: + * serial_cpumask [RW] - cpumask for serial workers + * parallel_cpumask [RW] - cpumask for parallel workers + */ +static struct attribute *padata_default_attrs[] = { + &serial_cpumask_attr.attr, + ¶llel_cpumask_attr.attr, + NULL, +}; + +static ssize_t padata_sysfs_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct padata_instance *pinst; + struct padata_sysfs_entry *pentry; + ssize_t ret = -EIO; + + pinst = kobj2pinst(kobj); + pentry = attr2pentry(attr); + if (pentry->show) + ret = pentry->show(pinst, attr, buf); + + return ret; +} + +static ssize_t padata_sysfs_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct padata_instance *pinst; + struct padata_sysfs_entry *pentry; + ssize_t ret = -EIO; + + pinst = kobj2pinst(kobj); + pentry = attr2pentry(attr); + if (pentry->show) + ret = pentry->store(pinst, attr, buf, count); + + return ret; +} + +static const struct sysfs_ops padata_sysfs_ops = { + .show = padata_sysfs_show, + .store = padata_sysfs_store, +}; + +static struct kobj_type padata_attr_type = { + .sysfs_ops = &padata_sysfs_ops, + .default_attrs = padata_default_attrs, + .release = padata_sysfs_release, +}; + /** * padata_alloc - Allocate and initialize padata instance. * Use default cpumask(cpu_possible_mask) @@ -989,6 +1133,7 @@ struct padata_instance *__padata_alloc(struct workqueue_struct *wq, put_online_cpus(); BLOCKING_INIT_NOTIFIER_HEAD(&pinst->cpumask_change_notifier); + kobject_init(&pinst->kobj, &padata_attr_type); mutex_init(&pinst->lock); return pinst; @@ -1011,14 +1156,6 @@ EXPORT_SYMBOL(__padata_alloc); */ void padata_free(struct padata_instance *pinst) { -#ifdef CONFIG_HOTPLUG_CPU - unregister_hotcpu_notifier(&pinst->cpu_notifier); -#endif - - padata_stop(pinst); - padata_free_pd(pinst->pd); - free_cpumask_var(pinst->cpumask.pcpu); - free_cpumask_var(pinst->cpumask.cbcpu); - kfree(pinst); + kobject_put(&pinst->kobj); } EXPORT_SYMBOL(padata_free); -- cgit v1.1 From a3fb1e330dd21d96c2e07dda513ad7e9af8b6ccf Mon Sep 17 00:00:00 2001 From: Dan Kruchinin Date: Wed, 14 Jul 2010 14:34:15 +0400 Subject: pcrypt: Added sysfs interface to pcrypt Added sysfs interface to pcrypt. Now pcrypt subsystem creates two sysfs directories with corresponding padata sysfs objects: /sys/kernel/pcrypt/[pencrypt|pdecrypt] Signed-off-by: Dan Kruchinin Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index c9662e2..7153a50 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -25,9 +25,11 @@ #include #include #include +#include #include struct pcrypt_instance { + const char *name; struct padata_instance *pinst; struct workqueue_struct *wq; @@ -55,7 +57,7 @@ struct pcrypt_instance { static struct pcrypt_instance pencrypt; static struct pcrypt_instance pdecrypt; - +static struct kset *pcrypt_kset; struct pcrypt_instance_ctx { struct crypto_spawn spawn; @@ -429,12 +431,25 @@ static int pcrypt_cpumask_change_notify(struct notifier_block *self, return 0; } +static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name) +{ + int ret; + + pinst->kobj.kset = pcrypt_kset; + ret = kobject_add(&pinst->kobj, NULL, name); + if (!ret) + kobject_uevent(&pinst->kobj, KOBJ_ADD); + + return ret; +} + static int __pcrypt_init_instance(struct pcrypt_instance *pcrypt, const char *name) { int ret = -ENOMEM; struct pcrypt_cpumask *mask; + pcrypt->name = name; pcrypt->wq = create_workqueue(name); if (!pcrypt->wq) goto err; @@ -459,7 +474,13 @@ static int __pcrypt_init_instance(struct pcrypt_instance *pcrypt, if (ret) goto err_free_cpumask; + ret = pcrypt_sysfs_add(pcrypt->pinst, name); + if (ret) + goto err_unregister_notifier; + return ret; +err_unregister_notifier: + padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); err_free_cpumask: free_cpumask_var(mask->mask); kfree(mask); @@ -473,6 +494,7 @@ err: static void __pcrypt_deinit_instance(struct pcrypt_instance *pcrypt) { + kobject_put(&pcrypt->pinst->kobj); free_cpumask_var(pcrypt->cb_cpumask->mask); kfree(pcrypt->cb_cpumask); @@ -491,11 +513,15 @@ static struct crypto_template pcrypt_tmpl = { static int __init pcrypt_init(void) { - int err; + int err = -ENOMEM; + + pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj); + if (!pcrypt_kset) + goto err; err = __pcrypt_init_instance(&pencrypt, "pencrypt"); if (err) - goto err; + goto err_unreg_kset; err = __pcrypt_init_instance(&pdecrypt, "pdecrypt"); if (err) @@ -508,6 +534,8 @@ static int __init pcrypt_init(void) err_deinit_pencrypt: __pcrypt_deinit_instance(&pencrypt); +err_unreg_kset: + kset_unregister(pcrypt_kset); err: return err; } @@ -517,6 +545,7 @@ static void __exit pcrypt_exit(void) __pcrypt_deinit_instance(&pencrypt); __pcrypt_deinit_instance(&pdecrypt); + kset_unregister(pcrypt_kset); crypto_unregister_template(&pcrypt_tmpl); } -- cgit v1.1 From 1fb1defbb00eb2954483a7d9a70f8a02edf51753 Mon Sep 17 00:00:00 2001 From: Peter Huewe Date: Thu, 15 Jul 2010 21:00:41 +0200 Subject: crypto: geode_aes - Convert pci_table entries to PCI_VDEVICE (if PCI_ANY_ID is used) This patch converts pci_table entries, where .subvendor=PCI_ANY_ID and .subdevice=PCI_ANY_ID, .class=0 and .class_mask=0, to use the PCI_VDEVICE macro, and thus improves readability. Signed-off-by: Peter Huewe Signed-off-by: Herbert Xu --- drivers/crypto/geode-aes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c index 09389dd..219d09c 100644 --- a/drivers/crypto/geode-aes.c +++ b/drivers/crypto/geode-aes.c @@ -573,7 +573,7 @@ geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id) } static struct pci_device_id geode_aes_tbl[] = { - { PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_LX_AES, PCI_ANY_ID, PCI_ANY_ID} , + { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), } , { 0, } }; -- cgit v1.1 From fad3a906d324c02b3c25ef51f702384154089846 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 20 Jul 2010 08:48:34 +0200 Subject: padata: Fix cpu index counting The counting of the cpu index got lost with a recent commit. This patch restores it. This fixes a hang of the parallel worker threads on cpu hotplug. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- kernel/padata.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/padata.c b/kernel/padata.c index 526f9ea..4287868 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -408,6 +408,7 @@ static void padata_init_pqueues(struct parallel_data *pd) pqueue = per_cpu_ptr(pd->pqueue, cpu); pqueue->pd = pd; pqueue->cpu_index = cpu_index; + cpu_index++; __padata_list_init(&pqueue->reorder); __padata_list_init(&pqueue->parallel); -- cgit v1.1 From b89661dff525a46edb7ee8a4423b5212068c05c0 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 20 Jul 2010 08:49:20 +0200 Subject: padata: Allocate cpumask dependend recources in any case The cpumask separation work assumes the cpumask dependend recources present regardless of valid or invalid cpumasks. With this patch we allocate the cpumask dependend recources in any case. This fixes two NULL pointer dereference crashes in padata_replace and in padata_get_cpumask. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- kernel/padata.c | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/kernel/padata.c b/kernel/padata.c index 4287868..6a51945 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -417,7 +417,7 @@ static void padata_init_pqueues(struct parallel_data *pd) } num_cpus = cpumask_weight(pd->cpumask.pcpu); - pd->max_seq_nr = (MAX_SEQ_NR / num_cpus) * num_cpus - 1; + pd->max_seq_nr = num_cpus ? (MAX_SEQ_NR / num_cpus) * num_cpus - 1 : 0; } /* Allocate and initialize the internal cpumask dependend resources. */ @@ -527,21 +527,19 @@ static void padata_replace(struct padata_instance *pinst, rcu_assign_pointer(pinst->pd, pd_new); synchronize_rcu(); - if (!pd_old) - goto out; - padata_flush_queues(pd_old); if (!cpumask_equal(pd_old->cpumask.pcpu, pd_new->cpumask.pcpu)) notification_mask |= PADATA_CPU_PARALLEL; if (!cpumask_equal(pd_old->cpumask.cbcpu, pd_new->cpumask.cbcpu)) notification_mask |= PADATA_CPU_SERIAL; + padata_flush_queues(pd_old); padata_free_pd(pd_old); + if (notification_mask) blocking_notifier_call_chain(&pinst->cpumask_change_notifier, notification_mask, pinst); -out: pinst->flags &= ~PADATA_RESET; } @@ -673,6 +671,7 @@ int __padata_set_cpumasks(struct padata_instance *pinst, struct parallel_data *pd = NULL; mutex_lock(&pinst->lock); + get_online_cpus(); valid = padata_validate_cpumask(pinst, pcpumask); if (!valid) { @@ -681,20 +680,16 @@ int __padata_set_cpumasks(struct padata_instance *pinst, } valid = padata_validate_cpumask(pinst, cbcpumask); - if (!valid) { + if (!valid) __padata_stop(pinst); - goto out_replace; - } - - get_online_cpus(); +out_replace: pd = padata_alloc_pd(pinst, pcpumask, cbcpumask); if (!pd) { err = -ENOMEM; goto out; } -out_replace: cpumask_copy(pinst->cpumask.pcpu, pcpumask); cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); @@ -705,7 +700,6 @@ out_replace: out: put_online_cpus(); - mutex_unlock(&pinst->lock); return err; @@ -776,11 +770,8 @@ static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) if (cpumask_test_cpu(cpu, cpu_online_mask)) { if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) || - !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) { + !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) __padata_stop(pinst); - padata_replace(pinst, pd); - goto out; - } pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu, pinst->cpumask.cbcpu); @@ -790,7 +781,6 @@ static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) padata_replace(pinst, pd); } -out: return 0; } -- cgit v1.1 From 7424713b83587006da72da84d7922471e366faba Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 20 Jul 2010 08:51:25 +0200 Subject: padata: Check for valid cpumasks Now that we allow to change the cpumasks from userspace, we have to check for valid cpumasks in padata_do_parallel. This patch adds the necessary check. This fixes a division by zero crash if the parallel cpumask contains no active cpu. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- kernel/padata.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/padata.c b/kernel/padata.c index 6a51945..7f895e2 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -114,7 +114,7 @@ int padata_do_parallel(struct padata_instance *pinst, pd = rcu_dereference(pinst->pd); err = -EINVAL; - if (!(pinst->flags & PADATA_INIT)) + if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID) goto out; if (!cpumask_test_cpu(cb_cpu, pd->cpumask.cbcpu)) -- cgit v1.1 From cc74f4bc111e9554bcd6445ad0fe1d90e5d2eb34 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 20 Jul 2010 08:52:20 +0200 Subject: crypto: pcrypt - Dont calulate a callback cpu on empty callback cpumask If the callback cpumask is empty, we crash with a division by zero when we try to calculate a callback cpu. So we don't update the callback cpu in pcrypt_do_parallel if the callback cpumask is empty. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 7153a50..794c172 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -82,6 +82,9 @@ static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, if (cpumask_test_cpu(cpu, cpumask->mask)) goto out; + if (!cpumask_weight(cpumask->mask)) + goto out; + cpu_index = cpu % cpumask_weight(cpumask->mask); cpu = cpumask_first(cpumask->mask); -- cgit v1.1 From e6cc11707661770ca2bd4db4b0256d28f48e7541 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 27 Jul 2010 07:14:28 +0200 Subject: padata: Rename padata_alloc functions We rename padata_alloc to padata_alloc_possible because this function allocates a padata_instance and uses the cpu_possible mask for parallel and serial workers. Also we rename __padata_alloc to padata_alloc to avoid to export underlined functions. Underlined functions are considered to be private to padata. Users are updated accordingly. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 2 +- include/linux/padata.h | 9 +++++---- kernel/padata.c | 24 ++++++++++++------------ 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 794c172..5546083 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -457,7 +457,7 @@ static int __pcrypt_init_instance(struct pcrypt_instance *pcrypt, if (!pcrypt->wq) goto err; - pcrypt->pinst = padata_alloc(pcrypt->wq); + pcrypt->pinst = padata_alloc_possible(pcrypt->wq); if (!pcrypt->pinst) goto err_destroy_workqueue; diff --git a/include/linux/padata.h b/include/linux/padata.h index 293ad46..71dfc9d 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -165,10 +165,11 @@ struct padata_instance { #define PADATA_INVALID 4 }; -extern struct padata_instance *padata_alloc(struct workqueue_struct *wq); -extern struct padata_instance *__padata_alloc(struct workqueue_struct *wq, - const struct cpumask *pcpumask, - const struct cpumask *cbcpumask); +extern struct padata_instance *padata_alloc_possible( + struct workqueue_struct *wq); +extern struct padata_instance *padata_alloc(struct workqueue_struct *wq, + const struct cpumask *pcpumask, + const struct cpumask *cbcpumask); extern void padata_free(struct padata_instance *pinst); extern int padata_do_parallel(struct padata_instance *pinst, struct padata_priv *padata, int cb_cpu); diff --git a/kernel/padata.c b/kernel/padata.c index 7f895e2..12860bc 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -1060,29 +1060,29 @@ static struct kobj_type padata_attr_type = { }; /** - * padata_alloc - Allocate and initialize padata instance. - * Use default cpumask(cpu_possible_mask) - * for serial and parallel workes. + * padata_alloc_possible - Allocate and initialize padata instance. + * Use the cpu_possible_mask for serial and + * parallel workers. * * @wq: workqueue to use for the allocated padata instance */ -struct padata_instance *padata_alloc(struct workqueue_struct *wq) +struct padata_instance *padata_alloc_possible(struct workqueue_struct *wq) { - return __padata_alloc(wq, cpu_possible_mask, cpu_possible_mask); + return padata_alloc(wq, cpu_possible_mask, cpu_possible_mask); } -EXPORT_SYMBOL(padata_alloc); +EXPORT_SYMBOL(padata_alloc_possible); /** - * __padata_alloc - allocate and initialize a padata instance - * and specify cpumasks for serial and parallel workers. + * padata_alloc - allocate and initialize a padata instance and specify + * cpumasks for serial and parallel workers. * * @wq: workqueue to use for the allocated padata instance * @pcpumask: cpumask that will be used for padata parallelization * @cbcpumask: cpumask that will be used for padata serialization */ -struct padata_instance *__padata_alloc(struct workqueue_struct *wq, - const struct cpumask *pcpumask, - const struct cpumask *cbcpumask) +struct padata_instance *padata_alloc(struct workqueue_struct *wq, + const struct cpumask *pcpumask, + const struct cpumask *cbcpumask) { struct padata_instance *pinst; struct parallel_data *pd = NULL; @@ -1138,7 +1138,7 @@ err_free_inst: err: return NULL; } -EXPORT_SYMBOL(__padata_alloc); +EXPORT_SYMBOL(padata_alloc); /** * padata_free - free a padata instance -- cgit v1.1 From 65ff577e6b6e482ee9de3569e058edebdc02f069 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 27 Jul 2010 07:15:06 +0200 Subject: padata: Rearrange set_cpumask functions padata_set_cpumask needs to be protected by a lock. We make __padata_set_cpumasks unlocked and static. So this function can be used by the exported and locked padata_set_cpumask and padata_set_cpumasks functions. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- include/linux/padata.h | 6 +-- kernel/padata.c | 117 ++++++++++++++++++++++++++++--------------------- 2 files changed, 70 insertions(+), 53 deletions(-) diff --git a/include/linux/padata.h b/include/linux/padata.h index 71dfc9d..bb0fc5d 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -178,9 +178,9 @@ extern int padata_get_cpumask(struct padata_instance *pinst, int cpumask_type, struct cpumask *out_mask); extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask); -extern int __padata_set_cpumasks(struct padata_instance *pinst, - cpumask_var_t pcpumask, - cpumask_var_t cbcpumask); +extern int padata_set_cpumasks(struct padata_instance *pinst, + cpumask_var_t pcpumask, + cpumask_var_t cbcpumask); extern int padata_add_cpu(struct padata_instance *pinst, int cpu, int mask); extern int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask); extern int padata_start(struct padata_instance *pinst); diff --git a/kernel/padata.c b/kernel/padata.c index 12860bc..4987203 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -623,6 +623,66 @@ int padata_get_cpumask(struct padata_instance *pinst, } EXPORT_SYMBOL(padata_get_cpumask); +static int __padata_set_cpumasks(struct padata_instance *pinst, + cpumask_var_t pcpumask, + cpumask_var_t cbcpumask) +{ + int valid; + struct parallel_data *pd; + + valid = padata_validate_cpumask(pinst, pcpumask); + if (!valid) { + __padata_stop(pinst); + goto out_replace; + } + + valid = padata_validate_cpumask(pinst, cbcpumask); + if (!valid) + __padata_stop(pinst); + +out_replace: + pd = padata_alloc_pd(pinst, pcpumask, cbcpumask); + if (!pd) + return -ENOMEM; + + cpumask_copy(pinst->cpumask.pcpu, pcpumask); + cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); + + padata_replace(pinst, pd); + + if (valid) + __padata_start(pinst); + + return 0; +} + +/** + * padata_set_cpumasks - Set both parallel and serial cpumasks. The first + * one is used by parallel workers and the second one + * by the wokers doing serialization. + * + * @pinst: padata instance + * @pcpumask: the cpumask to use for parallel workers + * @cbcpumask: the cpumsak to use for serial workers + */ +int padata_set_cpumasks(struct padata_instance *pinst, cpumask_var_t pcpumask, + cpumask_var_t cbcpumask) +{ + int err; + + mutex_lock(&pinst->lock); + get_online_cpus(); + + err = __padata_set_cpumasks(pinst, pcpumask, cbcpumask); + + put_online_cpus(); + mutex_unlock(&pinst->lock); + + return err; + +} +EXPORT_SYMBOL(padata_set_cpumasks); + /** * padata_set_cpumask: Sets specified by @cpumask_type cpumask to the value * equivalent to @cpumask. @@ -636,6 +696,10 @@ int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask) { struct cpumask *serial_mask, *parallel_mask; + int err = -EINVAL; + + mutex_lock(&pinst->lock); + get_online_cpus(); switch (cpumask_type) { case PADATA_CPU_PARALLEL: @@ -647,65 +711,18 @@ int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, serial_mask = cpumask; break; default: - return -EINVAL; + goto out; } - return __padata_set_cpumasks(pinst, parallel_mask, serial_mask); -} -EXPORT_SYMBOL(padata_set_cpumask); - -/** - * __padata_set_cpumasks - Set both parallel and serial cpumasks. The first - * one is used by parallel workers and the second one - * by the wokers doing serialization. - * - * @pinst: padata instance - * @pcpumask: the cpumask to use for parallel workers - * @cbcpumask: the cpumsak to use for serial workers - */ -int __padata_set_cpumasks(struct padata_instance *pinst, - cpumask_var_t pcpumask, cpumask_var_t cbcpumask) -{ - int valid; - int err = 0; - struct parallel_data *pd = NULL; - - mutex_lock(&pinst->lock); - get_online_cpus(); - - valid = padata_validate_cpumask(pinst, pcpumask); - if (!valid) { - __padata_stop(pinst); - goto out_replace; - } - - valid = padata_validate_cpumask(pinst, cbcpumask); - if (!valid) - __padata_stop(pinst); - -out_replace: - pd = padata_alloc_pd(pinst, pcpumask, cbcpumask); - if (!pd) { - err = -ENOMEM; - goto out; - } - - cpumask_copy(pinst->cpumask.pcpu, pcpumask); - cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); - - padata_replace(pinst, pd); - - if (valid) - __padata_start(pinst); + err = __padata_set_cpumasks(pinst, parallel_mask, serial_mask); out: put_online_cpus(); mutex_unlock(&pinst->lock); return err; - } -EXPORT_SYMBOL(__padata_set_cpumasks); +EXPORT_SYMBOL(padata_set_cpumask); static int __padata_add_cpu(struct padata_instance *pinst, int cpu) { -- cgit v1.1 From c635696c7c0fbc720698dbec34bb83e53df6a967 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 27 Jul 2010 07:15:50 +0200 Subject: padata: Pass the padata cpumasks to the cpumask_change_notifier chain We pass a pointer to the new padata cpumasks to the cpumask_change_notifier chain. So users can access the cpumasks without the need of an extra padata_get_cpumask function. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- include/linux/padata.h | 40 +++++++++++++++++++++------------------- kernel/padata.c | 3 ++- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/include/linux/padata.h b/include/linux/padata.h index bb0fc5d..43db792 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -98,6 +98,16 @@ struct padata_parallel_queue { int cpu_index; }; +/** + * struct padata_cpumask - The cpumasks for the parallel/serial workers + * + * @pcpu: cpumask for the parallel workers. + * @cbcpu: cpumask for the serial (callback) workers. + */ +struct padata_cpumask { + cpumask_var_t pcpu; + cpumask_var_t cbcpu; +}; /** * struct parallel_data - Internal control structure, covers everything @@ -110,8 +120,7 @@ struct padata_parallel_queue { * @reorder_objects: Number of objects waiting in the reorder queues. * @refcnt: Number of objects holding a reference on this parallel_data. * @max_seq_nr: Maximal used sequence number. - * @cpumask: Contains two cpumasks: pcpu and cbcpu for - * parallel and serial workers respectively. + * @cpumask: The cpumasks in use for parallel and serial workers. * @lock: Reorder lock. * @processed: Number of already processed objects. * @timer: Reorder timer. @@ -120,17 +129,14 @@ struct parallel_data { struct padata_instance *pinst; struct padata_parallel_queue *pqueue; struct padata_serial_queue *squeue; - atomic_t seq_nr; - atomic_t reorder_objects; - atomic_t refcnt; - unsigned int max_seq_nr; - struct { - cpumask_var_t pcpu; - cpumask_var_t cbcpu; - } cpumask; - spinlock_t lock ____cacheline_aligned; - unsigned int processed; - struct timer_list timer; + atomic_t seq_nr; + atomic_t reorder_objects; + atomic_t refcnt; + unsigned int max_seq_nr; + struct padata_cpumask cpumask; + spinlock_t lock ____cacheline_aligned; + unsigned int processed; + struct timer_list timer; }; /** @@ -139,8 +145,7 @@ struct parallel_data { * @cpu_notifier: cpu hotplug notifier. * @wq: The workqueue in use. * @pd: The internal control structure. - * @cpumask: User supplied cpumask. Contains two cpumasks: pcpu and - * cbcpu for parallel and serial works respectivly. + * @cpumask: User supplied cpumasks for parallel and serial works. * @cpumask_change_notifier: Notifiers chain for user-defined notify * callbacks that will be called when either @pcpu or @cbcpu * or both cpumasks change. @@ -152,10 +157,7 @@ struct padata_instance { struct notifier_block cpu_notifier; struct workqueue_struct *wq; struct parallel_data *pd; - struct { - cpumask_var_t pcpu; - cpumask_var_t cbcpu; - } cpumask; + struct padata_cpumask cpumask; struct blocking_notifier_head cpumask_change_notifier; struct kobject kobj; struct mutex lock; diff --git a/kernel/padata.c b/kernel/padata.c index 4987203..1c8c1d1 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -538,7 +538,8 @@ static void padata_replace(struct padata_instance *pinst, if (notification_mask) blocking_notifier_call_chain(&pinst->cpumask_change_notifier, - notification_mask, pinst); + notification_mask, + &pd_new->cpumask); pinst->flags &= ~PADATA_RESET; } -- cgit v1.1 From c57e842eff49b05c4642dd7cfb1e7aa62ab932fa Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 27 Jul 2010 07:16:33 +0200 Subject: crypto: pcrypt - Rename pcrypt_instance In the crypto-layer an instance refers usually to a crypto instance. The struct pcrypt_instance is not related to a crypto instance. It rather contains the padata informations, so we rename it to padata_pcrypt. The functions that handle this struct are renamed accordingly. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 5546083..7742553 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -28,8 +28,7 @@ #include #include -struct pcrypt_instance { - const char *name; +struct padata_pcrypt { struct padata_instance *pinst; struct workqueue_struct *wq; @@ -55,8 +54,8 @@ struct pcrypt_instance { struct notifier_block nblock; }; -static struct pcrypt_instance pencrypt; -static struct pcrypt_instance pdecrypt; +static struct padata_pcrypt pencrypt; +static struct padata_pcrypt pdecrypt; static struct kset *pcrypt_kset; struct pcrypt_instance_ctx { @@ -70,7 +69,7 @@ struct pcrypt_aead_ctx { }; static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, - struct pcrypt_instance *pcrypt) + struct padata_pcrypt *pcrypt) { unsigned int cpu_index, cpu, i; struct pcrypt_cpumask *cpumask; @@ -408,13 +407,13 @@ static void pcrypt_free(struct crypto_instance *inst) static int pcrypt_cpumask_change_notify(struct notifier_block *self, unsigned long val, void *data) { - struct pcrypt_instance *pcrypt; + struct padata_pcrypt *pcrypt; struct pcrypt_cpumask *new_mask, *old_mask; if (!(val & PADATA_CPU_SERIAL)) return 0; - pcrypt = container_of(self, struct pcrypt_instance, nblock); + pcrypt = container_of(self, struct padata_pcrypt, nblock); new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL); if (!new_mask) return -ENOMEM; @@ -446,13 +445,12 @@ static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name) return ret; } -static int __pcrypt_init_instance(struct pcrypt_instance *pcrypt, - const char *name) +static int pcrypt_init_padata(struct padata_pcrypt *pcrypt, + const char *name) { int ret = -ENOMEM; struct pcrypt_cpumask *mask; - pcrypt->name = name; pcrypt->wq = create_workqueue(name); if (!pcrypt->wq) goto err; @@ -495,7 +493,7 @@ err: return ret; } -static void __pcrypt_deinit_instance(struct pcrypt_instance *pcrypt) +static void pcrypt_fini_padata(struct padata_pcrypt *pcrypt) { kobject_put(&pcrypt->pinst->kobj); free_cpumask_var(pcrypt->cb_cpumask->mask); @@ -522,11 +520,11 @@ static int __init pcrypt_init(void) if (!pcrypt_kset) goto err; - err = __pcrypt_init_instance(&pencrypt, "pencrypt"); + err = pcrypt_init_padata(&pencrypt, "pencrypt"); if (err) goto err_unreg_kset; - err = __pcrypt_init_instance(&pdecrypt, "pdecrypt"); + err = pcrypt_init_padata(&pdecrypt, "pdecrypt"); if (err) goto err_deinit_pencrypt; @@ -536,7 +534,7 @@ static int __init pcrypt_init(void) return crypto_register_template(&pcrypt_tmpl); err_deinit_pencrypt: - __pcrypt_deinit_instance(&pencrypt); + pcrypt_fini_padata(&pencrypt); err_unreg_kset: kset_unregister(pcrypt_kset); err: @@ -545,8 +543,8 @@ err: static void __exit pcrypt_exit(void) { - __pcrypt_deinit_instance(&pencrypt); - __pcrypt_deinit_instance(&pdecrypt); + pcrypt_fini_padata(&pencrypt); + pcrypt_fini_padata(&pdecrypt); kset_unregister(pcrypt_kset); crypto_unregister_template(&pcrypt_tmpl); -- cgit v1.1 From d3f64e46aa21dd86a239274d218ec286461bfa68 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 27 Jul 2010 07:18:46 +0200 Subject: crypto: pcrypt - Update pcrypt cpumask according to the padata cpumask notifier The padata cpumask change notifier passes a padata_cpumask to the notifier chain. So we use this cpumask instead of asking padata for the cpumask. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 7742553..de30782 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -26,6 +26,7 @@ #include #include #include +#include #include struct padata_pcrypt { @@ -409,6 +410,7 @@ static int pcrypt_cpumask_change_notify(struct notifier_block *self, { struct padata_pcrypt *pcrypt; struct pcrypt_cpumask *new_mask, *old_mask; + struct padata_cpumask *cpumask = (struct padata_cpumask *)data; if (!(val & PADATA_CPU_SERIAL)) return 0; @@ -424,7 +426,7 @@ static int pcrypt_cpumask_change_notify(struct notifier_block *self, old_mask = pcrypt->cb_cpumask; - padata_get_cpumask(pcrypt->pinst, PADATA_CPU_SERIAL, new_mask->mask); + cpumask_copy(new_mask->mask, cpumask->cbcpu); rcu_assign_pointer(pcrypt->cb_cpumask, new_mask); synchronize_rcu_bh(); @@ -451,6 +453,8 @@ static int pcrypt_init_padata(struct padata_pcrypt *pcrypt, int ret = -ENOMEM; struct pcrypt_cpumask *mask; + get_online_cpus(); + pcrypt->wq = create_workqueue(name); if (!pcrypt->wq) goto err; @@ -467,7 +471,7 @@ static int pcrypt_init_padata(struct padata_pcrypt *pcrypt, goto err_free_padata; } - padata_get_cpumask(pcrypt->pinst, PADATA_CPU_SERIAL, mask->mask); + cpumask_and(mask->mask, cpu_possible_mask, cpu_active_mask); rcu_assign_pointer(pcrypt->cb_cpumask, mask); pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify; @@ -479,7 +483,10 @@ static int pcrypt_init_padata(struct padata_pcrypt *pcrypt, if (ret) goto err_unregister_notifier; + put_online_cpus(); + return ret; + err_unregister_notifier: padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); err_free_cpumask: @@ -490,6 +497,8 @@ err_free_padata: err_destroy_workqueue: destroy_workqueue(pcrypt->wq); err: + put_online_cpus(); + return ret; } -- cgit v1.1 From 0500e9b3f11ce84fc6ee48a3e29909145e58ba48 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 27 Jul 2010 07:19:27 +0200 Subject: padata: Remove padata_get_cpumask A function that copies the padata cpumasks to a user buffer is a bit error prone. The cpumask can change any time so we can't be sure to have the right cpumask when using this function. A user who is interested in the padata cpumasks should register to the padata cpumask notifier chain instead. Users of padata_get_cpumask are already updated, so we can remove it. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- include/linux/padata.h | 2 -- kernel/padata.c | 35 ----------------------------------- 2 files changed, 37 deletions(-) diff --git a/include/linux/padata.h b/include/linux/padata.h index 43db792..bdcd1e9 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -176,8 +176,6 @@ extern void padata_free(struct padata_instance *pinst); extern int padata_do_parallel(struct padata_instance *pinst, struct padata_priv *padata, int cb_cpu); extern void padata_do_serial(struct padata_priv *padata); -extern int padata_get_cpumask(struct padata_instance *pinst, - int cpumask_type, struct cpumask *out_mask); extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask); extern int padata_set_cpumasks(struct padata_instance *pinst, diff --git a/kernel/padata.c b/kernel/padata.c index 1c8c1d1..fd46792 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -589,41 +589,6 @@ static bool padata_validate_cpumask(struct padata_instance *pinst, return true; } -/** - * padata_get_cpumask: Fetch serial or parallel cpumask from the - * given padata instance and copy it to @out_mask - * - * @pinst: A pointer to padata instance - * @cpumask_type: Specifies which cpumask will be copied. - * Possible values are PADATA_CPU_SERIAL *or* PADATA_CPU_PARALLEL - * corresponding to serial and parallel cpumask respectively. - * @out_mask: A pointer to cpumask structure where selected - * cpumask will be copied. - */ -int padata_get_cpumask(struct padata_instance *pinst, - int cpumask_type, struct cpumask *out_mask) -{ - struct parallel_data *pd; - int ret = 0; - - rcu_read_lock_bh(); - pd = rcu_dereference(pinst->pd); - switch (cpumask_type) { - case PADATA_CPU_SERIAL: - cpumask_copy(out_mask, pd->cpumask.cbcpu); - break; - case PADATA_CPU_PARALLEL: - cpumask_copy(out_mask, pd->cpumask.pcpu); - break; - default: - ret = -EINVAL; - } - - rcu_read_unlock_bh(); - return ret; -} -EXPORT_SYMBOL(padata_get_cpumask); - static int __padata_set_cpumasks(struct padata_instance *pinst, cpumask_var_t pcpumask, cpumask_var_t cbcpumask) -- cgit v1.1 From 313910d3b98029a867bb4aa3ee552ae573db0458 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 27 Jul 2010 07:20:47 +0200 Subject: padata: update API documentation Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- Documentation/padata.txt | 76 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 12 deletions(-) diff --git a/Documentation/padata.txt b/Documentation/padata.txt index 93dd4e6..473ebf2 100644 --- a/Documentation/padata.txt +++ b/Documentation/padata.txt @@ -1,5 +1,5 @@ The padata parallel execution mechanism -Last updated for 2.6.34 +Last updated for 2.6.36 Padata is a mechanism by which the kernel can farm work out to be done in parallel on multiple CPUs while retaining the ordering of tasks. It was @@ -13,12 +13,28 @@ overall control of how tasks are to be run: #include - struct padata_instance *padata_alloc(const struct cpumask *cpumask, - struct workqueue_struct *wq); + struct padata_instance *padata_alloc(struct workqueue_struct *wq, + const struct cpumask *pcpumask, + const struct cpumask *cbcpumask); -The cpumask describes which processors will be used to execute work -submitted to this instance. The workqueue wq is where the work will -actually be done; it should be a multithreaded queue, naturally. +The pcpumask describes which processors will be used to execute work +submitted to this instance in parallel. The cbcpumask defines which +processors are allowed to use as the serialization callback processor. +The workqueue wq is where the work will actually be done; it should be +a multithreaded queue, naturally. + +To allocate a padata instance with the cpu_possible_mask for both +cpumasks this helper function can be used: + + struct padata_instance *padata_alloc_possible(struct workqueue_struct *wq); + +Note: Padata maintains two kinds of cpumasks internally. The user supplied +cpumasks, submitted by padata_alloc/padata_alloc_possible and the 'usable' +cpumasks. The usable cpumasks are always the subset of active cpus in the +user supplied cpumasks, these are the cpumasks padata actually use. So +it is legal to supply a cpumask to padata that contains offline cpus. +Once a offline cpu in the user supplied cpumask comes online, padata +is going to use it. There are functions for enabling and disabling the instance: @@ -34,13 +50,49 @@ is unused. The list of CPUs to be used can be adjusted with these functions: - int padata_set_cpumask(struct padata_instance *pinst, + int padata_set_cpumasks(struct padata_instance *pinst, + cpumask_var_t pcpumask, + cpumask_var_t cbcpumask); + int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask); - int padata_add_cpu(struct padata_instance *pinst, int cpu); - int padata_remove_cpu(struct padata_instance *pinst, int cpu); + int padata_add_cpu(struct padata_instance *pinst, int cpu, int mask); + int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask); + +Changing the CPU masks are expensive operations, though, so it should not be +done with great frequency. + +It's possible to change both cpumasks of a padata instance with +padata_set_cpumasks by specifying the cpumasks for parallel execution (pcpumask) +and for the serial callback function (cbcpumask). padata_set_cpumask is to +change just one of the cpumasks. Here cpumask_type is one of PADATA_CPU_SERIAL, +PADATA_CPU_PARALLEL and cpumask specifies the new cpumask to use. +To simply add or remove one cpu from a certain cpumask the functions +padata_add_cpu/padata_remove_cpu are used. cpu specifies the cpu to add or +remove and mask is one of PADATA_CPU_SERIAL, PADATA_CPU_PARALLEL. + +If a user is interested in padata cpumask changes, he can register to +the padata cpumask change notifier: + + int padata_register_cpumask_notifier(struct padata_instance *pinst, + struct notifier_block *nblock); + +To unregister from that notifier: + + int padata_unregister_cpumask_notifier(struct padata_instance *pinst, + struct notifier_block *nblock); + +The padata cpumask change notifier notifies about changes of the usable +cpumasks, i.e. the subset of active cpus in the user supplied cpumask. + +Padata calls the notifier chain with: + + blocking_notifier_call_chain(&pinst->cpumask_change_notifier, + notification_mask, + &pd_new->cpumask); -Changing the CPU mask has the look of an expensive operation, though, so it -probably should not be done with great frequency. +Here cpumask_change_notifier is registered notifier, notification_mask +is one of PADATA_CPU_SERIAL, PADATA_CPU_PARALLEL and cpumask is a pointer +to a struct padata_cpumask that contains the new cpumask informations. Actually submitting work to the padata instance requires the creation of a padata_priv structure: @@ -53,7 +105,7 @@ padata_priv structure: This structure will almost certainly be embedded within some larger structure specific to the work to be done. Most its fields are private to -padata, but the structure should be zeroed at initialization time, and the +padata, but the structure should be zeroed at initialisation time, and the parallel() and serial() functions should be provided. Those functions will be called in the process of getting the work done as we will see momentarily. -- cgit v1.1 From 4015d9a865e3bcc42d88bedc8ce1551000bab664 Mon Sep 17 00:00:00 2001 From: Richard Kennedy Date: Sat, 31 Jul 2010 19:58:00 +0800 Subject: random: Reorder struct entropy_store to remove padding on 64bits Re-order structure entropy_store to remove 8 bytes of padding on 64 bit builds, so shrinking this structure from 72 to 64 bytes and allowing it to fit into one cache line. Signed-off-by: Richard Kennedy Signed-off-by: Matt Mackall Signed-off-by: Herbert Xu --- drivers/char/random.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/char/random.c b/drivers/char/random.c index 8d85587..caef35a 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -407,8 +407,8 @@ struct entropy_store { struct poolinfo *poolinfo; __u32 *pool; const char *name; - int limit; struct entropy_store *pull; + int limit; /* read-write data: */ spinlock_t lock; -- cgit v1.1