388835/aesni-intel_glue.i.3

   1 struct ftrace_branch_data {
   2 };
   3 struct list_head {
   4 };
   5 int __static_cpu_has(unsigned short bit)
   6 {
   7         asm goto ("1: jmp %l[t_no]\n" "2:\n" ".section .altinstructions,\"a\"\n"
   8                   " " ".balign 4" " " "\n" " " ".long" " " "1b\n" " " ".long"
   9                   " " "0\n" " .word %P0\n" " .byte 2b - 1b\n" " .byte 0\n"
  10                   ".previous\n"::"i" (bit)::t_no);
  11         return 1;
  12       t_no:
  13 return 0;
  14 }
  15
  16 struct cpuinfo_x86 {
  17         unsigned int x86_capability[10];
  18 } __attribute__ ((__aligned__((1 << (6)))));
  19 extern struct cpuinfo_x86 boot_cpu_data;
  20 struct i387_fxsave_struct {
  21         unsigned short swd;
  22 } __attribute__ ((aligned(16)));
  23 struct xsave_hdr_struct {
  24         unsigned long long xstate_bv;
  25 } __attribute__ ((packed));
  26 struct xsave_struct {
  27         struct xsave_hdr_struct xsave_hdr;
  28 } __attribute__ ((packed, aligned(64)));
  29 union thread_xstate {
  30         struct i387_fxsave_struct fxsave;
  31         struct xsave_struct xsave;
  32 };
  33 struct fpu {
  34         union thread_xstate *state;
  35 };
  36 struct thread_struct {
  37         struct fpu fpu;
  38 } mm_segment_t;
  39 struct thread_info {
  40         struct task_struct *task;
  41         unsigned int status;
  42 };
  43 struct cipher_alg {
  44         unsigned int cia_min_keysize;
  45         unsigned int cia_max_keysize;
  46         int (*cia_setkey) (struct crypto_tfm * tfm, const unsigned char * key,
  47                            unsigned int keylen);
  48         void (*cia_encrypt) (struct crypto_tfm * tfm, unsigned char * dst, const unsigned char * src);
  49         void (*cia_decrypt) (struct crypto_tfm * tfm, unsigned char * dst, const unsigned char * src);
  50 };
  51 struct crypto_alg {
  52         struct list_head cra_list;
  53         unsigned int cra_flags;
  54         unsigned int cra_blocksize;
  55         unsigned int cra_ctxsize;
  56         unsigned int cra_alignmask;
  57         int cra_priority;
  58         char cra_name[64];
  59         char cra_driver_name[64];
  60         const struct crypto_type *cra_type;
  61         union {
  62                 struct cipher_alg cipher;
  63         } cra_u;
  64         struct module *cra_module;
  65 };
  66 extern const struct crypto_type crypto_ablkcipher_type;
  67 extern const struct crypto_type crypto_blkcipher_type;
  68 struct crypto_aes_ctx {
  69 };
  70 struct task_struct {
  71         struct thread_struct thread;
  72 };
  73 static __attribute__ ((always_inline))
  74     __attribute__ ((pure))
  75 int use_xsave(void)
  76 {
  77         return (
  78 __builtin_constant_p
  79                 ((__builtin_constant_p((154))
  80                   &&
  81                   (((4) == 0
  82                     && (1UL << (((154)) & 31) &
  83                         ((1 << ((0) & 31)) | 0 |
  84                          (1 << ((6) & 31)) | (1 << ((8) & 31))
  85                          | 0 | (1 << ((15) & 31)) | 0)))
  86                    || ((4) == 1
  87                        && (1UL << (((154)) & 31) & (0)))
  88                    || ((4) == 2
  89                        && (1UL << (((154)) & 31) & 0))
  90                    || ((4) == 3
  91                        && (1UL << (((154)) & 31) & (0)))
  92                    || ((4) == 4
  93                        && (1UL << (((154)) & 31) & 0))
  94                    || ((4) == 5
  95                        && (1UL << (((154)) & 31) & 0))
  96                    || ((4) == 6
  97                        && (1UL << (((154)) & 31) & 0))
  98                    || ((4) == 7
  99                        && (1UL << (((154)) & 31) & 0))
 100                    || ((4) == 8
 101                        && (1UL << (((154)) & 31) & 0))
 102                    || ((4) == 9
 103                        && (1UL << (((154)) & 31) & 0))) ? 1
 104                   :
 105 (__builtin_constant_p(154) ?
 106                      constant_test_bit(154,
 107                                        ((unsigned long *)((&boot_cpu_data)->
 108                                                           x86_capability))) :
 109                      variable_test_bit(((154))
 110 ,
 111                                        ((unsigned long *)((&boot_cpu_data)->
 112                                                           x86_capability))))))
 113                 ? (__builtin_constant_p((154))
 114                    &&
 115                    (((4) == 0
 116                      && (1UL << (((154)) & 31) &
 117                          ((1 << ((0) & 31)) | 0 |
 118                           (1 << ((6) & 31)) | (1 <<
 119                                                         ((0 * 32 +
 120                                                           8) & 31)) | 0 | (1
 121                                                                                <<
 122                                                                                ((15) & 31)) | 0))) || ((4) == 1 && (1UL << (((154)) & 31) & (0))) || ((4) == 2 && (1UL << (((154)) & 31) & 0)) || ((4) == 3 && (1UL << (((154)) & 31) & (0))) || ((4) == 4 && (1UL << (((154)) & 31) & 0)) || ((4) == 5 && (1UL << (((154)) & 31) & 0)) || ((4) == 6 && (1UL << (((154)) & 31) & 0)) || ((4) == 7 && (1UL << (((154)) & 31) & 0)) || ((4) == 8 && (1UL << (((154)) & 31) & 0)) || ((4) == 9 && (1UL << (((154)) & 31) & 0))) ? 1 : (__builtin_constant_p(((154))) ? constant_test_bit(((154)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((154)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))) : __builtin_constant_p((154)) ? __static_cpu_has((154)) : (__builtin_constant_p((154)) && (((4) == 0 && (1UL << (((154)) & 31) & ((1 << ((0) & 31)) | 0 | (1 << ((6) & 31)) | (1 << ((8) & 31)) | 0 | (1 << ((15) & 31)) | 0))) || ((4) == 1 && (1UL << (((154)) & 31) & (0))) || ((4) == 2 && (1UL << (((154)) & 31) & 0)) || ((4) == 3 && (1UL << (((154)) & 31) & (0))) || ((4) == 4 && (1UL << (((154)) & 31) & 0)) || ((4) == 5 && (1UL << (((154)) & 31) & 0)) || ((4) == 6 && (1UL << (((154)) & 31) & 0)) || ((4) == 7 && (1UL << (((154)) & 31) & 0)) || ((4) == 8 && (1UL << (((154)) & 31) & 0)) || ((4) == 9 && (1UL << (((154)) & 31) & 0))) ? 1 : (__builtin_constant_p(((154))) ? constant_test_bit(((154)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((154)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))));
 123 }
 124
 125 static inline __attribute__ ((always_inline)) __attribute__ ((always_inline))
 126     __attribute__ ((pure))
 127 int use_fxsr(void)
 128 {
 129         return (__builtin_constant_p
 130                 ((__builtin_constant_p((24))
 131                   &&
 132                   (((((24)) >> 5) == 0
 133                     && (1UL << (24) &
 134                         ((1 << ((0) & 31)) | 0 |
 135                          (1 << ((6) & 31)) | (1 << ((8) & 31))
 136                          | 0 | (1 << ((15) & 31)) | 0)))
 137                    || ((((24)) >> 5) == 1
 138                        && (1UL << (24) & (0)))
 139                    || ((((24)) >> 5) == 2
 140                        && (1UL << (24) & 0))
 141                    || ((((24)) >> 5) == 3
 142                        && (1UL << (24) & (0)))
 143                    || ((((24)) >> 5) == 4
 144                        && (1UL << (24) & 0))
 145                    || ((((24)) >> 5) == 5
 146                        && (1UL << (24) & 0))
 147                    || ((((24)) >> 5) == 6
 148                        && (1UL << (24) & 0))
 149                    || ((((24)) >> 5) == 7
 150                        && (1UL << (24) & 0))
 151                    || ((((24)) >> 5) == 8
 152                        && (1UL << (24) & 0))
 153                    || ((((24)) >> 5) == 9
 154                        && (1UL << (24) & 0))) ? 1
 155                   : (__builtin_constant_p(((24))) ?
 156                      constant_test_bit(((24)),
 157                                        ((unsigned long *)((&boot_cpu_data)->
 158                                                           x86_capability))) :
 159                      variable_test_bit(((24)),
 160                                        ((unsigned long *)((&boot_cpu_data)->
 161                                                           x86_capability))))))
 162                 ? (__builtin_constant_p((24))
 163                    &&
 164                    (((((24)) >> 5) == 0
 165                      && (1UL << (24) &
 166                          ((1 << ((0) & 31)) | 0 |
 167                           (1 << ((6) & 31)) | (1 <<
 168                                                         ((0 * 32 +
 169                                                           8) & 31)) | 0 | (1
 170                                                                                <<
 171                                                                                ((15) & 31)) | 0))) || ((((24)) >> 5) == 1 && (1UL << (24) & (0))) || ((((24)) >> 5) == 2 && (1UL << (24) & 0)) || ((((24)) >> 5) == 3 && (1UL << (24) & (0))) || ((((24)) >> 5) == 4 && (1UL << (24) & 0)) || ((((24)) >> 5) == 5 && (1UL << (24) & 0)) || ((((24)) >> 5) == 6 && (1UL << (24) & 0)) || ((((24)) >> 5) == 7 && (1UL << (24) & 0)) || ((((24)) >> 5) == 8 && (1UL << (24) & 0)) || ((((24)) >> 5) == 9 && (1UL << (24) & 0))) ? 1 : (__builtin_constant_p(((24))) ? constant_test_bit(((24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))) : __builtin_constant_p((24)) ? __static_cpu_has((24)) : (__builtin_constant_p((24)) && (((((24)) >> 5) == 0 && (1UL << (24) & ((1 << ((0) & 31)) | 0 | (1 << ((6) & 31)) | (1 << ((8) & 31)) | 0 | (1 << ((15) & 31)) | 0))) || ((((24)) >> 5) == 1 && (1UL << (24) & (0))) || ((((24)) >> 5) == 2 && (1UL << (24) & 0)) || ((((24)) >> 5) == 3 && (1UL << (24) & (0))) || ((((24)) >> 5) == 4 && (1UL << (24) & 0)) || ((((24)) >> 5) == 5 && (1UL << (24) & 0)) || ((((24)) >> 5) == 6 && (1UL << (24) & 0)) || ((((24)) >> 5) == 7 && (1UL << (24) & 0)) || ((((24)) >> 5) == 8 && (1UL << (24) & 0)) || ((((24)) >> 5) == 9 && (1UL << (24) & 0))) ? 1 : (__builtin_constant_p(((24))) ? constant_test_bit(((24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))));
 172 }
 173
 174 static inline __attribute__ ((always_inline))
 175 void fpu_save_init(struct fpu *fpu)
 176 {
 177         if (__builtin_constant_p(use_xsave())? !!use_xsave() : !!use_xsave()) {
 178                 if (__builtin_constant_p
 179                     (((!(fpu->state->xsave.xsave_hdr.
 180                          xstate_bv & 0x1)))) ? ! !((!(fpu->state->xsave.
 181                                                       xsave_hdr.
 182                                                       xstate_bv & 0x1))) : ( {
 183                                                                             ! !((!(fpu->state->xsave.xsave_hdr.xstate_bv & 0x1)));}
 184                     ))
 185                         return;
 186         } else if (!!use_fxsr()) {
 187         }
 188         if (__builtin_constant_p
 189             ((((__builtin_constant_p(fpu->state->fxsave.swd & (1 << 7)) ?
 190                 ! !(fpu->state->fxsave.swd & (1 << 7)) : ( {
 191                                                           __builtin_expect(! !
 192                                                                            (fpu->
 193                                                                             state->
 194                                                                             fxsave.
 195                                                                             swd
 196                                                                             & (1
 197                                                                                <<
 198                                                                                7)),
 199                                                                            1);}
 200                 )))))?
 201             ! !(((__builtin_constant_p(fpu->state->fxsave.swd & (1 << 7)) ?
 202                   ! !(fpu->state->fxsave.swd & (1 << 7)) : ( {
 203                                                             __builtin_expect(! !
 204                                                                              (fpu->
 205                                                                               state->
 206                                                                               fxsave.
 207                                                                               swd
 208                                                                               &
 209                                                                               (1
 210                                                                                <<
 211                                                                                7)),
 212                                                                              1);}
 213       )))):       ( {
 214                    int ______r;
 215                    ______r =
 216                    ! !(((__builtin_constant_p(fpu->state->fxsave.swd & (1 << 7))
 217                          ? ! !(fpu->state->fxsave.swd & (1 << 7)) : ( {
 218                                                                      ______r =
 219                                                                      __builtin_expect
 220                                                                      (! !
 221                                                                       (fpu->
 222                                                                        state->
 223                                                                        fxsave.
 224                                                                        swd & (1
 225                                                                               <<
 226                                                                               7)),
 227                                                                       1);}
 228                          ))));}
 229             ))
 230                 asm volatile ("fnclex");
 231 }
 232
 233 static
 234 __attribute__ ((always_inline))
 235 void __save_init_fpu(void)
 236 {
 237         fpu_save_init(0);
 238 }
 239
 240 static __attribute__ ((always_inline))
 241 void kernel_fpu_begin(void)
 242 {
 243         __save_init_fpu();
 244 }
 245
 246 static void aes_decrypt(struct crypto_tfm *tfm)
 247 {
 248         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 249         if (!!!irq_fpu_usable())
 250                 crypto_aes_decrypt_x86(ctx);
 251         else
 252                 kernel_fpu_begin();
 253 }
 254
 255 static struct crypto_alg aesni_alg = {
 256 .cra_list = {
 257                                                           &aesni_alg.cra_list,
 258                                                           &aesni_alg.cra_list}
 259         ,.cra_u = {
 260                    .cipher = {
 261 .cia_decrypt = aes_decrypt}
 262                    }
 263 };
 264
 265 static struct crypto_alg blk_ecb_alg = {
 266 .cra_list = {
 267                                           &blk_ecb_alg.cra_list,
 268                                           &blk_ecb_alg.cra_list}
 269 };
 270
 271 static struct crypto_alg ablk_ecb_alg = {
 272 .cra_list = {
 273                                                                                 &
 274                                                                                 ablk_ecb_alg.
 275                                                                                  cra_list,
 276                                                                                 &
 277                                                                                 ablk_ecb_alg.
 278                                                                                  cra_list}
 279 };
 280
 281 static struct crypto_alg ablk_cbc_alg = {
 282 .cra_list = {
 283                                                                                 &
 284                                                                                 ablk_cbc_alg.
 285                                                                                  cra_list,
 286                                                                                 &
 287                                                                                 ablk_cbc_alg.
 288                                                                                  cra_list}
 289 };
 290
 291 static struct crypto_alg ablk_pcbc_alg = {
 292 .cra_list = {
 293                                                                                 &
 294                                                                                 ablk_pcbc_alg.
 295                                                                                  cra_list,
 296                                                                                 &
 297                                                                                 ablk_pcbc_alg.
 298                                                                                  cra_list}
 299 };
 300
 301 int
 302 aesni_init(void)
 303 {
 304         if (!!crypto_register_alg(&aesni_alg))
 305                 crypto_unregister_alg(&ablk_ecb_alg);
 306 }