/* -*- c -*- */

/*
 * This file is for the definitions of simd vectorized operations.
 *
 * Currently contains sse2 functions that are built on amd64, x32 or
 * non-generic builds (CFLAGS=-march=...)
 * In future it may contain other instruction sets like AVX or NEON detected
 * at runtime in which case it needs to be included indirectly via a file
 * compiled with special options (or use gcc target attributes) so the binary
 * stays portable.
 */


#ifndef __NPY_SIMD_INC
#define __NPY_SIMD_INC

#include "lowlevel_strided_loops.h"
#include "numpy/npy_common.h"
/* for NO_FLOATING_POINT_SUPPORT */
#include "numpy/ufuncobject.h"
#ifdef NPY_HAVE_SSE2_INTRINSICS
#include <emmintrin.h>
#endif
#include <assert.h>
#include <stdlib.h>
#include <string.h> /* for memcpy */

int PyUFunc_getfperr(void);
void PyUFunc_clearfperr(void);

/*
 * stride is equal to element size and input and destination are equal or
 * don't overlap within one register
 */
#define IS_BLOCKABLE_UNARY(esize, vsize) \
    (steps[0] == (esize) && steps[0] == steps[1] && \
     (npy_is_aligned(args[0], esize) && npy_is_aligned(args[1], esize)) && \
     ((abs(args[1] - args[0]) >= (vsize)) || ((abs(args[1] - args[0]) == 0))))

#define IS_BLOCKABLE_REDUCE(esize, vsize) \
    (steps[1] == (esize) && abs(args[1] - args[0]) >= (vsize) && \
     npy_is_aligned(args[1], (esize)) && \
     npy_is_aligned(args[0], (esize)))

#define IS_BLOCKABLE_BINARY(esize, vsize) \
    (steps[0] == steps[1] && steps[1] == steps[2] && steps[2] == (esize) && \
     npy_is_aligned(args[2], (esize)) && npy_is_aligned(args[1], (esize)) && \
     npy_is_aligned(args[0], (esize)) && \
     (abs(args[2] - args[0]) >= (vsize) || abs(args[2] - args[0]) == 0) && \
     (abs(args[2] - args[1]) >= (vsize) || abs(args[2] - args[1]) >= 0))

#define IS_BLOCKABLE_BINARY_SCALAR1(esize, vsize) \
    (steps[0] == 0 && steps[1] == steps[2] && steps[2] == (esize) && \
     npy_is_aligned(args[2], (esize)) && npy_is_aligned(args[1], (esize)) && \
     ((abs(args[2] - args[1]) >= (vsize)) || (abs(args[2] - args[1]) == 0)) && \
     abs(args[2] - args[0]) >= (esize))

#define IS_BLOCKABLE_BINARY_SCALAR2(esize, vsize) \
    (steps[1] == 0 && steps[0] == steps[2] && steps[2] == (esize) && \
     npy_is_aligned(args[2], (esize)) && npy_is_aligned(args[0], (esize)) && \
     ((abs(args[2] - args[0]) >= (vsize)) || (abs(args[2] - args[0]) == 0)) && \
     abs(args[2] - args[1]) >= (esize))

#define IS_BLOCKABLE_BINARY_BOOL(esize, vsize) \
    (steps[0] == (esize) && steps[0] == steps[1] && steps[2] == (1) && \
     npy_is_aligned(args[1], (esize)) && \
     npy_is_aligned(args[0], (esize)))

#define IS_BLOCKABLE_BINARY_SCALAR1_BOOL(esize, vsize) \
    (steps[0] == 0 && steps[1] == (esize) && steps[2] == (1) && \
     npy_is_aligned(args[1], (esize)))

#define IS_BLOCKABLE_BINARY_SCALAR2_BOOL(esize, vsize) \
    (steps[0] == (esize) && steps[1] == 0 && steps[2] == (1) && \
     npy_is_aligned(args[0], (esize)))

/* align var to alignment */
#define LOOP_BLOCK_ALIGN_VAR(var, type, alignment)\
    npy_intp i, peel = npy_aligned_block_offset(var, sizeof(type),\
                                                alignment, n);\
    for(i = 0; i < peel; i++)

#define LOOP_BLOCKED(type, vsize)\
    for(; i < npy_blocked_end(peel, sizeof(type), vsize, n);\
            i += (vsize / sizeof(type)))

#define LOOP_BLOCKED_END\
    for (; i < n; i++)

/* fanout two bits to two bytes */
static const npy_int16 fanout_2[] = {
    0x0000,
    0x0001,
    0x0100,
    0x0101,
};

/* fanout four bits to four bytes */
static const npy_int32 fanout_4[] = {
    0x00000000,
    0x00000001,
    0x00000100,
    0x00000101,
    0x00010000,
    0x00010001,
    0x00010100,
    0x00010101,
    0x01000000,
    0x01000001,
    0x01000100,
    0x01000101,
    0x01010000,
    0x01010001,
    0x01010100,
    0x01010101
};


/*
 * Dispatcher functions
 * decide whether the operation can be vectorized and run it
 * if it was run returns true and false if nothing was done
 */

/*
 *****************************************************************************
 **                           FLOAT DISPATCHERS
 *****************************************************************************
 */

/**begin repeat
 * Float types
 *  #type = npy_float, npy_double, npy_longdouble#
 *  #TYPE = FLOAT, DOUBLE, LONGDOUBLE#
 *  #vector = 1, 1, 0#
 */

/**begin repeat1
 * #func = sqrt, absolute, minimum, maximum#
 * #check = IS_BLOCKABLE_UNARY, IS_BLOCKABLE_UNARY, IS_BLOCKABLE_REDUCE, IS_BLOCKABLE_REDUCE#
 * #name = unary, unary, unary_reduce, unary_reduce#
 */

#if @vector@ && defined NPY_HAVE_SSE2_INTRINSICS

/* prototypes */
static void
sse2_@func@_@TYPE@(@type@ *, @type@ *, const npy_intp n);

#endif

static NPY_INLINE int
run_@name@_simd_@func@_@TYPE@(char **args, npy_intp *dimensions, npy_intp *steps)
{
#if @vector@ && defined NPY_HAVE_SSE2_INTRINSICS
    if (@check@(sizeof(@type@), 16)) {
        sse2_@func@_@TYPE@((@type@*)args[1], (@type@*)args[0], dimensions[0]);
        return 1;
    }
#endif
    return 0;
}

/**end repeat1**/

/**begin repeat1
 * Arithmetic
 * # kind = add, subtract, multiply, divide#
 */

#if @vector@ && defined NPY_HAVE_SSE2_INTRINSICS

/* prototypes */
static void
sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2,
                          npy_intp n);
static void
sse2_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2,
                                  npy_intp n);
static void
sse2_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2,
                                  npy_intp n);

#endif

static NPY_INLINE int
run_binary_simd_@kind@_@TYPE@(char **args, npy_intp *dimensions, npy_intp *steps)
{
#if @vector@ && defined NPY_HAVE_SSE2_INTRINSICS
    @type@ * ip1 = (@type@ *)args[0];
    @type@ * ip2 = (@type@ *)args[1];
    @type@ * op = (@type@ *)args[2];
    npy_intp n = dimensions[0];
    /* argument one scalar */
    if (IS_BLOCKABLE_BINARY_SCALAR1(sizeof(@type@), 16)) {
        sse2_binary_scalar1_@kind@_@TYPE@(op, ip1, ip2, n);
        return 1;
    }
    /* argument two scalar */
    else if (IS_BLOCKABLE_BINARY_SCALAR2(sizeof(@type@), 16)) {
        sse2_binary_scalar2_@kind@_@TYPE@(op, ip1, ip2, n);
        return 1;
    }
    else if (IS_BLOCKABLE_BINARY(sizeof(@type@), 16)) {
        sse2_binary_@kind@_@TYPE@(op, ip1, ip2, n);
        return 1;
    }
#endif
    return 0;
}

/**end repeat1**/

/**begin repeat1
 * #kind = equal, not_equal, less, less_equal, greater, greater_equal,
 *         logical_and, logical_or#
 * #simd = 1, 1, 1, 1, 1, 1, 0, 0#
 */

#if @vector@ && @simd@ && defined NPY_HAVE_SSE2_INTRINSICS

/* prototypes */
static void
sse2_binary_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2,
                          npy_intp n);
static void
sse2_binary_scalar1_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2,
                                  npy_intp n);
static void
sse2_binary_scalar2_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2,
                                  npy_intp n);

#endif

static NPY_INLINE int
run_binary_simd_@kind@_@TYPE@(char **args, npy_intp *dimensions, npy_intp *steps)
{
#if @vector@ && @simd@ && defined NPY_HAVE_SSE2_INTRINSICS
    @type@ * ip1 = (@type@ *)args[0];
    @type@ * ip2 = (@type@ *)args[1];
    npy_bool * op = (npy_bool *)args[2];
    npy_intp n = dimensions[0];
    /* argument one scalar */
    if (IS_BLOCKABLE_BINARY_SCALAR1_BOOL(sizeof(@type@), 16)) {
        sse2_binary_scalar1_@kind@_@TYPE@(op, ip1, ip2, n);
        return 1;
    }
    /* argument two scalar */
    else if (IS_BLOCKABLE_BINARY_SCALAR2_BOOL(sizeof(@type@), 16)) {
        sse2_binary_scalar2_@kind@_@TYPE@(op, ip1, ip2, n);
        return 1;
    }
    else if (IS_BLOCKABLE_BINARY_BOOL(sizeof(@type@), 16)) {
        sse2_binary_@kind@_@TYPE@(op, ip1, ip2, n);
        return 1;
    }
#endif
    return 0;
}

/**end repeat1**/

/**end repeat**/

/*
 *****************************************************************************
 **                           BOOL DISPATCHERS
 *****************************************************************************
 */

/**begin repeat
 * # kind = logical_or, logical_and#
 */

static void
sse2_binary_@kind@_BOOL(npy_bool * op, npy_bool * ip1, npy_bool * ip2,
                        npy_intp n);

static NPY_INLINE int
run_binary_simd_@kind@_BOOL(char **args, npy_intp *dimensions, npy_intp *steps)
{
#if defined NPY_HAVE_SSE2_INTRINSICS
    if (sizeof(npy_bool) == 1 && IS_BLOCKABLE_BINARY(sizeof(npy_bool), 16)) {
        sse2_binary_@kind@_BOOL((npy_bool*)args[2], (npy_bool*)args[0],
                               (npy_bool*)args[1], dimensions[0]);
        return 1;
    }
#endif
    return 0;
}


static void
sse2_reduce_@kind@_BOOL(npy_bool * op, npy_bool * ip, npy_intp n);

static NPY_INLINE int
run_reduce_simd_@kind@_BOOL(char **args, npy_intp *dimensions, npy_intp *steps)
{
#if defined NPY_HAVE_SSE2_INTRINSICS
    if (sizeof(npy_bool) == 1 && IS_BLOCKABLE_REDUCE(sizeof(npy_bool), 16)) {
        sse2_reduce_@kind@_BOOL((npy_bool*)args[0], (npy_bool*)args[1],
                                dimensions[0]);
        return 1;
    }
#endif
    return 0;
}

/**end repeat**/

/**begin repeat
 * # kind = absolute, logical_not#
 */

static void
sse2_@kind@_BOOL(npy_bool *, npy_bool *, const npy_intp n);

static NPY_INLINE int
run_unary_simd_@kind@_BOOL(char **args, npy_intp *dimensions, npy_intp *steps)
{
#if defined NPY_HAVE_SSE2_INTRINSICS
    if (sizeof(npy_bool) == 1 && IS_BLOCKABLE_UNARY(sizeof(npy_bool), 16)) {
        sse2_@kind@_BOOL((npy_bool*)args[1], (npy_bool*)args[0], dimensions[0]);
        return 1;
    }
#endif
    return 0;
}

/**end repeat**/

#ifdef NPY_HAVE_SSE2_INTRINSICS

/*
 * Vectorized operations
 */
/*
 *****************************************************************************
 **                           FLOAT LOOPS
 *****************************************************************************
 */

/**begin repeat
* horizontal reductions on a vector
* # VOP = min, max#
*/

static NPY_INLINE npy_float sse2_horizontal_@VOP@___m128(__m128 v)
{
    npy_float r;
    __m128 tmp = _mm_movehl_ps(v, v);                   /* c     d     ... */
    __m128 m = _mm_@VOP@_ps(v, tmp);                    /* m(ac) m(bd) ... */
    tmp = _mm_shuffle_ps(m, m, _MM_SHUFFLE(1, 1, 1, 1));/* m(bd) m(bd) ... */
    _mm_store_ss(&r, _mm_@VOP@_ps(tmp, m));             /* m(acbd) ... */
    return r;
}

static NPY_INLINE npy_double sse2_horizontal_@VOP@___m128d(__m128d v)
{
    npy_double r;
    __m128d tmp = _mm_unpackhi_pd(v, v);    /* b     b */
    _mm_store_sd(&r, _mm_@VOP@_pd(tmp, v)); /* m(ab) m(bb) */
    return r;
}

/**end repeat**/

/**begin repeat
 *  #type = npy_float, npy_double#
 *  #TYPE = FLOAT, DOUBLE#
 *  #scalarf = npy_sqrtf, npy_sqrt#
 *  #c = f, #
 *  #vtype = __m128, __m128d#
 *  #vpre = _mm, _mm#
 *  #vsuf = ps, pd#
 *  #vsufs = ss, sd#
 *  #nan = NPY_NANF, NPY_NAN#
 *  #mtype = npy_int32, npy_int16#
 *  #fanout = fanout_4, fanout_2#
 */


/**begin repeat1
* Arithmetic
* # kind = add, subtract, multiply, divide#
* # OP = +, -, *, /#
* # VOP = add, sub, mul, div#
*/

static void
sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n)
{
    LOOP_BLOCK_ALIGN_VAR(op, @type@, 16)
        op[i] = ip1[i] @OP@ ip2[i];
    /* lots of specializations, to squeeze out max performance */
    if (npy_is_aligned(&ip1[i], 16) && npy_is_aligned(&ip2[i], 16)) {
        if (ip1 == ip2) {
            LOOP_BLOCKED(@type@, 16) {
                @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]);
                @vtype@ c = @vpre@_@VOP@_@vsuf@(a, a);
                @vpre@_store_@vsuf@(&op[i], c);
            }
        }
        else {
            LOOP_BLOCKED(@type@, 16) {
                @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]);
                @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]);
                @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b);
                @vpre@_store_@vsuf@(&op[i], c);
            }
        }
    }
    else if (npy_is_aligned(&ip1[i], 16)) {
        LOOP_BLOCKED(@type@, 16) {
            @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]);
            @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]);
            @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b);
            @vpre@_store_@vsuf@(&op[i], c);
        }
    }
    else if (npy_is_aligned(&ip2[i], 16)) {
        LOOP_BLOCKED(@type@, 16) {
            @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]);
            @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]);
            @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b);
            @vpre@_store_@vsuf@(&op[i], c);
        }
    }
    else {
        if (ip1 == ip2) {
            LOOP_BLOCKED(@type@, 16) {
                @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]);
                @vtype@ c = @vpre@_@VOP@_@vsuf@(a, a);
                @vpre@_store_@vsuf@(&op[i], c);
            }
        }
        else {
            LOOP_BLOCKED(@type@, 16) {
                @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]);
                @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]);
                @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b);
                @vpre@_store_@vsuf@(&op[i], c);
            }
        }
    }
    LOOP_BLOCKED_END {
        op[i] = ip1[i] @OP@ ip2[i];
    }
}


static void
sse2_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n)
{
    const @vtype@ a = @vpre@_set1_@vsuf@(ip1[0]);
    LOOP_BLOCK_ALIGN_VAR(op, @type@, 16)
        op[i] = ip1[0] @OP@ ip2[i];
    if (npy_is_aligned(&ip2[i], 16)) {
        LOOP_BLOCKED(@type@, 16) {
            @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]);
            @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b);
            @vpre@_store_@vsuf@(&op[i], c);
        }
    }
    else {
        LOOP_BLOCKED(@type@, 16) {
            @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]);
            @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b);
            @vpre@_store_@vsuf@(&op[i], c);
        }
    }
    LOOP_BLOCKED_END {
        op[i] = ip1[0] @OP@ ip2[i];
    }
}


static void
sse2_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n)
{
    const @vtype@ b = @vpre@_set1_@vsuf@(ip2[0]);
    LOOP_BLOCK_ALIGN_VAR(op, @type@, 16)
        op[i] = ip1[i] @OP@ ip2[0];
    if (npy_is_aligned(&ip1[i], 16)) {
        LOOP_BLOCKED(@type@, 16) {
            @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]);
            @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b);
            @vpre@_store_@vsuf@(&op[i], c);
        }
    }
    else {
        LOOP_BLOCKED(@type@, 16) {
            @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]);
            @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b);
            @vpre@_store_@vsuf@(&op[i], c);
        }
    }
    LOOP_BLOCKED_END {
        op[i] = ip1[i] @OP@ ip2[0];
    }
}

/**end repeat1**/

/**begin repeat1
 * #kind = equal, not_equal, less, less_equal, greater, greater_equal#
 * #OP = ==, !=, <, <=, >, >=#
 * #VOP = cmpeq, cmpneq, cmplt, cmple, cmpgt, cmpge#
*/

/* sets invalid fpu flag on QNaN for consistency with packed compare */
static NPY_INLINE int
sse2_ordered_cmp_@kind@_@TYPE@(const @type@ a, const @type@ b)
{
    @type@ tmp;
    @vtype@ v = @vpre@_@VOP@_@vsufs@(@vpre@_load_@vsufs@(&a),
                                     @vpre@_load_@vsufs@(&b));
    @vpre@_store_@vsufs@(&tmp, v);
    return sizeof(@type@) == 4 ?
        (*(npy_uint32 *)&tmp) & 1 : (*(npy_uint64 *)&tmp) & 1;
}

static void
sse2_binary_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2, npy_intp n)
{
    npy_bool * r;
    LOOP_BLOCK_ALIGN_VAR(ip1, @type@, 16) {
        op[i] = sse2_ordered_cmp_@kind@_@TYPE@(ip1[i], ip2[i]);
    }
    r = &op[i];
    LOOP_BLOCKED(@type@, 16) {
        @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]);
        @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]);
        @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b);
        @mtype@ ir = @fanout@[@vpre@_movemask_@vsuf@(c)];
        /* may be unaligned */
        memcpy(r, &ir, sizeof(ir));
        r += sizeof(ir);
    }
    LOOP_BLOCKED_END {
        op[i] = sse2_ordered_cmp_@kind@_@TYPE@(ip1[i], ip2[i]);
    }
}


static void
sse2_binary_scalar1_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2, npy_intp n)
{
    npy_bool * r;
    @vtype@ a = @vpre@_set1_@vsuf@(ip1[0]);
    LOOP_BLOCK_ALIGN_VAR(ip2, @type@, 16) {
        op[i] = sse2_ordered_cmp_@kind@_@TYPE@(ip1[0], ip2[i]);
    }
    r = &op[i];
    LOOP_BLOCKED(@type@, 16) {
        @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]);
        @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b);
        @mtype@ ir = @fanout@[@vpre@_movemask_@vsuf@(c)];
        /* may be unaligned */
        memcpy(r, &ir, sizeof(ir));
        r += sizeof(ir);
    }
    LOOP_BLOCKED_END {
        op[i] = sse2_ordered_cmp_@kind@_@TYPE@(ip1[0], ip2[i]);
    }
}


static void
sse2_binary_scalar2_@kind@_@TYPE@(npy_bool * op, @type@ * ip1, @type@ * ip2, npy_intp n)
{
    npy_bool * r;
    @vtype@ b = @vpre@_set1_@vsuf@(ip2[0]);
    LOOP_BLOCK_ALIGN_VAR(ip1, @type@, 16) {
        op[i] = sse2_ordered_cmp_@kind@_@TYPE@(ip1[i], ip2[0]);
    }
    r = &op[i];
    LOOP_BLOCKED(@type@, 16) {
        @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]);
        @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b);
        @mtype@ ir = @fanout@[@vpre@_movemask_@vsuf@(c)];
        /* may be unaligned */
        memcpy(r, &ir, sizeof(ir));
        r += sizeof(ir);
    }
    LOOP_BLOCKED_END {
        op[i] = sse2_ordered_cmp_@kind@_@TYPE@(ip1[i], ip2[0]);
    }
}
/**end repeat1**/

static void
sse2_sqrt_@TYPE@(@type@ * op, @type@ * ip, const npy_intp n)
{
    /* align output to 16 bytes */
    LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) {
        op[i] = @scalarf@(ip[i]);
    }
    assert(n < (16 / sizeof(@type@)) || npy_is_aligned(&op[i], 16));
    if (npy_is_aligned(&ip[i], 16)) {
        LOOP_BLOCKED(@type@, 16) {
            @vtype@ d = @vpre@_load_@vsuf@(&ip[i]);
            @vpre@_store_@vsuf@(&op[i], @vpre@_sqrt_@vsuf@(d));
        }
    }
    else {
        LOOP_BLOCKED(@type@, 16) {
            @vtype@ d = @vpre@_loadu_@vsuf@(&ip[i]);
            @vpre@_store_@vsuf@(&op[i], @vpre@_sqrt_@vsuf@(d));
        }
    }
    LOOP_BLOCKED_END {
        op[i] = @scalarf@(ip[i]);
    }
}


static void
sse2_absolute_@TYPE@(@type@ * op, @type@ * ip, const npy_intp n)
{
    /*
     * get 0x7FFFFFFF mask (everything but signbit set)
     * float & ~mask will remove the sign
     * this is equivalent to how the compiler implements fabs on amd64
     */
    const @vtype@ mask = @vpre@_set1_@vsuf@(-0.@c@);

    /* align output to 16 bytes */
    LOOP_BLOCK_ALIGN_VAR(op, @type@, 16) {
        const @type@ tmp = ip[i] > 0 ? ip[i]: -ip[i];
        /* add 0 to clear -0.0 */
        op[i] = tmp + 0;
    }
    assert(n < (16 / sizeof(@type@)) || npy_is_aligned(&op[i], 16));
    if (npy_is_aligned(&ip[i], 16)) {
        LOOP_BLOCKED(@type@, 16) {
            @vtype@ a = @vpre@_load_@vsuf@(&ip[i]);
            @vpre@_store_@vsuf@(&op[i], @vpre@_andnot_@vsuf@(mask, a));
        }
    }
    else {
        LOOP_BLOCKED(@type@, 16) {
            @vtype@ a = @vpre@_loadu_@vsuf@(&ip[i]);
            @vpre@_store_@vsuf@(&op[i], @vpre@_andnot_@vsuf@(mask, a));
        }
    }
    LOOP_BLOCKED_END {
        const @type@ tmp = ip[i] > 0 ? ip[i]: -ip[i];
        /* add 0 to clear -0.0 */
        op[i] = tmp + 0;
    }
}


/**begin repeat1
 * #kind = maximum, minimum#
 * #VOP = max, min#
 * #OP = >=, <=#
 **/
/* arguments swapped as unary reduce has the swapped compared to unary */
static void
sse2_@kind@_@TYPE@(@type@ * ip, @type@ * op, const npy_intp n)
{
    LOOP_BLOCK_ALIGN_VAR(ip, @type@, 16) {
        *op = (*op @OP@ ip[i] || npy_isnan(*op)) ? *op : ip[i];
    }
    assert(n < (16 / sizeof(@type@)) || npy_is_aligned(&ip[i], 16));
    if (i + 2 * 16 / sizeof(@type@) <= n) {
        /* load the first elements */
        @vtype@ c = @vpre@_load_@vsuf@((@type@*)&ip[i]);
#ifdef NO_FLOATING_POINT_SUPPORT
        @vtype@ cnan = @vpre@_cmpneq_@vsuf@(c, c);
#else
        /* minps/minpd will set invalid flag if nan is encountered */
        PyUFunc_clearfperr();
#endif
        i += 16 / sizeof(@type@);

        LOOP_BLOCKED(@type@, 16) {
            @vtype@ v = @vpre@_load_@vsuf@((@type@*)&ip[i]);
            c = @vpre@_@VOP@_@vsuf@(c, v);
#ifdef NO_FLOATING_POINT_SUPPORT
            /* check for nan, breaking the loop makes non nan case slow */
            cnan = @vpre@_or_@vsuf@(@vpre@_cmpneq_@vsuf@(v, v), cnan);
        }

        if (@vpre@_movemask_@vsuf@(cnan)) {
            *op = @nan@;
            return;
        }
#else
        }
#endif
        {
            @type@ tmp = sse2_horizontal_@VOP@_@vtype@(c);
            if (PyUFunc_getfperr() & UFUNC_FPE_INVALID)
                *op = @nan@;
            else
                *op  = (*op @OP@ tmp || npy_isnan(*op)) ? *op : tmp;
        }
    }
    LOOP_BLOCKED_END {
        *op  = (*op @OP@ ip[i] || npy_isnan(*op)) ? *op : ip[i];
    }
}
/**end repeat1**/

/**end repeat**/

/*
 *****************************************************************************
 **                           BOOL LOOPS
 *****************************************************************************
 */

/**begin repeat
 * # kind = logical_or, logical_and#
 * # and = 0, 1#
 * # op = ||, &&#
 * # sc = !=, ==#
 * # vpre = _mm*2#
 * # vsuf = si128*2#
 * # vtype = __m128i*2#
 * # type = npy_bool*2#
 * # vload = _mm_load_si128*2#
 * # vloadu = _mm_loadu_si128*2#
 * # vstore = _mm_store_si128*2#
 */

/*
 * convert any bit set to boolean true so vectorized and normal operations are
 * consistent, should not be required if bool is used correctly everywhere but
 * you never know
 */
#if !@and@
static NPY_INLINE @vtype@ byte_to_true(@vtype@ v)
{
    const @vtype@ zero = @vpre@_setzero_@vsuf@();
    const @vtype@ truemask = @vpre@_set1_epi8(1 == 1);
    /* get 0xFF for zeros */
    @vtype@ tmp = @vpre@_cmpeq_epi8(v, zero);
    /* filled with 0xFF/0x00, negate and mask to boolean true */
    return @vpre@_andnot_@vsuf@(tmp, truemask);
}
#endif

static void
sse2_binary_@kind@_BOOL(npy_bool * op, npy_bool * ip1, npy_bool * ip2, npy_intp n)
{
    LOOP_BLOCK_ALIGN_VAR(op, @type@, 16)
        op[i] = ip1[i] @op@ ip2[i];
    LOOP_BLOCKED(@type@, 16) {
        @vtype@ a = @vloadu@((@vtype@*)&ip1[i]);
        @vtype@ b = @vloadu@((@vtype@*)&ip2[i]);
#if @and@
        const @vtype@ zero = @vpre@_setzero_@vsuf@();
        /* get 0xFF for non zeros*/
        @vtype@ tmp = @vpre@_cmpeq_epi8(a, zero);
        /* andnot -> 0x00 for zeros xFF for non zeros, & with ip2 */
        tmp = @vpre@_andnot_@vsuf@(tmp, b);
#else
        @vtype@ tmp = @vpre@_or_@vsuf@(a, b);
#endif

        @vstore@((@vtype@*)&op[i], byte_to_true(tmp));
    }
    LOOP_BLOCKED_END {
        op[i] = (ip1[i] @op@ ip2[i]);
    }
}


static void
sse2_reduce_@kind@_BOOL(npy_bool * op, npy_bool * ip, const npy_intp n)
{
    const @vtype@ zero = @vpre@_setzero_@vsuf@();
    LOOP_BLOCK_ALIGN_VAR(ip, npy_bool, 16) {
        *op = *op @op@ ip[i];
        if (*op @sc@ 0) {
            return;
        }
    }
    /* unrolled once to replace a slow movmsk with a fast pmaxb */
    LOOP_BLOCKED(npy_bool, 32) {
        @vtype@ v = @vload@((@vtype@*)&ip[i]);
        @vtype@ v2 = @vload@((@vtype@*)&ip[i + 16]);
        v = @vpre@_cmpeq_epi8(v, zero);
        v2 = @vpre@_cmpeq_epi8(v2, zero);
#if @and@
        if ((@vpre@_movemask_epi8(@vpre@_max_epu8(v, v2)) != 0)) {
            *op = 0;
#else
        if ((@vpre@_movemask_epi8(@vpre@_min_epu8(v, v2)) != 0xFFFF)) {
            *op = 1;
#endif
            return;
        }
    }
    LOOP_BLOCKED_END {
        *op = *op @op@ ip[i];
        if (*op @sc@ 0) {
            return;
        }
    }
}

/**end repeat**/

/**begin repeat
 * # kind = absolute, logical_not#
 * # op = !=, ==#
 * # not = 0, 1#
 * # vpre = _mm*2#
 * # vsuf = si128*2#
 * # vtype = __m128i*2#
 * # type = npy_bool*2#
 * # vloadu = _mm_loadu_si128*2#
 * # vstore = _mm_store_si128*2#
 */

static void
sse2_@kind@_BOOL(@type@ * op, @type@ * ip, const npy_intp n)
{
    LOOP_BLOCK_ALIGN_VAR(op, @type@, 16)
        op[i] = (ip[i] @op@ 0);
    LOOP_BLOCKED(@type@, 16) {
        @vtype@ a = @vloadu@((@vtype@*)&ip[i]);
#if @not@
        const @vtype@ zero = @vpre@_setzero_@vsuf@();
        const @vtype@ truemask = @vpre@_set1_epi8(1 == 1);
        /* equivalent to byte_to_true but can skip the negation */
        a = @vpre@_cmpeq_epi8(a, zero);
        a = @vpre@_and_@vsuf@(a, truemask);
#else
        /* abs is kind of pointless but maybe its used for byte_to_true */
        a = byte_to_true(a);
#endif
        @vstore@((@vtype@*)&op[i], a);
    }
    LOOP_BLOCKED_END {
        op[i] = (ip[i] @op@ 0);
    }
}

/**end repeat**/

#endif /* NPY_HAVE_SSE2_INTRINSICS */

#endif
