6450 lines
157 KiB
C
6450 lines
157 KiB
C
/*
|
|
* Copyright © 2008 Rodrigo Kumpera
|
|
* Copyright © 2008 André Tupinambá
|
|
*
|
|
* Permission to use, copy, modify, distribute, and sell this software and its
|
|
* documentation for any purpose is hereby granted without fee, provided that
|
|
* the above copyright notice appear in all copies and that both that
|
|
* copyright notice and this permission notice appear in supporting
|
|
* documentation, and that the name of Red Hat not be used in advertising or
|
|
* publicity pertaining to distribution of the software without specific,
|
|
* written prior permission. Red Hat makes no representations about the
|
|
* suitability of this software for any purpose. It is provided "as is"
|
|
* without express or implied warranty.
|
|
*
|
|
* THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
|
|
* SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
|
* FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
|
* SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
|
|
* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
|
|
* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
|
|
* SOFTWARE.
|
|
*
|
|
* Author: Rodrigo Kumpera (kumpera@gmail.com)
|
|
* André Tupinambá (andrelrt@gmail.com)
|
|
*
|
|
* Based on work by Owen Taylor and Søren Sandmann
|
|
*/
|
|
#ifdef HAVE_CONFIG_H
|
|
#include <config.h>
|
|
#endif
|
|
|
|
#include <xmmintrin.h> /* for _mm_shuffle_pi16 and _MM_SHUFFLE */
|
|
#include <emmintrin.h> /* for SSE2 intrinsics */
|
|
#include "pixman-private.h"
|
|
#include "pixman-combine32.h"
|
|
#include "pixman-inlines.h"
|
|
|
|
static __m128i mask_0080;
|
|
static __m128i mask_00ff;
|
|
static __m128i mask_0101;
|
|
static __m128i mask_ffff;
|
|
static __m128i mask_ff000000;
|
|
static __m128i mask_alpha;
|
|
|
|
static __m128i mask_565_r;
|
|
static __m128i mask_565_g1, mask_565_g2;
|
|
static __m128i mask_565_b;
|
|
static __m128i mask_red;
|
|
static __m128i mask_green;
|
|
static __m128i mask_blue;
|
|
|
|
static __m128i mask_565_fix_rb;
|
|
static __m128i mask_565_fix_g;
|
|
|
|
static __m128i mask_565_rb;
|
|
static __m128i mask_565_pack_multiplier;
|
|
|
|
static force_inline __m128i
|
|
unpack_32_1x128 (uint32_t data)
|
|
{
|
|
return _mm_unpacklo_epi8 (_mm_cvtsi32_si128 (data), _mm_setzero_si128 ());
|
|
}
|
|
|
|
static force_inline void
|
|
unpack_128_2x128 (__m128i data, __m128i* data_lo, __m128i* data_hi)
|
|
{
|
|
*data_lo = _mm_unpacklo_epi8 (data, _mm_setzero_si128 ());
|
|
*data_hi = _mm_unpackhi_epi8 (data, _mm_setzero_si128 ());
|
|
}
|
|
|
|
static force_inline __m128i
|
|
unpack_565_to_8888 (__m128i lo)
|
|
{
|
|
__m128i r, g, b, rb, t;
|
|
|
|
r = _mm_and_si128 (_mm_slli_epi32 (lo, 8), mask_red);
|
|
g = _mm_and_si128 (_mm_slli_epi32 (lo, 5), mask_green);
|
|
b = _mm_and_si128 (_mm_slli_epi32 (lo, 3), mask_blue);
|
|
|
|
rb = _mm_or_si128 (r, b);
|
|
t = _mm_and_si128 (rb, mask_565_fix_rb);
|
|
t = _mm_srli_epi32 (t, 5);
|
|
rb = _mm_or_si128 (rb, t);
|
|
|
|
t = _mm_and_si128 (g, mask_565_fix_g);
|
|
t = _mm_srli_epi32 (t, 6);
|
|
g = _mm_or_si128 (g, t);
|
|
|
|
return _mm_or_si128 (rb, g);
|
|
}
|
|
|
|
static force_inline void
|
|
unpack_565_128_4x128 (__m128i data,
|
|
__m128i* data0,
|
|
__m128i* data1,
|
|
__m128i* data2,
|
|
__m128i* data3)
|
|
{
|
|
__m128i lo, hi;
|
|
|
|
lo = _mm_unpacklo_epi16 (data, _mm_setzero_si128 ());
|
|
hi = _mm_unpackhi_epi16 (data, _mm_setzero_si128 ());
|
|
|
|
lo = unpack_565_to_8888 (lo);
|
|
hi = unpack_565_to_8888 (hi);
|
|
|
|
unpack_128_2x128 (lo, data0, data1);
|
|
unpack_128_2x128 (hi, data2, data3);
|
|
}
|
|
|
|
static force_inline uint16_t
|
|
pack_565_32_16 (uint32_t pixel)
|
|
{
|
|
return (uint16_t) (((pixel >> 8) & 0xf800) |
|
|
((pixel >> 5) & 0x07e0) |
|
|
((pixel >> 3) & 0x001f));
|
|
}
|
|
|
|
static force_inline __m128i
|
|
pack_2x128_128 (__m128i lo, __m128i hi)
|
|
{
|
|
return _mm_packus_epi16 (lo, hi);
|
|
}
|
|
|
|
static force_inline __m128i
|
|
pack_565_2packedx128_128 (__m128i lo, __m128i hi)
|
|
{
|
|
__m128i rb0 = _mm_and_si128 (lo, mask_565_rb);
|
|
__m128i rb1 = _mm_and_si128 (hi, mask_565_rb);
|
|
|
|
__m128i t0 = _mm_madd_epi16 (rb0, mask_565_pack_multiplier);
|
|
__m128i t1 = _mm_madd_epi16 (rb1, mask_565_pack_multiplier);
|
|
|
|
__m128i g0 = _mm_and_si128 (lo, mask_green);
|
|
__m128i g1 = _mm_and_si128 (hi, mask_green);
|
|
|
|
t0 = _mm_or_si128 (t0, g0);
|
|
t1 = _mm_or_si128 (t1, g1);
|
|
|
|
/* Simulates _mm_packus_epi32 */
|
|
t0 = _mm_slli_epi32 (t0, 16 - 5);
|
|
t1 = _mm_slli_epi32 (t1, 16 - 5);
|
|
t0 = _mm_srai_epi32 (t0, 16);
|
|
t1 = _mm_srai_epi32 (t1, 16);
|
|
return _mm_packs_epi32 (t0, t1);
|
|
}
|
|
|
|
static force_inline __m128i
|
|
pack_565_2x128_128 (__m128i lo, __m128i hi)
|
|
{
|
|
__m128i data;
|
|
__m128i r, g1, g2, b;
|
|
|
|
data = pack_2x128_128 (lo, hi);
|
|
|
|
r = _mm_and_si128 (data, mask_565_r);
|
|
g1 = _mm_and_si128 (_mm_slli_epi32 (data, 3), mask_565_g1);
|
|
g2 = _mm_and_si128 (_mm_srli_epi32 (data, 5), mask_565_g2);
|
|
b = _mm_and_si128 (_mm_srli_epi32 (data, 3), mask_565_b);
|
|
|
|
return _mm_or_si128 (_mm_or_si128 (_mm_or_si128 (r, g1), g2), b);
|
|
}
|
|
|
|
static force_inline __m128i
|
|
pack_565_4x128_128 (__m128i* xmm0, __m128i* xmm1, __m128i* xmm2, __m128i* xmm3)
|
|
{
|
|
return _mm_packus_epi16 (pack_565_2x128_128 (*xmm0, *xmm1),
|
|
pack_565_2x128_128 (*xmm2, *xmm3));
|
|
}
|
|
|
|
static force_inline int
|
|
is_opaque (__m128i x)
|
|
{
|
|
__m128i ffs = _mm_cmpeq_epi8 (x, x);
|
|
|
|
return (_mm_movemask_epi8 (_mm_cmpeq_epi8 (x, ffs)) & 0x8888) == 0x8888;
|
|
}
|
|
|
|
static force_inline int
|
|
is_zero (__m128i x)
|
|
{
|
|
return _mm_movemask_epi8 (
|
|
_mm_cmpeq_epi8 (x, _mm_setzero_si128 ())) == 0xffff;
|
|
}
|
|
|
|
static force_inline int
|
|
is_transparent (__m128i x)
|
|
{
|
|
return (_mm_movemask_epi8 (
|
|
_mm_cmpeq_epi8 (x, _mm_setzero_si128 ())) & 0x8888) == 0x8888;
|
|
}
|
|
|
|
static force_inline __m128i
|
|
expand_pixel_32_1x128 (uint32_t data)
|
|
{
|
|
return _mm_shuffle_epi32 (unpack_32_1x128 (data), _MM_SHUFFLE (1, 0, 1, 0));
|
|
}
|
|
|
|
static force_inline __m128i
|
|
expand_alpha_1x128 (__m128i data)
|
|
{
|
|
return _mm_shufflehi_epi16 (_mm_shufflelo_epi16 (data,
|
|
_MM_SHUFFLE (3, 3, 3, 3)),
|
|
_MM_SHUFFLE (3, 3, 3, 3));
|
|
}
|
|
|
|
static force_inline void
|
|
expand_alpha_2x128 (__m128i data_lo,
|
|
__m128i data_hi,
|
|
__m128i* alpha_lo,
|
|
__m128i* alpha_hi)
|
|
{
|
|
__m128i lo, hi;
|
|
|
|
lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (3, 3, 3, 3));
|
|
hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (3, 3, 3, 3));
|
|
|
|
*alpha_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (3, 3, 3, 3));
|
|
*alpha_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (3, 3, 3, 3));
|
|
}
|
|
|
|
static force_inline void
|
|
expand_alpha_rev_2x128 (__m128i data_lo,
|
|
__m128i data_hi,
|
|
__m128i* alpha_lo,
|
|
__m128i* alpha_hi)
|
|
{
|
|
__m128i lo, hi;
|
|
|
|
lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (0, 0, 0, 0));
|
|
hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (0, 0, 0, 0));
|
|
*alpha_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (0, 0, 0, 0));
|
|
*alpha_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (0, 0, 0, 0));
|
|
}
|
|
|
|
static force_inline void
|
|
pix_multiply_2x128 (__m128i* data_lo,
|
|
__m128i* data_hi,
|
|
__m128i* alpha_lo,
|
|
__m128i* alpha_hi,
|
|
__m128i* ret_lo,
|
|
__m128i* ret_hi)
|
|
{
|
|
__m128i lo, hi;
|
|
|
|
lo = _mm_mullo_epi16 (*data_lo, *alpha_lo);
|
|
hi = _mm_mullo_epi16 (*data_hi, *alpha_hi);
|
|
lo = _mm_adds_epu16 (lo, mask_0080);
|
|
hi = _mm_adds_epu16 (hi, mask_0080);
|
|
*ret_lo = _mm_mulhi_epu16 (lo, mask_0101);
|
|
*ret_hi = _mm_mulhi_epu16 (hi, mask_0101);
|
|
}
|
|
|
|
static force_inline void
|
|
pix_add_multiply_2x128 (__m128i* src_lo,
|
|
__m128i* src_hi,
|
|
__m128i* alpha_dst_lo,
|
|
__m128i* alpha_dst_hi,
|
|
__m128i* dst_lo,
|
|
__m128i* dst_hi,
|
|
__m128i* alpha_src_lo,
|
|
__m128i* alpha_src_hi,
|
|
__m128i* ret_lo,
|
|
__m128i* ret_hi)
|
|
{
|
|
__m128i t1_lo, t1_hi;
|
|
__m128i t2_lo, t2_hi;
|
|
|
|
pix_multiply_2x128 (src_lo, src_hi, alpha_dst_lo, alpha_dst_hi, &t1_lo, &t1_hi);
|
|
pix_multiply_2x128 (dst_lo, dst_hi, alpha_src_lo, alpha_src_hi, &t2_lo, &t2_hi);
|
|
|
|
*ret_lo = _mm_adds_epu8 (t1_lo, t2_lo);
|
|
*ret_hi = _mm_adds_epu8 (t1_hi, t2_hi);
|
|
}
|
|
|
|
static force_inline void
|
|
negate_2x128 (__m128i data_lo,
|
|
__m128i data_hi,
|
|
__m128i* neg_lo,
|
|
__m128i* neg_hi)
|
|
{
|
|
*neg_lo = _mm_xor_si128 (data_lo, mask_00ff);
|
|
*neg_hi = _mm_xor_si128 (data_hi, mask_00ff);
|
|
}
|
|
|
|
static force_inline void
|
|
invert_colors_2x128 (__m128i data_lo,
|
|
__m128i data_hi,
|
|
__m128i* inv_lo,
|
|
__m128i* inv_hi)
|
|
{
|
|
__m128i lo, hi;
|
|
|
|
lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (3, 0, 1, 2));
|
|
hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (3, 0, 1, 2));
|
|
*inv_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (3, 0, 1, 2));
|
|
*inv_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (3, 0, 1, 2));
|
|
}
|
|
|
|
static force_inline void
|
|
over_2x128 (__m128i* src_lo,
|
|
__m128i* src_hi,
|
|
__m128i* alpha_lo,
|
|
__m128i* alpha_hi,
|
|
__m128i* dst_lo,
|
|
__m128i* dst_hi)
|
|
{
|
|
__m128i t1, t2;
|
|
|
|
negate_2x128 (*alpha_lo, *alpha_hi, &t1, &t2);
|
|
|
|
pix_multiply_2x128 (dst_lo, dst_hi, &t1, &t2, dst_lo, dst_hi);
|
|
|
|
*dst_lo = _mm_adds_epu8 (*src_lo, *dst_lo);
|
|
*dst_hi = _mm_adds_epu8 (*src_hi, *dst_hi);
|
|
}
|
|
|
|
static force_inline void
|
|
over_rev_non_pre_2x128 (__m128i src_lo,
|
|
__m128i src_hi,
|
|
__m128i* dst_lo,
|
|
__m128i* dst_hi)
|
|
{
|
|
__m128i lo, hi;
|
|
__m128i alpha_lo, alpha_hi;
|
|
|
|
expand_alpha_2x128 (src_lo, src_hi, &alpha_lo, &alpha_hi);
|
|
|
|
lo = _mm_or_si128 (alpha_lo, mask_alpha);
|
|
hi = _mm_or_si128 (alpha_hi, mask_alpha);
|
|
|
|
invert_colors_2x128 (src_lo, src_hi, &src_lo, &src_hi);
|
|
|
|
pix_multiply_2x128 (&src_lo, &src_hi, &lo, &hi, &lo, &hi);
|
|
|
|
over_2x128 (&lo, &hi, &alpha_lo, &alpha_hi, dst_lo, dst_hi);
|
|
}
|
|
|
|
static force_inline void
|
|
in_over_2x128 (__m128i* src_lo,
|
|
__m128i* src_hi,
|
|
__m128i* alpha_lo,
|
|
__m128i* alpha_hi,
|
|
__m128i* mask_lo,
|
|
__m128i* mask_hi,
|
|
__m128i* dst_lo,
|
|
__m128i* dst_hi)
|
|
{
|
|
__m128i s_lo, s_hi;
|
|
__m128i a_lo, a_hi;
|
|
|
|
pix_multiply_2x128 (src_lo, src_hi, mask_lo, mask_hi, &s_lo, &s_hi);
|
|
pix_multiply_2x128 (alpha_lo, alpha_hi, mask_lo, mask_hi, &a_lo, &a_hi);
|
|
|
|
over_2x128 (&s_lo, &s_hi, &a_lo, &a_hi, dst_lo, dst_hi);
|
|
}
|
|
|
|
/* load 4 pixels from a 16-byte boundary aligned address */
|
|
static force_inline __m128i
|
|
load_128_aligned (__m128i* src)
|
|
{
|
|
return _mm_load_si128 (src);
|
|
}
|
|
|
|
/* load 4 pixels from a unaligned address */
|
|
static force_inline __m128i
|
|
load_128_unaligned (const __m128i* src)
|
|
{
|
|
return _mm_loadu_si128 (src);
|
|
}
|
|
|
|
/* save 4 pixels using Write Combining memory on a 16-byte
|
|
* boundary aligned address
|
|
*/
|
|
static force_inline void
|
|
save_128_write_combining (__m128i* dst,
|
|
__m128i data)
|
|
{
|
|
_mm_stream_si128 (dst, data);
|
|
}
|
|
|
|
/* save 4 pixels on a 16-byte boundary aligned address */
|
|
static force_inline void
|
|
save_128_aligned (__m128i* dst,
|
|
__m128i data)
|
|
{
|
|
_mm_store_si128 (dst, data);
|
|
}
|
|
|
|
/* save 4 pixels on a unaligned address */
|
|
static force_inline void
|
|
save_128_unaligned (__m128i* dst,
|
|
__m128i data)
|
|
{
|
|
_mm_storeu_si128 (dst, data);
|
|
}
|
|
|
|
static force_inline __m128i
|
|
load_32_1x128 (uint32_t data)
|
|
{
|
|
return _mm_cvtsi32_si128 (data);
|
|
}
|
|
|
|
static force_inline __m128i
|
|
expand_alpha_rev_1x128 (__m128i data)
|
|
{
|
|
return _mm_shufflelo_epi16 (data, _MM_SHUFFLE (0, 0, 0, 0));
|
|
}
|
|
|
|
static force_inline __m128i
|
|
expand_pixel_8_1x128 (uint8_t data)
|
|
{
|
|
return _mm_shufflelo_epi16 (
|
|
unpack_32_1x128 ((uint32_t)data), _MM_SHUFFLE (0, 0, 0, 0));
|
|
}
|
|
|
|
static force_inline __m128i
|
|
pix_multiply_1x128 (__m128i data,
|
|
__m128i alpha)
|
|
{
|
|
return _mm_mulhi_epu16 (_mm_adds_epu16 (_mm_mullo_epi16 (data, alpha),
|
|
mask_0080),
|
|
mask_0101);
|
|
}
|
|
|
|
static force_inline __m128i
|
|
pix_add_multiply_1x128 (__m128i* src,
|
|
__m128i* alpha_dst,
|
|
__m128i* dst,
|
|
__m128i* alpha_src)
|
|
{
|
|
__m128i t1 = pix_multiply_1x128 (*src, *alpha_dst);
|
|
__m128i t2 = pix_multiply_1x128 (*dst, *alpha_src);
|
|
|
|
return _mm_adds_epu8 (t1, t2);
|
|
}
|
|
|
|
static force_inline __m128i
|
|
negate_1x128 (__m128i data)
|
|
{
|
|
return _mm_xor_si128 (data, mask_00ff);
|
|
}
|
|
|
|
static force_inline __m128i
|
|
invert_colors_1x128 (__m128i data)
|
|
{
|
|
return _mm_shufflelo_epi16 (data, _MM_SHUFFLE (3, 0, 1, 2));
|
|
}
|
|
|
|
static force_inline __m128i
|
|
over_1x128 (__m128i src, __m128i alpha, __m128i dst)
|
|
{
|
|
return _mm_adds_epu8 (src, pix_multiply_1x128 (dst, negate_1x128 (alpha)));
|
|
}
|
|
|
|
static force_inline __m128i
|
|
in_over_1x128 (__m128i* src, __m128i* alpha, __m128i* mask, __m128i* dst)
|
|
{
|
|
return over_1x128 (pix_multiply_1x128 (*src, *mask),
|
|
pix_multiply_1x128 (*alpha, *mask),
|
|
*dst);
|
|
}
|
|
|
|
static force_inline __m128i
|
|
over_rev_non_pre_1x128 (__m128i src, __m128i dst)
|
|
{
|
|
__m128i alpha = expand_alpha_1x128 (src);
|
|
|
|
return over_1x128 (pix_multiply_1x128 (invert_colors_1x128 (src),
|
|
_mm_or_si128 (alpha, mask_alpha)),
|
|
alpha,
|
|
dst);
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
pack_1x128_32 (__m128i data)
|
|
{
|
|
return _mm_cvtsi128_si32 (_mm_packus_epi16 (data, _mm_setzero_si128 ()));
|
|
}
|
|
|
|
static force_inline __m128i
|
|
expand565_16_1x128 (uint16_t pixel)
|
|
{
|
|
__m128i m = _mm_cvtsi32_si128 (pixel);
|
|
|
|
m = unpack_565_to_8888 (m);
|
|
|
|
return _mm_unpacklo_epi8 (m, _mm_setzero_si128 ());
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
core_combine_over_u_pixel_sse2 (uint32_t src, uint32_t dst)
|
|
{
|
|
uint8_t a;
|
|
__m128i xmms;
|
|
|
|
a = src >> 24;
|
|
|
|
if (a == 0xff)
|
|
{
|
|
return src;
|
|
}
|
|
else if (src)
|
|
{
|
|
xmms = unpack_32_1x128 (src);
|
|
return pack_1x128_32 (
|
|
over_1x128 (xmms, expand_alpha_1x128 (xmms),
|
|
unpack_32_1x128 (dst)));
|
|
}
|
|
|
|
return dst;
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
combine1 (const uint32_t *ps, const uint32_t *pm)
|
|
{
|
|
uint32_t s = *ps;
|
|
|
|
if (pm)
|
|
{
|
|
__m128i ms, mm;
|
|
|
|
mm = unpack_32_1x128 (*pm);
|
|
mm = expand_alpha_1x128 (mm);
|
|
|
|
ms = unpack_32_1x128 (s);
|
|
ms = pix_multiply_1x128 (ms, mm);
|
|
|
|
s = pack_1x128_32 (ms);
|
|
}
|
|
|
|
return s;
|
|
}
|
|
|
|
static force_inline __m128i
|
|
combine4 (const __m128i *ps, const __m128i *pm)
|
|
{
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_msk_lo, xmm_msk_hi;
|
|
__m128i s;
|
|
|
|
if (pm)
|
|
{
|
|
xmm_msk_lo = load_128_unaligned (pm);
|
|
|
|
if (is_transparent (xmm_msk_lo))
|
|
return _mm_setzero_si128 ();
|
|
}
|
|
|
|
s = load_128_unaligned (ps);
|
|
|
|
if (pm)
|
|
{
|
|
unpack_128_2x128 (s, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_msk_lo, &xmm_msk_lo, &xmm_msk_hi);
|
|
|
|
expand_alpha_2x128 (xmm_msk_lo, xmm_msk_hi, &xmm_msk_lo, &xmm_msk_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_msk_lo, &xmm_msk_hi,
|
|
&xmm_src_lo, &xmm_src_hi);
|
|
|
|
s = pack_2x128_128 (xmm_src_lo, xmm_src_hi);
|
|
}
|
|
|
|
return s;
|
|
}
|
|
|
|
static force_inline void
|
|
core_combine_over_u_sse2_mask (uint32_t * pd,
|
|
const uint32_t* ps,
|
|
const uint32_t* pm,
|
|
int w)
|
|
{
|
|
uint32_t s, d;
|
|
|
|
/* Align dst on a 16-byte boundary */
|
|
while (w && ((uintptr_t)pd & 15))
|
|
{
|
|
d = *pd;
|
|
s = combine1 (ps, pm);
|
|
|
|
if (s)
|
|
*pd = core_combine_over_u_pixel_sse2 (s, d);
|
|
pd++;
|
|
ps++;
|
|
pm++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
__m128i mask = load_128_unaligned ((__m128i *)pm);
|
|
|
|
if (!is_zero (mask))
|
|
{
|
|
__m128i src;
|
|
__m128i src_hi, src_lo;
|
|
__m128i mask_hi, mask_lo;
|
|
__m128i alpha_hi, alpha_lo;
|
|
|
|
src = load_128_unaligned ((__m128i *)ps);
|
|
|
|
if (is_opaque (_mm_and_si128 (src, mask)))
|
|
{
|
|
save_128_aligned ((__m128i *)pd, src);
|
|
}
|
|
else
|
|
{
|
|
__m128i dst = load_128_aligned ((__m128i *)pd);
|
|
__m128i dst_hi, dst_lo;
|
|
|
|
unpack_128_2x128 (mask, &mask_lo, &mask_hi);
|
|
unpack_128_2x128 (src, &src_lo, &src_hi);
|
|
|
|
expand_alpha_2x128 (mask_lo, mask_hi, &mask_lo, &mask_hi);
|
|
pix_multiply_2x128 (&src_lo, &src_hi,
|
|
&mask_lo, &mask_hi,
|
|
&src_lo, &src_hi);
|
|
|
|
unpack_128_2x128 (dst, &dst_lo, &dst_hi);
|
|
|
|
expand_alpha_2x128 (src_lo, src_hi,
|
|
&alpha_lo, &alpha_hi);
|
|
|
|
over_2x128 (&src_lo, &src_hi, &alpha_lo, &alpha_hi,
|
|
&dst_lo, &dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i *)pd,
|
|
pack_2x128_128 (dst_lo, dst_hi));
|
|
}
|
|
}
|
|
|
|
pm += 4;
|
|
ps += 4;
|
|
pd += 4;
|
|
w -= 4;
|
|
}
|
|
while (w)
|
|
{
|
|
d = *pd;
|
|
s = combine1 (ps, pm);
|
|
|
|
if (s)
|
|
*pd = core_combine_over_u_pixel_sse2 (s, d);
|
|
pd++;
|
|
ps++;
|
|
pm++;
|
|
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static force_inline void
|
|
core_combine_over_u_sse2_no_mask (uint32_t * pd,
|
|
const uint32_t* ps,
|
|
int w)
|
|
{
|
|
uint32_t s, d;
|
|
|
|
/* Align dst on a 16-byte boundary */
|
|
while (w && ((uintptr_t)pd & 15))
|
|
{
|
|
d = *pd;
|
|
s = *ps;
|
|
|
|
if (s)
|
|
*pd = core_combine_over_u_pixel_sse2 (s, d);
|
|
pd++;
|
|
ps++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
__m128i src;
|
|
__m128i src_hi, src_lo, dst_hi, dst_lo;
|
|
__m128i alpha_hi, alpha_lo;
|
|
|
|
src = load_128_unaligned ((__m128i *)ps);
|
|
|
|
if (!is_zero (src))
|
|
{
|
|
if (is_opaque (src))
|
|
{
|
|
save_128_aligned ((__m128i *)pd, src);
|
|
}
|
|
else
|
|
{
|
|
__m128i dst = load_128_aligned ((__m128i *)pd);
|
|
|
|
unpack_128_2x128 (src, &src_lo, &src_hi);
|
|
unpack_128_2x128 (dst, &dst_lo, &dst_hi);
|
|
|
|
expand_alpha_2x128 (src_lo, src_hi,
|
|
&alpha_lo, &alpha_hi);
|
|
over_2x128 (&src_lo, &src_hi, &alpha_lo, &alpha_hi,
|
|
&dst_lo, &dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i *)pd,
|
|
pack_2x128_128 (dst_lo, dst_hi));
|
|
}
|
|
}
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
w -= 4;
|
|
}
|
|
while (w)
|
|
{
|
|
d = *pd;
|
|
s = *ps;
|
|
|
|
if (s)
|
|
*pd = core_combine_over_u_pixel_sse2 (s, d);
|
|
pd++;
|
|
ps++;
|
|
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static force_inline void
|
|
sse2_combine_over_u (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
if (pm)
|
|
core_combine_over_u_sse2_mask (pd, ps, pm, w);
|
|
else
|
|
core_combine_over_u_sse2_no_mask (pd, ps, w);
|
|
}
|
|
|
|
static void
|
|
sse2_combine_over_reverse_u (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, d;
|
|
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_alpha_lo, xmm_alpha_hi;
|
|
|
|
/* Align dst on a 16-byte boundary */
|
|
while (w &&
|
|
((uintptr_t)pd & 15))
|
|
{
|
|
d = *pd;
|
|
s = combine1 (ps, pm);
|
|
|
|
*pd++ = core_combine_over_u_pixel_sse2 (d, s);
|
|
w--;
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
/* I'm loading unaligned because I'm not sure
|
|
* about the address alignment.
|
|
*/
|
|
xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
|
|
xmm_dst_hi = load_128_aligned ((__m128i*) pd);
|
|
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
|
|
over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_src_lo, &xmm_src_hi);
|
|
|
|
/* rebuid the 4 pixel data and save*/
|
|
save_128_aligned ((__m128i*)pd,
|
|
pack_2x128_128 (xmm_src_lo, xmm_src_hi));
|
|
|
|
w -= 4;
|
|
ps += 4;
|
|
pd += 4;
|
|
|
|
if (pm)
|
|
pm += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
d = *pd;
|
|
s = combine1 (ps, pm);
|
|
|
|
*pd++ = core_combine_over_u_pixel_sse2 (d, s);
|
|
ps++;
|
|
w--;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
core_combine_in_u_pixel_sse2 (uint32_t src, uint32_t dst)
|
|
{
|
|
uint32_t maska = src >> 24;
|
|
|
|
if (maska == 0)
|
|
{
|
|
return 0;
|
|
}
|
|
else if (maska != 0xff)
|
|
{
|
|
return pack_1x128_32 (
|
|
pix_multiply_1x128 (unpack_32_1x128 (dst),
|
|
expand_alpha_1x128 (unpack_32_1x128 (src))));
|
|
}
|
|
|
|
return dst;
|
|
}
|
|
|
|
static void
|
|
sse2_combine_in_u (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, d;
|
|
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
|
|
while (w && ((uintptr_t)pd & 15))
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_in_u_pixel_sse2 (d, s);
|
|
w--;
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*) pd);
|
|
xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*) pm);
|
|
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned ((__m128i*)pd,
|
|
pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
w -= 4;
|
|
if (pm)
|
|
pm += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_in_u_pixel_sse2 (d, s);
|
|
w--;
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_combine_in_reverse_u (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, d;
|
|
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
|
|
while (w && ((uintptr_t)pd & 15))
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_in_u_pixel_sse2 (s, d);
|
|
ps++;
|
|
w--;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*) pd);
|
|
xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*)pm);
|
|
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
w -= 4;
|
|
if (pm)
|
|
pm += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_in_u_pixel_sse2 (s, d);
|
|
w--;
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_combine_out_reverse_u (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
while (w && ((uintptr_t)pd & 15))
|
|
{
|
|
uint32_t s = combine1 (ps, pm);
|
|
uint32_t d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
unpack_32_1x128 (d), negate_1x128 (
|
|
expand_alpha_1x128 (unpack_32_1x128 (s)))));
|
|
|
|
if (pm)
|
|
pm++;
|
|
ps++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
|
|
xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
|
|
xmm_dst_hi = load_128_aligned ((__m128i*) pd);
|
|
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
negate_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
if (pm)
|
|
pm += 4;
|
|
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
uint32_t s = combine1 (ps, pm);
|
|
uint32_t d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
unpack_32_1x128 (d), negate_1x128 (
|
|
expand_alpha_1x128 (unpack_32_1x128 (s)))));
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_combine_out_u (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
while (w && ((uintptr_t)pd & 15))
|
|
{
|
|
uint32_t s = combine1 (ps, pm);
|
|
uint32_t d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
unpack_32_1x128 (s), negate_1x128 (
|
|
expand_alpha_1x128 (unpack_32_1x128 (d)))));
|
|
w--;
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
|
|
xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*)pm);
|
|
xmm_dst_hi = load_128_aligned ((__m128i*) pd);
|
|
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
negate_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
w -= 4;
|
|
if (pm)
|
|
pm += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
uint32_t s = combine1 (ps, pm);
|
|
uint32_t d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
unpack_32_1x128 (s), negate_1x128 (
|
|
expand_alpha_1x128 (unpack_32_1x128 (d)))));
|
|
w--;
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
core_combine_atop_u_pixel_sse2 (uint32_t src,
|
|
uint32_t dst)
|
|
{
|
|
__m128i s = unpack_32_1x128 (src);
|
|
__m128i d = unpack_32_1x128 (dst);
|
|
|
|
__m128i sa = negate_1x128 (expand_alpha_1x128 (s));
|
|
__m128i da = expand_alpha_1x128 (d);
|
|
|
|
return pack_1x128_32 (pix_add_multiply_1x128 (&s, &da, &d, &sa));
|
|
}
|
|
|
|
static void
|
|
sse2_combine_atop_u (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, d;
|
|
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
|
|
__m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
|
|
|
|
while (w && ((uintptr_t)pd & 15))
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_atop_u_pixel_sse2 (s, d);
|
|
w--;
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
|
|
xmm_dst_hi = load_128_aligned ((__m128i*) pd);
|
|
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_src_lo, &xmm_alpha_src_hi);
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
|
|
&xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
|
|
|
|
negate_2x128 (xmm_alpha_src_lo, xmm_alpha_src_hi,
|
|
&xmm_alpha_src_lo, &xmm_alpha_src_hi);
|
|
|
|
pix_add_multiply_2x128 (
|
|
&xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
w -= 4;
|
|
if (pm)
|
|
pm += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_atop_u_pixel_sse2 (s, d);
|
|
w--;
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
core_combine_reverse_atop_u_pixel_sse2 (uint32_t src,
|
|
uint32_t dst)
|
|
{
|
|
__m128i s = unpack_32_1x128 (src);
|
|
__m128i d = unpack_32_1x128 (dst);
|
|
|
|
__m128i sa = expand_alpha_1x128 (s);
|
|
__m128i da = negate_1x128 (expand_alpha_1x128 (d));
|
|
|
|
return pack_1x128_32 (pix_add_multiply_1x128 (&s, &da, &d, &sa));
|
|
}
|
|
|
|
static void
|
|
sse2_combine_atop_reverse_u (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, d;
|
|
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
|
|
__m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
|
|
|
|
while (w && ((uintptr_t)pd & 15))
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_reverse_atop_u_pixel_sse2 (s, d);
|
|
ps++;
|
|
w--;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
|
|
xmm_dst_hi = load_128_aligned ((__m128i*) pd);
|
|
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_src_lo, &xmm_alpha_src_hi);
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
|
|
&xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
|
|
|
|
negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
|
|
&xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
|
|
|
|
pix_add_multiply_2x128 (
|
|
&xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
w -= 4;
|
|
if (pm)
|
|
pm += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_reverse_atop_u_pixel_sse2 (s, d);
|
|
ps++;
|
|
w--;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
core_combine_xor_u_pixel_sse2 (uint32_t src,
|
|
uint32_t dst)
|
|
{
|
|
__m128i s = unpack_32_1x128 (src);
|
|
__m128i d = unpack_32_1x128 (dst);
|
|
|
|
__m128i neg_d = negate_1x128 (expand_alpha_1x128 (d));
|
|
__m128i neg_s = negate_1x128 (expand_alpha_1x128 (s));
|
|
|
|
return pack_1x128_32 (pix_add_multiply_1x128 (&s, &neg_d, &d, &neg_s));
|
|
}
|
|
|
|
static void
|
|
sse2_combine_xor_u (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * dst,
|
|
const uint32_t * src,
|
|
const uint32_t * mask,
|
|
int width)
|
|
{
|
|
int w = width;
|
|
uint32_t s, d;
|
|
uint32_t* pd = dst;
|
|
const uint32_t* ps = src;
|
|
const uint32_t* pm = mask;
|
|
|
|
__m128i xmm_src, xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
|
|
__m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
|
|
|
|
while (w && ((uintptr_t)pd & 15))
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_xor_u_pixel_sse2 (s, d);
|
|
w--;
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_src = combine4 ((__m128i*) ps, (__m128i*) pm);
|
|
xmm_dst = load_128_aligned ((__m128i*) pd);
|
|
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_src_lo, &xmm_alpha_src_hi);
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
|
|
&xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
|
|
|
|
negate_2x128 (xmm_alpha_src_lo, xmm_alpha_src_hi,
|
|
&xmm_alpha_src_lo, &xmm_alpha_src_hi);
|
|
negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
|
|
&xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
|
|
|
|
pix_add_multiply_2x128 (
|
|
&xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
w -= 4;
|
|
if (pm)
|
|
pm += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_xor_u_pixel_sse2 (s, d);
|
|
w--;
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
}
|
|
|
|
static force_inline void
|
|
sse2_combine_add_u (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * dst,
|
|
const uint32_t * src,
|
|
const uint32_t * mask,
|
|
int width)
|
|
{
|
|
int w = width;
|
|
uint32_t s, d;
|
|
uint32_t* pd = dst;
|
|
const uint32_t* ps = src;
|
|
const uint32_t* pm = mask;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
*pd++ = _mm_cvtsi128_si32 (
|
|
_mm_adds_epu8 (_mm_cvtsi32_si128 (s), _mm_cvtsi32_si128 (d)));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
__m128i s;
|
|
|
|
s = combine4 ((__m128i*)ps, (__m128i*)pm);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, _mm_adds_epu8 (s, load_128_aligned ((__m128i*)pd)));
|
|
|
|
pd += 4;
|
|
ps += 4;
|
|
if (pm)
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w--)
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
ps++;
|
|
*pd++ = _mm_cvtsi128_si32 (
|
|
_mm_adds_epu8 (_mm_cvtsi32_si128 (s), _mm_cvtsi32_si128 (d)));
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
core_combine_saturate_u_pixel_sse2 (uint32_t src,
|
|
uint32_t dst)
|
|
{
|
|
__m128i ms = unpack_32_1x128 (src);
|
|
__m128i md = unpack_32_1x128 (dst);
|
|
uint32_t sa = src >> 24;
|
|
uint32_t da = ~dst >> 24;
|
|
|
|
if (sa > da)
|
|
{
|
|
ms = pix_multiply_1x128 (
|
|
ms, expand_alpha_1x128 (unpack_32_1x128 (DIV_UN8 (da, sa) << 24)));
|
|
}
|
|
|
|
return pack_1x128_32 (_mm_adds_epu16 (md, ms));
|
|
}
|
|
|
|
static void
|
|
sse2_combine_saturate_u (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, d;
|
|
|
|
uint32_t pack_cmp;
|
|
__m128i xmm_src, xmm_dst;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
|
|
w--;
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i*)pd);
|
|
xmm_src = combine4 ((__m128i*)ps, (__m128i*)pm);
|
|
|
|
pack_cmp = _mm_movemask_epi8 (
|
|
_mm_cmpgt_epi32 (
|
|
_mm_srli_epi32 (xmm_src, 24),
|
|
_mm_srli_epi32 (_mm_xor_si128 (xmm_dst, mask_ff000000), 24)));
|
|
|
|
/* if some alpha src is grater than respective ~alpha dst */
|
|
if (pack_cmp)
|
|
{
|
|
s = combine1 (ps++, pm);
|
|
d = *pd;
|
|
*pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
|
|
if (pm)
|
|
pm++;
|
|
|
|
s = combine1 (ps++, pm);
|
|
d = *pd;
|
|
*pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
|
|
if (pm)
|
|
pm++;
|
|
|
|
s = combine1 (ps++, pm);
|
|
d = *pd;
|
|
*pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
|
|
if (pm)
|
|
pm++;
|
|
|
|
s = combine1 (ps++, pm);
|
|
d = *pd;
|
|
*pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
else
|
|
{
|
|
save_128_aligned ((__m128i*)pd, _mm_adds_epu8 (xmm_dst, xmm_src));
|
|
|
|
pd += 4;
|
|
ps += 4;
|
|
if (pm)
|
|
pm += 4;
|
|
}
|
|
|
|
w -= 4;
|
|
}
|
|
|
|
while (w--)
|
|
{
|
|
s = combine1 (ps, pm);
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
|
|
ps++;
|
|
if (pm)
|
|
pm++;
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_combine_src_ca (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, m;
|
|
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_mask_lo, xmm_mask_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (unpack_32_1x128 (s), unpack_32_1x128 (m)));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_src_hi = load_128_unaligned ((__m128i*)ps);
|
|
xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
|
|
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (unpack_32_1x128 (s), unpack_32_1x128 (m)));
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
core_combine_over_ca_pixel_sse2 (uint32_t src,
|
|
uint32_t mask,
|
|
uint32_t dst)
|
|
{
|
|
__m128i s = unpack_32_1x128 (src);
|
|
__m128i expAlpha = expand_alpha_1x128 (s);
|
|
__m128i unpk_mask = unpack_32_1x128 (mask);
|
|
__m128i unpk_dst = unpack_32_1x128 (dst);
|
|
|
|
return pack_1x128_32 (in_over_1x128 (&s, &expAlpha, &unpk_mask, &unpk_dst));
|
|
}
|
|
|
|
static void
|
|
sse2_combine_over_ca (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, m, d;
|
|
|
|
__m128i xmm_alpha_lo, xmm_alpha_hi;
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask_lo, xmm_mask_hi;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_over_ca_pixel_sse2 (s, m, d);
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*)pd);
|
|
xmm_src_hi = load_128_unaligned ((__m128i*)ps);
|
|
xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
|
|
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
|
|
in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_over_ca_pixel_sse2 (s, m, d);
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
core_combine_over_reverse_ca_pixel_sse2 (uint32_t src,
|
|
uint32_t mask,
|
|
uint32_t dst)
|
|
{
|
|
__m128i d = unpack_32_1x128 (dst);
|
|
|
|
return pack_1x128_32 (
|
|
over_1x128 (d, expand_alpha_1x128 (d),
|
|
pix_multiply_1x128 (unpack_32_1x128 (src),
|
|
unpack_32_1x128 (mask))));
|
|
}
|
|
|
|
static void
|
|
sse2_combine_over_reverse_ca (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, m, d;
|
|
|
|
__m128i xmm_alpha_lo, xmm_alpha_hi;
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask_lo, xmm_mask_hi;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_over_reverse_ca_pixel_sse2 (s, m, d);
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*)pd);
|
|
xmm_src_hi = load_128_unaligned ((__m128i*)ps);
|
|
xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
|
|
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_mask_lo, xmm_mask_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_over_reverse_ca_pixel_sse2 (s, m, d);
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_combine_in_ca (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, m, d;
|
|
|
|
__m128i xmm_alpha_lo, xmm_alpha_hi;
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask_lo, xmm_mask_hi;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
pix_multiply_1x128 (unpack_32_1x128 (s), unpack_32_1x128 (m)),
|
|
expand_alpha_1x128 (unpack_32_1x128 (d))));
|
|
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*)pd);
|
|
xmm_src_hi = load_128_unaligned ((__m128i*)ps);
|
|
xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
|
|
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
pix_multiply_1x128 (
|
|
unpack_32_1x128 (s), unpack_32_1x128 (m)),
|
|
expand_alpha_1x128 (unpack_32_1x128 (d))));
|
|
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_combine_in_reverse_ca (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, m, d;
|
|
|
|
__m128i xmm_alpha_lo, xmm_alpha_hi;
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask_lo, xmm_mask_hi;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
unpack_32_1x128 (d),
|
|
pix_multiply_1x128 (unpack_32_1x128 (m),
|
|
expand_alpha_1x128 (unpack_32_1x128 (s)))));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*)pd);
|
|
xmm_src_hi = load_128_unaligned ((__m128i*)ps);
|
|
xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
|
|
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
unpack_32_1x128 (d),
|
|
pix_multiply_1x128 (unpack_32_1x128 (m),
|
|
expand_alpha_1x128 (unpack_32_1x128 (s)))));
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_combine_out_ca (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, m, d;
|
|
|
|
__m128i xmm_alpha_lo, xmm_alpha_hi;
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask_lo, xmm_mask_hi;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
pix_multiply_1x128 (
|
|
unpack_32_1x128 (s), unpack_32_1x128 (m)),
|
|
negate_1x128 (expand_alpha_1x128 (unpack_32_1x128 (d)))));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*)pd);
|
|
xmm_src_hi = load_128_unaligned ((__m128i*)ps);
|
|
xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
|
|
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
negate_2x128 (xmm_alpha_lo, xmm_alpha_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
pix_multiply_1x128 (
|
|
unpack_32_1x128 (s), unpack_32_1x128 (m)),
|
|
negate_1x128 (expand_alpha_1x128 (unpack_32_1x128 (d)))));
|
|
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_combine_out_reverse_ca (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, m, d;
|
|
|
|
__m128i xmm_alpha_lo, xmm_alpha_hi;
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask_lo, xmm_mask_hi;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
unpack_32_1x128 (d),
|
|
negate_1x128 (pix_multiply_1x128 (
|
|
unpack_32_1x128 (m),
|
|
expand_alpha_1x128 (unpack_32_1x128 (s))))));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*)pd);
|
|
xmm_src_hi = load_128_unaligned ((__m128i*)ps);
|
|
xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
|
|
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
negate_2x128 (xmm_mask_lo, xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
unpack_32_1x128 (d),
|
|
negate_1x128 (pix_multiply_1x128 (
|
|
unpack_32_1x128 (m),
|
|
expand_alpha_1x128 (unpack_32_1x128 (s))))));
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
core_combine_atop_ca_pixel_sse2 (uint32_t src,
|
|
uint32_t mask,
|
|
uint32_t dst)
|
|
{
|
|
__m128i m = unpack_32_1x128 (mask);
|
|
__m128i s = unpack_32_1x128 (src);
|
|
__m128i d = unpack_32_1x128 (dst);
|
|
__m128i sa = expand_alpha_1x128 (s);
|
|
__m128i da = expand_alpha_1x128 (d);
|
|
|
|
s = pix_multiply_1x128 (s, m);
|
|
m = negate_1x128 (pix_multiply_1x128 (m, sa));
|
|
|
|
return pack_1x128_32 (pix_add_multiply_1x128 (&d, &m, &s, &da));
|
|
}
|
|
|
|
static void
|
|
sse2_combine_atop_ca (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, m, d;
|
|
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
|
|
__m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
|
|
__m128i xmm_mask_lo, xmm_mask_hi;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_atop_ca_pixel_sse2 (s, m, d);
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*)pd);
|
|
xmm_src_hi = load_128_unaligned ((__m128i*)ps);
|
|
xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
|
|
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_src_lo, &xmm_alpha_src_hi);
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
|
|
&xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_src_lo, &xmm_src_hi);
|
|
pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_alpha_src_lo, &xmm_alpha_src_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
negate_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
pix_add_multiply_2x128 (
|
|
&xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_atop_ca_pixel_sse2 (s, m, d);
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
core_combine_reverse_atop_ca_pixel_sse2 (uint32_t src,
|
|
uint32_t mask,
|
|
uint32_t dst)
|
|
{
|
|
__m128i m = unpack_32_1x128 (mask);
|
|
__m128i s = unpack_32_1x128 (src);
|
|
__m128i d = unpack_32_1x128 (dst);
|
|
|
|
__m128i da = negate_1x128 (expand_alpha_1x128 (d));
|
|
__m128i sa = expand_alpha_1x128 (s);
|
|
|
|
s = pix_multiply_1x128 (s, m);
|
|
m = pix_multiply_1x128 (m, sa);
|
|
|
|
return pack_1x128_32 (pix_add_multiply_1x128 (&d, &m, &s, &da));
|
|
}
|
|
|
|
static void
|
|
sse2_combine_atop_reverse_ca (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, m, d;
|
|
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
|
|
__m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
|
|
__m128i xmm_mask_lo, xmm_mask_hi;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_reverse_atop_ca_pixel_sse2 (s, m, d);
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*)pd);
|
|
xmm_src_hi = load_128_unaligned ((__m128i*)ps);
|
|
xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
|
|
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_src_lo, &xmm_alpha_src_hi);
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
|
|
&xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_src_lo, &xmm_src_hi);
|
|
pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_alpha_src_lo, &xmm_alpha_src_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
|
|
&xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
|
|
|
|
pix_add_multiply_2x128 (
|
|
&xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_reverse_atop_ca_pixel_sse2 (s, m, d);
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static force_inline uint32_t
|
|
core_combine_xor_ca_pixel_sse2 (uint32_t src,
|
|
uint32_t mask,
|
|
uint32_t dst)
|
|
{
|
|
__m128i a = unpack_32_1x128 (mask);
|
|
__m128i s = unpack_32_1x128 (src);
|
|
__m128i d = unpack_32_1x128 (dst);
|
|
|
|
__m128i alpha_dst = negate_1x128 (pix_multiply_1x128 (
|
|
a, expand_alpha_1x128 (s)));
|
|
__m128i dest = pix_multiply_1x128 (s, a);
|
|
__m128i alpha_src = negate_1x128 (expand_alpha_1x128 (d));
|
|
|
|
return pack_1x128_32 (pix_add_multiply_1x128 (&d,
|
|
&alpha_dst,
|
|
&dest,
|
|
&alpha_src));
|
|
}
|
|
|
|
static void
|
|
sse2_combine_xor_ca (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, m, d;
|
|
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
|
|
__m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
|
|
__m128i xmm_mask_lo, xmm_mask_hi;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_xor_ca_pixel_sse2 (s, m, d);
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*)pd);
|
|
xmm_src_hi = load_128_unaligned ((__m128i*)ps);
|
|
xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
|
|
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_src_lo, &xmm_alpha_src_hi);
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
|
|
&xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_src_lo, &xmm_src_hi);
|
|
pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_alpha_src_lo, &xmm_alpha_src_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
|
|
&xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
|
|
negate_2x128 (xmm_mask_lo, xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
pix_add_multiply_2x128 (
|
|
&xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = core_combine_xor_ca_pixel_sse2 (s, m, d);
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_combine_add_ca (pixman_implementation_t *imp,
|
|
pixman_op_t op,
|
|
uint32_t * pd,
|
|
const uint32_t * ps,
|
|
const uint32_t * pm,
|
|
int w)
|
|
{
|
|
uint32_t s, m, d;
|
|
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask_lo, xmm_mask_hi;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
_mm_adds_epu8 (pix_multiply_1x128 (unpack_32_1x128 (s),
|
|
unpack_32_1x128 (m)),
|
|
unpack_32_1x128 (d)));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_src_hi = load_128_unaligned ((__m128i*)ps);
|
|
xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
|
|
xmm_dst_hi = load_128_aligned ((__m128i*)pd);
|
|
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_src_lo, &xmm_src_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (
|
|
_mm_adds_epu8 (xmm_src_lo, xmm_dst_lo),
|
|
_mm_adds_epu8 (xmm_src_hi, xmm_dst_hi)));
|
|
|
|
ps += 4;
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *ps++;
|
|
m = *pm++;
|
|
d = *pd;
|
|
|
|
*pd++ = pack_1x128_32 (
|
|
_mm_adds_epu8 (pix_multiply_1x128 (unpack_32_1x128 (s),
|
|
unpack_32_1x128 (m)),
|
|
unpack_32_1x128 (d)));
|
|
w--;
|
|
}
|
|
}
|
|
|
|
static force_inline __m128i
|
|
create_mask_16_128 (uint16_t mask)
|
|
{
|
|
return _mm_set1_epi16 (mask);
|
|
}
|
|
|
|
/* Work around a code generation bug in Sun Studio 12. */
|
|
#if defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590)
|
|
# define create_mask_2x32_128(mask0, mask1) \
|
|
(_mm_set_epi32 ((mask0), (mask1), (mask0), (mask1)))
|
|
#else
|
|
static force_inline __m128i
|
|
create_mask_2x32_128 (uint32_t mask0,
|
|
uint32_t mask1)
|
|
{
|
|
return _mm_set_epi32 (mask0, mask1, mask0, mask1);
|
|
}
|
|
#endif
|
|
|
|
static void
|
|
sse2_composite_over_n_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t src;
|
|
uint32_t *dst_line, *dst, d;
|
|
int32_t w;
|
|
int dst_stride;
|
|
__m128i xmm_src, xmm_alpha;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
if (src == 0)
|
|
return;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
|
|
xmm_src = expand_pixel_32_1x128 (src);
|
|
xmm_alpha = expand_alpha_1x128 (xmm_src);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
|
|
dst_line += dst_stride;
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
d = *dst;
|
|
*dst++ = pack_1x128_32 (over_1x128 (xmm_src,
|
|
xmm_alpha,
|
|
unpack_32_1x128 (d)));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
over_2x128 (&xmm_src, &xmm_src,
|
|
&xmm_alpha, &xmm_alpha,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
/* rebuid the 4 pixel data and save*/
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
w -= 4;
|
|
dst += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
d = *dst;
|
|
*dst++ = pack_1x128_32 (over_1x128 (xmm_src,
|
|
xmm_alpha,
|
|
unpack_32_1x128 (d)));
|
|
w--;
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_n_0565 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t src;
|
|
uint16_t *dst_line, *dst, d;
|
|
int32_t w;
|
|
int dst_stride;
|
|
__m128i xmm_src, xmm_alpha;
|
|
__m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
if (src == 0)
|
|
return;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
|
|
|
|
xmm_src = expand_pixel_32_1x128 (src);
|
|
xmm_alpha = expand_alpha_1x128 (xmm_src);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
|
|
dst_line += dst_stride;
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
d = *dst;
|
|
|
|
*dst++ = pack_565_32_16 (
|
|
pack_1x128_32 (over_1x128 (xmm_src,
|
|
xmm_alpha,
|
|
expand565_16_1x128 (d))));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 8)
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
unpack_565_128_4x128 (xmm_dst,
|
|
&xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
|
|
|
|
over_2x128 (&xmm_src, &xmm_src,
|
|
&xmm_alpha, &xmm_alpha,
|
|
&xmm_dst0, &xmm_dst1);
|
|
over_2x128 (&xmm_src, &xmm_src,
|
|
&xmm_alpha, &xmm_alpha,
|
|
&xmm_dst2, &xmm_dst3);
|
|
|
|
xmm_dst = pack_565_4x128_128 (
|
|
&xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
|
|
|
|
save_128_aligned ((__m128i*)dst, xmm_dst);
|
|
|
|
dst += 8;
|
|
w -= 8;
|
|
}
|
|
|
|
while (w--)
|
|
{
|
|
d = *dst;
|
|
*dst++ = pack_565_32_16 (
|
|
pack_1x128_32 (over_1x128 (xmm_src, xmm_alpha,
|
|
expand565_16_1x128 (d))));
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t src;
|
|
uint32_t *dst_line, d;
|
|
uint32_t *mask_line, m;
|
|
uint32_t pack_cmp;
|
|
int dst_stride, mask_stride;
|
|
|
|
__m128i xmm_src;
|
|
__m128i xmm_dst;
|
|
__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
|
|
|
|
__m128i mmx_src, mmx_mask, mmx_dest;
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
if (src == 0)
|
|
return;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
|
|
|
|
xmm_src = _mm_unpacklo_epi8 (
|
|
create_mask_2x32_128 (src, src), _mm_setzero_si128 ());
|
|
mmx_src = xmm_src;
|
|
|
|
while (height--)
|
|
{
|
|
int w = width;
|
|
const uint32_t *pm = (uint32_t *)mask_line;
|
|
uint32_t *pd = (uint32_t *)dst_line;
|
|
|
|
dst_line += dst_stride;
|
|
mask_line += mask_stride;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
m = *pm++;
|
|
|
|
if (m)
|
|
{
|
|
d = *pd;
|
|
|
|
mmx_mask = unpack_32_1x128 (m);
|
|
mmx_dest = unpack_32_1x128 (d);
|
|
|
|
*pd = pack_1x128_32 (
|
|
_mm_adds_epu8 (pix_multiply_1x128 (mmx_mask, mmx_src),
|
|
mmx_dest));
|
|
}
|
|
|
|
pd++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_mask = load_128_unaligned ((__m128i*)pm);
|
|
|
|
pack_cmp =
|
|
_mm_movemask_epi8 (
|
|
_mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
|
|
|
|
/* if all bits in mask are zero, pack_cmp are equal to 0xffff */
|
|
if (pack_cmp != 0xffff)
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i*)pd);
|
|
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src, &xmm_src,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
xmm_mask_hi = pack_2x128_128 (xmm_mask_lo, xmm_mask_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, _mm_adds_epu8 (xmm_mask_hi, xmm_dst));
|
|
}
|
|
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
m = *pm++;
|
|
|
|
if (m)
|
|
{
|
|
d = *pd;
|
|
|
|
mmx_mask = unpack_32_1x128 (m);
|
|
mmx_dest = unpack_32_1x128 (d);
|
|
|
|
*pd = pack_1x128_32 (
|
|
_mm_adds_epu8 (pix_multiply_1x128 (mmx_mask, mmx_src),
|
|
mmx_dest));
|
|
}
|
|
|
|
pd++;
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t src;
|
|
uint32_t *dst_line, d;
|
|
uint32_t *mask_line, m;
|
|
uint32_t pack_cmp;
|
|
int dst_stride, mask_stride;
|
|
|
|
__m128i xmm_src, xmm_alpha;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
|
|
|
|
__m128i mmx_src, mmx_alpha, mmx_mask, mmx_dest;
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
if (src == 0)
|
|
return;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
|
|
|
|
xmm_src = _mm_unpacklo_epi8 (
|
|
create_mask_2x32_128 (src, src), _mm_setzero_si128 ());
|
|
xmm_alpha = expand_alpha_1x128 (xmm_src);
|
|
mmx_src = xmm_src;
|
|
mmx_alpha = xmm_alpha;
|
|
|
|
while (height--)
|
|
{
|
|
int w = width;
|
|
const uint32_t *pm = (uint32_t *)mask_line;
|
|
uint32_t *pd = (uint32_t *)dst_line;
|
|
|
|
dst_line += dst_stride;
|
|
mask_line += mask_stride;
|
|
|
|
while (w && (uintptr_t)pd & 15)
|
|
{
|
|
m = *pm++;
|
|
|
|
if (m)
|
|
{
|
|
d = *pd;
|
|
mmx_mask = unpack_32_1x128 (m);
|
|
mmx_dest = unpack_32_1x128 (d);
|
|
|
|
*pd = pack_1x128_32 (in_over_1x128 (&mmx_src,
|
|
&mmx_alpha,
|
|
&mmx_mask,
|
|
&mmx_dest));
|
|
}
|
|
|
|
pd++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_mask = load_128_unaligned ((__m128i*)pm);
|
|
|
|
pack_cmp =
|
|
_mm_movemask_epi8 (
|
|
_mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
|
|
|
|
/* if all bits in mask are zero, pack_cmp are equal to 0xffff */
|
|
if (pack_cmp != 0xffff)
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i*)pd);
|
|
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
in_over_2x128 (&xmm_src, &xmm_src,
|
|
&xmm_alpha, &xmm_alpha,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
|
|
pd += 4;
|
|
pm += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
m = *pm++;
|
|
|
|
if (m)
|
|
{
|
|
d = *pd;
|
|
mmx_mask = unpack_32_1x128 (m);
|
|
mmx_dest = unpack_32_1x128 (d);
|
|
|
|
*pd = pack_1x128_32 (
|
|
in_over_1x128 (&mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest));
|
|
}
|
|
|
|
pd++;
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_8888_n_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t *dst_line, *dst;
|
|
uint32_t *src_line, *src;
|
|
uint32_t mask;
|
|
int32_t w;
|
|
int dst_stride, src_stride;
|
|
|
|
__m128i xmm_mask;
|
|
__m128i xmm_src, xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_alpha_lo, xmm_alpha_hi;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
|
|
|
|
mask = _pixman_image_get_solid (imp, mask_image, PIXMAN_a8r8g8b8);
|
|
|
|
xmm_mask = create_mask_16_128 (mask >> 24);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
src = src_line;
|
|
src_line += src_stride;
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
uint32_t s = *src++;
|
|
|
|
if (s)
|
|
{
|
|
uint32_t d = *dst;
|
|
|
|
__m128i ms = unpack_32_1x128 (s);
|
|
__m128i alpha = expand_alpha_1x128 (ms);
|
|
__m128i dest = xmm_mask;
|
|
__m128i alpha_dst = unpack_32_1x128 (d);
|
|
|
|
*dst = pack_1x128_32 (
|
|
in_over_1x128 (&ms, &alpha, &dest, &alpha_dst));
|
|
}
|
|
dst++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_src = load_128_unaligned ((__m128i*)src);
|
|
|
|
if (!is_zero (xmm_src))
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
|
|
in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_mask, &xmm_mask,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
|
|
dst += 4;
|
|
src += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
uint32_t s = *src++;
|
|
|
|
if (s)
|
|
{
|
|
uint32_t d = *dst;
|
|
|
|
__m128i ms = unpack_32_1x128 (s);
|
|
__m128i alpha = expand_alpha_1x128 (ms);
|
|
__m128i mask = xmm_mask;
|
|
__m128i dest = unpack_32_1x128 (d);
|
|
|
|
*dst = pack_1x128_32 (
|
|
in_over_1x128 (&ms, &alpha, &mask, &dest));
|
|
}
|
|
|
|
dst++;
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_src_x888_0565 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint16_t *dst_line, *dst;
|
|
uint32_t *src_line, *src, s;
|
|
int dst_stride, src_stride;
|
|
int32_t w;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
src = src_line;
|
|
src_line += src_stride;
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
s = *src++;
|
|
*dst = convert_8888_to_0565 (s);
|
|
dst++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 8)
|
|
{
|
|
__m128i xmm_src0 = load_128_unaligned ((__m128i *)src + 0);
|
|
__m128i xmm_src1 = load_128_unaligned ((__m128i *)src + 1);
|
|
|
|
save_128_aligned ((__m128i*)dst, pack_565_2packedx128_128 (xmm_src0, xmm_src1));
|
|
|
|
w -= 8;
|
|
src += 8;
|
|
dst += 8;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *src++;
|
|
*dst = convert_8888_to_0565 (s);
|
|
dst++;
|
|
w--;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_composite_src_x888_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t *dst_line, *dst;
|
|
uint32_t *src_line, *src;
|
|
int32_t w;
|
|
int dst_stride, src_stride;
|
|
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
src = src_line;
|
|
src_line += src_stride;
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
*dst++ = *src++ | 0xff000000;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 16)
|
|
{
|
|
__m128i xmm_src1, xmm_src2, xmm_src3, xmm_src4;
|
|
|
|
xmm_src1 = load_128_unaligned ((__m128i*)src + 0);
|
|
xmm_src2 = load_128_unaligned ((__m128i*)src + 1);
|
|
xmm_src3 = load_128_unaligned ((__m128i*)src + 2);
|
|
xmm_src4 = load_128_unaligned ((__m128i*)src + 3);
|
|
|
|
save_128_aligned ((__m128i*)dst + 0, _mm_or_si128 (xmm_src1, mask_ff000000));
|
|
save_128_aligned ((__m128i*)dst + 1, _mm_or_si128 (xmm_src2, mask_ff000000));
|
|
save_128_aligned ((__m128i*)dst + 2, _mm_or_si128 (xmm_src3, mask_ff000000));
|
|
save_128_aligned ((__m128i*)dst + 3, _mm_or_si128 (xmm_src4, mask_ff000000));
|
|
|
|
dst += 16;
|
|
src += 16;
|
|
w -= 16;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
*dst++ = *src++ | 0xff000000;
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_x888_n_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t *dst_line, *dst;
|
|
uint32_t *src_line, *src;
|
|
uint32_t mask;
|
|
int dst_stride, src_stride;
|
|
int32_t w;
|
|
|
|
__m128i xmm_mask, xmm_alpha;
|
|
__m128i xmm_src, xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
|
|
|
|
mask = _pixman_image_get_solid (imp, mask_image, PIXMAN_a8r8g8b8);
|
|
|
|
xmm_mask = create_mask_16_128 (mask >> 24);
|
|
xmm_alpha = mask_00ff;
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
src = src_line;
|
|
src_line += src_stride;
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
uint32_t s = (*src++) | 0xff000000;
|
|
uint32_t d = *dst;
|
|
|
|
__m128i src = unpack_32_1x128 (s);
|
|
__m128i alpha = xmm_alpha;
|
|
__m128i mask = xmm_mask;
|
|
__m128i dest = unpack_32_1x128 (d);
|
|
|
|
*dst++ = pack_1x128_32 (
|
|
in_over_1x128 (&src, &alpha, &mask, &dest));
|
|
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_src = _mm_or_si128 (
|
|
load_128_unaligned ((__m128i*)src), mask_ff000000);
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_alpha, &xmm_alpha,
|
|
&xmm_mask, &xmm_mask,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
dst += 4;
|
|
src += 4;
|
|
w -= 4;
|
|
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
uint32_t s = (*src++) | 0xff000000;
|
|
uint32_t d = *dst;
|
|
|
|
__m128i src = unpack_32_1x128 (s);
|
|
__m128i alpha = xmm_alpha;
|
|
__m128i mask = xmm_mask;
|
|
__m128i dest = unpack_32_1x128 (d);
|
|
|
|
*dst++ = pack_1x128_32 (
|
|
in_over_1x128 (&src, &alpha, &mask, &dest));
|
|
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_8888_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
int dst_stride, src_stride;
|
|
uint32_t *dst_line, *dst;
|
|
uint32_t *src_line, *src;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
|
|
|
|
dst = dst_line;
|
|
src = src_line;
|
|
|
|
while (height--)
|
|
{
|
|
sse2_combine_over_u (imp, op, dst, src, NULL, width);
|
|
|
|
dst += dst_stride;
|
|
src += src_stride;
|
|
}
|
|
}
|
|
|
|
static force_inline uint16_t
|
|
composite_over_8888_0565pixel (uint32_t src, uint16_t dst)
|
|
{
|
|
__m128i ms;
|
|
|
|
ms = unpack_32_1x128 (src);
|
|
return pack_565_32_16 (
|
|
pack_1x128_32 (
|
|
over_1x128 (
|
|
ms, expand_alpha_1x128 (ms), expand565_16_1x128 (dst))));
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_8888_0565 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint16_t *dst_line, *dst, d;
|
|
uint32_t *src_line, *src, s;
|
|
int dst_stride, src_stride;
|
|
int32_t w;
|
|
|
|
__m128i xmm_alpha_lo, xmm_alpha_hi;
|
|
__m128i xmm_src, xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
src = src_line;
|
|
|
|
dst_line += dst_stride;
|
|
src_line += src_stride;
|
|
w = width;
|
|
|
|
/* Align dst on a 16-byte boundary */
|
|
while (w &&
|
|
((uintptr_t)dst & 15))
|
|
{
|
|
s = *src++;
|
|
d = *dst;
|
|
|
|
*dst++ = composite_over_8888_0565pixel (s, d);
|
|
w--;
|
|
}
|
|
|
|
/* It's a 8 pixel loop */
|
|
while (w >= 8)
|
|
{
|
|
/* I'm loading unaligned because I'm not sure
|
|
* about the address alignment.
|
|
*/
|
|
xmm_src = load_128_unaligned ((__m128i*) src);
|
|
xmm_dst = load_128_aligned ((__m128i*) dst);
|
|
|
|
/* Unpacking */
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_565_128_4x128 (xmm_dst,
|
|
&xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
|
|
/* I'm loading next 4 pixels from memory
|
|
* before to optimze the memory read.
|
|
*/
|
|
xmm_src = load_128_unaligned ((__m128i*) (src + 4));
|
|
|
|
over_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_dst0, &xmm_dst1);
|
|
|
|
/* Unpacking */
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
|
|
over_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_dst2, &xmm_dst3);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_565_4x128_128 (
|
|
&xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
|
|
|
|
w -= 8;
|
|
dst += 8;
|
|
src += 8;
|
|
}
|
|
|
|
while (w--)
|
|
{
|
|
s = *src++;
|
|
d = *dst;
|
|
|
|
*dst++ = composite_over_8888_0565pixel (s, d);
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_n_8_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t src, srca;
|
|
uint32_t *dst_line, *dst;
|
|
uint8_t *mask_line, *mask;
|
|
int dst_stride, mask_stride;
|
|
int32_t w;
|
|
uint32_t m, d;
|
|
|
|
__m128i xmm_src, xmm_alpha, xmm_def;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
|
|
|
|
__m128i mmx_src, mmx_alpha, mmx_mask, mmx_dest;
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
srca = src >> 24;
|
|
if (src == 0)
|
|
return;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
|
|
|
|
xmm_def = create_mask_2x32_128 (src, src);
|
|
xmm_src = expand_pixel_32_1x128 (src);
|
|
xmm_alpha = expand_alpha_1x128 (xmm_src);
|
|
mmx_src = xmm_src;
|
|
mmx_alpha = xmm_alpha;
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
mask = mask_line;
|
|
mask_line += mask_stride;
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
uint8_t m = *mask++;
|
|
|
|
if (m)
|
|
{
|
|
d = *dst;
|
|
mmx_mask = expand_pixel_8_1x128 (m);
|
|
mmx_dest = unpack_32_1x128 (d);
|
|
|
|
*dst = pack_1x128_32 (in_over_1x128 (&mmx_src,
|
|
&mmx_alpha,
|
|
&mmx_mask,
|
|
&mmx_dest));
|
|
}
|
|
|
|
w--;
|
|
dst++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
m = *((uint32_t*)mask);
|
|
|
|
if (srca == 0xff && m == 0xffffffff)
|
|
{
|
|
save_128_aligned ((__m128i*)dst, xmm_def);
|
|
}
|
|
else if (m)
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i*) dst);
|
|
xmm_mask = unpack_32_1x128 (m);
|
|
xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
|
|
|
|
/* Unpacking */
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
in_over_2x128 (&xmm_src, &xmm_src,
|
|
&xmm_alpha, &xmm_alpha,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
|
|
w -= 4;
|
|
dst += 4;
|
|
mask += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
uint8_t m = *mask++;
|
|
|
|
if (m)
|
|
{
|
|
d = *dst;
|
|
mmx_mask = expand_pixel_8_1x128 (m);
|
|
mmx_dest = unpack_32_1x128 (d);
|
|
|
|
*dst = pack_1x128_32 (in_over_1x128 (&mmx_src,
|
|
&mmx_alpha,
|
|
&mmx_mask,
|
|
&mmx_dest));
|
|
}
|
|
|
|
w--;
|
|
dst++;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
#if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
|
|
__attribute__((__force_align_arg_pointer__))
|
|
#endif
|
|
static pixman_bool_t
|
|
sse2_fill (pixman_implementation_t *imp,
|
|
uint32_t * bits,
|
|
int stride,
|
|
int bpp,
|
|
int x,
|
|
int y,
|
|
int width,
|
|
int height,
|
|
uint32_t filler)
|
|
{
|
|
uint32_t byte_width;
|
|
uint8_t *byte_line;
|
|
|
|
__m128i xmm_def;
|
|
|
|
if (bpp == 8)
|
|
{
|
|
uint8_t b;
|
|
uint16_t w;
|
|
|
|
stride = stride * (int) sizeof (uint32_t) / 1;
|
|
byte_line = (uint8_t *)(((uint8_t *)bits) + stride * y + x);
|
|
byte_width = width;
|
|
stride *= 1;
|
|
|
|
b = filler & 0xff;
|
|
w = (b << 8) | b;
|
|
filler = (w << 16) | w;
|
|
}
|
|
else if (bpp == 16)
|
|
{
|
|
stride = stride * (int) sizeof (uint32_t) / 2;
|
|
byte_line = (uint8_t *)(((uint16_t *)bits) + stride * y + x);
|
|
byte_width = 2 * width;
|
|
stride *= 2;
|
|
|
|
filler = (filler & 0xffff) * 0x00010001;
|
|
}
|
|
else if (bpp == 32)
|
|
{
|
|
stride = stride * (int) sizeof (uint32_t) / 4;
|
|
byte_line = (uint8_t *)(((uint32_t *)bits) + stride * y + x);
|
|
byte_width = 4 * width;
|
|
stride *= 4;
|
|
}
|
|
else
|
|
{
|
|
return FALSE;
|
|
}
|
|
|
|
xmm_def = create_mask_2x32_128 (filler, filler);
|
|
|
|
while (height--)
|
|
{
|
|
int w;
|
|
uint8_t *d = byte_line;
|
|
byte_line += stride;
|
|
w = byte_width;
|
|
|
|
if (w >= 1 && ((uintptr_t)d & 1))
|
|
{
|
|
*(uint8_t *)d = filler;
|
|
w -= 1;
|
|
d += 1;
|
|
}
|
|
|
|
while (w >= 2 && ((uintptr_t)d & 3))
|
|
{
|
|
*(uint16_t *)d = filler;
|
|
w -= 2;
|
|
d += 2;
|
|
}
|
|
|
|
while (w >= 4 && ((uintptr_t)d & 15))
|
|
{
|
|
*(uint32_t *)d = filler;
|
|
|
|
w -= 4;
|
|
d += 4;
|
|
}
|
|
|
|
while (w >= 128)
|
|
{
|
|
save_128_aligned ((__m128i*)(d), xmm_def);
|
|
save_128_aligned ((__m128i*)(d + 16), xmm_def);
|
|
save_128_aligned ((__m128i*)(d + 32), xmm_def);
|
|
save_128_aligned ((__m128i*)(d + 48), xmm_def);
|
|
save_128_aligned ((__m128i*)(d + 64), xmm_def);
|
|
save_128_aligned ((__m128i*)(d + 80), xmm_def);
|
|
save_128_aligned ((__m128i*)(d + 96), xmm_def);
|
|
save_128_aligned ((__m128i*)(d + 112), xmm_def);
|
|
|
|
d += 128;
|
|
w -= 128;
|
|
}
|
|
|
|
if (w >= 64)
|
|
{
|
|
save_128_aligned ((__m128i*)(d), xmm_def);
|
|
save_128_aligned ((__m128i*)(d + 16), xmm_def);
|
|
save_128_aligned ((__m128i*)(d + 32), xmm_def);
|
|
save_128_aligned ((__m128i*)(d + 48), xmm_def);
|
|
|
|
d += 64;
|
|
w -= 64;
|
|
}
|
|
|
|
if (w >= 32)
|
|
{
|
|
save_128_aligned ((__m128i*)(d), xmm_def);
|
|
save_128_aligned ((__m128i*)(d + 16), xmm_def);
|
|
|
|
d += 32;
|
|
w -= 32;
|
|
}
|
|
|
|
if (w >= 16)
|
|
{
|
|
save_128_aligned ((__m128i*)(d), xmm_def);
|
|
|
|
d += 16;
|
|
w -= 16;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
*(uint32_t *)d = filler;
|
|
|
|
w -= 4;
|
|
d += 4;
|
|
}
|
|
|
|
if (w >= 2)
|
|
{
|
|
*(uint16_t *)d = filler;
|
|
w -= 2;
|
|
d += 2;
|
|
}
|
|
|
|
if (w >= 1)
|
|
{
|
|
*(uint8_t *)d = filler;
|
|
w -= 1;
|
|
d += 1;
|
|
}
|
|
}
|
|
|
|
return TRUE;
|
|
}
|
|
|
|
static void
|
|
sse2_composite_src_n_8_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t src, srca;
|
|
uint32_t *dst_line, *dst;
|
|
uint8_t *mask_line, *mask;
|
|
int dst_stride, mask_stride;
|
|
int32_t w;
|
|
uint32_t m;
|
|
|
|
__m128i xmm_src, xmm_def;
|
|
__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
srca = src >> 24;
|
|
if (src == 0)
|
|
{
|
|
sse2_fill (imp, dest_image->bits.bits, dest_image->bits.rowstride,
|
|
PIXMAN_FORMAT_BPP (dest_image->bits.format),
|
|
dest_x, dest_y, width, height, 0);
|
|
return;
|
|
}
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
|
|
|
|
xmm_def = create_mask_2x32_128 (src, src);
|
|
xmm_src = expand_pixel_32_1x128 (src);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
mask = mask_line;
|
|
mask_line += mask_stride;
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
uint8_t m = *mask++;
|
|
|
|
if (m)
|
|
{
|
|
*dst = pack_1x128_32 (
|
|
pix_multiply_1x128 (xmm_src, expand_pixel_8_1x128 (m)));
|
|
}
|
|
else
|
|
{
|
|
*dst = 0;
|
|
}
|
|
|
|
w--;
|
|
dst++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
m = *((uint32_t*)mask);
|
|
|
|
if (srca == 0xff && m == 0xffffffff)
|
|
{
|
|
save_128_aligned ((__m128i*)dst, xmm_def);
|
|
}
|
|
else if (m)
|
|
{
|
|
xmm_mask = unpack_32_1x128 (m);
|
|
xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
|
|
|
|
/* Unpacking */
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src, &xmm_src,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_mask_lo, xmm_mask_hi));
|
|
}
|
|
else
|
|
{
|
|
save_128_aligned ((__m128i*)dst, _mm_setzero_si128 ());
|
|
}
|
|
|
|
w -= 4;
|
|
dst += 4;
|
|
mask += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
uint8_t m = *mask++;
|
|
|
|
if (m)
|
|
{
|
|
*dst = pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
xmm_src, expand_pixel_8_1x128 (m)));
|
|
}
|
|
else
|
|
{
|
|
*dst = 0;
|
|
}
|
|
|
|
w--;
|
|
dst++;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_n_8_0565 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t src;
|
|
uint16_t *dst_line, *dst, d;
|
|
uint8_t *mask_line, *mask;
|
|
int dst_stride, mask_stride;
|
|
int32_t w;
|
|
uint32_t m;
|
|
__m128i mmx_src, mmx_alpha, mmx_mask, mmx_dest;
|
|
|
|
__m128i xmm_src, xmm_alpha;
|
|
__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
|
|
__m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
if (src == 0)
|
|
return;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
|
|
|
|
xmm_src = expand_pixel_32_1x128 (src);
|
|
xmm_alpha = expand_alpha_1x128 (xmm_src);
|
|
mmx_src = xmm_src;
|
|
mmx_alpha = xmm_alpha;
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
mask = mask_line;
|
|
mask_line += mask_stride;
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
m = *mask++;
|
|
|
|
if (m)
|
|
{
|
|
d = *dst;
|
|
mmx_mask = expand_alpha_rev_1x128 (unpack_32_1x128 (m));
|
|
mmx_dest = expand565_16_1x128 (d);
|
|
|
|
*dst = pack_565_32_16 (
|
|
pack_1x128_32 (
|
|
in_over_1x128 (
|
|
&mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
|
|
}
|
|
|
|
w--;
|
|
dst++;
|
|
}
|
|
|
|
while (w >= 8)
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i*) dst);
|
|
unpack_565_128_4x128 (xmm_dst,
|
|
&xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
|
|
|
|
m = *((uint32_t*)mask);
|
|
mask += 4;
|
|
|
|
if (m)
|
|
{
|
|
xmm_mask = unpack_32_1x128 (m);
|
|
xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
|
|
|
|
/* Unpacking */
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
in_over_2x128 (&xmm_src, &xmm_src,
|
|
&xmm_alpha, &xmm_alpha,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst0, &xmm_dst1);
|
|
}
|
|
|
|
m = *((uint32_t*)mask);
|
|
mask += 4;
|
|
|
|
if (m)
|
|
{
|
|
xmm_mask = unpack_32_1x128 (m);
|
|
xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
|
|
|
|
/* Unpacking */
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
in_over_2x128 (&xmm_src, &xmm_src,
|
|
&xmm_alpha, &xmm_alpha,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst2, &xmm_dst3);
|
|
}
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_565_4x128_128 (
|
|
&xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
|
|
|
|
w -= 8;
|
|
dst += 8;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
m = *mask++;
|
|
|
|
if (m)
|
|
{
|
|
d = *dst;
|
|
mmx_mask = expand_alpha_rev_1x128 (unpack_32_1x128 (m));
|
|
mmx_dest = expand565_16_1x128 (d);
|
|
|
|
*dst = pack_565_32_16 (
|
|
pack_1x128_32 (
|
|
in_over_1x128 (
|
|
&mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
|
|
}
|
|
|
|
w--;
|
|
dst++;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint16_t *dst_line, *dst, d;
|
|
uint32_t *src_line, *src, s;
|
|
int dst_stride, src_stride;
|
|
int32_t w;
|
|
uint32_t opaque, zero;
|
|
|
|
__m128i ms;
|
|
__m128i xmm_src, xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
src = src_line;
|
|
src_line += src_stride;
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
s = *src++;
|
|
d = *dst;
|
|
|
|
ms = unpack_32_1x128 (s);
|
|
|
|
*dst++ = pack_565_32_16 (
|
|
pack_1x128_32 (
|
|
over_rev_non_pre_1x128 (ms, expand565_16_1x128 (d))));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 8)
|
|
{
|
|
/* First round */
|
|
xmm_src = load_128_unaligned ((__m128i*)src);
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
opaque = is_opaque (xmm_src);
|
|
zero = is_zero (xmm_src);
|
|
|
|
unpack_565_128_4x128 (xmm_dst,
|
|
&xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
|
|
/* preload next round*/
|
|
xmm_src = load_128_unaligned ((__m128i*)(src + 4));
|
|
|
|
if (opaque)
|
|
{
|
|
invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_dst0, &xmm_dst1);
|
|
}
|
|
else if (!zero)
|
|
{
|
|
over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_dst0, &xmm_dst1);
|
|
}
|
|
|
|
/* Second round */
|
|
opaque = is_opaque (xmm_src);
|
|
zero = is_zero (xmm_src);
|
|
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
|
|
if (opaque)
|
|
{
|
|
invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_dst2, &xmm_dst3);
|
|
}
|
|
else if (!zero)
|
|
{
|
|
over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_dst2, &xmm_dst3);
|
|
}
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_565_4x128_128 (
|
|
&xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
|
|
|
|
w -= 8;
|
|
src += 8;
|
|
dst += 8;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *src++;
|
|
d = *dst;
|
|
|
|
ms = unpack_32_1x128 (s);
|
|
|
|
*dst++ = pack_565_32_16 (
|
|
pack_1x128_32 (
|
|
over_rev_non_pre_1x128 (ms, expand565_16_1x128 (d))));
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t *dst_line, *dst, d;
|
|
uint32_t *src_line, *src, s;
|
|
int dst_stride, src_stride;
|
|
int32_t w;
|
|
uint32_t opaque, zero;
|
|
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
src = src_line;
|
|
src_line += src_stride;
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
s = *src++;
|
|
d = *dst;
|
|
|
|
*dst++ = pack_1x128_32 (
|
|
over_rev_non_pre_1x128 (
|
|
unpack_32_1x128 (s), unpack_32_1x128 (d)));
|
|
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_src_hi = load_128_unaligned ((__m128i*)src);
|
|
|
|
opaque = is_opaque (xmm_src_hi);
|
|
zero = is_zero (xmm_src_hi);
|
|
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
|
|
if (opaque)
|
|
{
|
|
invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
else if (!zero)
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*)dst);
|
|
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
|
|
w -= 4;
|
|
dst += 4;
|
|
src += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = *src++;
|
|
d = *dst;
|
|
|
|
*dst++ = pack_1x128_32 (
|
|
over_rev_non_pre_1x128 (
|
|
unpack_32_1x128 (s), unpack_32_1x128 (d)));
|
|
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t src;
|
|
uint16_t *dst_line, *dst, d;
|
|
uint32_t *mask_line, *mask, m;
|
|
int dst_stride, mask_stride;
|
|
int w;
|
|
uint32_t pack_cmp;
|
|
|
|
__m128i xmm_src, xmm_alpha;
|
|
__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
|
|
__m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
|
|
|
|
__m128i mmx_src, mmx_alpha, mmx_mask, mmx_dest;
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
if (src == 0)
|
|
return;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
|
|
|
|
xmm_src = expand_pixel_32_1x128 (src);
|
|
xmm_alpha = expand_alpha_1x128 (xmm_src);
|
|
mmx_src = xmm_src;
|
|
mmx_alpha = xmm_alpha;
|
|
|
|
while (height--)
|
|
{
|
|
w = width;
|
|
mask = mask_line;
|
|
dst = dst_line;
|
|
mask_line += mask_stride;
|
|
dst_line += dst_stride;
|
|
|
|
while (w && ((uintptr_t)dst & 15))
|
|
{
|
|
m = *(uint32_t *) mask;
|
|
|
|
if (m)
|
|
{
|
|
d = *dst;
|
|
mmx_mask = unpack_32_1x128 (m);
|
|
mmx_dest = expand565_16_1x128 (d);
|
|
|
|
*dst = pack_565_32_16 (
|
|
pack_1x128_32 (
|
|
in_over_1x128 (
|
|
&mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
|
|
}
|
|
|
|
w--;
|
|
dst++;
|
|
mask++;
|
|
}
|
|
|
|
while (w >= 8)
|
|
{
|
|
/* First round */
|
|
xmm_mask = load_128_unaligned ((__m128i*)mask);
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
pack_cmp = _mm_movemask_epi8 (
|
|
_mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
|
|
|
|
unpack_565_128_4x128 (xmm_dst,
|
|
&xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
/* preload next round */
|
|
xmm_mask = load_128_unaligned ((__m128i*)(mask + 4));
|
|
|
|
/* preload next round */
|
|
if (pack_cmp != 0xffff)
|
|
{
|
|
in_over_2x128 (&xmm_src, &xmm_src,
|
|
&xmm_alpha, &xmm_alpha,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst0, &xmm_dst1);
|
|
}
|
|
|
|
/* Second round */
|
|
pack_cmp = _mm_movemask_epi8 (
|
|
_mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
|
|
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
if (pack_cmp != 0xffff)
|
|
{
|
|
in_over_2x128 (&xmm_src, &xmm_src,
|
|
&xmm_alpha, &xmm_alpha,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst2, &xmm_dst3);
|
|
}
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_565_4x128_128 (
|
|
&xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
|
|
|
|
w -= 8;
|
|
dst += 8;
|
|
mask += 8;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
m = *(uint32_t *) mask;
|
|
|
|
if (m)
|
|
{
|
|
d = *dst;
|
|
mmx_mask = unpack_32_1x128 (m);
|
|
mmx_dest = expand565_16_1x128 (d);
|
|
|
|
*dst = pack_565_32_16 (
|
|
pack_1x128_32 (
|
|
in_over_1x128 (
|
|
&mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
|
|
}
|
|
|
|
w--;
|
|
dst++;
|
|
mask++;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_in_n_8_8 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint8_t *dst_line, *dst;
|
|
uint8_t *mask_line, *mask;
|
|
int dst_stride, mask_stride;
|
|
uint32_t d, m;
|
|
uint32_t src;
|
|
int32_t w;
|
|
|
|
__m128i xmm_alpha;
|
|
__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
mask = mask_line;
|
|
mask_line += mask_stride;
|
|
w = width;
|
|
|
|
while (w && ((uintptr_t)dst & 15))
|
|
{
|
|
m = (uint32_t) *mask++;
|
|
d = (uint32_t) *dst;
|
|
|
|
*dst++ = (uint8_t) pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
pix_multiply_1x128 (xmm_alpha,
|
|
unpack_32_1x128 (m)),
|
|
unpack_32_1x128 (d)));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 16)
|
|
{
|
|
xmm_mask = load_128_unaligned ((__m128i*)mask);
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
mask += 16;
|
|
dst += 16;
|
|
w -= 16;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
m = (uint32_t) *mask++;
|
|
d = (uint32_t) *dst;
|
|
|
|
*dst++ = (uint8_t) pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
pix_multiply_1x128 (
|
|
xmm_alpha, unpack_32_1x128 (m)),
|
|
unpack_32_1x128 (d)));
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_in_n_8 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint8_t *dst_line, *dst;
|
|
int dst_stride;
|
|
uint32_t d;
|
|
uint32_t src;
|
|
int32_t w;
|
|
|
|
__m128i xmm_alpha;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
|
|
|
|
src = src >> 24;
|
|
|
|
if (src == 0xff)
|
|
return;
|
|
|
|
if (src == 0x00)
|
|
{
|
|
pixman_fill (dest_image->bits.bits, dest_image->bits.rowstride,
|
|
8, dest_x, dest_y, width, height, src);
|
|
|
|
return;
|
|
}
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
w = width;
|
|
|
|
while (w && ((uintptr_t)dst & 15))
|
|
{
|
|
d = (uint32_t) *dst;
|
|
|
|
*dst++ = (uint8_t) pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
xmm_alpha,
|
|
unpack_32_1x128 (d)));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 16)
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
|
|
&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
dst += 16;
|
|
w -= 16;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
d = (uint32_t) *dst;
|
|
|
|
*dst++ = (uint8_t) pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
xmm_alpha,
|
|
unpack_32_1x128 (d)));
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_in_8_8 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint8_t *dst_line, *dst;
|
|
uint8_t *src_line, *src;
|
|
int src_stride, dst_stride;
|
|
int32_t w;
|
|
uint32_t s, d;
|
|
|
|
__m128i xmm_src, xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
src = src_line;
|
|
src_line += src_stride;
|
|
w = width;
|
|
|
|
while (w && ((uintptr_t)dst & 15))
|
|
{
|
|
s = (uint32_t) *src++;
|
|
d = (uint32_t) *dst;
|
|
|
|
*dst++ = (uint8_t) pack_1x128_32 (
|
|
pix_multiply_1x128 (
|
|
unpack_32_1x128 (s), unpack_32_1x128 (d)));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 16)
|
|
{
|
|
xmm_src = load_128_unaligned ((__m128i*)src);
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
src += 16;
|
|
dst += 16;
|
|
w -= 16;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
s = (uint32_t) *src++;
|
|
d = (uint32_t) *dst;
|
|
|
|
*dst++ = (uint8_t) pack_1x128_32 (
|
|
pix_multiply_1x128 (unpack_32_1x128 (s), unpack_32_1x128 (d)));
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_add_n_8_8 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint8_t *dst_line, *dst;
|
|
uint8_t *mask_line, *mask;
|
|
int dst_stride, mask_stride;
|
|
int32_t w;
|
|
uint32_t src;
|
|
uint32_t m, d;
|
|
|
|
__m128i xmm_alpha;
|
|
__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
mask = mask_line;
|
|
mask_line += mask_stride;
|
|
w = width;
|
|
|
|
while (w && ((uintptr_t)dst & 15))
|
|
{
|
|
m = (uint32_t) *mask++;
|
|
d = (uint32_t) *dst;
|
|
|
|
*dst++ = (uint8_t) pack_1x128_32 (
|
|
_mm_adds_epu16 (
|
|
pix_multiply_1x128 (
|
|
xmm_alpha, unpack_32_1x128 (m)),
|
|
unpack_32_1x128 (d)));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 16)
|
|
{
|
|
xmm_mask = load_128_unaligned ((__m128i*)mask);
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
xmm_dst_lo = _mm_adds_epu16 (xmm_mask_lo, xmm_dst_lo);
|
|
xmm_dst_hi = _mm_adds_epu16 (xmm_mask_hi, xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
|
|
mask += 16;
|
|
dst += 16;
|
|
w -= 16;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
m = (uint32_t) *mask++;
|
|
d = (uint32_t) *dst;
|
|
|
|
*dst++ = (uint8_t) pack_1x128_32 (
|
|
_mm_adds_epu16 (
|
|
pix_multiply_1x128 (
|
|
xmm_alpha, unpack_32_1x128 (m)),
|
|
unpack_32_1x128 (d)));
|
|
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_add_n_8 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint8_t *dst_line, *dst;
|
|
int dst_stride;
|
|
int32_t w;
|
|
uint32_t src;
|
|
|
|
__m128i xmm_src;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
src >>= 24;
|
|
|
|
if (src == 0x00)
|
|
return;
|
|
|
|
if (src == 0xff)
|
|
{
|
|
pixman_fill (dest_image->bits.bits, dest_image->bits.rowstride,
|
|
8, dest_x, dest_y, width, height, 0xff);
|
|
|
|
return;
|
|
}
|
|
|
|
src = (src << 24) | (src << 16) | (src << 8) | src;
|
|
xmm_src = _mm_set_epi32 (src, src, src, src);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
w = width;
|
|
|
|
while (w && ((uintptr_t)dst & 15))
|
|
{
|
|
*dst = (uint8_t)_mm_cvtsi128_si32 (
|
|
_mm_adds_epu8 (
|
|
xmm_src,
|
|
_mm_cvtsi32_si128 (*dst)));
|
|
|
|
w--;
|
|
dst++;
|
|
}
|
|
|
|
while (w >= 16)
|
|
{
|
|
save_128_aligned (
|
|
(__m128i*)dst, _mm_adds_epu8 (xmm_src, load_128_aligned ((__m128i*)dst)));
|
|
|
|
dst += 16;
|
|
w -= 16;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
*dst = (uint8_t)_mm_cvtsi128_si32 (
|
|
_mm_adds_epu8 (
|
|
xmm_src,
|
|
_mm_cvtsi32_si128 (*dst)));
|
|
|
|
w--;
|
|
dst++;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_add_8_8 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint8_t *dst_line, *dst;
|
|
uint8_t *src_line, *src;
|
|
int dst_stride, src_stride;
|
|
int32_t w;
|
|
uint16_t t;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
src = src_line;
|
|
|
|
dst_line += dst_stride;
|
|
src_line += src_stride;
|
|
w = width;
|
|
|
|
/* Small head */
|
|
while (w && (uintptr_t)dst & 3)
|
|
{
|
|
t = (*dst) + (*src++);
|
|
*dst++ = t | (0 - (t >> 8));
|
|
w--;
|
|
}
|
|
|
|
sse2_combine_add_u (imp, op,
|
|
(uint32_t*)dst, (uint32_t*)src, NULL, w >> 2);
|
|
|
|
/* Small tail */
|
|
dst += w & 0xfffc;
|
|
src += w & 0xfffc;
|
|
|
|
w &= 3;
|
|
|
|
while (w)
|
|
{
|
|
t = (*dst) + (*src++);
|
|
*dst++ = t | (0 - (t >> 8));
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_add_8888_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t *dst_line, *dst;
|
|
uint32_t *src_line, *src;
|
|
int dst_stride, src_stride;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
src = src_line;
|
|
src_line += src_stride;
|
|
|
|
sse2_combine_add_u (imp, op, dst, src, NULL, width);
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_composite_add_n_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t *dst_line, *dst, src;
|
|
int dst_stride;
|
|
|
|
__m128i xmm_src;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
if (src == 0)
|
|
return;
|
|
|
|
if (src == ~0)
|
|
{
|
|
pixman_fill (dest_image->bits.bits, dest_image->bits.rowstride, 32,
|
|
dest_x, dest_y, width, height, ~0);
|
|
|
|
return;
|
|
}
|
|
|
|
xmm_src = _mm_set_epi32 (src, src, src, src);
|
|
while (height--)
|
|
{
|
|
int w = width;
|
|
uint32_t d;
|
|
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
d = *dst;
|
|
*dst++ =
|
|
_mm_cvtsi128_si32 ( _mm_adds_epu8 (xmm_src, _mm_cvtsi32_si128 (d)));
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
save_128_aligned
|
|
((__m128i*)dst,
|
|
_mm_adds_epu8 (xmm_src, load_128_aligned ((__m128i*)dst)));
|
|
|
|
dst += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w--)
|
|
{
|
|
d = *dst;
|
|
*dst++ =
|
|
_mm_cvtsi128_si32 (_mm_adds_epu8 (xmm_src,
|
|
_mm_cvtsi32_si128 (d)));
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
sse2_composite_add_n_8_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t *dst_line, *dst;
|
|
uint8_t *mask_line, *mask;
|
|
int dst_stride, mask_stride;
|
|
int32_t w;
|
|
uint32_t src;
|
|
|
|
__m128i xmm_src;
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
if (src == 0)
|
|
return;
|
|
xmm_src = expand_pixel_32_1x128 (src);
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
mask = mask_line;
|
|
mask_line += mask_stride;
|
|
w = width;
|
|
|
|
while (w && ((uintptr_t)dst & 15))
|
|
{
|
|
uint8_t m = *mask++;
|
|
if (m)
|
|
{
|
|
*dst = pack_1x128_32
|
|
(_mm_adds_epu16
|
|
(pix_multiply_1x128 (xmm_src, expand_pixel_8_1x128 (m)),
|
|
unpack_32_1x128 (*dst)));
|
|
}
|
|
dst++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
uint32_t m = *(uint32_t*)mask;
|
|
if (m)
|
|
{
|
|
__m128i xmm_mask_lo, xmm_mask_hi;
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
|
|
__m128i xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
__m128i xmm_mask =
|
|
_mm_unpacklo_epi8 (unpack_32_1x128(m),
|
|
_mm_setzero_si128 ());
|
|
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
pix_multiply_2x128 (&xmm_src, &xmm_src,
|
|
&xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
xmm_dst_lo = _mm_adds_epu16 (xmm_mask_lo, xmm_dst_lo);
|
|
xmm_dst_hi = _mm_adds_epu16 (xmm_mask_hi, xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
|
|
w -= 4;
|
|
dst += 4;
|
|
mask += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
uint8_t m = *mask++;
|
|
if (m)
|
|
{
|
|
*dst = pack_1x128_32
|
|
(_mm_adds_epu16
|
|
(pix_multiply_1x128 (xmm_src, expand_pixel_8_1x128 (m)),
|
|
unpack_32_1x128 (*dst)));
|
|
}
|
|
dst++;
|
|
w--;
|
|
}
|
|
}
|
|
}
|
|
|
|
static pixman_bool_t
|
|
sse2_blt (pixman_implementation_t *imp,
|
|
uint32_t * src_bits,
|
|
uint32_t * dst_bits,
|
|
int src_stride,
|
|
int dst_stride,
|
|
int src_bpp,
|
|
int dst_bpp,
|
|
int src_x,
|
|
int src_y,
|
|
int dest_x,
|
|
int dest_y,
|
|
int width,
|
|
int height)
|
|
{
|
|
uint8_t * src_bytes;
|
|
uint8_t * dst_bytes;
|
|
int byte_width;
|
|
|
|
if (src_bpp != dst_bpp)
|
|
return FALSE;
|
|
|
|
if (src_bpp == 16)
|
|
{
|
|
src_stride = src_stride * (int) sizeof (uint32_t) / 2;
|
|
dst_stride = dst_stride * (int) sizeof (uint32_t) / 2;
|
|
src_bytes =(uint8_t *)(((uint16_t *)src_bits) + src_stride * (src_y) + (src_x));
|
|
dst_bytes = (uint8_t *)(((uint16_t *)dst_bits) + dst_stride * (dest_y) + (dest_x));
|
|
byte_width = 2 * width;
|
|
src_stride *= 2;
|
|
dst_stride *= 2;
|
|
}
|
|
else if (src_bpp == 32)
|
|
{
|
|
src_stride = src_stride * (int) sizeof (uint32_t) / 4;
|
|
dst_stride = dst_stride * (int) sizeof (uint32_t) / 4;
|
|
src_bytes = (uint8_t *)(((uint32_t *)src_bits) + src_stride * (src_y) + (src_x));
|
|
dst_bytes = (uint8_t *)(((uint32_t *)dst_bits) + dst_stride * (dest_y) + (dest_x));
|
|
byte_width = 4 * width;
|
|
src_stride *= 4;
|
|
dst_stride *= 4;
|
|
}
|
|
else
|
|
{
|
|
return FALSE;
|
|
}
|
|
|
|
while (height--)
|
|
{
|
|
int w;
|
|
uint8_t *s = src_bytes;
|
|
uint8_t *d = dst_bytes;
|
|
src_bytes += src_stride;
|
|
dst_bytes += dst_stride;
|
|
w = byte_width;
|
|
|
|
while (w >= 2 && ((uintptr_t)d & 3))
|
|
{
|
|
*(uint16_t *)d = *(uint16_t *)s;
|
|
w -= 2;
|
|
s += 2;
|
|
d += 2;
|
|
}
|
|
|
|
while (w >= 4 && ((uintptr_t)d & 15))
|
|
{
|
|
*(uint32_t *)d = *(uint32_t *)s;
|
|
|
|
w -= 4;
|
|
s += 4;
|
|
d += 4;
|
|
}
|
|
|
|
while (w >= 64)
|
|
{
|
|
__m128i xmm0, xmm1, xmm2, xmm3;
|
|
|
|
xmm0 = load_128_unaligned ((__m128i*)(s));
|
|
xmm1 = load_128_unaligned ((__m128i*)(s + 16));
|
|
xmm2 = load_128_unaligned ((__m128i*)(s + 32));
|
|
xmm3 = load_128_unaligned ((__m128i*)(s + 48));
|
|
|
|
save_128_aligned ((__m128i*)(d), xmm0);
|
|
save_128_aligned ((__m128i*)(d + 16), xmm1);
|
|
save_128_aligned ((__m128i*)(d + 32), xmm2);
|
|
save_128_aligned ((__m128i*)(d + 48), xmm3);
|
|
|
|
s += 64;
|
|
d += 64;
|
|
w -= 64;
|
|
}
|
|
|
|
while (w >= 16)
|
|
{
|
|
save_128_aligned ((__m128i*)d, load_128_unaligned ((__m128i*)s) );
|
|
|
|
w -= 16;
|
|
d += 16;
|
|
s += 16;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
*(uint32_t *)d = *(uint32_t *)s;
|
|
|
|
w -= 4;
|
|
s += 4;
|
|
d += 4;
|
|
}
|
|
|
|
if (w >= 2)
|
|
{
|
|
*(uint16_t *)d = *(uint16_t *)s;
|
|
w -= 2;
|
|
s += 2;
|
|
d += 2;
|
|
}
|
|
}
|
|
|
|
return TRUE;
|
|
}
|
|
|
|
static void
|
|
sse2_composite_copy_area (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
sse2_blt (imp, src_image->bits.bits,
|
|
dest_image->bits.bits,
|
|
src_image->bits.rowstride,
|
|
dest_image->bits.rowstride,
|
|
PIXMAN_FORMAT_BPP (src_image->bits.format),
|
|
PIXMAN_FORMAT_BPP (dest_image->bits.format),
|
|
src_x, src_y, dest_x, dest_y, width, height);
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_x888_8_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t *src, *src_line, s;
|
|
uint32_t *dst, *dst_line, d;
|
|
uint8_t *mask, *mask_line;
|
|
uint32_t m;
|
|
int src_stride, mask_stride, dst_stride;
|
|
int32_t w;
|
|
__m128i ms;
|
|
|
|
__m128i xmm_src, xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
|
|
|
|
while (height--)
|
|
{
|
|
src = src_line;
|
|
src_line += src_stride;
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
mask = mask_line;
|
|
mask_line += mask_stride;
|
|
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
s = 0xff000000 | *src++;
|
|
m = (uint32_t) *mask++;
|
|
d = *dst;
|
|
ms = unpack_32_1x128 (s);
|
|
|
|
if (m != 0xff)
|
|
{
|
|
__m128i ma = expand_alpha_rev_1x128 (unpack_32_1x128 (m));
|
|
__m128i md = unpack_32_1x128 (d);
|
|
|
|
ms = in_over_1x128 (&ms, &mask_00ff, &ma, &md);
|
|
}
|
|
|
|
*dst++ = pack_1x128_32 (ms);
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
m = *(uint32_t*) mask;
|
|
xmm_src = _mm_or_si128 (
|
|
load_128_unaligned ((__m128i*)src), mask_ff000000);
|
|
|
|
if (m == 0xffffffff)
|
|
{
|
|
save_128_aligned ((__m128i*)dst, xmm_src);
|
|
}
|
|
else
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
xmm_mask = _mm_unpacklo_epi16 (unpack_32_1x128 (m), _mm_setzero_si128());
|
|
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_rev_2x128 (
|
|
xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&mask_00ff, &mask_00ff, &xmm_mask_lo, &xmm_mask_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
|
|
src += 4;
|
|
dst += 4;
|
|
mask += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
m = (uint32_t) *mask++;
|
|
|
|
if (m)
|
|
{
|
|
s = 0xff000000 | *src;
|
|
|
|
if (m == 0xff)
|
|
{
|
|
*dst = s;
|
|
}
|
|
else
|
|
{
|
|
__m128i ma, md, ms;
|
|
|
|
d = *dst;
|
|
|
|
ma = expand_alpha_rev_1x128 (unpack_32_1x128 (m));
|
|
md = unpack_32_1x128 (d);
|
|
ms = unpack_32_1x128 (s);
|
|
|
|
*dst = pack_1x128_32 (in_over_1x128 (&ms, &mask_00ff, &ma, &md));
|
|
}
|
|
|
|
}
|
|
|
|
src++;
|
|
dst++;
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_8888_8_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t *src, *src_line, s;
|
|
uint32_t *dst, *dst_line, d;
|
|
uint8_t *mask, *mask_line;
|
|
uint32_t m;
|
|
int src_stride, mask_stride, dst_stride;
|
|
int32_t w;
|
|
|
|
__m128i xmm_src, xmm_src_lo, xmm_src_hi, xmm_srca_lo, xmm_srca_hi;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
|
|
|
|
while (height--)
|
|
{
|
|
src = src_line;
|
|
src_line += src_stride;
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
mask = mask_line;
|
|
mask_line += mask_stride;
|
|
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
uint32_t sa;
|
|
|
|
s = *src++;
|
|
m = (uint32_t) *mask++;
|
|
d = *dst;
|
|
|
|
sa = s >> 24;
|
|
|
|
if (m)
|
|
{
|
|
if (sa == 0xff && m == 0xff)
|
|
{
|
|
*dst = s;
|
|
}
|
|
else
|
|
{
|
|
__m128i ms, md, ma, msa;
|
|
|
|
ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
|
|
ms = unpack_32_1x128 (s);
|
|
md = unpack_32_1x128 (d);
|
|
|
|
msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
|
|
|
|
*dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
|
|
}
|
|
}
|
|
|
|
dst++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
m = *(uint32_t *) mask;
|
|
|
|
if (m)
|
|
{
|
|
xmm_src = load_128_unaligned ((__m128i*)src);
|
|
|
|
if (m == 0xffffffff && is_opaque (xmm_src))
|
|
{
|
|
save_128_aligned ((__m128i *)dst, xmm_src);
|
|
}
|
|
else
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i *)dst);
|
|
|
|
xmm_mask = _mm_unpacklo_epi16 (unpack_32_1x128 (m), _mm_setzero_si128());
|
|
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi);
|
|
expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
}
|
|
|
|
src += 4;
|
|
dst += 4;
|
|
mask += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
uint32_t sa;
|
|
|
|
s = *src++;
|
|
m = (uint32_t) *mask++;
|
|
d = *dst;
|
|
|
|
sa = s >> 24;
|
|
|
|
if (m)
|
|
{
|
|
if (sa == 0xff && m == 0xff)
|
|
{
|
|
*dst = s;
|
|
}
|
|
else
|
|
{
|
|
__m128i ms, md, ma, msa;
|
|
|
|
ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
|
|
ms = unpack_32_1x128 (s);
|
|
md = unpack_32_1x128 (d);
|
|
|
|
msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
|
|
|
|
*dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
|
|
}
|
|
}
|
|
|
|
dst++;
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_reverse_n_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t src;
|
|
uint32_t *dst_line, *dst;
|
|
__m128i xmm_src;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_dsta_hi, xmm_dsta_lo;
|
|
int dst_stride;
|
|
int32_t w;
|
|
|
|
src = _pixman_image_get_solid (imp, src_image, dest_image->bits.format);
|
|
|
|
if (src == 0)
|
|
return;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
|
|
xmm_src = expand_pixel_32_1x128 (src);
|
|
|
|
while (height--)
|
|
{
|
|
dst = dst_line;
|
|
|
|
dst_line += dst_stride;
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
__m128i vd;
|
|
|
|
vd = unpack_32_1x128 (*dst);
|
|
|
|
*dst = pack_1x128_32 (over_1x128 (vd, expand_alpha_1x128 (vd),
|
|
xmm_src));
|
|
w--;
|
|
dst++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
__m128i tmp_lo, tmp_hi;
|
|
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dsta_lo, &xmm_dsta_hi);
|
|
|
|
tmp_lo = xmm_src;
|
|
tmp_hi = xmm_src;
|
|
|
|
over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
|
|
&xmm_dsta_lo, &xmm_dsta_hi,
|
|
&tmp_lo, &tmp_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (tmp_lo, tmp_hi));
|
|
|
|
w -= 4;
|
|
dst += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
__m128i vd;
|
|
|
|
vd = unpack_32_1x128 (*dst);
|
|
|
|
*dst = pack_1x128_32 (over_1x128 (vd, expand_alpha_1x128 (vd),
|
|
xmm_src));
|
|
w--;
|
|
dst++;
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
static void
|
|
sse2_composite_over_8888_8888_8888 (pixman_implementation_t *imp,
|
|
pixman_composite_info_t *info)
|
|
{
|
|
PIXMAN_COMPOSITE_ARGS (info);
|
|
uint32_t *src, *src_line, s;
|
|
uint32_t *dst, *dst_line, d;
|
|
uint32_t *mask, *mask_line;
|
|
uint32_t m;
|
|
int src_stride, mask_stride, dst_stride;
|
|
int32_t w;
|
|
|
|
__m128i xmm_src, xmm_src_lo, xmm_src_hi, xmm_srca_lo, xmm_srca_hi;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
|
|
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
dest_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
|
|
PIXMAN_IMAGE_GET_LINE (
|
|
src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
|
|
|
|
while (height--)
|
|
{
|
|
src = src_line;
|
|
src_line += src_stride;
|
|
dst = dst_line;
|
|
dst_line += dst_stride;
|
|
mask = mask_line;
|
|
mask_line += mask_stride;
|
|
|
|
w = width;
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
uint32_t sa;
|
|
|
|
s = *src++;
|
|
m = (*mask++) >> 24;
|
|
d = *dst;
|
|
|
|
sa = s >> 24;
|
|
|
|
if (m)
|
|
{
|
|
if (sa == 0xff && m == 0xff)
|
|
{
|
|
*dst = s;
|
|
}
|
|
else
|
|
{
|
|
__m128i ms, md, ma, msa;
|
|
|
|
ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
|
|
ms = unpack_32_1x128 (s);
|
|
md = unpack_32_1x128 (d);
|
|
|
|
msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
|
|
|
|
*dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
|
|
}
|
|
}
|
|
|
|
dst++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
xmm_mask = load_128_unaligned ((__m128i*)mask);
|
|
|
|
if (!is_transparent (xmm_mask))
|
|
{
|
|
xmm_src = load_128_unaligned ((__m128i*)src);
|
|
|
|
if (is_opaque (xmm_mask) && is_opaque (xmm_src))
|
|
{
|
|
save_128_aligned ((__m128i *)dst, xmm_src);
|
|
}
|
|
else
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i *)dst);
|
|
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi);
|
|
expand_alpha_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
}
|
|
|
|
src += 4;
|
|
dst += 4;
|
|
mask += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
uint32_t sa;
|
|
|
|
s = *src++;
|
|
m = (*mask++) >> 24;
|
|
d = *dst;
|
|
|
|
sa = s >> 24;
|
|
|
|
if (m)
|
|
{
|
|
if (sa == 0xff && m == 0xff)
|
|
{
|
|
*dst = s;
|
|
}
|
|
else
|
|
{
|
|
__m128i ms, md, ma, msa;
|
|
|
|
ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
|
|
ms = unpack_32_1x128 (s);
|
|
md = unpack_32_1x128 (d);
|
|
|
|
msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
|
|
|
|
*dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
|
|
}
|
|
}
|
|
|
|
dst++;
|
|
w--;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
/* A variant of 'sse2_combine_over_u' with minor tweaks */
|
|
static force_inline void
|
|
scaled_nearest_scanline_sse2_8888_8888_OVER (uint32_t* pd,
|
|
const uint32_t* ps,
|
|
int32_t w,
|
|
pixman_fixed_t vx,
|
|
pixman_fixed_t unit_x,
|
|
pixman_fixed_t src_width_fixed,
|
|
pixman_bool_t fully_transparent_src)
|
|
{
|
|
uint32_t s, d;
|
|
const uint32_t* pm = NULL;
|
|
|
|
__m128i xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_alpha_lo, xmm_alpha_hi;
|
|
|
|
if (fully_transparent_src)
|
|
return;
|
|
|
|
/* Align dst on a 16-byte boundary */
|
|
while (w && ((uintptr_t)pd & 15))
|
|
{
|
|
d = *pd;
|
|
s = combine1 (ps + pixman_fixed_to_int (vx), pm);
|
|
vx += unit_x;
|
|
while (vx >= 0)
|
|
vx -= src_width_fixed;
|
|
|
|
*pd++ = core_combine_over_u_pixel_sse2 (s, d);
|
|
if (pm)
|
|
pm++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
__m128i tmp;
|
|
uint32_t tmp1, tmp2, tmp3, tmp4;
|
|
|
|
tmp1 = *(ps + pixman_fixed_to_int (vx));
|
|
vx += unit_x;
|
|
while (vx >= 0)
|
|
vx -= src_width_fixed;
|
|
tmp2 = *(ps + pixman_fixed_to_int (vx));
|
|
vx += unit_x;
|
|
while (vx >= 0)
|
|
vx -= src_width_fixed;
|
|
tmp3 = *(ps + pixman_fixed_to_int (vx));
|
|
vx += unit_x;
|
|
while (vx >= 0)
|
|
vx -= src_width_fixed;
|
|
tmp4 = *(ps + pixman_fixed_to_int (vx));
|
|
vx += unit_x;
|
|
while (vx >= 0)
|
|
vx -= src_width_fixed;
|
|
|
|
tmp = _mm_set_epi32 (tmp4, tmp3, tmp2, tmp1);
|
|
|
|
xmm_src_hi = combine4 ((__m128i*)&tmp, (__m128i*)pm);
|
|
|
|
if (is_opaque (xmm_src_hi))
|
|
{
|
|
save_128_aligned ((__m128i*)pd, xmm_src_hi);
|
|
}
|
|
else if (!is_zero (xmm_src_hi))
|
|
{
|
|
xmm_dst_hi = load_128_aligned ((__m128i*) pd);
|
|
|
|
unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_2x128 (
|
|
xmm_src_lo, xmm_src_hi, &xmm_alpha_lo, &xmm_alpha_hi);
|
|
|
|
over_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
/* rebuid the 4 pixel data and save*/
|
|
save_128_aligned ((__m128i*)pd,
|
|
pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
|
|
w -= 4;
|
|
pd += 4;
|
|
if (pm)
|
|
pm += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
d = *pd;
|
|
s = combine1 (ps + pixman_fixed_to_int (vx), pm);
|
|
vx += unit_x;
|
|
while (vx >= 0)
|
|
vx -= src_width_fixed;
|
|
|
|
*pd++ = core_combine_over_u_pixel_sse2 (s, d);
|
|
if (pm)
|
|
pm++;
|
|
|
|
w--;
|
|
}
|
|
}
|
|
|
|
FAST_NEAREST_MAINLOOP (sse2_8888_8888_cover_OVER,
|
|
scaled_nearest_scanline_sse2_8888_8888_OVER,
|
|
uint32_t, uint32_t, COVER)
|
|
FAST_NEAREST_MAINLOOP (sse2_8888_8888_none_OVER,
|
|
scaled_nearest_scanline_sse2_8888_8888_OVER,
|
|
uint32_t, uint32_t, NONE)
|
|
FAST_NEAREST_MAINLOOP (sse2_8888_8888_pad_OVER,
|
|
scaled_nearest_scanline_sse2_8888_8888_OVER,
|
|
uint32_t, uint32_t, PAD)
|
|
FAST_NEAREST_MAINLOOP (sse2_8888_8888_normal_OVER,
|
|
scaled_nearest_scanline_sse2_8888_8888_OVER,
|
|
uint32_t, uint32_t, NORMAL)
|
|
|
|
static force_inline void
|
|
scaled_nearest_scanline_sse2_8888_n_8888_OVER (const uint32_t * mask,
|
|
uint32_t * dst,
|
|
const uint32_t * src,
|
|
int32_t w,
|
|
pixman_fixed_t vx,
|
|
pixman_fixed_t unit_x,
|
|
pixman_fixed_t src_width_fixed,
|
|
pixman_bool_t zero_src)
|
|
{
|
|
__m128i xmm_mask;
|
|
__m128i xmm_src, xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_alpha_lo, xmm_alpha_hi;
|
|
|
|
if (zero_src || (*mask >> 24) == 0)
|
|
return;
|
|
|
|
xmm_mask = create_mask_16_128 (*mask >> 24);
|
|
|
|
while (w && (uintptr_t)dst & 15)
|
|
{
|
|
uint32_t s = *(src + pixman_fixed_to_int (vx));
|
|
vx += unit_x;
|
|
while (vx >= 0)
|
|
vx -= src_width_fixed;
|
|
|
|
if (s)
|
|
{
|
|
uint32_t d = *dst;
|
|
|
|
__m128i ms = unpack_32_1x128 (s);
|
|
__m128i alpha = expand_alpha_1x128 (ms);
|
|
__m128i dest = xmm_mask;
|
|
__m128i alpha_dst = unpack_32_1x128 (d);
|
|
|
|
*dst = pack_1x128_32 (
|
|
in_over_1x128 (&ms, &alpha, &dest, &alpha_dst));
|
|
}
|
|
dst++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
uint32_t tmp1, tmp2, tmp3, tmp4;
|
|
|
|
tmp1 = *(src + pixman_fixed_to_int (vx));
|
|
vx += unit_x;
|
|
while (vx >= 0)
|
|
vx -= src_width_fixed;
|
|
tmp2 = *(src + pixman_fixed_to_int (vx));
|
|
vx += unit_x;
|
|
while (vx >= 0)
|
|
vx -= src_width_fixed;
|
|
tmp3 = *(src + pixman_fixed_to_int (vx));
|
|
vx += unit_x;
|
|
while (vx >= 0)
|
|
vx -= src_width_fixed;
|
|
tmp4 = *(src + pixman_fixed_to_int (vx));
|
|
vx += unit_x;
|
|
while (vx >= 0)
|
|
vx -= src_width_fixed;
|
|
|
|
xmm_src = _mm_set_epi32 (tmp4, tmp3, tmp2, tmp1);
|
|
|
|
if (!is_zero (xmm_src))
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
|
|
in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_mask, &xmm_mask,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned (
|
|
(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
|
|
dst += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
uint32_t s = *(src + pixman_fixed_to_int (vx));
|
|
vx += unit_x;
|
|
while (vx >= 0)
|
|
vx -= src_width_fixed;
|
|
|
|
if (s)
|
|
{
|
|
uint32_t d = *dst;
|
|
|
|
__m128i ms = unpack_32_1x128 (s);
|
|
__m128i alpha = expand_alpha_1x128 (ms);
|
|
__m128i mask = xmm_mask;
|
|
__m128i dest = unpack_32_1x128 (d);
|
|
|
|
*dst = pack_1x128_32 (
|
|
in_over_1x128 (&ms, &alpha, &mask, &dest));
|
|
}
|
|
|
|
dst++;
|
|
w--;
|
|
}
|
|
|
|
}
|
|
|
|
FAST_NEAREST_MAINLOOP_COMMON (sse2_8888_n_8888_cover_OVER,
|
|
scaled_nearest_scanline_sse2_8888_n_8888_OVER,
|
|
uint32_t, uint32_t, uint32_t, COVER, TRUE, TRUE)
|
|
FAST_NEAREST_MAINLOOP_COMMON (sse2_8888_n_8888_pad_OVER,
|
|
scaled_nearest_scanline_sse2_8888_n_8888_OVER,
|
|
uint32_t, uint32_t, uint32_t, PAD, TRUE, TRUE)
|
|
FAST_NEAREST_MAINLOOP_COMMON (sse2_8888_n_8888_none_OVER,
|
|
scaled_nearest_scanline_sse2_8888_n_8888_OVER,
|
|
uint32_t, uint32_t, uint32_t, NONE, TRUE, TRUE)
|
|
FAST_NEAREST_MAINLOOP_COMMON (sse2_8888_n_8888_normal_OVER,
|
|
scaled_nearest_scanline_sse2_8888_n_8888_OVER,
|
|
uint32_t, uint32_t, uint32_t, NORMAL, TRUE, TRUE)
|
|
|
|
#if BILINEAR_INTERPOLATION_BITS < 8
|
|
# define BILINEAR_DECLARE_VARIABLES \
|
|
const __m128i xmm_wt = _mm_set_epi16 (wt, wt, wt, wt, wt, wt, wt, wt); \
|
|
const __m128i xmm_wb = _mm_set_epi16 (wb, wb, wb, wb, wb, wb, wb, wb); \
|
|
const __m128i xmm_addc = _mm_set_epi16 (0, 1, 0, 1, 0, 1, 0, 1); \
|
|
const __m128i xmm_ux = _mm_set_epi16 (unit_x, -unit_x, unit_x, -unit_x, \
|
|
unit_x, -unit_x, unit_x, -unit_x); \
|
|
const __m128i xmm_zero = _mm_setzero_si128 (); \
|
|
__m128i xmm_x = _mm_set_epi16 (vx, -(vx + 1), vx, -(vx + 1), \
|
|
vx, -(vx + 1), vx, -(vx + 1))
|
|
#else
|
|
# define BILINEAR_DECLARE_VARIABLES \
|
|
const __m128i xmm_wt = _mm_set_epi16 (wt, wt, wt, wt, wt, wt, wt, wt); \
|
|
const __m128i xmm_wb = _mm_set_epi16 (wb, wb, wb, wb, wb, wb, wb, wb); \
|
|
const __m128i xmm_addc = _mm_set_epi16 (0, 0, 0, 0, 1, 1, 1, 1); \
|
|
const __m128i xmm_ux = _mm_set_epi16 (unit_x, unit_x, unit_x, unit_x, \
|
|
-unit_x, -unit_x, -unit_x, -unit_x); \
|
|
const __m128i xmm_zero = _mm_setzero_si128 (); \
|
|
__m128i xmm_x = _mm_set_epi16 (vx, vx, vx, vx, \
|
|
-(vx + 1), -(vx + 1), -(vx + 1), -(vx + 1))
|
|
#endif
|
|
|
|
#define BILINEAR_INTERPOLATE_ONE_PIXEL(pix) \
|
|
do { \
|
|
__m128i xmm_wh, xmm_lo, xmm_hi, a; \
|
|
/* fetch 2x2 pixel block into sse2 registers */ \
|
|
__m128i tltr = _mm_loadl_epi64 ( \
|
|
(__m128i *)&src_top[pixman_fixed_to_int (vx)]); \
|
|
__m128i blbr = _mm_loadl_epi64 ( \
|
|
(__m128i *)&src_bottom[pixman_fixed_to_int (vx)]); \
|
|
vx += unit_x; \
|
|
/* vertical interpolation */ \
|
|
a = _mm_add_epi16 (_mm_mullo_epi16 (_mm_unpacklo_epi8 (tltr, xmm_zero), \
|
|
xmm_wt), \
|
|
_mm_mullo_epi16 (_mm_unpacklo_epi8 (blbr, xmm_zero), \
|
|
xmm_wb)); \
|
|
if (BILINEAR_INTERPOLATION_BITS < 8) \
|
|
{ \
|
|
/* calculate horizontal weights */ \
|
|
xmm_wh = _mm_add_epi16 (xmm_addc, _mm_srli_epi16 (xmm_x, \
|
|
16 - BILINEAR_INTERPOLATION_BITS)); \
|
|
xmm_x = _mm_add_epi16 (xmm_x, xmm_ux); \
|
|
/* horizontal interpolation */ \
|
|
a = _mm_madd_epi16 (_mm_unpackhi_epi16 (_mm_shuffle_epi32 ( \
|
|
a, _MM_SHUFFLE (1, 0, 3, 2)), a), xmm_wh); \
|
|
} \
|
|
else \
|
|
{ \
|
|
/* calculate horizontal weights */ \
|
|
xmm_wh = _mm_add_epi16 (xmm_addc, _mm_srli_epi16 (xmm_x, \
|
|
16 - BILINEAR_INTERPOLATION_BITS)); \
|
|
xmm_x = _mm_add_epi16 (xmm_x, xmm_ux); \
|
|
/* horizontal interpolation */ \
|
|
xmm_lo = _mm_mullo_epi16 (a, xmm_wh); \
|
|
xmm_hi = _mm_mulhi_epu16 (a, xmm_wh); \
|
|
a = _mm_add_epi32 (_mm_unpacklo_epi16 (xmm_lo, xmm_hi), \
|
|
_mm_unpackhi_epi16 (xmm_lo, xmm_hi)); \
|
|
} \
|
|
/* shift and pack the result */ \
|
|
a = _mm_srli_epi32 (a, BILINEAR_INTERPOLATION_BITS * 2); \
|
|
a = _mm_packs_epi32 (a, a); \
|
|
a = _mm_packus_epi16 (a, a); \
|
|
pix = _mm_cvtsi128_si32 (a); \
|
|
} while (0)
|
|
|
|
#define BILINEAR_SKIP_ONE_PIXEL() \
|
|
do { \
|
|
vx += unit_x; \
|
|
xmm_x = _mm_add_epi16 (xmm_x, xmm_ux); \
|
|
} while(0)
|
|
|
|
static force_inline void
|
|
scaled_bilinear_scanline_sse2_8888_8888_SRC (uint32_t * dst,
|
|
const uint32_t * mask,
|
|
const uint32_t * src_top,
|
|
const uint32_t * src_bottom,
|
|
int32_t w,
|
|
int wt,
|
|
int wb,
|
|
pixman_fixed_t vx,
|
|
pixman_fixed_t unit_x,
|
|
pixman_fixed_t max_vx,
|
|
pixman_bool_t zero_src)
|
|
{
|
|
BILINEAR_DECLARE_VARIABLES;
|
|
uint32_t pix1, pix2, pix3, pix4;
|
|
|
|
while ((w -= 4) >= 0)
|
|
{
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix2);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix3);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix4);
|
|
*dst++ = pix1;
|
|
*dst++ = pix2;
|
|
*dst++ = pix3;
|
|
*dst++ = pix4;
|
|
}
|
|
|
|
if (w & 2)
|
|
{
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix2);
|
|
*dst++ = pix1;
|
|
*dst++ = pix2;
|
|
}
|
|
|
|
if (w & 1)
|
|
{
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
|
|
*dst = pix1;
|
|
}
|
|
|
|
}
|
|
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_cover_SRC,
|
|
scaled_bilinear_scanline_sse2_8888_8888_SRC,
|
|
uint32_t, uint32_t, uint32_t,
|
|
COVER, FLAG_NONE)
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_pad_SRC,
|
|
scaled_bilinear_scanline_sse2_8888_8888_SRC,
|
|
uint32_t, uint32_t, uint32_t,
|
|
PAD, FLAG_NONE)
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_none_SRC,
|
|
scaled_bilinear_scanline_sse2_8888_8888_SRC,
|
|
uint32_t, uint32_t, uint32_t,
|
|
NONE, FLAG_NONE)
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_normal_SRC,
|
|
scaled_bilinear_scanline_sse2_8888_8888_SRC,
|
|
uint32_t, uint32_t, uint32_t,
|
|
NORMAL, FLAG_NONE)
|
|
|
|
static force_inline void
|
|
scaled_bilinear_scanline_sse2_8888_8888_OVER (uint32_t * dst,
|
|
const uint32_t * mask,
|
|
const uint32_t * src_top,
|
|
const uint32_t * src_bottom,
|
|
int32_t w,
|
|
int wt,
|
|
int wb,
|
|
pixman_fixed_t vx,
|
|
pixman_fixed_t unit_x,
|
|
pixman_fixed_t max_vx,
|
|
pixman_bool_t zero_src)
|
|
{
|
|
BILINEAR_DECLARE_VARIABLES;
|
|
uint32_t pix1, pix2, pix3, pix4;
|
|
|
|
while (w && ((uintptr_t)dst & 15))
|
|
{
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
|
|
|
|
if (pix1)
|
|
{
|
|
pix2 = *dst;
|
|
*dst = core_combine_over_u_pixel_sse2 (pix1, pix2);
|
|
}
|
|
|
|
w--;
|
|
dst++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
__m128i xmm_src;
|
|
__m128i xmm_src_hi, xmm_src_lo, xmm_dst_hi, xmm_dst_lo;
|
|
__m128i xmm_alpha_hi, xmm_alpha_lo;
|
|
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix2);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix3);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix4);
|
|
|
|
xmm_src = _mm_set_epi32 (pix4, pix3, pix2, pix1);
|
|
|
|
if (!is_zero (xmm_src))
|
|
{
|
|
if (is_opaque (xmm_src))
|
|
{
|
|
save_128_aligned ((__m128i *)dst, xmm_src);
|
|
}
|
|
else
|
|
{
|
|
__m128i xmm_dst = load_128_aligned ((__m128i *)dst);
|
|
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_alpha_lo, &xmm_alpha_hi);
|
|
over_2x128 (&xmm_src_lo, &xmm_src_hi, &xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned ((__m128i *)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
}
|
|
|
|
w -= 4;
|
|
dst += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
|
|
|
|
if (pix1)
|
|
{
|
|
pix2 = *dst;
|
|
*dst = core_combine_over_u_pixel_sse2 (pix1, pix2);
|
|
}
|
|
|
|
w--;
|
|
dst++;
|
|
}
|
|
}
|
|
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_cover_OVER,
|
|
scaled_bilinear_scanline_sse2_8888_8888_OVER,
|
|
uint32_t, uint32_t, uint32_t,
|
|
COVER, FLAG_NONE)
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_pad_OVER,
|
|
scaled_bilinear_scanline_sse2_8888_8888_OVER,
|
|
uint32_t, uint32_t, uint32_t,
|
|
PAD, FLAG_NONE)
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_none_OVER,
|
|
scaled_bilinear_scanline_sse2_8888_8888_OVER,
|
|
uint32_t, uint32_t, uint32_t,
|
|
NONE, FLAG_NONE)
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_normal_OVER,
|
|
scaled_bilinear_scanline_sse2_8888_8888_OVER,
|
|
uint32_t, uint32_t, uint32_t,
|
|
NORMAL, FLAG_NONE)
|
|
|
|
static force_inline void
|
|
scaled_bilinear_scanline_sse2_8888_8_8888_OVER (uint32_t * dst,
|
|
const uint8_t * mask,
|
|
const uint32_t * src_top,
|
|
const uint32_t * src_bottom,
|
|
int32_t w,
|
|
int wt,
|
|
int wb,
|
|
pixman_fixed_t vx,
|
|
pixman_fixed_t unit_x,
|
|
pixman_fixed_t max_vx,
|
|
pixman_bool_t zero_src)
|
|
{
|
|
BILINEAR_DECLARE_VARIABLES;
|
|
uint32_t pix1, pix2, pix3, pix4;
|
|
uint32_t m;
|
|
|
|
while (w && ((uintptr_t)dst & 15))
|
|
{
|
|
uint32_t sa;
|
|
|
|
m = (uint32_t) *mask++;
|
|
|
|
if (m)
|
|
{
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
|
|
sa = pix1 >> 24;
|
|
|
|
if (sa == 0xff && m == 0xff)
|
|
{
|
|
*dst = pix1;
|
|
}
|
|
else
|
|
{
|
|
__m128i ms, md, ma, msa;
|
|
|
|
pix2 = *dst;
|
|
ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
|
|
ms = unpack_32_1x128 (pix1);
|
|
md = unpack_32_1x128 (pix2);
|
|
|
|
msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
|
|
|
|
*dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
BILINEAR_SKIP_ONE_PIXEL ();
|
|
}
|
|
|
|
w--;
|
|
dst++;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
__m128i xmm_src, xmm_src_lo, xmm_src_hi, xmm_srca_lo, xmm_srca_hi;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
|
|
|
|
m = *(uint32_t*)mask;
|
|
|
|
if (m)
|
|
{
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix2);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix3);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix4);
|
|
|
|
xmm_src = _mm_set_epi32 (pix4, pix3, pix2, pix1);
|
|
|
|
if (m == 0xffffffff && is_opaque (xmm_src))
|
|
{
|
|
save_128_aligned ((__m128i *)dst, xmm_src);
|
|
}
|
|
else
|
|
{
|
|
xmm_dst = load_128_aligned ((__m128i *)dst);
|
|
|
|
xmm_mask = _mm_unpacklo_epi16 (unpack_32_1x128 (m), _mm_setzero_si128());
|
|
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi);
|
|
expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
|
|
|
|
in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi,
|
|
&xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
BILINEAR_SKIP_ONE_PIXEL ();
|
|
BILINEAR_SKIP_ONE_PIXEL ();
|
|
BILINEAR_SKIP_ONE_PIXEL ();
|
|
BILINEAR_SKIP_ONE_PIXEL ();
|
|
}
|
|
|
|
w -= 4;
|
|
dst += 4;
|
|
mask += 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
uint32_t sa;
|
|
|
|
m = (uint32_t) *mask++;
|
|
|
|
if (m)
|
|
{
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
|
|
sa = pix1 >> 24;
|
|
|
|
if (sa == 0xff && m == 0xff)
|
|
{
|
|
*dst = pix1;
|
|
}
|
|
else
|
|
{
|
|
__m128i ms, md, ma, msa;
|
|
|
|
pix2 = *dst;
|
|
ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
|
|
ms = unpack_32_1x128 (pix1);
|
|
md = unpack_32_1x128 (pix2);
|
|
|
|
msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
|
|
|
|
*dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
BILINEAR_SKIP_ONE_PIXEL ();
|
|
}
|
|
|
|
w--;
|
|
dst++;
|
|
}
|
|
}
|
|
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8_8888_cover_OVER,
|
|
scaled_bilinear_scanline_sse2_8888_8_8888_OVER,
|
|
uint32_t, uint8_t, uint32_t,
|
|
COVER, FLAG_HAVE_NON_SOLID_MASK)
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8_8888_pad_OVER,
|
|
scaled_bilinear_scanline_sse2_8888_8_8888_OVER,
|
|
uint32_t, uint8_t, uint32_t,
|
|
PAD, FLAG_HAVE_NON_SOLID_MASK)
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8_8888_none_OVER,
|
|
scaled_bilinear_scanline_sse2_8888_8_8888_OVER,
|
|
uint32_t, uint8_t, uint32_t,
|
|
NONE, FLAG_HAVE_NON_SOLID_MASK)
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8_8888_normal_OVER,
|
|
scaled_bilinear_scanline_sse2_8888_8_8888_OVER,
|
|
uint32_t, uint8_t, uint32_t,
|
|
NORMAL, FLAG_HAVE_NON_SOLID_MASK)
|
|
|
|
static force_inline void
|
|
scaled_bilinear_scanline_sse2_8888_n_8888_OVER (uint32_t * dst,
|
|
const uint32_t * mask,
|
|
const uint32_t * src_top,
|
|
const uint32_t * src_bottom,
|
|
int32_t w,
|
|
int wt,
|
|
int wb,
|
|
pixman_fixed_t vx,
|
|
pixman_fixed_t unit_x,
|
|
pixman_fixed_t max_vx,
|
|
pixman_bool_t zero_src)
|
|
{
|
|
BILINEAR_DECLARE_VARIABLES;
|
|
uint32_t pix1, pix2, pix3, pix4;
|
|
__m128i xmm_mask;
|
|
|
|
if (zero_src || (*mask >> 24) == 0)
|
|
return;
|
|
|
|
xmm_mask = create_mask_16_128 (*mask >> 24);
|
|
|
|
while (w && ((uintptr_t)dst & 15))
|
|
{
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
|
|
if (pix1)
|
|
{
|
|
uint32_t d = *dst;
|
|
|
|
__m128i ms = unpack_32_1x128 (pix1);
|
|
__m128i alpha = expand_alpha_1x128 (ms);
|
|
__m128i dest = xmm_mask;
|
|
__m128i alpha_dst = unpack_32_1x128 (d);
|
|
|
|
*dst = pack_1x128_32
|
|
(in_over_1x128 (&ms, &alpha, &dest, &alpha_dst));
|
|
}
|
|
|
|
dst++;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix2);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix3);
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix4);
|
|
|
|
if (pix1 | pix2 | pix3 | pix4)
|
|
{
|
|
__m128i xmm_src, xmm_src_lo, xmm_src_hi;
|
|
__m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
|
|
__m128i xmm_alpha_lo, xmm_alpha_hi;
|
|
|
|
xmm_src = _mm_set_epi32 (pix4, pix3, pix2, pix1);
|
|
|
|
xmm_dst = load_128_aligned ((__m128i*)dst);
|
|
|
|
unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
|
|
unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
|
|
expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi);
|
|
|
|
in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
|
|
&xmm_alpha_lo, &xmm_alpha_hi,
|
|
&xmm_mask, &xmm_mask,
|
|
&xmm_dst_lo, &xmm_dst_hi);
|
|
|
|
save_128_aligned
|
|
((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
|
|
}
|
|
|
|
dst += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
BILINEAR_INTERPOLATE_ONE_PIXEL (pix1);
|
|
if (pix1)
|
|
{
|
|
uint32_t d = *dst;
|
|
|
|
__m128i ms = unpack_32_1x128 (pix1);
|
|
__m128i alpha = expand_alpha_1x128 (ms);
|
|
__m128i dest = xmm_mask;
|
|
__m128i alpha_dst = unpack_32_1x128 (d);
|
|
|
|
*dst = pack_1x128_32
|
|
(in_over_1x128 (&ms, &alpha, &dest, &alpha_dst));
|
|
}
|
|
|
|
dst++;
|
|
w--;
|
|
}
|
|
}
|
|
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_n_8888_cover_OVER,
|
|
scaled_bilinear_scanline_sse2_8888_n_8888_OVER,
|
|
uint32_t, uint32_t, uint32_t,
|
|
COVER, FLAG_HAVE_SOLID_MASK)
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_n_8888_pad_OVER,
|
|
scaled_bilinear_scanline_sse2_8888_n_8888_OVER,
|
|
uint32_t, uint32_t, uint32_t,
|
|
PAD, FLAG_HAVE_SOLID_MASK)
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_n_8888_none_OVER,
|
|
scaled_bilinear_scanline_sse2_8888_n_8888_OVER,
|
|
uint32_t, uint32_t, uint32_t,
|
|
NONE, FLAG_HAVE_SOLID_MASK)
|
|
FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_n_8888_normal_OVER,
|
|
scaled_bilinear_scanline_sse2_8888_n_8888_OVER,
|
|
uint32_t, uint32_t, uint32_t,
|
|
NORMAL, FLAG_HAVE_SOLID_MASK)
|
|
|
|
static const pixman_fast_path_t sse2_fast_paths[] =
|
|
{
|
|
/* PIXMAN_OP_OVER */
|
|
PIXMAN_STD_FAST_PATH (OVER, solid, a8, r5g6b5, sse2_composite_over_n_8_0565),
|
|
PIXMAN_STD_FAST_PATH (OVER, solid, a8, b5g6r5, sse2_composite_over_n_8_0565),
|
|
PIXMAN_STD_FAST_PATH (OVER, solid, null, a8r8g8b8, sse2_composite_over_n_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, solid, null, x8r8g8b8, sse2_composite_over_n_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, solid, null, r5g6b5, sse2_composite_over_n_0565),
|
|
PIXMAN_STD_FAST_PATH (OVER, solid, null, b5g6r5, sse2_composite_over_n_0565),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, a8r8g8b8, sse2_composite_over_8888_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, x8r8g8b8, sse2_composite_over_8888_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, a8b8g8r8, sse2_composite_over_8888_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, x8b8g8r8, sse2_composite_over_8888_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, r5g6b5, sse2_composite_over_8888_0565),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, b5g6r5, sse2_composite_over_8888_0565),
|
|
PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8r8g8b8, sse2_composite_over_n_8_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8r8g8b8, sse2_composite_over_n_8_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8b8g8r8, sse2_composite_over_n_8_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8b8g8r8, sse2_composite_over_n_8_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, a8r8g8b8, sse2_composite_over_8888_8888_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8, x8r8g8b8, sse2_composite_over_8888_8_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8, a8r8g8b8, sse2_composite_over_8888_8_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8, x8b8g8r8, sse2_composite_over_8888_8_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8, a8b8g8r8, sse2_composite_over_8888_8_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, a8, x8r8g8b8, sse2_composite_over_x888_8_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, a8, a8r8g8b8, sse2_composite_over_x888_8_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, a8, x8b8g8r8, sse2_composite_over_x888_8_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, a8, a8b8g8r8, sse2_composite_over_x888_8_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, solid, a8r8g8b8, sse2_composite_over_x888_n_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, solid, x8r8g8b8, sse2_composite_over_x888_n_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, solid, a8b8g8r8, sse2_composite_over_x888_n_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, solid, x8b8g8r8, sse2_composite_over_x888_n_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid, a8r8g8b8, sse2_composite_over_8888_n_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid, x8r8g8b8, sse2_composite_over_8888_n_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid, a8b8g8r8, sse2_composite_over_8888_n_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid, x8b8g8r8, sse2_composite_over_8888_n_8888),
|
|
PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, a8r8g8b8, sse2_composite_over_n_8888_8888_ca),
|
|
PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, x8r8g8b8, sse2_composite_over_n_8888_8888_ca),
|
|
PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, a8b8g8r8, sse2_composite_over_n_8888_8888_ca),
|
|
PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, x8b8g8r8, sse2_composite_over_n_8888_8888_ca),
|
|
PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, r5g6b5, sse2_composite_over_n_8888_0565_ca),
|
|
PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, b5g6r5, sse2_composite_over_n_8888_0565_ca),
|
|
PIXMAN_STD_FAST_PATH (OVER, pixbuf, pixbuf, a8r8g8b8, sse2_composite_over_pixbuf_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, pixbuf, pixbuf, x8r8g8b8, sse2_composite_over_pixbuf_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, a8b8g8r8, sse2_composite_over_pixbuf_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, x8b8g8r8, sse2_composite_over_pixbuf_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER, pixbuf, pixbuf, r5g6b5, sse2_composite_over_pixbuf_0565),
|
|
PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, b5g6r5, sse2_composite_over_pixbuf_0565),
|
|
PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
|
|
PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
|
|
|
|
/* PIXMAN_OP_OVER_REVERSE */
|
|
PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8r8g8b8, sse2_composite_over_reverse_n_8888),
|
|
PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8b8g8r8, sse2_composite_over_reverse_n_8888),
|
|
|
|
/* PIXMAN_OP_ADD */
|
|
PIXMAN_STD_FAST_PATH_CA (ADD, solid, a8r8g8b8, a8r8g8b8, sse2_composite_add_n_8888_8888_ca),
|
|
PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, sse2_composite_add_8_8),
|
|
PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, null, a8r8g8b8, sse2_composite_add_8888_8888),
|
|
PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, null, a8b8g8r8, sse2_composite_add_8888_8888),
|
|
PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8, sse2_composite_add_n_8_8),
|
|
PIXMAN_STD_FAST_PATH (ADD, solid, null, a8, sse2_composite_add_n_8),
|
|
PIXMAN_STD_FAST_PATH (ADD, solid, null, x8r8g8b8, sse2_composite_add_n_8888),
|
|
PIXMAN_STD_FAST_PATH (ADD, solid, null, a8r8g8b8, sse2_composite_add_n_8888),
|
|
PIXMAN_STD_FAST_PATH (ADD, solid, null, x8b8g8r8, sse2_composite_add_n_8888),
|
|
PIXMAN_STD_FAST_PATH (ADD, solid, null, a8b8g8r8, sse2_composite_add_n_8888),
|
|
PIXMAN_STD_FAST_PATH (ADD, solid, a8, x8r8g8b8, sse2_composite_add_n_8_8888),
|
|
PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8r8g8b8, sse2_composite_add_n_8_8888),
|
|
PIXMAN_STD_FAST_PATH (ADD, solid, a8, x8b8g8r8, sse2_composite_add_n_8_8888),
|
|
PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8b8g8r8, sse2_composite_add_n_8_8888),
|
|
|
|
/* PIXMAN_OP_SRC */
|
|
PIXMAN_STD_FAST_PATH (SRC, solid, a8, a8r8g8b8, sse2_composite_src_n_8_8888),
|
|
PIXMAN_STD_FAST_PATH (SRC, solid, a8, x8r8g8b8, sse2_composite_src_n_8_8888),
|
|
PIXMAN_STD_FAST_PATH (SRC, solid, a8, a8b8g8r8, sse2_composite_src_n_8_8888),
|
|
PIXMAN_STD_FAST_PATH (SRC, solid, a8, x8b8g8r8, sse2_composite_src_n_8_8888),
|
|
PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, r5g6b5, sse2_composite_src_x888_0565),
|
|
PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, b5g6r5, sse2_composite_src_x888_0565),
|
|
PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, r5g6b5, sse2_composite_src_x888_0565),
|
|
PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, b5g6r5, sse2_composite_src_x888_0565),
|
|
PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, a8r8g8b8, sse2_composite_src_x888_8888),
|
|
PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, a8b8g8r8, sse2_composite_src_x888_8888),
|
|
PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, a8r8g8b8, sse2_composite_copy_area),
|
|
PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, a8b8g8r8, sse2_composite_copy_area),
|
|
PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
|
|
PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
|
|
PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
|
|
PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
|
|
PIXMAN_STD_FAST_PATH (SRC, r5g6b5, null, r5g6b5, sse2_composite_copy_area),
|
|
PIXMAN_STD_FAST_PATH (SRC, b5g6r5, null, b5g6r5, sse2_composite_copy_area),
|
|
|
|
/* PIXMAN_OP_IN */
|
|
PIXMAN_STD_FAST_PATH (IN, a8, null, a8, sse2_composite_in_8_8),
|
|
PIXMAN_STD_FAST_PATH (IN, solid, a8, a8, sse2_composite_in_n_8_8),
|
|
PIXMAN_STD_FAST_PATH (IN, solid, null, a8, sse2_composite_in_n_8),
|
|
|
|
SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_NORMAL (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_NORMAL (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_NORMAL (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_NEAREST_FAST_PATH_NORMAL (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
|
|
|
|
SIMPLE_NEAREST_SOLID_MASK_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_n_8888),
|
|
SIMPLE_NEAREST_SOLID_MASK_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_n_8888),
|
|
SIMPLE_NEAREST_SOLID_MASK_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_n_8888),
|
|
SIMPLE_NEAREST_SOLID_MASK_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_n_8888),
|
|
SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_NORMAL (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_n_8888),
|
|
SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_NORMAL (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_n_8888),
|
|
SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_NORMAL (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_n_8888),
|
|
SIMPLE_NEAREST_SOLID_MASK_FAST_PATH_NORMAL (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_n_8888),
|
|
|
|
SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_BILINEAR_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_BILINEAR_FAST_PATH (SRC, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
|
|
SIMPLE_BILINEAR_FAST_PATH (SRC, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
|
|
SIMPLE_BILINEAR_FAST_PATH (SRC, x8b8g8r8, x8b8g8r8, sse2_8888_8888),
|
|
|
|
SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_BILINEAR_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
|
|
SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
|
|
SIMPLE_BILINEAR_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
|
|
|
|
SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_n_8888),
|
|
SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_n_8888),
|
|
SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_n_8888),
|
|
SIMPLE_BILINEAR_SOLID_MASK_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_n_8888),
|
|
|
|
SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8_8888),
|
|
SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8_8888),
|
|
SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8_8888),
|
|
SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8_8888),
|
|
|
|
{ PIXMAN_OP_NONE },
|
|
};
|
|
|
|
static uint32_t *
|
|
sse2_fetch_x8r8g8b8 (pixman_iter_t *iter, const uint32_t *mask)
|
|
{
|
|
int w = iter->width;
|
|
__m128i ff000000 = mask_ff000000;
|
|
uint32_t *dst = iter->buffer;
|
|
uint32_t *src = (uint32_t *)iter->bits;
|
|
|
|
iter->bits += iter->stride;
|
|
|
|
while (w && ((uintptr_t)dst) & 0x0f)
|
|
{
|
|
*dst++ = (*src++) | 0xff000000;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 4)
|
|
{
|
|
save_128_aligned (
|
|
(__m128i *)dst, _mm_or_si128 (
|
|
load_128_unaligned ((__m128i *)src), ff000000));
|
|
|
|
dst += 4;
|
|
src += 4;
|
|
w -= 4;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
*dst++ = (*src++) | 0xff000000;
|
|
w--;
|
|
}
|
|
|
|
return iter->buffer;
|
|
}
|
|
|
|
static uint32_t *
|
|
sse2_fetch_r5g6b5 (pixman_iter_t *iter, const uint32_t *mask)
|
|
{
|
|
int w = iter->width;
|
|
uint32_t *dst = iter->buffer;
|
|
uint16_t *src = (uint16_t *)iter->bits;
|
|
__m128i ff000000 = mask_ff000000;
|
|
|
|
iter->bits += iter->stride;
|
|
|
|
while (w && ((uintptr_t)dst) & 0x0f)
|
|
{
|
|
uint16_t s = *src++;
|
|
|
|
*dst++ = convert_0565_to_8888 (s);
|
|
w--;
|
|
}
|
|
|
|
while (w >= 8)
|
|
{
|
|
__m128i lo, hi, s;
|
|
|
|
s = _mm_loadu_si128 ((__m128i *)src);
|
|
|
|
lo = unpack_565_to_8888 (_mm_unpacklo_epi16 (s, _mm_setzero_si128 ()));
|
|
hi = unpack_565_to_8888 (_mm_unpackhi_epi16 (s, _mm_setzero_si128 ()));
|
|
|
|
save_128_aligned ((__m128i *)(dst + 0), _mm_or_si128 (lo, ff000000));
|
|
save_128_aligned ((__m128i *)(dst + 4), _mm_or_si128 (hi, ff000000));
|
|
|
|
dst += 8;
|
|
src += 8;
|
|
w -= 8;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
uint16_t s = *src++;
|
|
|
|
*dst++ = convert_0565_to_8888 (s);
|
|
w--;
|
|
}
|
|
|
|
return iter->buffer;
|
|
}
|
|
|
|
static uint32_t *
|
|
sse2_fetch_a8 (pixman_iter_t *iter, const uint32_t *mask)
|
|
{
|
|
int w = iter->width;
|
|
uint32_t *dst = iter->buffer;
|
|
uint8_t *src = iter->bits;
|
|
__m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6;
|
|
|
|
iter->bits += iter->stride;
|
|
|
|
while (w && (((uintptr_t)dst) & 15))
|
|
{
|
|
*dst++ = *(src++) << 24;
|
|
w--;
|
|
}
|
|
|
|
while (w >= 16)
|
|
{
|
|
xmm0 = _mm_loadu_si128((__m128i *)src);
|
|
|
|
xmm1 = _mm_unpacklo_epi8 (_mm_setzero_si128(), xmm0);
|
|
xmm2 = _mm_unpackhi_epi8 (_mm_setzero_si128(), xmm0);
|
|
xmm3 = _mm_unpacklo_epi16 (_mm_setzero_si128(), xmm1);
|
|
xmm4 = _mm_unpackhi_epi16 (_mm_setzero_si128(), xmm1);
|
|
xmm5 = _mm_unpacklo_epi16 (_mm_setzero_si128(), xmm2);
|
|
xmm6 = _mm_unpackhi_epi16 (_mm_setzero_si128(), xmm2);
|
|
|
|
_mm_store_si128(((__m128i *)(dst + 0)), xmm3);
|
|
_mm_store_si128(((__m128i *)(dst + 4)), xmm4);
|
|
_mm_store_si128(((__m128i *)(dst + 8)), xmm5);
|
|
_mm_store_si128(((__m128i *)(dst + 12)), xmm6);
|
|
|
|
dst += 16;
|
|
src += 16;
|
|
w -= 16;
|
|
}
|
|
|
|
while (w)
|
|
{
|
|
*dst++ = *(src++) << 24;
|
|
w--;
|
|
}
|
|
|
|
return iter->buffer;
|
|
}
|
|
|
|
typedef struct
|
|
{
|
|
pixman_format_code_t format;
|
|
pixman_iter_get_scanline_t get_scanline;
|
|
} fetcher_info_t;
|
|
|
|
static const fetcher_info_t fetchers[] =
|
|
{
|
|
{ PIXMAN_x8r8g8b8, sse2_fetch_x8r8g8b8 },
|
|
{ PIXMAN_r5g6b5, sse2_fetch_r5g6b5 },
|
|
{ PIXMAN_a8, sse2_fetch_a8 },
|
|
{ PIXMAN_null }
|
|
};
|
|
|
|
static pixman_bool_t
|
|
sse2_src_iter_init (pixman_implementation_t *imp, pixman_iter_t *iter)
|
|
{
|
|
pixman_image_t *image = iter->image;
|
|
|
|
#define FLAGS \
|
|
(FAST_PATH_STANDARD_FLAGS | FAST_PATH_ID_TRANSFORM | \
|
|
FAST_PATH_BITS_IMAGE | FAST_PATH_SAMPLES_COVER_CLIP_NEAREST)
|
|
|
|
if ((iter->iter_flags & ITER_NARROW) &&
|
|
(iter->image_flags & FLAGS) == FLAGS)
|
|
{
|
|
const fetcher_info_t *f;
|
|
|
|
for (f = &fetchers[0]; f->format != PIXMAN_null; f++)
|
|
{
|
|
if (image->common.extended_format_code == f->format)
|
|
{
|
|
uint8_t *b = (uint8_t *)image->bits.bits;
|
|
int s = image->bits.rowstride * 4;
|
|
|
|
iter->bits = b + s * iter->y + iter->x * PIXMAN_FORMAT_BPP (f->format) / 8;
|
|
iter->stride = s;
|
|
|
|
iter->get_scanline = f->get_scanline;
|
|
return TRUE;
|
|
}
|
|
}
|
|
}
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
#if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
|
|
__attribute__((__force_align_arg_pointer__))
|
|
#endif
|
|
pixman_implementation_t *
|
|
_pixman_implementation_create_sse2 (pixman_implementation_t *fallback)
|
|
{
|
|
pixman_implementation_t *imp = _pixman_implementation_create (fallback, sse2_fast_paths);
|
|
|
|
/* SSE2 constants */
|
|
mask_565_r = create_mask_2x32_128 (0x00f80000, 0x00f80000);
|
|
mask_565_g1 = create_mask_2x32_128 (0x00070000, 0x00070000);
|
|
mask_565_g2 = create_mask_2x32_128 (0x000000e0, 0x000000e0);
|
|
mask_565_b = create_mask_2x32_128 (0x0000001f, 0x0000001f);
|
|
mask_red = create_mask_2x32_128 (0x00f80000, 0x00f80000);
|
|
mask_green = create_mask_2x32_128 (0x0000fc00, 0x0000fc00);
|
|
mask_blue = create_mask_2x32_128 (0x000000f8, 0x000000f8);
|
|
mask_565_fix_rb = create_mask_2x32_128 (0x00e000e0, 0x00e000e0);
|
|
mask_565_fix_g = create_mask_2x32_128 (0x0000c000, 0x0000c000);
|
|
mask_0080 = create_mask_16_128 (0x0080);
|
|
mask_00ff = create_mask_16_128 (0x00ff);
|
|
mask_0101 = create_mask_16_128 (0x0101);
|
|
mask_ffff = create_mask_16_128 (0xffff);
|
|
mask_ff000000 = create_mask_2x32_128 (0xff000000, 0xff000000);
|
|
mask_alpha = create_mask_2x32_128 (0x00ff0000, 0x00000000);
|
|
mask_565_rb = create_mask_2x32_128 (0x00f800f8, 0x00f800f8);
|
|
mask_565_pack_multiplier = create_mask_2x32_128 (0x20000004, 0x20000004);
|
|
|
|
/* Set up function pointers */
|
|
imp->combine_32[PIXMAN_OP_OVER] = sse2_combine_over_u;
|
|
imp->combine_32[PIXMAN_OP_OVER_REVERSE] = sse2_combine_over_reverse_u;
|
|
imp->combine_32[PIXMAN_OP_IN] = sse2_combine_in_u;
|
|
imp->combine_32[PIXMAN_OP_IN_REVERSE] = sse2_combine_in_reverse_u;
|
|
imp->combine_32[PIXMAN_OP_OUT] = sse2_combine_out_u;
|
|
imp->combine_32[PIXMAN_OP_OUT_REVERSE] = sse2_combine_out_reverse_u;
|
|
imp->combine_32[PIXMAN_OP_ATOP] = sse2_combine_atop_u;
|
|
imp->combine_32[PIXMAN_OP_ATOP_REVERSE] = sse2_combine_atop_reverse_u;
|
|
imp->combine_32[PIXMAN_OP_XOR] = sse2_combine_xor_u;
|
|
imp->combine_32[PIXMAN_OP_ADD] = sse2_combine_add_u;
|
|
|
|
imp->combine_32[PIXMAN_OP_SATURATE] = sse2_combine_saturate_u;
|
|
|
|
imp->combine_32_ca[PIXMAN_OP_SRC] = sse2_combine_src_ca;
|
|
imp->combine_32_ca[PIXMAN_OP_OVER] = sse2_combine_over_ca;
|
|
imp->combine_32_ca[PIXMAN_OP_OVER_REVERSE] = sse2_combine_over_reverse_ca;
|
|
imp->combine_32_ca[PIXMAN_OP_IN] = sse2_combine_in_ca;
|
|
imp->combine_32_ca[PIXMAN_OP_IN_REVERSE] = sse2_combine_in_reverse_ca;
|
|
imp->combine_32_ca[PIXMAN_OP_OUT] = sse2_combine_out_ca;
|
|
imp->combine_32_ca[PIXMAN_OP_OUT_REVERSE] = sse2_combine_out_reverse_ca;
|
|
imp->combine_32_ca[PIXMAN_OP_ATOP] = sse2_combine_atop_ca;
|
|
imp->combine_32_ca[PIXMAN_OP_ATOP_REVERSE] = sse2_combine_atop_reverse_ca;
|
|
imp->combine_32_ca[PIXMAN_OP_XOR] = sse2_combine_xor_ca;
|
|
imp->combine_32_ca[PIXMAN_OP_ADD] = sse2_combine_add_ca;
|
|
|
|
imp->blt = sse2_blt;
|
|
imp->fill = sse2_fill;
|
|
|
|
imp->src_iter_init = sse2_src_iter_init;
|
|
|
|
return imp;
|
|
}
|