diff --git a/ggml/src/iqk/iqk_common.h b/ggml/src/iqk/iqk_common.h index f4ef0c8c..488259a6 100644 --- a/ggml/src/iqk/iqk_common.h +++ b/ggml/src/iqk/iqk_common.h @@ -80,6 +80,7 @@ struct Perf { #ifdef __AVX2__ #define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1) +#define MM256_SET1_M128I(x) _mm256_broadcastsi128_si256(x) #endif typedef struct { @@ -281,7 +282,7 @@ static inline __m128i load_iq4nl_values_128() { static inline __m256i load_iq4nl_values_256() { auto val128 = load_iq4nl_values_128(); - return MM256_SET_M128I(val128, val128); + return MM256_SET1_M128I(val128); } #ifdef HAVE_FANCY_SIMD @@ -297,7 +298,7 @@ static inline __m128i load_iq4k_values_128() { static inline __m256i load_iq4k_values_256() { auto val128 = load_iq4k_values_128(); - return MM256_SET_M128I(val128, val128); + return MM256_SET1_M128I(val128); } template struct Q8 { diff --git a/ggml/src/iqk/iqk_gemm_1bit.cpp b/ggml/src/iqk/iqk_gemm_1bit.cpp index 66e57f72..8e59c12f 100644 --- a/ggml/src/iqk/iqk_gemm_1bit.cpp +++ b/ggml/src/iqk/iqk_gemm_1bit.cpp @@ -817,7 +817,7 @@ void mul_mat_iq1_s_q8_K(int n, const void * vx, size_t bx, const DataInfo& info, auto deltas_l = _mm_unpacklo_epi16(deltas128, deltas128); auto deltas_h = _mm_unpackhi_epi16(deltas128, deltas128); auto deltas = MM256_SET_M128I(deltas_h, deltas_l); // blocks 0,0, 1,1, 2,2, ..., 7,7 - auto all_scales = MM256_SET_M128I(scales128, scales128); + auto all_scales = MM256_SET1_M128I(scales128); auto shuffle = shuffle0; for (int ib64 = 0; ib64 < QK_K/64; ++ib64) { scales[ib64] = _mm256_shuffle_epi8(all_scales, shuffle); @@ -883,7 +883,7 @@ void mul_mat_iq1_m_q8_K(int n, const void * vx, size_t bx, const DataInfo& info, auto qs = iq1m[ibl].qs; auto qh = iq1m[ibl].qh; auto aux = _mm_loadl_epi64((const __m128i *)iq1m[ibl].scales); - auto sc16 = _mm256_shuffle_epi8(MM256_SET_M128I(aux, aux), scale_shuffle); + auto sc16 = _mm256_shuffle_epi8(MM256_SET1_M128I(aux), scale_shuffle); sc16 = _mm256_and_si256(sc16, _mm256_set1_epi64x(0x0e0001c000380007)); sc16 = _mm256_mullo_epi16(sc16, _mm256_set1_epi64x(0x0001000800400200)); helper.vec = _mm256_add_epi8(_mm256_srli_epi16(sc16, 8), _mm256_set1_epi16(1)); @@ -1036,7 +1036,7 @@ static void mul_mat_iq1_s_r4_q8_1(int n, const void * vx, size_t bx, const DataI auto delta4 = _mm_mul_ps(_mm_set1_ps(0.0625f), _mm_cvtepi32_ps(_mm_cvtepi16_epi32(signs))); auto delta = _mm256_set_m128(delta4, delta4); scales4 = _mm_unpacklo_epi16(scales4, scales4); // 0,0, 1,1, 2,2, 3,3 - auto scales = MM256_SET_M128I(scales4, scales4); + auto scales = MM256_SET1_M128I(scales4); auto idxl = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)x[4*ib+k].qs)); idxh = _mm256_sllv_epi64(idxh, _mm256_set_epi64x(0, 2, 5, 8)); idxh = _mm256_srlv_epi64(idxh, _mm256_set_epi64x(1, 0, 0, 0)); @@ -1118,7 +1118,7 @@ static void mul_mat_iq1_m_r4_q8_0(int n, const void * vx, size_t bx, const DataI auto signs128 = _mm_or_si128(_mm_cmpeq_epi8(_mm_and_si128(idxh, ms), ms), _mm_set1_epi8(1)); signs128 = _mm_add_epi8(_mm_set1_epi8(-8), signs128); - auto signs = MM256_SET_M128I(signs128, signs128); + auto signs = MM256_SET1_M128I(signs128); auto idxl = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)x[4*ib+k].qs)); idxh = _mm_and_si128(idxh, _mm_set1_epi8(0x07)); helper.vec = _mm256_or_si256(idxl, _mm256_slli_epi16(_mm256_cvtepu8_epi16(idxh), 8)); @@ -1228,7 +1228,7 @@ struct DequantizerIQ1BN { IQK_ALWAYS_INLINE void prepare_iq1bn_quants(const block_iq1_bn * x, __m256i& v1, __m256i& v2) const { auto data128 = _mm_loadu_si128((const __m128i *)x); // Note: we load 16 instead of 13 bytes! - auto data = MM256_SET_M128I(data128, data128); + auto data = MM256_SET1_M128I(data128); auto val1 = _mm256_mulhi_epu16(_mm256_mullo_epi16(_mm256_shuffle_epi8(data, shuff[0]), mult[0]), m3); auto val2 = _mm256_mulhi_epu16(_mm256_mullo_epi16(_mm256_shuffle_epi8(data, shuff[1]), mult[1]), m3); auto val3 = _mm256_mulhi_epu16(_mm256_mullo_epi16(_mm256_shuffle_epi8(data, shuff[2]), mult[2]), m3); @@ -1476,7 +1476,7 @@ static void mul_mat_q1_0_g128_q8_0(int n, const void * vx, size_t bx, const Data } #else auto bits128 = _mm_loadu_si128((const __m128i *)x[ib].qs); - auto bits = MM256_SET_M128I(bits128, bits128); + auto bits = MM256_SET1_M128I(bits128); for (int k = 0; k < 4; ++k) { qx[k] = _mm256_shuffle_epi8(bits, shuffle[k]); qx[k] = _mm256_cmpeq_epi8(_mm256_and_si256(qx[k], mask), mask); diff --git a/ggml/src/iqk/iqk_gemm_floats.cpp b/ggml/src/iqk/iqk_gemm_floats.cpp index a2a3c3c6..9bd5e837 100644 --- a/ggml/src/iqk/iqk_gemm_floats.cpp +++ b/ggml/src/iqk/iqk_gemm_floats.cpp @@ -352,7 +352,7 @@ static void mul_mat_bf16_r16_bf16(int n, const void * vx, size_t bx, const DataI static_for([&](const int iy) { auto y128 = _mm_loadu_si128((const __m128i*)y[iy]+ib); //auto y = _mm512_broadcast_i32x4(y128); - auto y256 = MM256_SET_M128I(y128, y128); + auto y256 = MM256_SET1_M128I(y128); auto y = _mm512_inserti32x8(_mm512_castsi256_si512(y256), y256, 1); acc[2*iy+0] = _mm512_dpbf16_ps(acc[2*iy+0], qx[0], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); acc[2*iy+0] = _mm512_dpbf16_ps(acc[2*iy+0], qx[1], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); @@ -380,7 +380,7 @@ static void mul_mat_bf16_r16_bf16(int n, const void * vx, size_t bx, const DataI qx[3] = (__m512bh)_mm512_loadu_si512((const __m512i *)b8+4*ib+3); static_for([&](const int iy) { auto y128 = _mm_loadu_si128((const __m128i*)y[iy]+ib); - auto y256 = MM256_SET_M128I(y128, y128); + auto y256 = MM256_SET1_M128I(y128); auto y = _mm512_inserti32x8(_mm512_castsi256_si512(y256), y256, 1); acc[iy] = _mm512_dpbf16_ps(acc[iy], qx[0], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); acc[iy] = _mm512_dpbf16_ps(acc[iy], qx[1], (__m512bh)_mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); diff --git a/ggml/src/iqk/iqk_gemm_iqk_quants.cpp b/ggml/src/iqk/iqk_gemm_iqk_quants.cpp index e4e7a1f8..e2a58c15 100644 --- a/ggml/src/iqk/iqk_gemm_iqk_quants.cpp +++ b/ggml/src/iqk/iqk_gemm_iqk_quants.cpp @@ -24,7 +24,7 @@ struct IQXKScales { const __m256i prod = _mm256_madd_epi16(scales16, q8.load_bsums(iy, i)); accm[iy] = _mm256_fmadd_ps(_mm256_set1_ps(d * q8.scale(iy, i)), _mm256_cvtepi32_ps(prod), accm[iy]); } - scales16 = MM256_SET_M128I(scales8, scales8); + scales16 = MM256_SET1_M128I(scales8); scales[0] = _mm512_cvtepi8_epi16(_mm256_shuffle_epi8(scales16, shuffle1)); scales[1] = _mm512_cvtepi8_epi16(_mm256_shuffle_epi8(scales16, shuffle2)); } @@ -50,8 +50,8 @@ struct IQXKScales2 { const __m256i prod = _mm256_madd_epi16(scales_s, q8.load_bsums(iy, i)); accm[iy] = _mm256_fmadd_ps(_mm256_set1_ps(d * q8.scale(iy, i)), _mm256_cvtepi32_ps(prod), accm[iy]); } - auto aux_1 = MM256_SET_M128I(_mm256_castsi256_si128(scales16), _mm256_castsi256_si128(scales16)); - auto aux_2 = MM256_SET_M128I(_mm256_extracti128_si256(scales16, 1), _mm256_extracti128_si256(scales16, 1)); + auto aux_1 = MM256_SET1_M128I(_mm256_castsi256_si128(scales16)); + auto aux_2 = MM256_SET1_M128I(_mm256_extracti128_si256(scales16, 1)); auto scales16_1 = _mm512_inserti32x8(_mm512_castsi256_si512(aux_1), aux_1, 1); auto scales16_2 = _mm512_inserti32x8(_mm512_castsi256_si512(aux_2), aux_2, 1); scales[0] = _mm512_shuffle_epi8(scales16_1, shuffles[0]); @@ -81,7 +81,7 @@ struct DequantizerIQ2KS final : public BaseDequantizer auto shifts = _mm_and_si128(_mm_cmpeq_epi8(_mm_and_si128(_mm_set1_epi8(x[i].extra), hmask), hmask), m5); auto mins128 = _mm_mullo_epi16(scales128, _mm_cvtepi8_epi16(_mm_add_epi8(m32, shifts))); auto mins = MM256_SET_M128I(_mm_shuffle_epi8(mins128, s8k.shuffles[1]), _mm_shuffle_epi8(mins128, s8k.shuffles[0])); - auto scales256 = MM256_SET_M128I(scales128, scales128); + auto scales256 = MM256_SET1_M128I(scales128); auto all_scales = _mm512_inserti32x8(_mm512_castsi256_si512(scales256), scales256, 1); __m512i scales[4]; for (int k = 0; k < 4; ++k) scales[k] = _mm512_shuffle_epi8(all_scales, shuffles[k]); @@ -106,7 +106,7 @@ struct DequantizerIQ2KS final : public BaseDequantizer static inline __m512i load_values() { static const uint8_t kvalues_iq2nl[16] = {1, 19, 33, 49, 0, 0, 0, 0, 6, 24, 38, 54, 0, 0, 0, 0}; auto val128 = _mm_loadu_si128((const __m128i *)kvalues_iq2nl); - auto val256 = MM256_SET_M128I(val128, val128); + auto val256 = MM256_SET1_M128I(val128); return _mm512_inserti32x8(_mm512_castsi256_si512(val256), val256, 1); } inline __m128i make_scales(const uint8_t * scales_l, uint8_t scales_h) const { @@ -154,7 +154,7 @@ struct DequantizerIQ2K final : public BaseDequantizer { static inline __m512i load_values() { static const uint8_t kvalues_iq2nl[16] = {1, 19, 33, 49, 0, 0, 0, 0, 6, 24, 38, 54, 0, 0, 0, 0}; auto val128 = _mm_loadu_si128((const __m128i *)kvalues_iq2nl); - auto val256 = MM256_SET_M128I(val128, val128); + auto val256 = MM256_SET1_M128I(val128); return _mm512_inserti32x8(_mm512_castsi256_si512(val256), val256, 1); } inline __m128i make_scales(const uint8_t * scales_l) const { @@ -193,7 +193,7 @@ struct DequantizerIQ3K final : public BaseDequantizer { static inline __m512i load_values() { static const uint8_t kvalues_iq3nl[16] = {1, 24, 41, 54, 65, 77, 92, 111, 5, 28, 45, 58, 69, 81, 96, 115}; auto val128 = _mm_loadu_si128((const __m128i *)kvalues_iq3nl); - auto val256 = MM256_SET_M128I(val128, val128); + auto val256 = MM256_SET1_M128I(val128); return _mm512_inserti32x8(_mm512_castsi256_si512(val256), val256, 1); } inline __m128i make_scales(uint16_t signs, const uint8_t * scales_l) const { @@ -227,7 +227,7 @@ struct DequantizerIQ3KS final : public BaseDequantizer auto shifts = _mm_mask_add_epi16(m64, __mmask8(x[i].extra >> 8), m64, _mm_set1_epi16(4)); auto mins128 = _mm_mullo_epi16(scales128, shifts); auto mins = MM256_SET_M128I(_mm_shuffle_epi8(mins128, s8k.shuffles[1]), _mm_shuffle_epi8(mins128, s8k.shuffles[0])); - auto scales256 = MM256_SET_M128I(scales128, scales128); + auto scales256 = MM256_SET1_M128I(scales128); auto all_scales = _mm512_inserti32x8(_mm512_castsi256_si512(scales256), scales256, 1); __m512i scales[4]; for (int k = 0; k < 4; ++k) scales[k] = _mm512_shuffle_epi8(all_scales, shuffles[k]); @@ -259,7 +259,7 @@ struct DequantizerIQ3KS final : public BaseDequantizer static inline __m512i load_values() { static const uint8_t kvalues_iq3nl[16] = {1, 24, 41, 54, 65, 77, 92, 111, 5, 28, 45, 58, 69, 81, 96, 115}; auto val128 = _mm_loadu_si128((const __m128i *)kvalues_iq3nl); - auto val256 = MM256_SET_M128I(val128, val128); + auto val256 = MM256_SET1_M128I(val128); return _mm512_inserti32x8(_mm512_castsi256_si512(val256), val256, 1); } @@ -310,7 +310,7 @@ struct DequantizerIQ4KSS final : public BaseDequantizer { scales128 = _mm_add_epi16(_mm_and_si128(scales128, mask), m127); auto scales_s = _mm_mullo_epi16(scales128, _mm_add_epi16(m128, shifts)); s8k.accum_mins(scales_s, q8, i, d, accm); - auto scales256 = MM256_SET_M128I(scales128, scales128); + auto scales256 = MM256_SET1_M128I(scales128); auto all_scales = _mm512_inserti32x8(_mm512_castsi256_si512(scales256), scales256, 1); scales[0] = _mm512_shuffle_epi8(all_scales, shuffles[0]); scales[1] = _mm512_shuffle_epi8(all_scales, shuffles[1]); @@ -351,7 +351,7 @@ struct DequantizerIQ2KL final : public BaseDequantizer auto scales128 = make_scales(i); auto mins128 = _mm_mullo_epi16(scales128, _mm_set1_epi16(-64)); auto mins = MM256_SET_M128I(_mm_shuffle_epi8(mins128, s8k.shuffles[1]), _mm_shuffle_epi8(mins128, s8k.shuffles[0])); - auto scales256 = MM256_SET_M128I(scales128, scales128); + auto scales256 = MM256_SET1_M128I(scales128); auto all_scales = _mm512_inserti32x8(_mm512_castsi256_si512(scales256), scales256, 1); __m512i scales[4]; for (int k = 0; k < 4; ++k) scales[k] = _mm512_shuffle_epi8(all_scales, shuffles[k]); @@ -406,7 +406,7 @@ struct DequantizerIQ2KL final : public BaseDequantizer }; for (int k = 0; k < 4; ++k) { auto v128 = _mm_loadu_si128((const __m128i *)k_values + k); - auto v256 = MM256_SET_M128I(v128, v128); + auto v256 = MM256_SET1_M128I(v128); values[k] = _mm512_inserti32x8(_mm512_castsi256_si512(v256), v256, 1); } } @@ -437,7 +437,7 @@ struct DequantizerIQ4KS final : public BaseDequantizer { scales128 = _mm_add_epi16(_mm_and_si128(scales128, mask), m127); auto scales_s = _mm_mullo_epi16(scales128, _mm_add_epi16(m128, shifts)); s8k.accum_mins(scales_s, q8, i, d, accm); - auto scales256 = MM256_SET_M128I(scales128, scales128); + auto scales256 = MM256_SET1_M128I(scales128); auto all_scales = _mm512_inserti32x8(_mm512_castsi256_si512(scales256), scales256, 1); scales[0] = _mm512_shuffle_epi8(all_scales, shuffles[0]); scales[1] = _mm512_shuffle_epi8(all_scales, shuffles[1]); @@ -452,7 +452,7 @@ struct DequantizerIQ4KS final : public BaseDequantizer { scales128 = _mm_add_epi16(_mm_and_si128(scales128, mask), m127); auto mins128 = _mm_mullo_epi16(scales128, _mm_add_epi16(m128, shifts)); auto mins = MM256_SET_M128I(_mm_shuffle_epi8(mins128, s8k.shuffles[1]), _mm_shuffle_epi8(mins128, s8k.shuffles[0])); - auto scales256 = MM256_SET_M128I(scales128, scales128); + auto scales256 = MM256_SET1_M128I(scales128); auto all_scales = _mm512_inserti32x8(_mm512_castsi256_si512(scales256), scales256, 1); __m512i scales[4]; for (int k = 0; k < 4; ++k) scales[k] = _mm512_shuffle_epi8(all_scales, shuffles[k]); @@ -548,7 +548,7 @@ struct DequantizerIQ5KS final : public BaseDequantizer { scales128 = _mm_add_epi16(_mm_and_si128(scales128, mask), m127); auto scales_s = _mm_mullo_epi16(scales128, _mm_add_epi16(m128, shifts)); s8k.accum_mins(scales_s, q8, i, d, accm); - auto scales256 = MM256_SET_M128I(scales128, scales128); + auto scales256 = MM256_SET1_M128I(scales128); auto all_scales = _mm512_inserti32x8(_mm512_castsi256_si512(scales256), scales256, 1); scales[0] = _mm512_shuffle_epi8(all_scales, shuffles[0]); scales[1] = _mm512_shuffle_epi8(all_scales, shuffles[1]); @@ -563,7 +563,7 @@ struct DequantizerIQ5KS final : public BaseDequantizer { scales128 = _mm_add_epi16(_mm_and_si128(scales128, mask), m127); auto mins128 = _mm_mullo_epi16(scales128, _mm_add_epi16(m128, shifts)); auto mins = MM256_SET_M128I(_mm_shuffle_epi8(mins128, s8k.shuffles[1]), _mm_shuffle_epi8(mins128, s8k.shuffles[0])); - auto scales256 = MM256_SET_M128I(scales128, scales128); + auto scales256 = MM256_SET1_M128I(scales128); auto all_scales = _mm512_inserti32x8(_mm512_castsi256_si512(scales256), scales256, 1); __m512i scales[4]; for (int k = 0; k < 4; ++k) scales[k] = _mm512_shuffle_epi8(all_scales, shuffles[k]); @@ -600,8 +600,8 @@ struct DequantizerIQ5KS final : public BaseDequantizer { }; auto values128_1 = _mm_loadu_si128((const __m128i *)kvalues_iq5nl + 0); auto values128_2 = _mm_loadu_si128((const __m128i *)kvalues_iq5nl + 1); - auto values256_1 = MM256_SET_M128I(values128_1, values128_1); - auto values256_2 = MM256_SET_M128I(values128_2, values128_2); + auto values256_1 = MM256_SET1_M128I(values128_1); + auto values256_2 = MM256_SET1_M128I(values128_2); values[0] = _mm512_inserti32x8(_mm512_castsi256_si512(values256_1), values256_1, 1); values[1] = _mm512_inserti32x8(_mm512_castsi256_si512(values256_2), values256_2, 1); } @@ -671,8 +671,8 @@ struct DequantizerIQ5K final : public BaseDequantizer { }; auto values128_1 = _mm_loadu_si128((const __m128i *)kvalues_iq5nl + 0); auto values128_2 = _mm_loadu_si128((const __m128i *)kvalues_iq5nl + 1); - auto values256_1 = MM256_SET_M128I(values128_1, values128_1); - auto values256_2 = MM256_SET_M128I(values128_2, values128_2); + auto values256_1 = MM256_SET1_M128I(values128_1); + auto values256_2 = MM256_SET1_M128I(values128_2); values[0] = _mm512_inserti32x8(_mm512_castsi256_si512(values256_1), values256_1, 1); values[1] = _mm512_inserti32x8(_mm512_castsi256_si512(values256_2), values256_2, 1); } @@ -734,7 +734,7 @@ struct DequantizerIQ6K final : public BaseDequantizer { }; for (int k = 0; k < 4; ++k) { auto values128 = _mm_loadu_si128((const __m128i *)kvalues_iq6nl + k); - auto values256 = MM256_SET_M128I(values128, values128); + auto values256 = MM256_SET1_M128I(values128); values[k] = _mm512_inserti32x8(_mm512_castsi256_si512(values256), values256, 1); } } @@ -924,8 +924,8 @@ static void mul_mat_qX_K_q8_K_AVX512(int n, const void * vx, size_t bx, const Da inline void prepare_scales_16(const __m256i& all_scales, __m256i * scales) { const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0); const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1); - scales[0] = MM256_SET_M128I(l_scales, l_scales); - scales[1] = MM256_SET_M128I(h_scales, h_scales); + scales[0] = MM256_SET1_M128I(l_scales); + scales[1] = MM256_SET1_M128I(h_scales); } struct IQXKScales { @@ -964,7 +964,7 @@ struct DequantizerIQ2KS final : public BaseDequantizer auto shifts = _mm_and_si128(_mm_cmpeq_epi8(_mm_and_si128(_mm_set1_epi8(x[i].extra), hmask), hmask), m5); auto scales_s = _mm_mullo_epi16(scales128, _mm_cvtepi8_epi16(_mm_add_epi8(m32, shifts))); s8k.accum_mins(scales_s, q8, i, d, accm); - return MM256_SET_M128I(scales128, scales128); + return MM256_SET1_M128I(scales128); } inline void prepare(int i, int j) { bits.prepare(x[i].qs, j); @@ -976,7 +976,7 @@ struct DequantizerIQ2KS final : public BaseDequantizer static inline __m256i load_values() { static const uint8_t kvalues_iq2nl[16] = {1, 19, 33, 49, 0, 0, 0, 0, 6, 24, 38, 54, 0, 0, 0, 0}; auto val128 = _mm_loadu_si128((const __m128i *)kvalues_iq2nl); - return MM256_SET_M128I(val128, val128); + return MM256_SET1_M128I(val128); } inline __m128i make_scales(const uint8_t * scales_l, uint8_t scales_h) const { const uint16_t * scales = (const uint16_t *)scales_l; @@ -1008,7 +1008,7 @@ struct DequantizerIQ2KL final : public BaseDequantizer auto scales128 = make_scales(i); auto scales_s = _mm_mullo_epi16(scales128, _mm_set1_epi16(-64)); s8k.accum_mins(scales_s, q8, i, d, accm); - return MM256_SET_M128I(scales128, scales128); + return MM256_SET1_M128I(scales128); } inline void prepare(int i, int j) { __m256i ql[2], mask[2]; @@ -1050,7 +1050,7 @@ struct DequantizerIQ2KL final : public BaseDequantizer }; for (int k = 0; k < 4; ++k) { auto v128 = _mm_loadu_si128((const __m128i *)k_values + k); - values[k] = MM256_SET_M128I(v128, v128); + values[k] = MM256_SET1_M128I(v128); } } struct { __m256i values[4]; } bits; @@ -1078,7 +1078,7 @@ struct DequantizerIQ2K final : public BaseDequantizer { static inline __m256i load_values() { static const uint8_t kvalues_iq2nl[16] = {1, 19, 33, 49, 0, 0, 0, 0, 6, 24, 38, 54, 0, 0, 0, 0}; auto val128 = _mm_loadu_si128((const __m128i *)kvalues_iq2nl); - return MM256_SET_M128I(val128, val128); + return MM256_SET1_M128I(val128); } inline __m128i make_scales(const uint8_t * scales_l) const { uint64_t aux64; std::memcpy(&aux64, scales_l, 8); @@ -1116,7 +1116,7 @@ struct DequantizerIQ3K final : public BaseDequantizer { static inline __m256i load_values() { static const uint8_t kvalues_iq3nl[16] = {1, 24, 41, 54, 65, 77, 92, 111, 5, 28, 45, 58, 69, 81, 96, 115}; auto val128 = _mm_loadu_si128((const __m128i *)kvalues_iq3nl); - return MM256_SET_M128I(val128, val128); + return MM256_SET1_M128I(val128); } inline __m128i make_scales(uint16_t signs, const uint8_t * scales_l) const { uint64_t aux64; std::memcpy(&aux64, scales_l, 8); @@ -1147,7 +1147,7 @@ struct DequantizerIQ3KS final : public BaseDequantizer auto sch = _mm_cmpeq_epi16(_mm_and_si128(_mm_set1_epi16(x[i].extra), mask), mask); auto scales128 = _mm_add_epi16(scl, _mm_and_si128(sch, _mm_set1_epi16(16))); scales128 = _mm_sub_epi16(scales128, _mm_set1_epi16(16)); - return MM256_SET_M128I(scales128, scales128); + return MM256_SET1_M128I(scales128); } inline void prepare(int i, int j) { uint8_t extra = x[i].extra >> (8 + 4*j); @@ -1161,7 +1161,7 @@ struct DequantizerIQ3KS final : public BaseDequantizer } inline __m256i load_values() { auto v = _mm_loadu_si128((const __m128i *)iq3nl_values); - return MM256_SET_M128I(v, v); + return MM256_SET1_M128I(v); } @@ -1193,7 +1193,7 @@ struct DequantizerIQ4KSS final : public BaseDequantizer { scales128 = _mm_add_epi16(_mm_and_si128(scales128, mask), m127); auto scales_s = _mm_mullo_epi16(scales128, _mm_add_epi16(m128, shifts)); s8k.accum_mins(scales_s, q8, i, d, accd); - return MM256_SET_M128I(scales128, scales128); + return MM256_SET1_M128I(scales128); } inline void prepare(int, int j) { for (int k = 0; k < 2; ++k) { @@ -1226,7 +1226,7 @@ struct DequantizerIQ4KS final : public BaseDequantizer { inline __m256i new_block(int i, [[maybe_unused]] const Q8& q8, [[maybe_unused]] __m256 * accd) { auto scales128 = _mm_cvtepu8_epi16(_mm_loadl_epi64((const __m128i *)x[i].scales)); scales128 = _mm_add_epi16(_mm_and_si128(scales128, mask), m127); - return MM256_SET_M128I(scales128, scales128); + return MM256_SET1_M128I(scales128); } inline void prepare(int i, int j) { bits.prepare16(x[i].qs, j); @@ -1238,8 +1238,8 @@ struct DequantizerIQ4KS final : public BaseDequantizer { void load_values() { auto v1 = _mm_loadu_si128((const __m128i *)iq4k_values+0); auto v2 = _mm_loadu_si128((const __m128i *)iq4k_values+1); - values[0] = MM256_SET_M128I(v1, v1); - values[1] = MM256_SET_M128I(v2, v2); + values[0] = MM256_SET1_M128I(v1); + values[1] = MM256_SET1_M128I(v2); } @@ -1278,10 +1278,10 @@ struct DequantizerIQ4K final : public BaseDequantizer { void load_values() { auto v1 = _mm_loadu_si128((const __m128i *)iq4k_values+0); auto v2 = _mm_loadu_si128((const __m128i *)iq4k_values+1); - values[0] = MM256_SET_M128I(v1, v1); + values[0] = MM256_SET1_M128I(v1); values[1] = MM256_SET_M128I(v1, v2); values[2] = MM256_SET_M128I(v2, v1); - values[3] = MM256_SET_M128I(v2, v2); + values[3] = MM256_SET1_M128I(v2); } Q4Bits bits; @@ -1302,7 +1302,7 @@ struct DequantizerIQ5KS final : public BaseDequantizer { scales128 = _mm_add_epi16(_mm_and_si128(scales128, mask), m127); auto scales_s = _mm_mullo_epi16(scales128, _mm_add_epi16(m128, shifts)); s8k.accum_mins(scales_s, q8, i, d, accd); - return MM256_SET_M128I(scales128, scales128); + return MM256_SET1_M128I(scales128); } inline void prepare(int i, int j) { bits.prepare(x[i].qs, j); @@ -1321,8 +1321,8 @@ struct DequantizerIQ5KS final : public BaseDequantizer { }; auto values128_1 = _mm_loadu_si128((const __m128i *)kvalues_iq5nl + 0); auto values128_2 = _mm_loadu_si128((const __m128i *)kvalues_iq5nl + 1); - values[0] = MM256_SET_M128I(values128_1, values128_1); - values[1] = MM256_SET_M128I(values128_2, values128_2); + values[0] = MM256_SET1_M128I(values128_1); + values[1] = MM256_SET1_M128I(values128_2); } Q4Bits bits; @@ -1369,8 +1369,8 @@ struct DequantizerIQ5K final : public BaseDequantizer { static void load_values(__m256i * values) { auto values128_1 = _mm_loadu_si128((const __m128i *)iq5nl_values + 0); auto values128_2 = _mm_loadu_si128((const __m128i *)iq5nl_values + 1); - values[0] = MM256_SET_M128I(values128_1, values128_1); - values[1] = MM256_SET_M128I(values128_2, values128_2); + values[0] = MM256_SET1_M128I(values128_1); + values[1] = MM256_SET1_M128I(values128_2); } Q4Bits bits; @@ -1414,7 +1414,7 @@ struct DequantizerIQ6K final : public BaseDequantizer { static void load_values(__m256i * values) { for (int k = 0; k < 4; ++k) { auto values128 = _mm_loadu_si128((const __m128i *)iq6nl_values + k); - values[k] = MM256_SET_M128I(values128, values128); + values[k] = MM256_SET1_M128I(values128); } } @@ -1740,7 +1740,7 @@ static void mul_mat_iq3_k_r4_q8_k(int n, const void * vx, size_t bx, const DataI auto smask = _mm256_set_epi64x(0x0808080808080808, 0x0404040404040404, 0x0202020202020202, 0x0101010101010101); auto shift_shuffle = _mm256_set_epi64x(0x0707070706060606, 0x0505050504040404, 0x0303030302020202, 0x0101010100000000); auto values128 = _mm_loadu_si128((const __m128i *)iq3nl_values); - auto values = MM256_SET_M128I(values128, values128); + auto values = MM256_SET1_M128I(values128); values = _mm256_add_epi8(values, _mm256_set1_epi8(64)); static const uint8_t k_shuff[32] = {0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15, 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15}; auto shuff = _mm256_loadu_si256((const __m256i *)k_shuff); @@ -1835,7 +1835,7 @@ static void mul_mat_iq4_k_r4_q8_k(int n, const void * vx, size_t bx, const DataI #else auto s_shuffle = _mm256_set_epi64x(0x0f0e0f0e0d0c0d0c, 0x0b0a0b0a09080908, 0x0706070605040504, 0x0302030201000100); auto values128 = _mm_loadu_si128((const __m128i *)iq4k_values); - auto values = MM256_SET_M128I(values128, values128); + auto values = MM256_SET1_M128I(values128); #endif int nbl = n / QK_K; __m256 acc[nrc_y] = {}; @@ -1934,8 +1934,8 @@ static void mul_mat_iq5_k_r4_q8_k(int n, const void * vx, size_t bx, const DataI { auto val1 = _mm_loadu_si128((const __m128i *)iq5nl_values+0); auto val2 = _mm_loadu_si128((const __m128i *)iq5nl_values+1); - values[0] = MM256_SET_M128I(val1, val1); - values[1] = MM256_SET_M128I(val2, val2); + values[0] = MM256_SET1_M128I(val1); + values[1] = MM256_SET1_M128I(val2); #ifdef HAVE_FANCY_SIMD values[0] = _mm256_sub_epi8(values[0], _mm256_set1_epi8(-128)); values[1] = _mm256_sub_epi8(values[1], _mm256_set1_epi8(-128)); @@ -2053,7 +2053,7 @@ static void mul_mat_iq4_ks_r4_q8_k(int n, const void * vx, size_t bx, const Data #ifndef HAVE_FANCY_SIMD auto s_shuffle = _mm256_set_epi64x(0x0f0e0f0e0d0c0d0c, 0x0b0a0b0a09080908, 0x0706070605040504, 0x0302030201000100); auto values128 = _mm_loadu_si128((const __m128i *)iq4k_values); - auto values = MM256_SET_M128I(values128, values128); + auto values = MM256_SET1_M128I(values128); #else auto values = load_iq4nl_values_256(); #endif @@ -2163,8 +2163,8 @@ static void mul_mat_iq5_ks_r4_q8_k(int n, const void * vx, size_t bx, const Data { auto val1 = _mm_loadu_si128((const __m128i *)iq5nl_values+0); auto val2 = _mm_loadu_si128((const __m128i *)iq5nl_values+1); - values[0] = MM256_SET_M128I(val1, val1); - values[1] = MM256_SET_M128I(val2, val2); + values[0] = MM256_SET1_M128I(val1); + values[1] = MM256_SET1_M128I(val2); #ifdef HAVE_FANCY_SIMD values[0] = _mm256_sub_epi8(values[0], _mm256_set1_epi8(-128)); values[1] = _mm256_sub_epi8(values[1], _mm256_set1_epi8(-128)); @@ -2325,7 +2325,7 @@ void iqk_convert_iq2_ks_q8_k_r8(int n, const void * vx, size_t bx, void * vy, in __m256i values; { auto v = _mm_loadl_epi64((const __m128i *)iq2nl_values); - values = MM256_SET_M128I(v, v); + values = MM256_SET1_M128I(v); } ggml_half dh[k_nr]; @@ -2404,7 +2404,7 @@ void iqk_convert_iq2_k_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int __m256i values; { auto v = _mm_loadl_epi64((const __m128i *)iq2nl_values); - values = MM256_SET_M128I(v, v); + values = MM256_SET1_M128I(v); } __m256i xv[8]; @@ -2485,7 +2485,7 @@ void iqk_convert_iq2_kl_q8_k_r8(int n, const void * vx, size_t bx, void * vy, in }; for (int k = 0; k < 4; ++k) { auto v = _mm_loadu_si128((const __m128i *)k_values + k); - values[k] = MM256_SET_M128I(v, v); + values[k] = MM256_SET1_M128I(v); } } @@ -2584,7 +2584,7 @@ void iqk_convert_iq3_ks_q8_k_r8(int n, const void * vx, size_t bx, void * vy, in __m256i values; { auto v = _mm_loadu_si128((const __m128i *)iq3nl_values); - values = MM256_SET_M128I(v, v); + values = MM256_SET1_M128I(v); } ggml_half drow[k_nr]; @@ -2662,7 +2662,7 @@ void iqk_convert_iq3_k_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int __m256i values; { auto v = _mm_loadu_si128((const __m128i *)iq3nl_values); - values = MM256_SET_M128I(v, v); + values = MM256_SET1_M128I(v); } __m256i xv[8]; @@ -2751,8 +2751,8 @@ void iqk_convert_iq4_kss_q8_k_r8(int n, const void * vx, size_t bx, void * vy, i { auto v1 = _mm_loadu_si128((const __m128i *)iq4k_values+0); auto v2 = _mm_loadu_si128((const __m128i *)iq4k_values+1); - values[0] = MM256_SET_M128I(v1, v1); - values[1] = MM256_SET_M128I(v2, v2); + values[0] = MM256_SET1_M128I(v1); + values[1] = MM256_SET1_M128I(v2); } float drow[k_nr]; @@ -2824,8 +2824,8 @@ void iqk_convert_iq4_ks_q8_k_r8(int n, const void * vx, size_t bx, void * vy, in { auto v1 = _mm_loadu_si128((const __m128i *)iq4k_values+0); auto v2 = _mm_loadu_si128((const __m128i *)iq4k_values+1); - values[0] = MM256_SET_M128I(v1, v1); - values[1] = MM256_SET_M128I(v2, v2); + values[0] = MM256_SET1_M128I(v1); + values[1] = MM256_SET1_M128I(v2); } float drow[k_nr]; @@ -2892,10 +2892,10 @@ void iqk_convert_iq4_k_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int { auto v1 = _mm_loadu_si128((const __m128i *)iq4k_values+0); auto v2 = _mm_loadu_si128((const __m128i *)iq4k_values+1); - values[0] = MM256_SET_M128I(v1, v1); + values[0] = MM256_SET1_M128I(v1); values[1] = MM256_SET_M128I(v1, v2); values[2] = MM256_SET_M128I(v2, v1); - values[3] = MM256_SET_M128I(v2, v2); + values[3] = MM256_SET1_M128I(v2); } __m256i xv[8]; @@ -2964,8 +2964,8 @@ void iqk_convert_iq5_ks_q8_k_r8(int n, const void * vx, size_t bx, void * vy, in { auto v1 = _mm_loadu_si128((const __m128i *)iq5nl_values+0); auto v2 = _mm_loadu_si128((const __m128i *)iq5nl_values+1); - values[0] = MM256_SET_M128I(v1, v1); - values[1] = MM256_SET_M128I(v2, v2); + values[0] = MM256_SET1_M128I(v1); + values[1] = MM256_SET1_M128I(v2); } float drow[k_nr]; @@ -3050,8 +3050,8 @@ void iqk_convert_iq5_k_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int { auto v1 = _mm_loadu_si128((const __m128i *)iq5nl_values+0); auto v2 = _mm_loadu_si128((const __m128i *)iq5nl_values+1); - values[0] = MM256_SET_M128I(v1, v1); - values[1] = MM256_SET_M128I(v2, v2); + values[0] = MM256_SET1_M128I(v1); + values[1] = MM256_SET1_M128I(v2); } __m256i xv[8]; @@ -3118,8 +3118,8 @@ void iqk_convert_iq5_k_q8_0_r8(int n, const void * vx, size_t bx, void * vy, int { auto v1 = _mm_loadu_si128((const __m128i *)iq5nl_values+0); auto v2 = _mm_loadu_si128((const __m128i *)iq5nl_values+1); - values[0] = MM256_SET_M128I(v1, v1); - values[1] = MM256_SET_M128I(v2, v2); + values[0] = MM256_SET1_M128I(v1); + values[1] = MM256_SET1_M128I(v2); } __m256i xv[8]; @@ -3229,7 +3229,7 @@ void iqk_convert_iq6_k_q8_k_r8(int n, const void * vx, size_t bx, void * vy, int __m256i values[4]; for (int k = 0; k < 4; ++k) { auto values128 = _mm_loadu_si128((const __m128i *)iq6nl_values + k); - values[k] = MM256_SET_M128I(values128, values128); + values[k] = MM256_SET1_M128I(values128); } __m256i xv[8]; diff --git a/ggml/src/iqk/iqk_gemm_iquants.cpp b/ggml/src/iqk/iqk_gemm_iquants.cpp index 2ffe0310..de07199b 100644 --- a/ggml/src/iqk/iqk_gemm_iquants.cpp +++ b/ggml/src/iqk/iqk_gemm_iquants.cpp @@ -126,7 +126,7 @@ struct SignHelper { values[3] = _mm256_mask_sub_epi8(values[3], mask[3], _mm256_setzero_si256(), values[3]); #else auto s128 = _mm_loadu_si128((const __m128i *)sign_bits); - auto s256 = MM256_SET_M128I(s128, s128); + auto s256 = MM256_SET1_M128I(s128); __m256i aux256; auto shuffle = mask1; auto step = _mm256_set1_epi8(4); @@ -164,7 +164,7 @@ struct DequantizerIQ2XXS final : public BaseDequantizer { inline void new_block(int i, __m256i * scales) { auto sc16 = load_scales(i); - scales[0] = MM256_SET_M128I(sc16, sc16); + scales[0] = MM256_SET1_M128I(sc16); } inline void new_block_f(int i, __m256 * scales) { auto sc16 = load_scales(i); @@ -179,7 +179,7 @@ struct DequantizerIQ2XXS final : public BaseDequantizer { inline float new_block(int i, __m256i * scales, __m256i& mins) { auto sc16 = load_scales(i); mins = scb.shuffle(sc16); - scales[0] = MM256_SET_M128I(sc16, sc16); + scales[0] = MM256_SET1_M128I(sc16); return -d*minv; } @@ -252,8 +252,8 @@ struct DequantizerIQ2XS final : public BaseDequantizer { inline static void prepare_scales(const __m256i& all, __m256i * scales) { auto scales_l = _mm256_castsi256_si128(all); auto scales_h = _mm256_extractf128_si256(all, 1); - scales[0] = MM256_SET_M128I(scales_l, scales_l); - scales[1] = MM256_SET_M128I(scales_h, scales_h); + scales[0] = MM256_SET1_M128I(scales_l); + scales[1] = MM256_SET1_M128I(scales_h); } inline void new_block(int i, __m256i * scales) { @@ -321,8 +321,8 @@ struct DequantizerIQ2XS final : public BaseDequantizer { auto full = _mm256_or_si256(psb1, oddb); auto full_l = _mm256_castsi256_si128(full); auto full_h = _mm256_extractf128_si256(full, 1); - auto full_1 = MM256_SET_M128I(full_l, full_l); - auto full_2 = MM256_SET_M128I(full_h, full_h); + auto full_1 = MM256_SET1_M128I(full_l); + auto full_2 = MM256_SET1_M128I(full_h); sign_value(full_1, helper.shuff1, helper.mask, helper.mone, values[0]); sign_value(full_1, helper.shuff2, helper.mask, helper.mone, values[1]); sign_value(full_2, helper.shuff1, helper.mask, helper.mone, values[2]); @@ -397,8 +397,8 @@ struct DequantizerIQ2S final : public BaseDequantizer { inline static void prepare_scales(const __m256i& all, __m256i * scales) { auto scales_l = _mm256_castsi256_si128(all); auto scales_h = _mm256_extractf128_si256(all, 1); - scales[0] = MM256_SET_M128I(scales_l, scales_l); - scales[1] = MM256_SET_M128I(scales_h, scales_h); + scales[0] = MM256_SET1_M128I(scales_l); + scales[1] = MM256_SET1_M128I(scales_h); } inline void new_block(int i, __m256i * scales) { @@ -506,7 +506,7 @@ struct DequantizerIQ3XXS final : public BaseDequantizer { inline void new_block(int i, __m256i * scales) { auto scales16 = prepare_scales(i); - scales[0] = MM256_SET_M128I(scales16, scales16); + scales[0] = MM256_SET1_M128I(scales16); } inline void new_block_f(int i, __m256 * scales) { auto sc16 = prepare_scales(i); @@ -520,7 +520,7 @@ struct DequantizerIQ3XXS final : public BaseDequantizer { inline float new_block(int i, __m256i * scales, __m256i& mins) { auto scales16 = prepare_scales(i); mins = scb.shuffle(scales16); - scales[0] = MM256_SET_M128I(scales16, scales16); + scales[0] = MM256_SET1_M128I(scales16); return -d*minv; } @@ -638,7 +638,7 @@ struct DequantizerIQ3S final : public BaseDequantizer { } inline void new_block(int i, __m256i * scales) { auto scales16 = make_scales(i, d); - scales[0] = MM256_SET_M128I(scales16, scales16); + scales[0] = MM256_SET1_M128I(scales16); } inline void new_block_f(int i, __m256 * scales) { auto sc16 = make_scales(i, d); @@ -652,7 +652,7 @@ struct DequantizerIQ3S final : public BaseDequantizer { inline float new_block(int i, __m256i * scales, __m256i& mins) { auto scales16 = make_scales(i, d); mins = scb.shuffle(scales16); - scales[0] = MM256_SET_M128I(scales16, scales16); + scales[0] = MM256_SET1_M128I(scales16); return -minv*d; } @@ -1056,7 +1056,7 @@ static void mul_mat_iq2_xxs_r4_q8_k(int n, const void * vx, size_t bx, const Dat scales = _mm_maddubs_epi16(scales, _mm_set1_epi32(0x10080402)); scales = _mm_add_epi32(_mm_madd_epi16(_mm_set1_epi16(1), scales), _mm_set1_epi32(1)); #endif - auto scales32 = MM256_SET_M128I(scales, scales); + auto scales32 = MM256_SET1_M128I(scales); auto signs128 = _mm_and_si128(sas, _mm_set1_epi8(-2)); // 0xfe = -2 as signed. Needed to shutup compiler warning. signs128 = _mm_xor_si128(signs128, _mm_srli_epi16(signs128, 1)); #ifdef HAVE_FANCY_SIMD @@ -1073,7 +1073,7 @@ static void mul_mat_iq2_xxs_r4_q8_k(int n, const void * vx, size_t bx, const Dat isum[iy] = _mm256_add_epi32(isum[iy], _mm256_mullo_epi32(scales32, sumi)); } #else - auto signs = MM256_SET_M128I(signs128, signs128); + auto signs = MM256_SET1_M128I(signs128); auto shuffle = sign_shuffle; auto s1 = _mm256_or_si256(_mm256_cmpeq_epi8(_mm256_and_si256(_mm256_shuffle_epi8(signs, shuffle), smask), smask), _mm256_set1_epi8(1)); shuffle = _mm256_add_epi8(shuffle, m4); @@ -1173,7 +1173,7 @@ static void mul_mat_iq2_xs_r4_q8_k(int n, const void * vx, size_t bx, const Data isum[2*iy+1] = _mm256_add_epi32(isum[2*iy+1], _mm256_madd_epi16(scs[1], s34)); } #else - auto signs = MM256_SET_M128I(signs128, signs128); + auto signs = MM256_SET1_M128I(signs128); auto shuffle = sign_shuffle; auto s1 = _mm256_or_si256(_mm256_cmpeq_epi8(_mm256_and_si256(_mm256_shuffle_epi8(signs, shuffle), smask), smask), _mm256_set1_epi8(1)); shuffle = _mm256_add_epi8(shuffle, m4); @@ -1335,7 +1335,7 @@ static void mul_mat_iq2_xs_r4_q8_k_16(int n, const void * vx, size_t bx, const D isum[2*iy+1] = _mm256_add_epi32(isum[2*iy+1], _mm256_madd_epi16(scs[1], s34)); } #else - auto signs = MM256_SET_M128I(signs128, signs128); + auto signs = MM256_SET1_M128I(signs128); auto shuffle = sign_shuffle; auto s = _mm256_or_si256(_mm256_cmpeq_epi8(_mm256_and_si256(_mm256_shuffle_epi8(signs, shuffle), smask), smask), _mm256_set1_epi8(1)); shuffle = _mm256_add_epi8(shuffle, m4); @@ -1454,7 +1454,7 @@ static void mul_mat_iq2_s_r4_q8_k(int n, const void * vx, size_t bx, const DataI isum[2*iy+1] = _mm256_add_epi32(isum[2*iy+1], _mm256_madd_epi16(scs[1], s34)); } #else - auto signs = MM256_SET_M128I(signs128, signs128); + auto signs = MM256_SET1_M128I(signs128); auto shuffle = sign_shuffle; auto s1 = _mm256_or_si256(_mm256_cmpeq_epi8(_mm256_and_si256(_mm256_shuffle_epi8(signs, shuffle), smask), smask), _mm256_set1_epi8(1)); shuffle = _mm256_add_epi8(shuffle, m4); @@ -1613,7 +1613,7 @@ static void mul_mat_iq2_s_r4_q8_k_16(int n, const void * vx, size_t bx, const Da isum[2*iy+1] = _mm256_add_epi32(isum[2*iy+1], _mm256_madd_epi16(scs[1], s34)); } #else - auto signs = MM256_SET_M128I(signs128, signs128); + auto signs = MM256_SET1_M128I(signs128); auto shuffle = sign_shuffle; auto s = _mm256_or_si256(_mm256_cmpeq_epi8(_mm256_and_si256(_mm256_shuffle_epi8(signs, shuffle), smask), smask), _mm256_set1_epi8(1)); shuffle = _mm256_add_epi8(shuffle, m4); @@ -1703,7 +1703,7 @@ static void mul_mat_iq3_xxs_r4_q8_k(int n, const void * vx, size_t bx, const Dat //auto t2 = _mm_or_si128(_mm_srli_epi32(_mm_and_si128(scales, _mm_set1_epi32(0x00010000)), 14), _mm_srli_epi32(_mm_and_si128(scales, _mm_set1_epi32(0x01000000)), 21)); //scales = _mm_or_si128(_mm_slli_epi32(_mm_or_si128(t1, t2), 1), _mm_set1_epi32(1)); #endif - auto scales32 = MM256_SET_M128I(scales, scales); + auto scales32 = MM256_SET1_M128I(scales); auto signs128 = _mm_and_si128(sas, _mm_set1_epi8(-2)); // 0xfe = -2 as signed. Needed to shutup compiler warning. signs128 = _mm_xor_si128(signs128, _mm_srli_epi16(signs128, 1)); #ifdef HAVE_FANCY_SIMD @@ -1720,7 +1720,7 @@ static void mul_mat_iq3_xxs_r4_q8_k(int n, const void * vx, size_t bx, const Dat isum[iy] = _mm256_add_epi32(isum[iy], _mm256_mullo_epi32(scales32, sumi)); } #else - auto signs = MM256_SET_M128I(signs128, signs128); + auto signs = MM256_SET1_M128I(signs128); auto shuffle = sign_shuffle; auto s1 = _mm256_or_si256(_mm256_cmpeq_epi8(_mm256_and_si256(_mm256_shuffle_epi8(signs, shuffle), smask), smask), _mm256_set1_epi8(1)); shuffle = _mm256_add_epi8(shuffle, m4); diff --git a/ggml/src/iqk/iqk_gemm_kquants.cpp b/ggml/src/iqk/iqk_gemm_kquants.cpp index 88a5d191..8b481b10 100644 --- a/ggml/src/iqk/iqk_gemm_kquants.cpp +++ b/ggml/src/iqk/iqk_gemm_kquants.cpp @@ -22,7 +22,7 @@ struct Scales8K { const __m128i mins128 = _mm256_extracti128_si256(mins_and_scales, 1); accum_mins(mins128, q8, i, c, accd); const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0); - return MM256_SET_M128I(sc128, sc128); + return MM256_SET1_M128I(sc128); } #ifdef HAVE_FANCY_SIMD template @@ -58,8 +58,8 @@ inline void process_mins_16(const __m256i& all_scales, const Q8& q8, int i, floa inline void prepare_scales_16(const __m256i& all_scales, __m256i * scales) { const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0); const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1); - scales[0] = MM256_SET_M128I(l_scales, l_scales); - scales[1] = MM256_SET_M128I(h_scales, h_scales); + scales[0] = MM256_SET1_M128I(l_scales); + scales[1] = MM256_SET1_M128I(h_scales); } // Handles q3_K scales @@ -81,7 +81,7 @@ struct ScaleQ3 { struct Scale16 { inline void make_scales(const __m128i& scales8, __m512i * scales) const { - auto all_scales8 = MM256_SET_M128I(scales8, scales8); + auto all_scales8 = MM256_SET1_M128I(scales8); auto scales1 = _mm256_shuffle_epi8(all_scales8, shuffle1); auto scales2 = _mm256_shuffle_epi8(all_scales8, shuffle2); scales[0] = _mm512_cvtepi8_epi16(scales1); @@ -297,7 +297,7 @@ struct DequantizerIQ4XS final : public BaseDequantizer { prepare(x[i].qs); auto scales128 = siq4.make_scales(*(const uint32_t *)x[i].scales_l, x[i].scales_h); s8k.accum_mins(scales128, q8, i, -128.f*d, accd); - auto scales256 = MM256_SET_M128I(scales128, scales128); + auto scales256 = MM256_SET1_M128I(scales128); auto all_scales = _mm512_inserti32x8(_mm512_castsi256_si512(scales256), scales256, 1); scales[0] = _mm512_shuffle_epi8(all_scales, shuffles[0]); scales[1] = _mm512_shuffle_epi8(all_scales, shuffles[1]); @@ -610,7 +610,7 @@ struct DequantizerIQ4XS final : public BaseDequantizer { d = GGML_FP16_TO_FP32(x[i].d); auto scales128 = siq4.make_scales(*(const uint32_t *)x[i].scales_l, x[i].scales_h); s8k.accum_mins(scales128, q8, i, -128.f*d, accd); - return MM256_SET_M128I(scales128, scales128); + return MM256_SET1_M128I(scales128); } inline void prepare(int i, int j) { bits.prepare16(x[i].qs, j); @@ -727,7 +727,7 @@ static void mul_mat_qY_K_q8_K_T(int n, const void * vx, size_t bx, const DataInf // const __m128i mins128 = _mm256_extracti128_si256(mins_and_scales, 1); // accum_mins(mins128, q8, i, c, accd); // const __m128i sc128 = _mm256_extracti128_si256(mins_and_scales, 0); -// return MM256_SET_M128I(sc128, sc128); +// return MM256_SET1_M128I(sc128); // } // // inline void new_block(int i, const Q8& q8, __m256 * accd, __m512i * scales) { @@ -1042,7 +1042,7 @@ static void mul_mat_iq4_xs_r8_q8_k_avx2(int n, const void * vx, size_t bx, const #ifndef HAVE_VNNI256 auto s_shuffle = _mm256_set_epi64x(0x0f0e0f0e0d0c0d0c, 0x0b0a0b0a09080908, 0x0706070605040504, 0x0302030201000100); auto values128 = _mm_loadu_si128((const __m128i *)iq4k_values); - auto values = MM256_SET_M128I(values128, values128); + auto values = MM256_SET1_M128I(values128); #else auto values = load_iq4nl_values_256(); #endif @@ -1089,7 +1089,7 @@ static void mul_mat_iq4_xs_r8_q8_k_avx2(int n, const void * vx, size_t bx, const #endif for (int iy = 0; iy < nrc_y; ++iy) { auto y128 = _mm_loadu_si128((const __m128i*)q8.y[iy][ibl].qs+2*ib+0); - auto y = MM256_SET_M128I(y128, y128); + auto y = MM256_SET1_M128I(y128); #ifdef HAVE_VNNI256 auto sumi = _mm256_setzero_si256(); sumi = ggml_mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); @@ -1121,7 +1121,7 @@ static void mul_mat_iq4_xs_r8_q8_k_avx2(int n, const void * vx, size_t bx, const #endif for (int iy = 0; iy < nrc_y; ++iy) { auto y128 = _mm_loadu_si128((const __m128i*)q8.y[iy][ibl].qs+2*ib+1); - auto y = MM256_SET_M128I(y128, y128); + auto y = MM256_SET1_M128I(y128); #ifdef HAVE_VNNI256 auto sumi = _mm256_setzero_si256(); sumi = ggml_mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); @@ -1562,7 +1562,7 @@ static void mul_mat_q4_k_r4_q8_k(int n, const void * vx, size_t bx, const DataIn #else auto aux = _mm_set1_epi32(hd.val[ib]); aux = _mm_cvtepu8_epi16(_mm_unpacklo_epi8(aux, aux)); - auto scales_d = MM256_SET_M128I(aux, aux); + auto scales_d = MM256_SET1_M128I(aux); #endif auto bits1 = _mm256_loadu_si256((const __m256i *)iq4[ibl].qs+2*ib+0); auto bits2 = _mm256_loadu_si256((const __m256i *)iq4[ibl].qs+2*ib+1); @@ -1631,7 +1631,7 @@ static void mul_mat_q5_k_r4_q8_k(int n, const void * vx, size_t bx, const DataIn #else auto aux = _mm_set1_epi32(hd.val[ib]); aux = _mm_cvtepu8_epi16(_mm_unpacklo_epi8(aux, aux)); - auto scales_d = MM256_SET_M128I(aux, aux); + auto scales_d = MM256_SET1_M128I(aux); #endif auto lbits1 = _mm256_loadu_si256((const __m256i *)iq5[ibl].qs+2*ib+0); auto lbits2 = _mm256_loadu_si256((const __m256i *)iq5[ibl].qs+2*ib+1); @@ -1859,7 +1859,7 @@ static void mul_mat_q8_k_r8_q8_k(int n, const void * vx, size_t bx, const DataIn auto s3 = _mm256_sign_epi8(qx[3], qx[3]); for (int iy = 0; iy < nrc_y; ++iy) { auto y128 = _mm_loadu_si128((const __m128i*)q8.y[iy][ibl].qs+ib); - auto y = MM256_SET_M128I(y128, y128); + auto y = MM256_SET1_M128I(y128); #ifdef HAVE_VNNI256 isum[iy] = ggml_mm256_dpbusd_epi32(isum[iy], s0, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qx[0])); isum[iy] = ggml_mm256_dpbusd_epi32(isum[iy], s1, _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x55), qx[1])); @@ -1908,7 +1908,7 @@ static void mul_mat_q8_k_r16_q8_k(int n, const void * vx, size_t bx, const DataI qx[3] = _mm512_loadu_si512((const __m512i *)iq16[ibl].qs+4*ib+3); for (int iy = 0; iy < nrc_y; ++iy) { auto y128 = _mm_loadu_si128((const __m128i*)q8.y[iy][ibl].qs+ib); - auto y256 = MM256_SET_M128I(y128, y128); + auto y256 = MM256_SET1_M128I(y128); auto y = _mm512_inserti32x8(_mm512_castsi256_si512(y256), y256, 1); isum[iy] = _mm512_dpbusd_epi32(isum[iy], qx[0], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x00))); isum[iy] = _mm512_dpbusd_epi32(isum[iy], qx[1], _mm512_shuffle_epi32(y, _MM_PERM_ENUM(0x55))); @@ -2059,7 +2059,7 @@ static void mul_mat_q8_KV_r8_q8_KV(int n, const void * vx, size_t bx, const Data #endif for (int iy = 0; iy < nrc_y; ++iy) { auto y128 = _mm_loadu_si128((const __m128i*)q8y[iy]+ib); - auto y = MM256_SET_M128I(y128, y128); + auto y = MM256_SET1_M128I(y128); #ifdef HAVE_FANCY_SIMD acc[iy] = _mm256_dpbusd_epi32(acc[iy], qx[0], _mm256_shuffle_epi32(y, 0x00)); acc[iy] = _mm256_dpbusd_epi32(acc[iy], qx[1], _mm256_shuffle_epi32(y, 0x55)); @@ -2634,7 +2634,7 @@ void iqk_convert_iq4_xs_q8_k_r8(int n, const void * vx, size_t bx, void * vy, in block_q8_k_r * y = (block_q8_k_r *)vy; auto values128 = _mm_loadu_si128((const __m128i *)iq4k_values); - auto values = MM256_SET_M128I(values128, values128); + auto values = MM256_SET1_M128I(values128); int16_t ls[16]; float dnew[k_nr]; diff --git a/ggml/src/iqk/iqk_gemm_ktquants.cpp b/ggml/src/iqk/iqk_gemm_ktquants.cpp index 57702199..581b3cb2 100644 --- a/ggml/src/iqk/iqk_gemm_ktquants.cpp +++ b/ggml/src/iqk/iqk_gemm_ktquants.cpp @@ -220,13 +220,13 @@ struct Trellis3 { tmp[1] = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(val, 1)); for (int k = 0; k < 2; ++k) { auto vl = _mm256_castsi256_si128(tmp[k]); - auto v = MM256_SET_M128I(vl, vl); + auto v = MM256_SET1_M128I(vl); aux[8*k+0] = _mm256_shuffle_epi32(v, 0x00); aux[8*k+1] = _mm256_shuffle_epi32(v, 0x55); aux[8*k+2] = _mm256_shuffle_epi32(v, 0xaa); aux[8*k+3] = _mm256_shuffle_epi32(v, 0xff); auto vh = _mm256_extracti128_si256(tmp[k], 1); - v = MM256_SET_M128I(vh, vh); + v = MM256_SET1_M128I(vh); aux[8*k+4] = _mm256_shuffle_epi32(v, 0x00); aux[8*k+5] = _mm256_shuffle_epi32(v, 0x55); aux[8*k+6] = _mm256_shuffle_epi32(v, 0xaa); @@ -267,7 +267,7 @@ struct Trellis3 { __m256i aux[16]; for (int k = 0; k < 4; ++k) { auto v128 = _mm_add_epi32(_mm_cvtepu16_epi32(_mm_loadl_epi64((const __m128i *)(val + 4*k))), _mm_set1_epi32(v0)); - auto v = MM256_SET_M128I(v128, v128); + auto v = MM256_SET1_M128I(v128); aux[4*k+0] = _mm256_shuffle_epi32(v, 0x00); aux[4*k+1] = _mm256_shuffle_epi32(v, 0x55); aux[4*k+2] = _mm256_shuffle_epi32(v, 0xaa); diff --git a/ggml/src/iqk/iqk_gemm_legacy_quants.cpp b/ggml/src/iqk/iqk_gemm_legacy_quants.cpp index 4955d472..adbe8914 100644 --- a/ggml/src/iqk/iqk_gemm_legacy_quants.cpp +++ b/ggml/src/iqk/iqk_gemm_legacy_quants.cpp @@ -619,7 +619,7 @@ static inline __m128i load_unsigned_mxfp4_values_128() { static inline __m256i load_unsigned_mxfp4_values_256() { auto val128 = load_unsigned_mxfp4_values_128(); - return MM256_SET_M128I(val128, val128); + return MM256_SET1_M128I(val128); } #ifdef HAVE_FANCY_SIMD @@ -635,7 +635,7 @@ static inline __m128i load_mxfp4_values_128() { static inline __m256i load_mxfp4_values_256() { auto val128 = load_mxfp4_values_128(); - return MM256_SET_M128I(val128, val128); + return MM256_SET1_M128I(val128); } struct MXFP4_Dequantizer { @@ -896,7 +896,7 @@ static void mul_mat_iq4_nl_r4_q8_2(int n, const void * vx, size_t bx, const Data auto m1 = _mm256_set1_epi16(1); #endif auto values128 = _mm_loadu_si128((const __m128i *)iq4k_values); - auto values = MM256_SET_M128I(values128, values128); + auto values = MM256_SET1_M128I(values128); int nb = n / QK4_NL; __m256 acc[nrc_y] = {}; __m256i qs[4]; @@ -989,8 +989,8 @@ inline void prepare_q4_0_quants_avx2(const uint8_t * qs, __m256i * v, const __m2 inline __m256i accum_q4_0_quants(const __m256i * v, const int8_t * qs) { auto y4l = _mm_loadu_si128((const __m128i*)qs+0); auto y4h = _mm_loadu_si128((const __m128i*)qs+1); - auto yl = MM256_SET_M128I(y4l, y4l); - auto yh = MM256_SET_M128I(y4h, y4h); + auto yl = MM256_SET1_M128I(y4l); + auto yh = MM256_SET1_M128I(y4h); #ifdef HAVE_VNNI256 auto sumi = _mm256_setzero_si256(); sumi = ggml_mm256_dpbusd_epi32(sumi, v[0], _mm256_shuffle_epi32(yl, 0x00)); @@ -1135,8 +1135,8 @@ static void mul_mat_q4_0_r8_q8_2(int n, const void * vx, size_t bx, const DataIn auto dot = [&qx] (const int8_t * qy) { auto y4l = _mm_loadu_si128((const __m128i*)qy+0); auto y4h = _mm_loadu_si128((const __m128i*)qy+1); - auto y8l = MM256_SET_M128I(y4l, y4l); - auto y8h = MM256_SET_M128I(y4h, y4h); + auto y8l = MM256_SET1_M128I(y4l); + auto y8h = MM256_SET1_M128I(y4h); auto yl = _mm512_inserti32x8(_mm512_castsi256_si512(y8l), y8l, 1); auto yh = _mm512_inserti32x8(_mm512_castsi256_si512(y8h), y8h, 1); auto sumi = _mm512_setzero_si512(); @@ -1539,8 +1539,8 @@ static void mul_mat_q6_0_r4_q8_2(int n, const void * vx, size_t bx, const DataIn inline __m512i qx_r8_q8_dot_product(const __m512i * qx, const int8_t * y) { auto y4l = _mm_loadu_si128((const __m128i*)y+0); auto y4h = _mm_loadu_si128((const __m128i*)y+1); - auto y8l = MM256_SET_M128I(y4l, y4l); - auto y8h = MM256_SET_M128I(y4h, y4h); + auto y8l = MM256_SET1_M128I(y4l); + auto y8h = MM256_SET1_M128I(y4h); auto yl = _mm512_inserti32x8(_mm512_castsi256_si512(y8l), y8l, 1); auto yh = _mm512_inserti32x8(_mm512_castsi256_si512(y8h), y8h, 1); auto sumi = _mm512_setzero_si512(); @@ -1557,8 +1557,8 @@ inline __m512i qx_r8_q8_dot_product(const __m512i * qx, const int8_t * y) { inline __m256i qx_r8_q8_dot_product(const __m256i * qx, const int8_t * y) { auto y4l = _mm_loadu_si128((const __m128i*)y+0); auto y4h = _mm_loadu_si128((const __m128i*)y+1); - auto yl = MM256_SET_M128I(y4l, y4l); - auto yh = MM256_SET_M128I(y4h, y4h); + auto yl = MM256_SET1_M128I(y4l); + auto yh = MM256_SET1_M128I(y4h); auto sumi = _mm256_setzero_si256(); sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(yl, 0x00)); sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(yl, 0x55)); @@ -1677,7 +1677,7 @@ static void mul_mat_q8_0_r8_q8_2(int n, const void * vx, size_t bx, const DataIn __m256i qx[4], sx[4]; auto dot = [&qx, &sx, &m1] (const int8_t * qy) { auto y128 = _mm_loadu_si128((const __m128i*)qy); - auto y = MM256_SET_M128I(y128, y128); + auto y = MM256_SET1_M128I(y128); #ifdef HAVE_VNNI256 auto sumi = _mm256_setzero_si256(); sumi = ggml_mm256_dpbusd_epi32(sumi, sx[0], _mm256_sign_epi8(_mm256_shuffle_epi32(y, 0x00), qx[0])); @@ -1772,7 +1772,7 @@ static void mul_mat_q8_1_r8_q8_2(int n, const void * vx, size_t bx, const DataIn __m256i qx[4]; auto dot = [&qx] (const int8_t * qy) { auto y128 = _mm_loadu_si128((const __m128i*)qy); - auto y = MM256_SET_M128I(y128, y128); + auto y = MM256_SET1_M128I(y128); auto sumi = _mm256_setzero_si256(); sumi = _mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00)); sumi = _mm256_dpbusd_epi32(sumi, qx[1], _mm256_shuffle_epi32(y, 0x55)); @@ -1876,7 +1876,7 @@ static void mul_mat_q8_1_r8_q8_2(int n, const void * vx, size_t bx, const DataIn __m256i qx[4]; auto dot = [&qx] (const int8_t * qy) { auto y128 = _mm_loadu_si128((const __m128i*)qy); - auto y = MM256_SET_M128I(y128, y128); + auto y = MM256_SET1_M128I(y128); #ifdef HAVE_VNNI256 auto sumi = _mm256_setzero_si256(); sumi = ggml_mm256_dpbusd_epi32(sumi, qx[0], _mm256_shuffle_epi32(y, 0x00));