The Gaudi Framework  master (37c0b60a)
Classes | Macros | Functions | Variables
instrset.h File Reference
#include <stdint.h>
#include <stdlib.h>
Include dependency graph for instrset.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes

class  Const_int_t< n >
 
class  Const_uint_t< n >
 
struct  EList< T, N >
 

Macros

#define INSTRSET_H   20102
 
#define ALLOW_FP_PERMUTE   true
 
#define INSTRSET   0
 
#define const_int(n)   ( Const_int_t<n>() )
 
#define const_uint(n)   ( Const_uint_t<n>() )
 

Functions

int instrset_detect (void)
 
bool hasFMA3 (void)
 
bool hasFMA4 (void)
 
bool hasXOP (void)
 
bool hasAVX512ER (void)
 
bool hasAVX512VBMI (void)
 
bool hasAVX512VBMI2 (void)
 
int physicalProcessors (int *logical_processors=0)
 
constexpr int bit_scan_reverse_const (uint64_t const n)
 
template<typename V >
constexpr auto get_inttype ()
 
template<int N>
constexpr auto zero_mask (int const (&a)[N])
 
template<typename V >
constexpr auto zero_mask_broad (int const (&A)[V::size()])
 
template<int N, int B>
constexpr uint64_t make_bit_mask (int const (&a)[N])
 
template<typename V >
constexpr auto make_broad_mask (uint64_t const m)
 
template<typename V >
constexpr auto perm_mask_broad (int const (&A)[V::size()])
 
template<typename V >
constexpr uint64_t perm_flags (int const (&a)[V::size()])
 
template<int N>
constexpr uint64_t compress_mask (int const (&a)[N])
 
template<int N>
constexpr uint64_t expand_mask (int const (&a)[N])
 
template<typename V >
constexpr uint64_t perm16_flags (int const (&a)[V::size()])
 
template<typename V , int oppos = 0>
constexpr auto pshufb_mask (int const (&A)[V::size()])
 
template<int N>
constexpr EList< int, N/2 > largeblock_perm (int const (&a)[N])
 
template<typename V >
constexpr uint64_t blend_flags (int const (&a)[V::size()])
 
template<int N, int dozero>
constexpr EList< int, 2 *N > blend_perm_indexes (int const (&a)[N])
 
template<int N>
constexpr EList< int, N/2 > largeblock_indexes (int const (&a)[N])
 
template<typename dummy >
void blend2 ()
 
template<typename dummy >
void blend4 ()
 
template<typename dummy >
void blend8 ()
 
template<typename dummy >
void blend16 ()
 
template<typename dummy >
void blend32 ()
 
template<int N, int dozero, int src1, int src2>
constexpr EList< int, N > blend_half_indexes (int const (&a)[N])
 
template<typename W , int... i0>
auto blend_half (W const &a, W const &b)
 

Variables

constexpr int V_DC = -256
 
const int perm_zeroing = 1
 
const int perm_perm = 2
 
const int perm_allzero = 4
 
const int perm_largeblock = 8
 
const int perm_addz = 0x10
 
const int perm_addz2 = 0x20
 
const int perm_cross_lane = 0x40
 
const int perm_same_pattern = 0x80
 
const int perm_punpckh = 0x100
 
const int perm_punpckl = 0x200
 
const int perm_rotate
 
const int perm_shright
 
const int perm_shleft
 
const int perm_rotate_big
 
const int perm_broadcast = 0x8000
 
const int perm_zext = 0x10000
 
const int perm_compress = 0x20000
 
const int perm_expand = 0x40000
 
const int perm_outofrange = 0x10000000
 
const int perm_rot_count = 32
 
const int perm_ipattern
 
const int blend_zeroing = 1
 
const int blend_allzero = 2
 
const int blend_largeblock = 4
 
const int blend_addz = 8
 
const int blend_a = 0x10
 
const int blend_b = 0x20
 
const int blend_perma = 0x40
 
const int blend_permb = 0x80
 
const int blend_cross_lane = 0x100
 
const int blend_same_pattern = 0x200
 
const int blend_punpckhab = 0x1000
 
const int blend_punpckhba = 0x2000
 
const int blend_punpcklab = 0x4000
 
const int blend_punpcklba = 0x8000
 
const int blend_rotateab = 0x10000
 
const int blend_rotateba = 0x20000
 
const int blend_shufab = 0x40000
 
const int blend_shufba = 0x80000
 
const int blend_rotate_big = 0x100000
 
const int blend_outofrange = 0x10000000
 
const int blend_shufpattern = 32
 
const int blend_rotpattern = 40
 

Macro Definition Documentation

◆ ALLOW_FP_PERMUTE

#define ALLOW_FP_PERMUTE   true

Definition at line 30 of file instrset.h.

◆ const_int

#define const_int (   n)    ( Const_int_t<n>() )

Definition at line 406 of file instrset.h.

◆ const_uint

#define const_uint (   n)    ( Const_uint_t<n>() )

Definition at line 407 of file instrset.h.

◆ INSTRSET

#define INSTRSET   0

Definition at line 78 of file instrset.h.

◆ INSTRSET_H

#define INSTRSET_H   20102

Definition at line 25 of file instrset.h.

Function Documentation

◆ bit_scan_reverse_const()

constexpr int bit_scan_reverse_const ( uint64_t const  n)
constexpr

Definition at line 380 of file instrset.h.

380  {
381  if ( n == 0 ) return -1;
382  uint64_t a = n, b = 0, j = 64, k = 0;
383  do {
384  j >>= 1;
385  k = (uint64_t)1 << j;
386  if ( a >= k ) {
387  a >>= j;
388  b += j;
389  }
390  } while ( j > 0 );
391  return int( b );
392  }

◆ blend16()

template<typename dummy >
void blend16 ( )

Definition at line 1297 of file instrset.h.

1297 {}

◆ blend2()

template<typename dummy >
void blend2 ( )

Definition at line 1291 of file instrset.h.

1291 {}

◆ blend32()

template<typename dummy >
void blend32 ( )

Definition at line 1299 of file instrset.h.

1299 {}

◆ blend4()

template<typename dummy >
void blend4 ( )

Definition at line 1293 of file instrset.h.

1293 {}

◆ blend8()

template<typename dummy >
void blend8 ( )

Definition at line 1295 of file instrset.h.

1295 {}

◆ blend_flags()

template<typename V >
constexpr uint64_t blend_flags ( int const (&)  a[V::size()])
constexpr

Definition at line 1036 of file instrset.h.

1036  {
1037  // a is a reference to a constexpr array of permutation indexes
1038  // V is a vector class
1039  constexpr int N = V::size(); // number of elements
1040  uint64_t r = blend_largeblock | blend_same_pattern | blend_allzero; // return value
1041  uint32_t iu = 0; // loop counter
1042  int32_t ii = 0; // loop counter
1043  int ix = 0; // index number i
1044  const uint32_t nlanes = sizeof( V ) / 16; // number of 128-bit lanes
1045  const uint32_t lanesize = N / nlanes; // elements per lane
1046  uint32_t lane = 0; // current lane
1047  uint32_t rot = 999; // rotate left count
1048  int lanepattern[lanesize] = { 0 }; // pattern in each lane
1049  if ( lanesize == 2 && N <= 8 ) {
1050  r |= blend_shufab | blend_shufba; // check if it fits shufpd
1051  }
1052 
1053  for ( ii = 0; ii < N; ii++ ) { // loop through indexes
1054  ix = a[ii]; // index
1055  if ( ix < 0 ) {
1056  if ( ix == -1 )
1057  r |= blend_zeroing; // set to zero
1058  else if ( ix != V_DC ) {
1059  r = blend_outofrange;
1060  break; // illegal index
1061  }
1062  } else { // ix >= 0
1063  r &= ~blend_allzero;
1064  if ( ix < N ) {
1065  r |= blend_a; // data from a
1066  if ( ix != ii ) r |= blend_perma; // permutation of a
1067  } else if ( ix < 2 * N ) {
1068  r |= blend_b; // data from b
1069  if ( ix != ii + N ) r |= blend_permb; // permutation of b
1070  } else {
1071  r = blend_outofrange;
1072  break; // illegal index
1073  }
1074  }
1075  // check if pattern fits a larger block size:
1076  // even indexes must be even, odd indexes must fit the preceding even index + 1
1077  if ( ( ii & 1 ) == 0 ) { // even index
1078  if ( ix >= 0 && ( ix & 1 ) ) r &= ~blend_largeblock; // not even. does not fit larger block size
1079  int iy = a[ii + 1]; // next odd index
1080  if ( iy >= 0 && ( iy & 1 ) == 0 ) r &= ~blend_largeblock; // not odd. does not fit larger block size
1081  if ( ix >= 0 && iy >= 0 && iy != ix + 1 ) r &= ~blend_largeblock; // does not fit preceding index + 1
1082  if ( ix == -1 && iy >= 0 ) r |= blend_addz; // needs additional zeroing at current block size
1083  if ( iy == -1 && ix >= 0 ) r |= blend_addz; // needs additional zeroing at current block size
1084  }
1085  lane = (uint32_t)ii / lanesize; // current lane
1086  if ( lane == 0 ) { // first lane, or no pattern yet
1087  lanepattern[ii] = ix; // save pattern
1088  }
1089  // check if crossing lanes
1090  if ( ix >= 0 ) {
1091  uint32_t lanei = uint32_t( ix & ~N ) / lanesize; // source lane
1092  if ( lanei != lane ) {
1093  r |= blend_cross_lane; // crossing lane
1094  }
1095  if ( lanesize == 2 ) { // check if it fits pshufd
1096  if ( lanei != lane ) r &= ~( blend_shufab | blend_shufba );
1097  if ( ( ( ( ix & N ) != 0 ) ^ ii ) & 1 )
1098  r &= ~blend_shufab;
1099  else
1100  r &= ~blend_shufba;
1101  }
1102  }
1103  // check if same pattern in all lanes
1104  if ( lane != 0 && ix >= 0 ) { // not first lane
1105  int j = ii - int( lane * lanesize ); // index into lanepattern
1106  int jx = ix - int( lane * lanesize ); // pattern within lane
1107  if ( jx < 0 || ( jx & ~N ) >= (int)lanesize ) r &= ~blend_same_pattern; // source is in another lane
1108  if ( lanepattern[j] < 0 ) {
1109  lanepattern[j] = jx; // pattern not known from previous lane
1110  } else {
1111  if ( lanepattern[j] != jx ) r &= ~blend_same_pattern; // not same pattern
1112  }
1113  }
1114  }
1115  if ( !( r & blend_largeblock ) ) r &= ~blend_addz; // remove irrelevant flag
1116  if ( r & blend_cross_lane ) r &= ~blend_same_pattern; // remove irrelevant flag
1117  if ( !( r & ( blend_perma | blend_permb ) ) ) {
1118  return r; // no permutation. more checks are superfluous
1119  }
1120  if ( r & blend_same_pattern ) {
1121  // same pattern in all lanes. check if it fits unpack patterns
1122  r |= blend_punpckhab | blend_punpckhba | blend_punpcklab | blend_punpcklba;
1123  for ( iu = 0; iu < lanesize; iu++ ) { // loop through lanepattern
1124  ix = lanepattern[iu];
1125  if ( ix >= 0 ) {
1126  if ( (uint32_t)ix != iu / 2 + ( iu & 1 ) * N ) r &= ~blend_punpcklab;
1127  if ( (uint32_t)ix != iu / 2 + ( ( iu & 1 ) ^ 1 ) * N ) r &= ~blend_punpcklba;
1128  if ( (uint32_t)ix != ( iu + lanesize ) / 2 + ( iu & 1 ) * N ) r &= ~blend_punpckhab;
1129  if ( (uint32_t)ix != ( iu + lanesize ) / 2 + ( ( iu & 1 ) ^ 1 ) * N ) r &= ~blend_punpckhba;
1130  }
1131  }
1132 #if INSTRSET >= 4 // SSSE3. check if it fits palignr
1133  for ( iu = 0; iu < lanesize; iu++ ) {
1134  ix = lanepattern[iu];
1135  if ( ix >= 0 ) {
1136  uint32_t t = ix & ~N;
1137  if ( ix & N ) t += lanesize;
1138  uint32_t tb = ( t + 2 * lanesize - iu ) % ( lanesize * 2 );
1139  if ( rot == 999 ) {
1140  rot = tb;
1141  } else { // check if fit
1142  if ( rot != tb ) rot = 1000;
1143  }
1144  }
1145  }
1146  if ( rot < 999 ) { // firs palignr
1147  if ( rot < lanesize ) {
1148  r |= blend_rotateba;
1149  } else {
1150  r |= blend_rotateab;
1151  }
1152  const uint32_t elementsize = sizeof( V ) / N;
1153  r |= uint64_t( ( rot & ( lanesize - 1 ) ) * elementsize ) << blend_rotpattern;
1154  }
1155 #endif
1156  if ( lanesize == 4 ) {
1157  // check if it fits shufps
1158  r |= blend_shufab | blend_shufba;
1159  for ( ii = 0; ii < 2; ii++ ) {
1160  ix = lanepattern[ii];
1161  if ( ix >= 0 ) {
1162  if ( ix & N )
1163  r &= ~blend_shufab;
1164  else
1165  r &= ~blend_shufba;
1166  }
1167  }
1168  for ( ; ii < 4; ii++ ) {
1169  ix = lanepattern[ii];
1170  if ( ix >= 0 ) {
1171  if ( ix & N )
1172  r &= ~blend_shufba;
1173  else
1174  r &= ~blend_shufab;
1175  }
1176  }
1177  if ( r & ( blend_shufab | blend_shufba ) ) { // fits shufps/shufpd
1178  uint8_t shufpattern = 0; // get pattern
1179  for ( iu = 0; iu < lanesize; iu++ ) { shufpattern |= ( lanepattern[iu] & 3 ) << iu * 2; }
1180  r |= (uint64_t)shufpattern << blend_shufpattern; // return pattern
1181  }
1182  }
1183  } else if ( nlanes > 1 ) { // not same pattern in all lanes
1184  rot = 999; // check if it fits big rotate
1185  for ( ii = 0; ii < N; ii++ ) {
1186  ix = a[ii];
1187  if ( ix >= 0 ) {
1188  uint32_t rot2 = ( ix + 2 * N - ii ) % ( 2 * N ); // rotate count
1189  if ( rot == 999 ) {
1190  rot = rot2; // save rotate count
1191  } else if ( rot != rot2 ) {
1192  rot = 1000;
1193  break; // does not fit big rotate
1194  }
1195  }
1196  }
1197  if ( rot < 2 * N ) { // fits big rotate
1198  r |= blend_rotate_big | (uint64_t)rot << blend_rotpattern;
1199  }
1200  }
1201  if ( lanesize == 2 && ( r & ( blend_shufab | blend_shufba ) ) ) { // fits shufpd. Get pattern
1202  for ( ii = 0; ii < N; ii++ ) { r |= uint64_t( a[ii] & 1 ) << ( blend_shufpattern + ii ); }
1203  }
1204  return r;
1205  }

◆ blend_half()

template<typename W , int... i0>
auto blend_half ( W const &  a,
W const &  b 
)

Definition at line 1352 of file instrset.h.

1352  {
1353  typedef decltype( a.get_low() ) V; // type for half-size vector
1354  constexpr int N = V::size(); // size of half-size vector
1355  static_assert( sizeof...( i0 ) == N, "wrong number of indexes in blend_half" );
1356  constexpr int ind[N] = { i0... }; // array of indexes
1357 
1358  // lambda to find which of the four possible sources are used
1359  // return: EList<int, 5> containing a list of up to 4 sources. The last element is the number of sources used
1360  auto listsources = []( int const n, int const( &ind )[N] ) constexpr {
1361  bool source_used[4] = { false, false, false, false }; // list of sources used
1362  int i = 0;
1363  for ( i = 0; i < n; i++ ) {
1364  int ix = ind[i]; // index
1365  if ( ix >= 0 ) {
1366  int src = ix / n; // source used
1367  source_used[src & 3] = true;
1368  }
1369  }
1370  // return a list of sources used. The last element is the number of sources used
1371  EList<int, 5> sources = { { 0 } };
1372  int nsrc = 0; // number of sources
1373  for ( i = 0; i < 4; i++ ) {
1374  if ( source_used[i] ) { sources.a[nsrc++] = i; }
1375  }
1376  sources.a[4] = nsrc;
1377  return sources;
1378  };
1379  // list of sources used
1380  constexpr EList<int, 5> sources = listsources( N, ind );
1381  constexpr int nsrc = sources.a[4]; // number of sources used
1382 
1383  if constexpr ( nsrc == 0 ) { // no sources
1384  return V( 0 );
1385  }
1386  // get indexes for the first one or two sources
1387  constexpr int uindex = ( nsrc > 2 ) ? 1 : 2; // unused elements set to zero if two blends are combined
1388  constexpr EList<int, N> L = blend_half_indexes<N, uindex, sources.a[0], sources.a[1]>( ind );
1389  V x0;
1390  V src0 = selectblend<W, sources.a[0]>( a, b ); // first source
1391  V src1 = selectblend<W, sources.a[1]>( a, b ); // second source
1392  if constexpr ( N == 2 ) {
1393  x0 = blend2<L.a[0], L.a[1]>( src0, src1 );
1394  } else if constexpr ( N == 4 ) {
1395  x0 = blend4<L.a[0], L.a[1], L.a[2], L.a[3]>( src0, src1 );
1396  } else if constexpr ( N == 8 ) {
1397  x0 = blend8<L.a[0], L.a[1], L.a[2], L.a[3], L.a[4], L.a[5], L.a[6], L.a[7]>( src0, src1 );
1398  } else if constexpr ( N == 16 ) {
1399  x0 = blend16<L.a[0], L.a[1], L.a[2], L.a[3], L.a[4], L.a[5], L.a[6], L.a[7], L.a[8], L.a[9], L.a[10], L.a[11],
1400  L.a[12], L.a[13], L.a[14], L.a[15]>( src0, src1 );
1401  } else if constexpr ( N == 32 ) {
1402  x0 = blend32<L.a[0], L.a[1], L.a[2], L.a[3], L.a[4], L.a[5], L.a[6], L.a[7], L.a[8], L.a[9], L.a[10], L.a[11],
1403  L.a[12], L.a[13], L.a[14], L.a[15], L.a[16], L.a[17], L.a[18], L.a[19], L.a[20], L.a[21], L.a[22],
1404  L.a[23], L.a[24], L.a[25], L.a[26], L.a[27], L.a[28], L.a[29], L.a[30], L.a[31]>( src0, src1 );
1405  }
1406  if constexpr ( nsrc > 2 ) { // get last one or two sources
1407  constexpr EList<int, N> M = blend_half_indexes<N, 1, sources.a[2], sources.a[3]>( ind );
1408  V x1;
1409  V src2 = selectblend<W, sources.a[2]>( a, b ); // third source
1410  V src3 = selectblend<W, sources.a[3]>( a, b ); // fourth source
1411  if constexpr ( N == 2 ) {
1412  x1 = blend2<M.a[0], M.a[1]>( src0, src1 );
1413  } else if constexpr ( N == 4 ) {
1414  x1 = blend4<M.a[0], M.a[1], M.a[2], M.a[3]>( src2, src3 );
1415  } else if constexpr ( N == 8 ) {
1416  x1 = blend8<M.a[0], M.a[1], M.a[2], M.a[3], M.a[4], M.a[5], M.a[6], M.a[7]>( src2, src3 );
1417  } else if constexpr ( N == 16 ) {
1418  x1 = blend16<M.a[0], M.a[1], M.a[2], M.a[3], M.a[4], M.a[5], M.a[6], M.a[7], M.a[8], M.a[9], M.a[10], M.a[11],
1419  M.a[12], M.a[13], M.a[14], M.a[15]>( src2, src3 );
1420  } else if constexpr ( N == 32 ) {
1421  x1 = blend32<M.a[0], M.a[1], M.a[2], M.a[3], M.a[4], M.a[5], M.a[6], M.a[7], M.a[8], M.a[9], M.a[10], M.a[11],
1422  M.a[12], M.a[13], M.a[14], M.a[15], M.a[16], M.a[17], M.a[18], M.a[19], M.a[20], M.a[21], M.a[22],
1423  M.a[23], M.a[24], M.a[25], M.a[26], M.a[27], M.a[28], M.a[29], M.a[30], M.a[31]>( src2, src3 );
1424  }
1425  x0 |= x1; // combine result of two blends. Unused elements are zero
1426  }
1427  return x0;
1428  }

◆ blend_half_indexes()

template<int N, int dozero, int src1, int src2>
constexpr EList<int, N> blend_half_indexes ( int const (&)  a[N])
constexpr

Definition at line 1308 of file instrset.h.

1308  {
1309  // a is a reference to a constexpr array of permutation indexes
1310  EList<int, N> list = { { 0 } }; // list to return
1311  int u = dozero ? -1 : V_DC; // value to use for unused entries
1312  int j = 0; // loop counter
1313 
1314  for ( j = 0; j < N; j++ ) { // loop through indexes
1315  int ix = a[j]; // current index
1316  if ( ix < 0 ) { // zero or don't care
1317  list.a[j] = ( dozero == 2 ) ? ix : u;
1318  } else {
1319  int src = ix / N; // source
1320  if ( src == src1 ) {
1321  list.a[j] = ix & ( N - 1 );
1322  } else if ( src == src2 ) {
1323  list.a[j] = ( ix & ( N - 1 ) ) + N;
1324  } else
1325  list.a[j] = u;
1326  }
1327  }
1328  return list;
1329  }

◆ blend_perm_indexes()

template<int N, int dozero>
constexpr EList<int, 2 * N> blend_perm_indexes ( int const (&)  a[N])
constexpr

Definition at line 1213 of file instrset.h.

1213  {
1214  // a is a reference to a constexpr array of permutation indexes
1215  EList<int, 2 * N> list = { { 0 } }; // list to return
1216  int u = dozero ? -1 : V_DC; // value to use for unused entries
1217  int j = 0;
1218 
1219  for ( j = 0; j < N; j++ ) { // loop through indexes
1220  int ix = a[j]; // current index
1221  if ( ix < 0 ) { // zero or don't care
1222  if ( dozero == 2 ) {
1223  // list.a[j] = list.a[j + N] = ix; // fails in gcc in complicated cases
1224  list.a[j] = ix;
1225  list.a[j + N] = ix;
1226  } else {
1227  // list.a[j] = list.a[j + N] = u;
1228  list.a[j] = u;
1229  list.a[j + N] = u;
1230  }
1231  } else if ( ix < N ) { // value from a
1232  list.a[j] = ix;
1233  list.a[j + N] = u;
1234  } else {
1235  list.a[j] = u; // value from b
1236  list.a[j + N] = ix - N;
1237  }
1238  }
1239  return list;
1240  }

◆ compress_mask()

template<int N>
constexpr uint64_t compress_mask ( int const (&)  a[N])
constexpr

Definition at line 823 of file instrset.h.

823  {
824  // a is a reference to a constexpr array of permutation indexes
825  int ix = 0, lasti = -1, lastp = -1;
826  uint64_t m = 0;
827  int i = 0;
828  int j = 1; // loop counters
829  for ( i = 0; i < N; i++ ) {
830  ix = a[i]; // permutation index
831  if ( ix >= 0 ) {
832  m |= (uint64_t)1 << ix; // mask for compression source
833  for ( j = 1; j < i - lastp; j++ ) {
834  m |= (uint64_t)1 << ( lasti + j ); // dummy filling source
835  }
836  lastp = i;
837  lasti = ix;
838  }
839  }
840  return m;
841  }

◆ expand_mask()

template<int N>
constexpr uint64_t expand_mask ( int const (&)  a[N])
constexpr

Definition at line 847 of file instrset.h.

847  {
848  // a is a reference to a constexpr array of permutation indexes
849  int ix = 0, lasti = -1, lastp = -1;
850  uint64_t m = 0;
851  int i = 0;
852  int j = 1;
853  for ( i = 0; i < N; i++ ) {
854  ix = a[i]; // permutation index
855  if ( ix >= 0 ) {
856  m |= (uint64_t)1 << i; // mask for expansion destination
857  for ( j = 1; j < ix - lasti; j++ ) {
858  m |= (uint64_t)1 << ( lastp + j ); // dummy filling destination
859  }
860  lastp = i;
861  lasti = ix;
862  }
863  }
864  return m;
865  }

◆ get_inttype()

template<typename V >
constexpr auto get_inttype ( )
constexpr

Definition at line 468 of file instrset.h.

468  {
469  constexpr int elementsize = sizeof( V ) / V::size(); // size of vector elements
470 
471  if constexpr ( elementsize >= 8 ) {
472  return -int64_t( 1 );
473  } else if constexpr ( elementsize >= 4 ) {
474  return int32_t( -1 );
475  } else if constexpr ( elementsize >= 2 ) {
476  return int16_t( -1 );
477  } else {
478  return int8_t( -1 );
479  }
480  }

◆ hasAVX512ER()

bool hasAVX512ER ( void  )

Definition at line 142 of file instrset_detect.cpp.

142  {
143  if ( instrset_detect() < 9 ) return false; // must have AVX512F
144  int abcd[4]; // cpuid results
145  cpuid( abcd, 7 ); // call cpuid function 7
146  return ( ( abcd[1] & ( 1 << 27 ) ) != 0 ); // ebx bit 27 indicates AVX512ER
147  }

◆ hasAVX512VBMI()

bool hasAVX512VBMI ( void  )

Definition at line 150 of file instrset_detect.cpp.

150  {
151  if ( instrset_detect() < 10 ) return false; // must have AVX512BW
152  int abcd[4]; // cpuid results
153  cpuid( abcd, 7 ); // call cpuid function 7
154  return ( ( abcd[2] & ( 1 << 1 ) ) != 0 ); // ecx bit 1 indicates AVX512VBMI
155  }

◆ hasAVX512VBMI2()

bool hasAVX512VBMI2 ( void  )

Definition at line 158 of file instrset_detect.cpp.

158  {
159  if ( instrset_detect() < 10 ) return false; // must have AVX512BW
160  int abcd[4]; // cpuid results
161  cpuid( abcd, 7 ); // call cpuid function 7
162  return ( ( abcd[2] & ( 1 << 6 ) ) != 0 ); // ecx bit 6 indicates AVX512VBMI2
163  }

◆ hasFMA3()

bool hasFMA3 ( void  )

Definition at line 110 of file instrset_detect.cpp.

110  {
111  if ( instrset_detect() < 7 ) return false; // must have AVX
112  int abcd[4]; // cpuid results
113  cpuid( abcd, 1 ); // call cpuid function 1
114  return ( ( abcd[2] & ( 1 << 12 ) ) != 0 ); // ecx bit 12 indicates FMA3
115  }

◆ hasFMA4()

bool hasFMA4 ( void  )

Definition at line 118 of file instrset_detect.cpp.

118  {
119  if ( instrset_detect() < 7 ) return false; // must have AVX
120  int abcd[4]; // cpuid results
121  cpuid( abcd, 0x80000001 ); // call cpuid function 0x80000001
122  return ( ( abcd[2] & ( 1 << 16 ) ) != 0 ); // ecx bit 16 indicates FMA4
123  }

◆ hasXOP()

bool hasXOP ( void  )

Definition at line 126 of file instrset_detect.cpp.

126  {
127  if ( instrset_detect() < 7 ) return false; // must have AVX
128  int abcd[4]; // cpuid results
129  cpuid( abcd, 0x80000001 ); // call cpuid function 0x80000001
130  return ( ( abcd[2] & ( 1 << 11 ) ) != 0 ); // ecx bit 11 indicates XOP
131  }

◆ instrset_detect()

int instrset_detect ( void  )

Definition at line 63 of file instrset_detect.cpp.

63  {
64 
65  static int iset = -1; // remember value for next call
66  if ( iset >= 0 ) {
67  return iset; // called before
68  }
69  iset = 0; // default value
70  int abcd[4] = { 0, 0, 0, 0 }; // cpuid results
71  cpuid( abcd, 0 ); // call cpuid function 0
72  if ( abcd[0] == 0 ) return iset; // no further cpuid function supported
73  cpuid( abcd, 1 ); // call cpuid function 1 for feature flags
74  if ( ( abcd[3] & ( 1 << 0 ) ) == 0 ) return iset; // no floating point
75  if ( ( abcd[3] & ( 1 << 23 ) ) == 0 ) return iset; // no MMX
76  if ( ( abcd[3] & ( 1 << 15 ) ) == 0 ) return iset; // no conditional move
77  if ( ( abcd[3] & ( 1 << 24 ) ) == 0 ) return iset; // no FXSAVE
78  if ( ( abcd[3] & ( 1 << 25 ) ) == 0 ) return iset; // no SSE
79  iset = 1; // 1: SSE supported
80  if ( ( abcd[3] & ( 1 << 26 ) ) == 0 ) return iset; // no SSE2
81  iset = 2; // 2: SSE2 supported
82  if ( ( abcd[2] & ( 1 << 0 ) ) == 0 ) return iset; // no SSE3
83  iset = 3; // 3: SSE3 supported
84  if ( ( abcd[2] & ( 1 << 9 ) ) == 0 ) return iset; // no SSSE3
85  iset = 4; // 4: SSSE3 supported
86  if ( ( abcd[2] & ( 1 << 19 ) ) == 0 ) return iset; // no SSE4.1
87  iset = 5; // 5: SSE4.1 supported
88  if ( ( abcd[2] & ( 1 << 23 ) ) == 0 ) return iset; // no POPCNT
89  if ( ( abcd[2] & ( 1 << 20 ) ) == 0 ) return iset; // no SSE4.2
90  iset = 6; // 6: SSE4.2 supported
91  if ( ( abcd[2] & ( 1 << 27 ) ) == 0 ) return iset; // no OSXSAVE
92  if ( ( xgetbv( 0 ) & 6 ) != 6 ) return iset; // AVX not enabled in O.S.
93  if ( ( abcd[2] & ( 1 << 28 ) ) == 0 ) return iset; // no AVX
94  iset = 7; // 7: AVX supported
95  cpuid( abcd, 7 ); // call cpuid leaf 7 for feature flags
96  if ( ( abcd[1] & ( 1 << 5 ) ) == 0 ) return iset; // no AVX2
97  iset = 8;
98  if ( ( abcd[1] & ( 1 << 16 ) ) == 0 ) return iset; // no AVX512
99  cpuid( abcd, 0xD ); // call cpuid leaf 0xD for feature flags
100  if ( ( abcd[0] & 0x60 ) != 0x60 ) return iset; // no AVX512
101  iset = 9;
102  cpuid( abcd, 7 ); // call cpuid leaf 7 for feature flags
103  if ( ( abcd[1] & ( 1 << 31 ) ) == 0 ) return iset; // no AVX512VL
104  if ( ( abcd[1] & 0x40020000 ) != 0x40020000 ) return iset; // no AVX512BW, AVX512DQ
105  iset = 10;
106  return iset;
107  }

◆ largeblock_indexes()

template<int N>
constexpr EList<int, N / 2> largeblock_indexes ( int const (&)  a[N])
constexpr

Definition at line 1248 of file instrset.h.

1248  {
1249  // Parameter a is a reference to a constexpr array of N permutation indexes
1250  EList<int, N / 2> list = { { 0 } }; // list to return
1251 
1252  bool fit_addz = false; // additional zeroing needed at the lower block level
1253  int ix = 0; // even index
1254  int iy = 0; // odd index
1255  int iz = 0; // combined index
1256  int i = 0; // loop counter
1257 
1258  for ( i = 0; i < N; i += 2 ) {
1259  ix = a[i]; // even index
1260  iy = a[i + 1]; // odd index
1261  if ( ix >= 0 ) {
1262  iz = ix / 2; // half index
1263  } else if ( iy >= 0 ) {
1264  iz = iy / 2; // half index
1265  } else
1266  iz = ix | iy; // -1 or V_DC. -1 takes precedence
1267  list.a[i / 2] = iz; // save to list
1268  // check if additional zeroing is needed at current block size
1269  if ( ( ix == -1 && iy >= 0 ) || ( iy == -1 && ix >= 0 ) ) { fit_addz = true; }
1270  }
1271  // replace -1 by V_DC if fit_addz
1272  if ( fit_addz ) {
1273  for ( i = 0; i < N / 2; i++ ) {
1274  if ( list.a[i] < 0 ) list.a[i] = V_DC;
1275  }
1276  }
1277  return list;
1278  }

◆ largeblock_perm()

template<int N>
constexpr EList<int, N / 2> largeblock_perm ( int const (&)  a[N])
constexpr

Definition at line 977 of file instrset.h.

977  {
978  // Parameter a is a reference to a constexpr array of permutation indexes
979  EList<int, N / 2> list = { { 0 } }; // result indexes
980  int ix = 0; // even index
981  int iy = 0; // odd index
982  int iz = 0; // combined index
983  bool fit_addz = false; // additional zeroing needed at the lower block level
984  int i = 0; // loop counter
985 
986  // check if additional zeroing is needed at current block size
987  for ( i = 0; i < N; i += 2 ) {
988  ix = a[i]; // even index
989  iy = a[i + 1]; // odd index
990  if ( ( ix == -1 && iy >= 0 ) || ( iy == -1 && ix >= 0 ) ) { fit_addz = true; }
991  }
992 
993  // loop through indexes
994  for ( i = 0; i < N; i += 2 ) {
995  ix = a[i]; // even index
996  iy = a[i + 1]; // odd index
997  if ( ix >= 0 ) {
998  iz = ix / 2; // half index
999  } else if ( iy >= 0 ) {
1000  iz = iy / 2;
1001  } else {
1002  iz = ix | iy; // -1 or V_DC. -1 takes precedence
1003  if ( fit_addz ) iz = V_DC; // V_DC, because result will be zeroed later
1004  }
1005  list.a[i / 2] = iz; // save to list
1006  }
1007  return list;
1008  }

◆ make_bit_mask()

template<int N, int B>
constexpr uint64_t make_bit_mask ( int const (&)  a[N])
constexpr

Definition at line 524 of file instrset.h.

524  {
525  uint64_t r = 0; // return value
526  uint8_t j = uint8_t( B & 0xFF ); // index to selected bit
527  uint64_t s = 0; // bit number i in r
528  uint64_t f = 0; // 1 if bit not flipped
529  int i = 0;
530  for ( i = 0; i < N; i++ ) {
531  int ix = a[i];
532  if ( ix < 0 ) { // -1 or V_DC
533  s = ( B >> 10 ) & 1;
534  } else {
535  s = ( (uint32_t)ix >> j ) & 1; // extract selected bit
536  if ( i < N / 2 ) {
537  f = ( B >> 8 ) & 1; // lower half
538  } else {
539  f = ( B >> 9 ) & 1; // upper half
540  }
541  s ^= f ^ 1; // flip bit if needed
542  }
543  r |= uint64_t( s ) << i; // set bit in return value
544  }
545  return r;
546  }

◆ make_broad_mask()

template<typename V >
constexpr auto make_broad_mask ( uint64_t const  m)
constexpr

Definition at line 551 of file instrset.h.

551  {
552  constexpr int N = V::size(); // number of vector elements
553  typedef decltype( get_inttype<V>() ) Etype; // element type
554  EList<Etype, N> u = { { 0 } }; // list for returning
555  int i = 0;
556  for ( i = 0; i < N; i++ ) { u.a[i] = ( ( m >> i ) & 1 ) != 0 ? get_inttype<V>() : 0; }
557  return u; // return encapsulated array
558  }

◆ perm16_flags()

template<typename V >
constexpr uint64_t perm16_flags ( int const (&)  a[V::size()])
constexpr

Definition at line 875 of file instrset.h.

875  {
876  // a is a reference to a constexpr array of permutation indexes
877  // V is a vector class
878  constexpr int N = V::size(); // number of elements
879 
880  uint64_t retval = 0; // return value
881  uint32_t pat[4] = { 0, 0, 0, 0 }; // permute patterns
882  uint32_t i = 0; // loop counter
883  int ix = 0; // index number i
884  const uint32_t lanesize = 8; // elements per lane
885  uint32_t lane = 0; // current lane
886  int lanepattern[lanesize] = { 0 }; // pattern in each lane
887 
888  for ( i = 0; i < N; i++ ) {
889  ix = a[i];
890  lane = i / lanesize; // current lane
891  if ( lane == 0 ) {
892  lanepattern[i] = ix; // save pattern
893  } else if ( ix >= 0 ) { // not first lane
894  uint32_t j = i - lane * lanesize; // index into lanepattern
895  int jx = ix - lane * lanesize; // pattern within lane
896  if ( lanepattern[j] < 0 ) {
897  lanepattern[j] = jx; // pattern not known from previous lane
898  }
899  }
900  }
901  // four patterns: low2low, high2high, high2low, low2high
902  for ( i = 0; i < 4; i++ ) {
903  // loop through low pattern
904  if ( lanepattern[i] >= 0 ) {
905  if ( lanepattern[i] < 4 ) { // low2low
906  retval |= 1;
907  pat[0] |= uint32_t( lanepattern[i] & 3 ) << ( 2 * i );
908  } else { // high2low
909  retval |= 4;
910  pat[2] |= uint32_t( lanepattern[i] & 3 ) << ( 2 * i );
911  }
912  }
913  // loop through high pattern
914  if ( lanepattern[i + 4] >= 0 ) {
915  if ( lanepattern[i + 4] < 4 ) { // low2high
916  retval |= 8;
917  pat[3] |= uint32_t( lanepattern[i + 4] & 3 ) << ( 2 * i );
918  } else { // high2high
919  retval |= 2;
920  pat[1] |= uint32_t( lanepattern[i + 4] & 3 ) << ( 2 * i );
921  }
922  }
923  }
924  // join return data
925  for ( i = 0; i < 4; i++ ) { retval |= (uint64_t)pat[i] << ( 32 + i * 8 ); }
926  return retval;
927  }

◆ perm_flags()

template<typename V >
constexpr uint64_t perm_flags ( int const (&)  a[V::size()])
constexpr

Definition at line 602 of file instrset.h.

602  {
603  // a is a reference to a constexpr array of permutation indexes
604  // V is a vector class
605  constexpr int N = V::size(); // number of elements
606  uint64_t r = perm_largeblock | perm_same_pattern | perm_allzero; // return value
607  uint32_t i = 0; // loop counter
608  int j = 0; // loop counter
609  int ix = 0; // index number i
610  const uint32_t nlanes = sizeof( V ) / 16; // number of 128-bit lanes
611  const uint32_t lanesize = N / nlanes; // elements per lane
612  const uint32_t elementsize = sizeof( V ) / N; // size of each vector element
613  uint32_t lane = 0; // current lane
614  uint32_t rot = 999; // rotate left count
615  int32_t broadc = 999; // index to broadcasted element
616  uint32_t patfail = 0; // remember certain patterns that do not fit
617  uint32_t addz2 = 0; // remember certain patterns need extra zeroing
618  int32_t compresslasti = -1; // last index in perm_compress fit
619  int32_t compresslastp = -1; // last position in perm_compress fit
620  int32_t expandlasti = -1; // last index in perm_expand fit
621  int32_t expandlastp = -1; // last position in perm_expand fit
622 
623  int lanepattern[lanesize] = { 0 }; // pattern in each lane
624 
625  for ( i = 0; i < N; i++ ) { // loop through indexes
626  ix = a[i]; // current index
627  // meaning of ix: -1 = set to zero, V_DC = don't care, non-negative value = permute.
628  if ( ix == -1 ) {
629  r |= perm_zeroing; // zeroing requested
630  } else if ( ix != V_DC && uint32_t( ix ) >= N ) {
631  r |= perm_outofrange; // index out of range
632  }
633  if ( ix >= 0 ) {
634  r &= ~perm_allzero; // not all zero
635  if ( ix != (int)i ) r |= perm_perm; // needs permutation
636  if ( broadc == 999 )
637  broadc = ix; // remember broadcast index
638  else if ( broadc != ix )
639  broadc = 1000; // does not fit broadcast
640  }
641  // check if pattern fits a larger block size:
642  // even indexes must be even, odd indexes must fit the preceding even index + 1
643  if ( ( i & 1 ) == 0 ) { // even index
644  if ( ix >= 0 && ( ix & 1 ) ) r &= ~perm_largeblock; // not even. does not fit larger block size
645  int iy = a[i + 1]; // next odd index
646  if ( iy >= 0 && ( iy & 1 ) == 0 ) r &= ~perm_largeblock; // not odd. does not fit larger block size
647  if ( ix >= 0 && iy >= 0 && iy != ix + 1 ) r &= ~perm_largeblock; // does not fit preceding index + 1
648  if ( ix == -1 && iy >= 0 ) r |= perm_addz; // needs additional zeroing at current block size
649  if ( iy == -1 && ix >= 0 ) r |= perm_addz; // needs additional zeroing at current block size
650  }
651  lane = i / lanesize; // current lane
652  if ( lane == 0 ) { // first lane, or no pattern yet
653  lanepattern[i] = ix; // save pattern
654  }
655  // check if crossing lanes
656  if ( ix >= 0 ) {
657  uint32_t lanei = (uint32_t)ix / lanesize; // source lane
658  if ( lanei != lane ) r |= perm_cross_lane; // crossing lane
659  }
660  // check if same pattern in all lanes
661  if ( lane != 0 && ix >= 0 ) { // not first lane
662  int j1 = i - int( lane * lanesize ); // index into lanepattern
663  int jx = ix - int( lane * lanesize ); // pattern within lane
664  if ( jx < 0 || jx >= (int)lanesize ) r &= ~perm_same_pattern; // source is in another lane
665  if ( lanepattern[j1] < 0 ) {
666  lanepattern[j1] = jx; // pattern not known from previous lane
667  } else {
668  if ( lanepattern[j1] != jx ) r &= ~perm_same_pattern; // not same pattern
669  }
670  }
671  if ( ix >= 0 ) {
672  // check if pattern fits zero extension (perm_zext)
673  if ( uint32_t( ix * 2 ) != i ) {
674  patfail |= 1; // does not fit zero extension
675  }
676  // check if pattern fits compress (perm_compress)
677  if ( ix > compresslasti && ix - compresslasti >= (int)i - compresslastp ) {
678  if ( (int)i - compresslastp > 1 ) addz2 |= 2; // perm_compress may need additional zeroing
679  compresslasti = ix;
680  compresslastp = i;
681  } else {
682  patfail |= 2; // does not fit perm_compress
683  }
684  // check if pattern fits expand (perm_expand)
685  if ( ix > expandlasti && ix - expandlasti <= (int)i - expandlastp ) {
686  if ( ix - expandlasti > 1 ) addz2 |= 4; // perm_expand may need additional zeroing
687  expandlasti = ix;
688  expandlastp = i;
689  } else {
690  patfail |= 4; // does not fit perm_compress
691  }
692  } else if ( ix == -1 ) {
693  if ( ( i & 1 ) == 0 ) addz2 |= 1; // zero extension needs additional zeroing
694  }
695  }
696  if ( !( r & perm_perm ) ) return r; // more checks are superfluous
697 
698  if ( !( r & perm_largeblock ) ) r &= ~perm_addz; // remove irrelevant flag
699  if ( r & perm_cross_lane ) r &= ~perm_same_pattern; // remove irrelevant flag
700  if ( ( patfail & 1 ) == 0 ) {
701  r |= perm_zext; // fits zero extension
702  if ( ( addz2 & 1 ) != 0 ) r |= perm_addz2;
703  } else if ( ( patfail & 2 ) == 0 ) {
704  r |= perm_compress; // fits compression
705  if ( ( addz2 & 2 ) != 0 ) { // check if additional zeroing needed
706  for ( j = 0; j < compresslastp; j++ ) {
707  if ( a[j] == -1 ) r |= perm_addz2;
708  }
709  }
710  } else if ( ( patfail & 4 ) == 0 ) {
711  r |= perm_expand; // fits expansion
712  if ( ( addz2 & 4 ) != 0 ) { // check if additional zeroing needed
713  for ( j = 0; j < expandlastp; j++ ) {
714  if ( a[j] == -1 ) r |= perm_addz2;
715  }
716  }
717  }
718 
719  if ( r & perm_same_pattern ) {
720  // same pattern in all lanes. check if it fits specific patterns
721  bool fit = true;
722  // fit shift or rotate
723  for ( i = 0; i < lanesize; i++ ) {
724  if ( lanepattern[i] >= 0 ) {
725  uint32_t rot1 = uint32_t( lanepattern[i] + lanesize - i ) % lanesize;
726  if ( rot == 999 ) {
727  rot = rot1;
728  } else { // check if fit
729  if ( rot != rot1 ) fit = false;
730  }
731  }
732  }
733  rot &= lanesize - 1; // prevent out of range values
734  if ( fit ) { // fits rotate, and possibly shift
735  uint64_t rot2 = ( rot * elementsize ) & 0xF; // rotate right count in bytes
736  r |= rot2 << perm_rot_count; // put shift/rotate count in output bit 16-19
737 #if INSTRSET >= 4 // SSSE3
738  r |= perm_rotate; // allow palignr
739 #endif
740  // fit shift left
741  fit = true;
742  for ( i = 0; i < lanesize - rot; i++ ) { // check if first rot elements are zero or don't care
743  if ( lanepattern[i] >= 0 ) fit = false;
744  }
745  if ( fit ) {
746  r |= perm_shleft;
747  for ( ; i < lanesize; i++ )
748  if ( lanepattern[i] == -1 ) r |= perm_addz; // additional zeroing needed
749  }
750  // fit shift right
751  fit = true;
752  for ( i = lanesize - (uint32_t)rot; i < lanesize;
753  i++ ) { // check if last (lanesize-rot) elements are zero or don't care
754  if ( lanepattern[i] >= 0 ) fit = false;
755  }
756  if ( fit ) {
757  r |= perm_shright;
758  for ( i = 0; i < lanesize - rot; i++ ) {
759  if ( lanepattern[i] == -1 ) r |= perm_addz; // additional zeroing needed
760  }
761  }
762  }
763  // fit punpckhi
764  fit = true;
765  uint32_t j2 = lanesize / 2;
766  for ( i = 0; i < lanesize; i++ ) {
767  if ( lanepattern[i] >= 0 && lanepattern[i] != (int)j2 ) fit = false;
768  if ( ( i & 1 ) != 0 ) j2++;
769  }
770  if ( fit ) r |= perm_punpckh;
771  // fit punpcklo
772  fit = true;
773  j2 = 0;
774  for ( i = 0; i < lanesize; i++ ) {
775  if ( lanepattern[i] >= 0 && lanepattern[i] != (int)j2 ) fit = false;
776  if ( ( i & 1 ) != 0 ) j2++;
777  }
778  if ( fit ) r |= perm_punpckl;
779  // fit pshufd
780  if ( elementsize >= 4 ) {
781  uint64_t p = 0;
782  for ( i = 0; i < lanesize; i++ ) {
783  if ( lanesize == 4 ) {
784  p |= ( lanepattern[i] & 3 ) << 2 * i;
785  } else { // lanesize = 2
786  p |= ( ( lanepattern[i] & 1 ) * 10 + 4 ) << 4 * i;
787  }
788  }
789  r |= p << perm_ipattern;
790  }
791  }
792 #if INSTRSET >= 7
793  else { // not same pattern in all lanes
794  if constexpr ( nlanes > 1 ) { // Try if it fits big rotate
795  for ( i = 0; i < N; i++ ) {
796  ix = a[i];
797  if ( ix >= 0 ) {
798  uint32_t rot2 = ( ix + N - i ) % N; // rotate count
799  if ( rot == 999 ) {
800  rot = rot2; // save rotate count
801  } else if ( rot != rot2 ) {
802  rot = 1000;
803  break; // does not fit big rotate
804  }
805  }
806  }
807  if ( rot < N ) { // fits big rotate
808  r |= perm_rotate_big | (uint64_t)rot << perm_rot_count;
809  }
810  }
811  }
812 #endif
813  if ( broadc < 999 && ( r & ( perm_rotate | perm_shright | perm_shleft | perm_rotate_big ) ) == 0 ) {
814  r |= perm_broadcast | (uint64_t)broadc << perm_rot_count; // fits broadcast
815  }
816  return r;
817  }

◆ perm_mask_broad()

template<typename V >
constexpr auto perm_mask_broad ( int const (&)  A[V::size()])
constexpr

Definition at line 563 of file instrset.h.

563  {
564  constexpr int N = V::size(); // number of vector elements
565  typedef decltype( get_inttype<V>() ) Etype; // vector element type
566  EList<Etype, N> u = { { 0 } }; // list for returning
567  int i = 0;
568  for ( i = 0; i < N; i++ ) { u.a[i] = Etype( A[i] ); }
569  return u; // return encapsulated array
570  }

◆ physicalProcessors()

int physicalProcessors ( int *  logical_processors = 0)

◆ pshufb_mask()

template<typename V , int oppos = 0>
constexpr auto pshufb_mask ( int const (&)  A[V::size()])
constexpr

Definition at line 934 of file instrset.h.

934  {
935  // Parameter a is a reference to a constexpr array of permutation indexes
936  // V is a vector class
937  // oppos = 1 for data from the opposite 128-bit lane in 256-bit vectors
938  constexpr uint32_t N = V::size(); // number of vector elements
939  constexpr uint32_t elementsize = sizeof( V ) / N; // size of each vector element
940  constexpr uint32_t nlanes = sizeof( V ) / 16; // number of 128 bit lanes in vector
941  constexpr uint32_t elements_per_lane = N / nlanes; // number of vector elements per lane
942 
943  EList<int8_t, sizeof( V )> u = { { 0 } }; // list for returning
944 
945  uint32_t i = 0; // loop counters
946  uint32_t j = 0;
947  int m = 0;
948  int k = 0;
949  uint32_t lane = 0;
950 
951  for ( lane = 0; lane < nlanes; lane++ ) { // loop through lanes
952  for ( i = 0; i < elements_per_lane; i++ ) { // loop through elements in lane
953  // permutation index for element within lane
954  int8_t p = -1;
955  int ix = A[m];
956  if ( ix >= 0 ) {
957  ix ^= oppos * elements_per_lane; // flip bit if opposite lane
958  }
959  ix -= int( lane * elements_per_lane ); // index relative to lane
960  if ( ix >= 0 && ix < (int)elements_per_lane ) { // index points to desired lane
961  p = ix * elementsize;
962  }
963  for ( j = 0; j < elementsize; j++ ) { // loop through bytes in element
964  u.a[k++] = p < 0 ? -1 : p + j; // store byte permutation index
965  }
966  m++;
967  }
968  }
969  return u; // return encapsulated array
970  }

◆ zero_mask()

template<int N>
constexpr auto zero_mask ( int const (&)  a[N])
constexpr

Definition at line 485 of file instrset.h.

485  {
486  uint64_t mask = 0;
487  int i = 0;
488 
489  for ( i = 0; i < N; i++ ) {
490  if ( a[i] >= 0 ) mask |= uint64_t( 1 ) << i;
491  }
492  if constexpr ( N <= 8 )
493  return uint8_t( mask );
494  else if constexpr ( N <= 16 )
495  return uint16_t( mask );
496  else if constexpr ( N <= 32 )
497  return uint32_t( mask );
498  else
499  return mask;
500  }

◆ zero_mask_broad()

template<typename V >
constexpr auto zero_mask_broad ( int const (&)  A[V::size()])
constexpr

Definition at line 505 of file instrset.h.

505  {
506  constexpr int N = V::size(); // number of vector elements
507  typedef decltype( get_inttype<V>() ) Etype; // element type
508  EList<Etype, N> u = { { 0 } }; // list for return
509  int i = 0;
510  for ( i = 0; i < N; i++ ) { u.a[i] = A[i] >= 0 ? get_inttype<V>() : 0; }
511  return u; // return encapsulated array
512  }

Variable Documentation

◆ blend_a

const int blend_a = 0x10

Definition at line 1016 of file instrset.h.

◆ blend_addz

const int blend_addz = 8

Definition at line 1015 of file instrset.h.

◆ blend_allzero

const int blend_allzero = 2

Definition at line 1013 of file instrset.h.

◆ blend_b

const int blend_b = 0x20

Definition at line 1017 of file instrset.h.

◆ blend_cross_lane

const int blend_cross_lane = 0x100

Definition at line 1020 of file instrset.h.

◆ blend_largeblock

const int blend_largeblock = 4

Definition at line 1014 of file instrset.h.

◆ blend_outofrange

const int blend_outofrange = 0x10000000

Definition at line 1031 of file instrset.h.

◆ blend_perma

const int blend_perma = 0x40

Definition at line 1018 of file instrset.h.

◆ blend_permb

const int blend_permb = 0x80

Definition at line 1019 of file instrset.h.

◆ blend_punpckhab

const int blend_punpckhab = 0x1000

Definition at line 1022 of file instrset.h.

◆ blend_punpckhba

const int blend_punpckhba = 0x2000

Definition at line 1023 of file instrset.h.

◆ blend_punpcklab

const int blend_punpcklab = 0x4000

Definition at line 1024 of file instrset.h.

◆ blend_punpcklba

const int blend_punpcklba = 0x8000

Definition at line 1025 of file instrset.h.

◆ blend_rotate_big

const int blend_rotate_big = 0x100000

Definition at line 1030 of file instrset.h.

◆ blend_rotateab

const int blend_rotateab = 0x10000

Definition at line 1026 of file instrset.h.

◆ blend_rotateba

const int blend_rotateba = 0x20000

Definition at line 1027 of file instrset.h.

◆ blend_rotpattern

const int blend_rotpattern = 40

Definition at line 1033 of file instrset.h.

◆ blend_same_pattern

const int blend_same_pattern = 0x200

Definition at line 1021 of file instrset.h.

◆ blend_shufab

const int blend_shufab = 0x40000

Definition at line 1028 of file instrset.h.

◆ blend_shufba

const int blend_shufba = 0x80000

Definition at line 1029 of file instrset.h.

◆ blend_shufpattern

const int blend_shufpattern = 32

Definition at line 1032 of file instrset.h.

◆ blend_zeroing

const int blend_zeroing = 1

Definition at line 1012 of file instrset.h.

◆ perm_addz

const int perm_addz = 0x10

Definition at line 578 of file instrset.h.

◆ perm_addz2

const int perm_addz2 = 0x20

Definition at line 579 of file instrset.h.

◆ perm_allzero

const int perm_allzero = 4

Definition at line 576 of file instrset.h.

◆ perm_broadcast

const int perm_broadcast = 0x8000

Definition at line 592 of file instrset.h.

◆ perm_compress

const int perm_compress = 0x20000

Definition at line 594 of file instrset.h.

◆ perm_cross_lane

const int perm_cross_lane = 0x40

Definition at line 580 of file instrset.h.

◆ perm_expand

const int perm_expand = 0x40000

Definition at line 595 of file instrset.h.

◆ perm_ipattern

const int perm_ipattern
Initial value:
=
40

Definition at line 598 of file instrset.h.

◆ perm_largeblock

const int perm_largeblock = 8

Definition at line 577 of file instrset.h.

◆ perm_outofrange

const int perm_outofrange = 0x10000000

Definition at line 596 of file instrset.h.

◆ perm_perm

const int perm_perm = 2

Definition at line 575 of file instrset.h.

◆ perm_punpckh

const int perm_punpckh = 0x100

Definition at line 582 of file instrset.h.

◆ perm_punpckl

const int perm_punpckl = 0x200

Definition at line 583 of file instrset.h.

◆ perm_rot_count

const int perm_rot_count = 32

Definition at line 597 of file instrset.h.

◆ perm_rotate

const int perm_rotate
Initial value:
=
0x400

Definition at line 584 of file instrset.h.

◆ perm_rotate_big

const int perm_rotate_big
Initial value:
=
0x4000

Definition at line 590 of file instrset.h.

◆ perm_same_pattern

const int perm_same_pattern = 0x80

Definition at line 581 of file instrset.h.

◆ perm_shleft

const int perm_shleft
Initial value:
=
0x2000

Definition at line 588 of file instrset.h.

◆ perm_shright

const int perm_shright
Initial value:
=
0x1000

Definition at line 586 of file instrset.h.

◆ perm_zeroing

const int perm_zeroing = 1

Definition at line 574 of file instrset.h.

◆ perm_zext

const int perm_zext = 0x10000

Definition at line 593 of file instrset.h.

◆ V_DC

constexpr int V_DC = -256
constexpr

Definition at line 220 of file instrset.h.

blend4
void blend4()
Definition: instrset.h:1293
blend8
void blend8()
Definition: instrset.h:1295
details::size
constexpr auto size(const T &, Args &&...) noexcept
Definition: AnyDataWrapper.h:23
IOTest.N
N
Definition: IOTest.py:112
perm_punpckl
const int perm_punpckl
Definition: instrset.h:583
perm_shleft
const int perm_shleft
Definition: instrset.h:588
perm_broadcast
const int perm_broadcast
Definition: instrset.h:592
perm_largeblock
const int perm_largeblock
Definition: instrset.h:577
blend_rotateab
const int blend_rotateab
Definition: instrset.h:1026
gaudirun.s
string s
Definition: gaudirun.py:346
perm_rot_count
const int perm_rot_count
Definition: instrset.h:597
blend_perma
const int blend_perma
Definition: instrset.h:1018
blend32
void blend32()
Definition: instrset.h:1299
perm_ipattern
const int perm_ipattern
Definition: instrset.h:598
perm_zeroing
const int perm_zeroing
Definition: instrset.h:574
blend_addz
const int blend_addz
Definition: instrset.h:1015
blend_rotate_big
const int blend_rotate_big
Definition: instrset.h:1030
blend_punpckhab
const int blend_punpckhab
Definition: instrset.h:1022
perm_rotate
const int perm_rotate
Definition: instrset.h:584
perm_shright
const int perm_shright
Definition: instrset.h:586
bug_34121.t
t
Definition: bug_34121.py:31
blend_rotateba
const int blend_rotateba
Definition: instrset.h:1027
blend_same_pattern
const int blend_same_pattern
Definition: instrset.h:1021
Gaudi::Units::m
constexpr double m
Definition: SystemOfUnits.h:108
perm_zext
const int perm_zext
Definition: instrset.h:593
ProduceConsume.j
j
Definition: ProduceConsume.py:104
blend_a
const int blend_a
Definition: instrset.h:1016
blend_zeroing
const int blend_zeroing
Definition: instrset.h:1012
blend_allzero
const int blend_allzero
Definition: instrset.h:1013
blend2
void blend2()
Definition: instrset.h:1291
blend_shufpattern
const int blend_shufpattern
Definition: instrset.h:1032
blend_shufba
const int blend_shufba
Definition: instrset.h:1029
EList
Definition: instrset.h:461
perm_perm
const int perm_perm
Definition: instrset.h:575
cpluginsvc.n
n
Definition: cpluginsvc.py:234
blend_rotpattern
const int blend_rotpattern
Definition: instrset.h:1033
perm_compress
const int perm_compress
Definition: instrset.h:594
blend_punpcklba
const int blend_punpcklba
Definition: instrset.h:1025
perm_cross_lane
const int perm_cross_lane
Definition: instrset.h:580
perm_addz2
const int perm_addz2
Definition: instrset.h:579
perm_punpckh
const int perm_punpckh
Definition: instrset.h:582
blend_punpcklab
const int blend_punpcklab
Definition: instrset.h:1024
perm_rotate_big
const int perm_rotate_big
Definition: instrset.h:590
blend_punpckhba
const int blend_punpckhba
Definition: instrset.h:1023
EList::a
T a[N]
Definition: instrset.h:478
compareRootHistos.retval
retval
Definition: compareRootHistos.py:499
blend_b
const int blend_b
Definition: instrset.h:1017
V_DC
constexpr int V_DC
Definition: instrset.h:220
instrset_detect
int instrset_detect(void)
Definition: instrset_detect.cpp:63
perm_same_pattern
const int perm_same_pattern
Definition: instrset.h:581
blend_largeblock
const int blend_largeblock
Definition: instrset.h:1014
perm_allzero
const int perm_allzero
Definition: instrset.h:576
blend_half
auto blend_half(W const &a, W const &b)
Definition: instrset.h:1352
Gaudi::Units::L
constexpr double L
Definition: SystemOfUnits.h:118
perm_outofrange
const int perm_outofrange
Definition: instrset.h:596
blend_shufab
const int blend_shufab
Definition: instrset.h:1028
blend_half_indexes
constexpr EList< int, N > blend_half_indexes(int const (&a)[N])
Definition: instrset.h:1308
blend16
void blend16()
Definition: instrset.h:1297
cpuid
#define cpuid(func, ax, bx, cx, dx)
Definition: PerfMonAuditor.cpp:83
perm_addz
const int perm_addz
Definition: instrset.h:578
blend_cross_lane
const int blend_cross_lane
Definition: instrset.h:1020
blend_outofrange
const int blend_outofrange
Definition: instrset.h:1031
blend_permb
const int blend_permb
Definition: instrset.h:1019
perm_expand
const int perm_expand
Definition: instrset.h:595