3
H4W^/I  ã               @   s<  d dl mZmZmZmZ d dlZd dlZddddœZdd„ Zd	d
„ Z	ej
ƒ Zejdddd� ejddd� ejddd� ejƒ ZejrŽejZn(ejr¦ejr dZq¶dZnejr²dZndZddddddgddddddgddddddgddddddgddddddgddddddggZg Zejdƒ ejdƒ ejd ƒ ejd!jejd  ƒƒ ejd"ƒ ejd#ƒ ejd$ƒ ejd%ƒ ejd&ƒ �xreD �]hZe\ZZZZZZej�r¢d'nd(Zejd)ƒ ej�rÊd*jeeeeƒZnd+jeeeeƒZd,Z d-e e  Z!eje!d. ƒ g Z"e"jd/ƒ e"jd0ƒ e"jd1ƒ e"jd2ƒ e"jd3e d4 ƒ e"jd3e d5 ƒ ej�r\e"jd6ƒ n
e"jd7ƒ e"jd8ƒ ej�s‚e"jd9ƒ e"jd:ƒ e"jd;e d< ƒ ee"7 Zejd=e d> ƒ ej�rÌd?ee  nd Z#ejd@jee#ƒƒ ej�röejdAƒ nejdBe dC ƒ ejdDƒ eedEeeedFdGejejƒ7 ZejdHƒ eed?eeedFdGejejƒ7 ZejdIƒ eedeeedFdGejejƒ7 ZejdJƒ eedeeedFdGejejƒ7 ZejdKƒ ejdLƒ ee	eeedFdGejejƒ7 ZejdMƒ ejdNƒ ejdOƒ xædgD ]ÞZ$ejdRe dS e$ e  d. ƒ ee"7 ZejdTe e  dU e$ dV ƒ ejdWƒ ejdXƒ ejdYƒ ejdZƒ ejd[ƒ ejd\ƒ ej�r”ejd]ƒ n
ejd^ƒ ejd_ƒ ej�sºejd`ƒ ejdaƒ ejdbƒ ejdOƒ �qüW ejd(ƒ �q€W ejdcƒ e%eddƒ�$Z&xeD ]Z'e&j(e'de ƒ �q
W W dQ R X e)dfe ƒ dS )hé    )Úabsolute_importÚdivisionÚprint_functionÚunicode_literalsNé   é   é   )Úfloatzat::HalfÚuint8_tc             C   s$  dd„ }g }	|	j dt| ƒ d ƒ |	j d| d ƒ |	j d| d ƒ x0td	| ƒD ]"}
d
|
 }|	j dt|ƒ d ƒ qRW |rœ|	j d;ƒ |	j dƒ |	j d=ƒ n|	j d?ƒ |	j d| d ƒ |	j d| d ƒ |	j dAƒ |dk�rf|	j d| d ƒ |	j d| d ƒ |	j dƒ |	j d ƒ |	j dƒ |�rF|	j d!ƒ |	j d"ƒ |	j d#ƒ n|	j d$ƒ |	j d%ƒ |	j d&ƒ n0|	j d| d ƒ |	j dƒ |	j d ƒ |	j dƒ |	j d'ƒ |	j d(j|ƒƒ |	j d)j|ƒƒ |	j d| d* ƒ |	j dCƒ |	j d,j|ƒƒ xNtd	| ƒD ]@}
d
|
 }d-}t| | }|| d	k}|	j||||||ƒƒ �qøW |	j dƒ |�rX|	j d.ƒ n
|	j d/ƒ x>td	| ƒD ]0}
d
|
 }|	j d0t|ƒ d1 t|ƒ d2 ƒ �qnW |	j d3ƒ |�r¾|	j d4ƒ n
|	j d5ƒ xBtd	| ƒD ]4}
d
|
 }|	j d0t|ƒ d6 d7 t|ƒ d8 ƒ �qÔW |	j dƒ |	j d9ƒ |	S )DNc             S   sŒ   g }|dkr"|j d| | | f ƒ nD|dkr@|j d| | | f ƒ n&|dkr^|j d| | | f ƒ ndsft‚|rz|j d|  ƒ n|j d	|  ƒ |S )
Nr	   zI        vop%d = _mm256_fmadd_ps(vwgt, _mm256_loadu_ps(ip + (%d)), vop%d);zat::Halfz²        vop%d = _mm256_fmadd_ps(
            vwgt,
            _mm256_cvtph_ps(
                _mm_loadu_si128(reinterpret_cast<const __m128i*>(ip + (%d)))),
            vop%d);r
   zà        vop%d = _mm256_fmadd_ps(
            vwgt,
            _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(
                _mm_loadl_epi64(reinterpret_cast<const __m128i*>(ip + (%d))))),
            _mm256_add_ps(vop%d, vbio));Fz_        _mm_prefetch(
            reinterpret_cast<const char*>(&ip_next_T0[%d]), _MM_HINT_T0);z9        // skip unnecessary prefetch of (&ip_next_T0[%d]))ÚappendÚAssertionError)ZregidÚInTypeÚuse_weightsÚisaÚprefetchÚcode© r   úH/tmp/pip-build-e66j97b2/torch/caffe2/perfkernels/hp_emblookup_codegen.pyÚcompute   s*    

zunroll.<locals>.computez    // unrolling z timesz	    for (z: rangeIndex = 0; rangeIndex < output_size; ++rangeIndex) {z      z%* op = &out[rangeIndex * block_size];r   é   z      __m256 vopz = _mm256_setzero_ps();z,      if (dataInd != offsets[rangeIndex]) {
z        return false;
z      }zÎ      int64_t end_offset =
          (rangeIndex == output_size - 1 ? index_size
                                         : offsets[rangeIndex + 1]);
      int64_t length = end_offset - offsets[rangeIndex];z      for (Úint64_tz4 start = dataInd; dataInd < end_offset; ++dataInd) {z8      if (dataInd + lengths[rangeIndex] > index_size) {
zP start = dataInd; dataInd < start + lengths[rangeIndex];
           ++dataInd) {z        const z idx = indices[dataInd];z+        if (idx < 0 || idx >= data_size) {
z          return false;
z	        }r
   z        z wgt = 1.f;z bio;z        if (weights) {zL          wgt = weights[IS_WEIGHT_POSITIONAL ? (dataInd - start) : dataInd];z{        const float* scale_bias = reinterpret_cast<const float*>(
            &input[idx * fused_block_size + block_size]);z"        bio = wgt * scale_bias[1];z"        wgt = wgt * scale_bias[0];z,        bio = wgt * scale_bias[2 * idx + 1];z(        wgt = wgt * scale_bias[2 * idx];z*        __m256 vbio = _mm256_set1_ps(bio);z*        __m256 vwgt = _mm256_set1_ps(wgt);z6        const {}* ip = &input[idx * fused_block_size];z|        const {} next_T0 = (dataInd < index_size - prefdist_T0)
            ? (dataInd + prefdist_T0)
            : dataInd;z  idx_pref_T0 = indices[next_T0];z;        if (idx_pref_T0 < 0 || idx_pref_T0 >= data_size) {
zF        const {}* ip_next_T0 = &input[idx_pref_T0 * fused_block_size];é@   z1      if (!normalize_by_lengths || length == 0) {z>      if (!normalize_by_lengths || lengths[rangeIndex] == 0) {z        _mm256_storeu_ps(&op[z], vopz);z      } else {z8        __m256 vlen_inv = _mm256_set1_ps(1.0f / length);zE        __m256 vlen_inv = _mm256_set1_ps(1.0f / lengths[rangeIndex]);z], _mm256_mul_ps(Zvopz, vlen_inv));z    }zB      if (dataInd != offsets[rangeIndex]) {
        return false;
zI      if (dataInd != offsets[rangeIndex]) {
        return false;
      }z      for (int64_tzF      for (int64_t start = dataInd; dataInd < end_offset; ++dataInd) {zN      if (dataInd + lengths[rangeIndex] > index_size) {
        return false;
zU      if (dataInd + lengths[rangeIndex] > index_size) {
        return false;
      }zC        if (idx < 0 || idx >= data_size) {
          return false;
zL        if (idx < 0 || idx >= data_size) {
          return false;
        }zS        if (idx_pref_T0 < 0 || idx_pref_T0 >= data_size) {
          return false;
z\        if (idx_pref_T0 < 0 || idx_pref_T0 >= data_size) {
          return false;
        })r   ÚstrÚrangeÚformatÚsizeofÚextend)ZufÚ	IndexTyper   ÚOutTyper   r   ÚfusedÚuse_offsetsr   r   ÚiÚjZcachelinesizeZ
byteoffsetr   r   r   r   Úunroll
   sœ    (










(

(

r#   c       	      C   sB  dd„ }g }|dkr|j dƒ |j d|  d ƒ |j d| d ƒ |j d	ƒ |j d
ƒ |j dƒ |j dƒ |j dƒ |j dƒ |j dƒ |r¬|j dAƒ |j dƒ |j dCƒ n|j dEƒ |j d|  d ƒ |j d|  d ƒ |j dGƒ |dk�rv|j d| d ƒ |j d| d ƒ |j d ƒ |j d!ƒ |j dƒ |�rV|j d"ƒ |j d#ƒ |j d$ƒ n|j d%ƒ |j d&ƒ |j d'ƒ n0|j d| d ƒ |j d ƒ |j d!ƒ |j dƒ |j d(ƒ |j d)j|ƒƒ |j d*j| ƒƒ |j d|  d+ ƒ |j dIƒ |j d-j|ƒƒ |j d.ƒ |j d/ƒ |j||||ƒƒ |j dƒ |j d0ƒ |d1k�rL|j d2ƒ nJ|dk�rv|j d3ƒ |j d4ƒ |j d5ƒ n |dk�rŒ|j d6ƒ n
d7�s–t‚|j dƒ |j dƒ |�rÆ|j d8ƒ |j d9ƒ n|j d:ƒ |j d;ƒ |j d<ƒ |j d.ƒ |j d/ƒ |j d=ƒ |j dƒ |j d0ƒ |j d>ƒ |j dƒ |j dƒ |j d?ƒ |S )JNc             S   sV   g }| dkr|j dƒ n0| dkr,|j dƒ n| dkr@|j dƒ ndsHt‚|j dƒ |S )	Nr	   zœ          _mm256_storeu_ps(
              &op[j],
              _mm256_fmadd_ps(
                  vwgt, _mm256_loadu_ps(&ip[j]), _mm256_loadu_ps(&op[j])));zat::Halfa
            _mm256_storeu_ps(
              &op[j],
              _mm256_fmadd_ps(
                  vwgt,
                  _mm256_cvtph_ps(_mm_loadu_si128(
                      reinterpret_cast<const __m128i*>(&ip[j]))),
                  _mm256_loadu_ps(&op[j])));r
   a8            _mm256_storeu_ps(
              &op[j],
              _mm256_fmadd_ps(
                  vwgt,
                  _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(_mm_loadl_epi64(
                      reinterpret_cast<const __m128i*>(&ip[j])))),
                  _mm256_add_ps(_mm256_loadu_ps(&op[j]), vbio)));Fzb          _mm_prefetch(
              reinterpret_cast<const char*>(&ip_next_T0[j]), _MM_HINT_T0);)r   r   )r   r   r   r   r   r   r   r   ·   s    	zgeneric.<locals>.computezat::Halfz(    alignas(64) at::Half vtmp1[8] = {0};z	    for (z: rangeIndex = 0; rangeIndex < output_size; ++rangeIndex) {z      z%* op = &out[rangeIndex * block_size];z      int64_t j = 0;z+      for (; j + 8 <= block_size; j += 8) {z6        _mm256_storeu_ps(op + j, _mm256_setzero_ps());z      }z#      for (; j < block_size; j++) {z        op[j] = 0.0f;z,      if (dataInd != offsets[rangeIndex]) {
z        return false;
zÆ      int end_offset =
          (rangeIndex == output_size - 1 ? index_size
                                         : offsets[rangeIndex + 1]);
      int length = end_offset - offsets[rangeIndex];z      for (r   z4 start = dataInd; dataInd < end_offset; ++dataInd) {z8      if (dataInd + lengths[rangeIndex] > index_size) {
zP start = dataInd; dataInd < start + lengths[rangeIndex];
           ++dataInd) {z        const z idx = indices[dataInd];z+        if (idx < 0 || idx >= data_size) {
z          return false;
z	        }r
   z        z wgt = 1.f;z bio;z        if (weights) {zL          wgt = weights[IS_WEIGHT_POSITIONAL ? (dataInd - start) : dataInd];z{        const float* scale_bias = reinterpret_cast<const float*>(
            &input[idx * fused_block_size + block_size]);z"        bio = wgt * scale_bias[1];z"        wgt = wgt * scale_bias[0];z,        bio = wgt * scale_bias[2 * idx + 1];z(        wgt = wgt * scale_bias[2 * idx];z*        __m256 vbio = _mm256_set1_ps(bio);z*        __m256 vwgt = _mm256_set1_ps(wgt);z6        const {}* ip = &input[idx * fused_block_size];z|        const {} next_T0 = (dataInd < index_size - prefdist_T0)
            ? (dataInd + prefdist_T0)
            : dataInd;z  idx_pref_T0 = indices[next_T0];z;        if (idx_pref_T0 < 0 || idx_pref_T0 >= data_size) {
zF        const {}* ip_next_T0 = &input[idx_pref_T0 * fused_block_size];z        j = 0;z-        for (; j + 8 <= block_size; j += 8) {z%        for (; j < block_size; j++) {r	   z.          op[j] = std::fma(wgt, ip[j], op[j]);z          vtmp1[0] = ip[j];zc          __m256 vtmp2 =
              _mm256_cvtph_ps(*(reinterpret_cast<const __m128i*>(vtmp1)));z>          op[j] = std::fma(wgt, ((float*)(&vtmp2))[0], op[j]);z;          op[j] = std::fma(wgt, (float)ip[j], bio + op[j]);Fz+      if (normalize_by_lengths && length) {z&        float len_inv = 1.0f / length;z8      if (normalize_by_lengths && lengths[rangeIndex]) {z3        float len_inv = 1.0f / lengths[rangeIndex];z2        __m256 vlen_inv = _mm256_set1_ps(len_inv);zd          _mm256_storeu_ps(
              &op[j], _mm256_mul_ps(_mm256_loadu_ps(&op[j]), vlen_inv));z"          op[j] = len_inv * op[j];z    }zB      if (dataInd != offsets[rangeIndex]) {
        return false;
zI      if (dataInd != offsets[rangeIndex]) {
        return false;
      }z      for (int64_tzF      for (int64_t start = dataInd; dataInd < end_offset; ++dataInd) {zN      if (dataInd + lengths[rangeIndex] > index_size) {
        return false;
zU      if (dataInd + lengths[rangeIndex] > index_size) {
        return false;
      }zC        if (idx < 0 || idx >= data_size) {
          return false;
zL        if (idx < 0 || idx >= data_size) {
          return false;
        }zS        if (idx_pref_T0 < 0 || idx_pref_T0 >= data_size) {
          return false;
z\        if (idx_pref_T0 < 0 || idx_pref_T0 >= data_size) {
          return false;
        })r   r   r   r   )	r   r   r   r   r   r   r    r   r   r   r   r   Úgeneric¶   sº    '







































r$   z-fz
--filenamez	file name)Úhelpz--fusedÚ
store_true)Úactionz--use-offsetsz/embedding_lookup_fused_8bit_rowwise_idx_avx2.ccz+embedding_lookup_fused_8bit_rowwise_avx2.cczembedding_lookup_idx_avx2.cczembedding_lookup_avx2.ccZint32_tÚintr	   r   Zhalfzat::Halfr
   z//// --------------------------z//// ATTENTION:z//// THIS CODE IS AUTOGENERATEDz
//// BY {}z//// DO NOT MODIFY!!!z //// --------------------------
z#include <c10/util/Half.h>z#include <immintrin.h>znamespace caffe2 {
ZFused8BitRowwiseÚ z$template <bool IS_WEIGHT_POSITIONAL>z{}EmbeddingLookupIdx_{}_{}_{}z{}EmbeddingLookup_{}_{}_{}Z
__avx2_fmazstatic bool ú(z    const int64_t block_size,z    const int64_t output_size,z    const int64_t index_size,z    const int64_t data_size,z
    const z* input,z
* indices,z    const int64_t* offsets,z    const int* lengths,z    const float* weights,z    const float* scale_bias,z    bool normalize_by_lengths,z    z* out) {z  const z prefdist_T0 = 16;r   z.  const {} fused_block_size = block_size + {};z  int64_t dataInd = 0;z  z dataInd = 0;z  if (block_size == 128) {é   TZAVX2z   } else if (block_size == 64) {z   } else if (block_size == 32) {z   } else if (block_size == 16) {z
  } else {z    // generic codez  }z  return dataInd == index_size;Ú}ÚfalseÚtruezbool Ú_z	  return ú<z>(z      block_size,z      output_size,z      index_size,z      data_size,z      input,z      indices,z      offsets,z      lengths,z      weights,z      scale_bias,z      normalize_by_lengths,z      out);z} // namespace caffe2ÚwÚ
zCreated )r-   r.   )*Ú
__future__r   r   r   r   ÚargparseÚsysr   r#   r$   ÚArgumentParserÚparserÚadd_argumentÚ
parse_argsÚoptsÚfilenamer   r    Úoptionsr   r   r   ÚargvÚoZIndexTypeNamer   Z
InTypeNamer   ZOutTypeNamer   ÚprefixZfn_baseÚsuffixÚfnÚargsÚoffsetZis_weight_positionalÚopenZfoutÚcÚwriteÚprintr   r   r   r   Ú<module>   sÞ    - ?







































